diff --git a/BUILD b/BUILD new file mode 100644 index 000000000..2ce21e408 --- /dev/null +++ b/BUILD @@ -0,0 +1,78 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +filegroup( + name = "config", + srcs = glob([ + "**/*.yaml", + "**/*.yml", + "**/*.json", + ]) + [ + "pod", + ], +) + +filegroup( + name = "sources", + srcs = glob([ + "**/*", + ]), +) + +go_library( + name = "go_default_library", + srcs = ["doc.go"], + tags = ["automanaged"], +) + +go_test( + name = "go_default_xtest", + srcs = ["examples_test.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/testapi:go_default_library", + "//pkg/api/validation:go_default_library", + "//pkg/apis/apps:go_default_library", + "//pkg/apis/apps/validation:go_default_library", + "//pkg/apis/batch:go_default_library", + "//pkg/apis/extensions:go_default_library", + "//pkg/apis/extensions/validation:go_default_library", + "//pkg/capabilities:go_default_library", + "//pkg/registry/batch/job:go_default_library", + "//plugin/pkg/scheduler/api:go_default_library", + "//plugin/pkg/scheduler/api/latest:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//examples/explorer:all-srcs", + "//examples/guestbook-go:all-srcs", + "//examples/https-nginx:all-srcs", + "//examples/sharing-clusters:all-srcs", + ], + tags = ["automanaged"], +) diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000..d74aa8a61 --- /dev/null +++ b/OWNERS @@ -0,0 +1,9 @@ +reviewers: + - brendandburns + - thockin + - zmerlynn +approvers: + - brendandburns + - eparis + - thockin + - zmerlynn diff --git a/README.md b/README.md index 2568d6b25..896fead2b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,29 @@ -# examples -Kubernetes application example tutorials +# Kubernetes Examples: releases.k8s.io/HEAD -Home for examples to be moved from kubernetes/kubernetes/examples +This directory contains a number of examples of how to run +real applications with Kubernetes. + +Demonstrations of how to use specific Kubernetes features can be found in our [documents](../docs/). + + +### Maintained Examples + +Maintained Examples are expected to be updated with every Kubernetes +release, to use the latest and greatest features, current guidelines +and best practices, and to refresh command syntax, output, changed +prerequisites, as needed. + +|Name | Description | Notable Features Used | Complexity Level| +------------- | ------------- | ------------ | ------------ | +|[Guestbook](guestbook/) | PHP app with Redis | Replication Controller, Service | Beginner | +|[WordPress](mysql-wordpress-pd/) | WordPress with MySQL | Deployment, Persistent Volume with Claim | Beginner| +|[Cassandra](storage/cassandra/) | Cloud Native Cassandra | Daemon Set | Intermediate + +* Note: Please add examples to the list above that are maintained. + +See [Example Guidelines](guidelines.md) for a description of what goes +in this directory, and what examples should contain. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/README.md?pixel)]() + diff --git a/cluster-dns/README.md b/cluster-dns/README.md new file mode 100644 index 000000000..11f6ba62e --- /dev/null +++ b/cluster-dns/README.md @@ -0,0 +1,182 @@ +## Kubernetes DNS example + +This is a toy example demonstrating how to use kubernetes DNS. + +### Step Zero: Prerequisites + +This example assumes that you have forked the repository and [turned up a Kubernetes cluster](../../docs/getting-started-guides/). Make sure DNS is enabled in your setup, see [DNS doc](https://github.com/kubernetes/dns). + +```sh +$ cd kubernetes +$ hack/dev-build-and-up.sh +``` + +### Step One: Create two namespaces + +We'll see how cluster DNS works across multiple [namespaces](../../docs/user-guide/namespaces.md), first we need to create two namespaces: + +```sh +$ kubectl create -f examples/cluster-dns/namespace-dev.yaml +$ kubectl create -f examples/cluster-dns/namespace-prod.yaml +``` + +Now list all namespaces: + +```sh +$ kubectl get namespaces +NAME LABELS STATUS +default Active +development name=development Active +production name=production Active +``` + +For kubectl client to work with each namespace, we define two contexts: + +```sh +$ kubectl config set-context dev --namespace=development --cluster=${CLUSTER_NAME} --user=${USER_NAME} +$ kubectl config set-context prod --namespace=production --cluster=${CLUSTER_NAME} --user=${USER_NAME} +``` + +You can view your cluster name and user name in kubernetes config at ~/.kube/config. + +### Step Two: Create backend replication controller in each namespace + +Use the file [`examples/cluster-dns/dns-backend-rc.yaml`](dns-backend-rc.yaml) to create a backend server [replication controller](../../docs/user-guide/replication-controller.md) in each namespace. + +```sh +$ kubectl config use-context dev +$ kubectl create -f examples/cluster-dns/dns-backend-rc.yaml +``` + +Once that's up you can list the pod in the cluster: + +```sh +$ kubectl get rc +CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS +dns-backend dns-backend ddysher/dns-backend name=dns-backend 1 +``` + +Now repeat the above commands to create a replication controller in prod namespace: + +```sh +$ kubectl config use-context prod +$ kubectl create -f examples/cluster-dns/dns-backend-rc.yaml +$ kubectl get rc +CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS +dns-backend dns-backend ddysher/dns-backend name=dns-backend 1 +``` + +### Step Three: Create backend service + +Use the file [`examples/cluster-dns/dns-backend-service.yaml`](dns-backend-service.yaml) to create +a [service](../../docs/user-guide/services.md) for the backend server. + +```sh +$ kubectl config use-context dev +$ kubectl create -f examples/cluster-dns/dns-backend-service.yaml +``` + +Once that's up you can list the service in the cluster: + +```sh +$ kubectl get service dns-backend +NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE +dns-backend 10.0.2.3 8000/TCP name=dns-backend 1d +``` + +Again, repeat the same process for prod namespace: + +```sh +$ kubectl config use-context prod +$ kubectl create -f examples/cluster-dns/dns-backend-service.yaml +$ kubectl get service dns-backend +NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE +dns-backend 10.0.2.4 8000/TCP name=dns-backend 1d +``` + +### Step Four: Create client pod in one namespace + +Use the file [`examples/cluster-dns/dns-frontend-pod.yaml`](dns-frontend-pod.yaml) to create a client [pod](../../docs/user-guide/pods.md) in dev namespace. The client pod will make a connection to backend and exit. Specifically, it tries to connect to address `http://dns-backend.development.cluster.local:8000`. + +```sh +$ kubectl config use-context dev +$ kubectl create -f examples/cluster-dns/dns-frontend-pod.yaml +``` + +Once that's up you can list the pod in the cluster: + +```sh +$ kubectl get pods dns-frontend +NAME READY STATUS RESTARTS AGE +dns-frontend 0/1 ExitCode:0 0 1m +``` + +Wait until the pod succeeds, then we can see the output from the client pod: + +```sh +$ kubectl logs dns-frontend +2015-05-07T20:13:54.147664936Z 10.0.236.129 +2015-05-07T20:13:54.147721290Z Send request to: http://dns-backend.development.cluster.local:8000 +2015-05-07T20:13:54.147733438Z +2015-05-07T20:13:54.147738295Z Hello World! +``` + +Please refer to the [source code](images/frontend/client.py) about the log. First line prints out the ip address associated with the service in dev namespace; remaining lines print out our request and server response. + +If we switch to prod namespace with the same pod config, we'll see the same result, i.e. dns will resolve across namespace. + +```sh +$ kubectl config use-context prod +$ kubectl create -f examples/cluster-dns/dns-frontend-pod.yaml +$ kubectl logs dns-frontend +2015-05-07T20:13:54.147664936Z 10.0.236.129 +2015-05-07T20:13:54.147721290Z Send request to: http://dns-backend.development.cluster.local:8000 +2015-05-07T20:13:54.147733438Z +2015-05-07T20:13:54.147738295Z Hello World! +``` + + +#### Note about default namespace + +If you prefer not using namespace, then all your services can be addressed using `default` namespace, e.g. `http://dns-backend.default.svc.cluster.local:8000`, or shorthand version `http://dns-backend:8000` + + +### tl; dr; + +For those of you who are impatient, here is the summary of the commands we ran in this tutorial. Remember to set first `$CLUSTER_NAME` and `$USER_NAME` to the values found in `~/.kube/config`. + +```sh +# create dev and prod namespaces +kubectl create -f examples/cluster-dns/namespace-dev.yaml +kubectl create -f examples/cluster-dns/namespace-prod.yaml + +# create two contexts +kubectl config set-context dev --namespace=development --cluster=${CLUSTER_NAME} --user=${USER_NAME} +kubectl config set-context prod --namespace=production --cluster=${CLUSTER_NAME} --user=${USER_NAME} + +# create two backend replication controllers +kubectl config use-context dev +kubectl create -f examples/cluster-dns/dns-backend-rc.yaml +kubectl config use-context prod +kubectl create -f examples/cluster-dns/dns-backend-rc.yaml + +# create backend services +kubectl config use-context dev +kubectl create -f examples/cluster-dns/dns-backend-service.yaml +kubectl config use-context prod +kubectl create -f examples/cluster-dns/dns-backend-service.yaml + +# create a pod in each namespace and get its output +kubectl config use-context dev +kubectl create -f examples/cluster-dns/dns-frontend-pod.yaml +kubectl logs dns-frontend + +kubectl config use-context prod +kubectl create -f examples/cluster-dns/dns-frontend-pod.yaml +kubectl logs dns-frontend +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/cluster-dns/README.md?pixel)]() + diff --git a/cluster-dns/dns-backend-rc.yaml b/cluster-dns/dns-backend-rc.yaml new file mode 100644 index 000000000..9649d367b --- /dev/null +++ b/cluster-dns/dns-backend-rc.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: dns-backend + labels: + name: dns-backend +spec: + replicas: 1 + selector: + name: dns-backend + template: + metadata: + labels: + name: dns-backend + spec: + containers: + - name: dns-backend + image: gcr.io/google_containers/example-dns-backend:v1 + ports: + - name: backend-port + containerPort: 8000 diff --git a/cluster-dns/dns-backend-service.yaml b/cluster-dns/dns-backend-service.yaml new file mode 100644 index 000000000..0a814ce18 --- /dev/null +++ b/cluster-dns/dns-backend-service.yaml @@ -0,0 +1,9 @@ +kind: Service +apiVersion: v1 +metadata: + name: dns-backend +spec: + ports: + - port: 8000 + selector: + name: dns-backend diff --git a/cluster-dns/dns-frontend-pod.yaml b/cluster-dns/dns-frontend-pod.yaml new file mode 100644 index 000000000..443e4aa20 --- /dev/null +++ b/cluster-dns/dns-frontend-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dns-frontend + labels: + name: dns-frontend +spec: + containers: + - name: dns-frontend + image: gcr.io/google_containers/example-dns-frontend:v1 + command: + - python + - client.py + - http://dns-backend.development.cluster.local:8000 + imagePullPolicy: Always + restartPolicy: Never diff --git a/cluster-dns/images/backend/Dockerfile b/cluster-dns/images/backend/Dockerfile new file mode 100644 index 000000000..c2a4e7cf6 --- /dev/null +++ b/cluster-dns/images/backend/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2.7-slim + +COPY . /dns-backend +WORKDIR /dns-backend + +CMD ["python", "server.py"] diff --git a/cluster-dns/images/backend/Makefile b/cluster-dns/images/backend/Makefile new file mode 100644 index 000000000..67992ec26 --- /dev/null +++ b/cluster-dns/images/backend/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TAG = v1 +PREFIX = gcr.io/google_containers +IMAGE = example-dns-backend + +all: push + +image: + docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . + +push: image + gcloud docker -- push $(PREFIX)/$(IMAGE) + +clean: diff --git a/cluster-dns/images/backend/server.py b/cluster-dns/images/backend/server.py new file mode 100644 index 000000000..5dcfb93b9 --- /dev/null +++ b/cluster-dns/images/backend/server.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +PORT_NUMBER = 8000 + +# This class will handles any incoming request. +class HTTPHandler(BaseHTTPRequestHandler): + # Handler for the GET requests + def do_GET(self): + self.send_response(200) + self.send_header('Content-type','text/html') + self.end_headers() + self.wfile.write("Hello World!") + +try: + # Create a web server and define the handler to manage the incoming request. + server = HTTPServer(('', PORT_NUMBER), HTTPHandler) + print 'Started httpserver on port ' , PORT_NUMBER + server.serve_forever() +except KeyboardInterrupt: + print '^C received, shutting down the web server' + server.socket.close() diff --git a/cluster-dns/images/frontend/Dockerfile b/cluster-dns/images/frontend/Dockerfile new file mode 100644 index 000000000..56da6eb71 --- /dev/null +++ b/cluster-dns/images/frontend/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2.7-slim + +RUN pip install requests + +COPY . /dns-frontend +WORKDIR /dns-frontend + +CMD ["python", "client.py"] diff --git a/cluster-dns/images/frontend/Makefile b/cluster-dns/images/frontend/Makefile new file mode 100644 index 000000000..2f6337545 --- /dev/null +++ b/cluster-dns/images/frontend/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TAG = v1 +PREFIX = gcr.io/google_containers +IMAGE = example-dns-frontend + +all: push + +image: + docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . + +push: image + gcloud docker -- push $(PREFIX)/$(IMAGE) + +clean: diff --git a/cluster-dns/images/frontend/client.py b/cluster-dns/images/frontend/client.py new file mode 100644 index 000000000..1a56df504 --- /dev/null +++ b/cluster-dns/images/frontend/client.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import requests +import socket + +from urlparse import urlparse + + +def CheckServiceAddress(address): + hostname = urlparse(address).hostname + service_address = socket.gethostbyname(hostname) + print service_address + + +def GetServerResponse(address): + print 'Send request to:', address + response = requests.get(address) + print response + print response.content + + +def Main(): + parser = argparse.ArgumentParser() + parser.add_argument('address') + args = parser.parse_args() + CheckServiceAddress(args.address) + GetServerResponse(args.address) + + +if __name__ == "__main__": + Main() diff --git a/cluster-dns/namespace-dev.yaml b/cluster-dns/namespace-dev.yaml new file mode 100644 index 000000000..149c2f665 --- /dev/null +++ b/cluster-dns/namespace-dev.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "development" + labels: + name: "development" diff --git a/cluster-dns/namespace-prod.yaml b/cluster-dns/namespace-prod.yaml new file mode 100644 index 000000000..04bb44c1d --- /dev/null +++ b/cluster-dns/namespace-prod.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "production" + labels: + name: "production" diff --git a/cockroachdb/OWNERS b/cockroachdb/OWNERS new file mode 100644 index 000000000..9b4cd15cb --- /dev/null +++ b/cockroachdb/OWNERS @@ -0,0 +1,4 @@ +reviewers: + - a-robinson +approvers: + - a-robinson diff --git a/cockroachdb/README.md b/cockroachdb/README.md new file mode 100644 index 000000000..db5561f96 --- /dev/null +++ b/cockroachdb/README.md @@ -0,0 +1,125 @@ +# CockroachDB on Kubernetes as a StatefulSet + +This example deploys [CockroachDB](https://cockroachlabs.com) on Kubernetes as +a StatefulSet. CockroachDB is a distributed, scalable NewSQL database. Please see +[the homepage](https://cockroachlabs.com) and the +[documentation](https://www.cockroachlabs.com/docs/) for details. + +## Limitations + +### StatefulSet limitations + +Standard StatefulSet limitations apply: There is currently no possibility to use +node-local storage (outside of single-node tests), and so there is likely +a performance hit associated with running CockroachDB on some external storage. +Note that CockroachDB already does replication and thus it is unnecessary to +deploy it onto persistent volumes which already replicate internally. +For this reason, high-performance use cases on a private Kubernetes cluster +may want to consider a DaemonSet deployment until Stateful Sets support node-local +storage (see #7562). + +### Recovery after persistent storage failure + +A persistent storage failure (e.g. losing the hard drive) is gracefully handled +by CockroachDB as long as enough replicas survive (two out of three by +default). Due to the bootstrapping in this deployment, a storage failure of the +first node is special in that the administrator must manually prepopulate the +"new" storage medium by running an instance of CockroachDB with the `--join` +parameter. If this is not done, the first node will bootstrap a new cluster, +which will lead to a lot of trouble. + +### Dynamic volume provisioning + +The deployment is written for a use case in which dynamic volume provisioning is +available. When that is not the case, the persistent volume claims need +to be created manually. See [minikube.sh](minikube.sh) for the necessary +steps. If you're on GCE or AWS, where dynamic provisioning is supported, no +manual work is needed to create the persistent volumes. + +## Testing locally on minikube + +Follow the steps in [minikube.sh](minikube.sh) (or simply run that file). + +## Testing in the cloud on GCE or AWS + +Once you have a Kubernetes cluster running, just run +`kubectl create -f cockroachdb-statefulset.yaml` to create your cockroachdb cluster. +This works because GCE and AWS support dynamic volume provisioning by default, +so persistent volumes will be created for the CockroachDB pods as needed. + +## Accessing the database + +Along with our StatefulSet configuration, we expose a standard Kubernetes service +that offers a load-balanced virtual IP for clients to access the database +with. In our example, we've called this service `cockroachdb-public`. + +Start up a client pod and open up an interactive, (mostly) Postgres-flavor +SQL shell using: + +```console +$ kubectl run -it --rm cockroach-client --image=cockroachdb/cockroach --restart=Never --command -- ./cockroach sql --host cockroachdb-public +``` + +You can see example SQL statements for inserting and querying data in the +included [demo script](demo.sh), but can use almost any Postgres-style SQL +commands. Some more basic examples can be found within +[CockroachDB's documentation](https://www.cockroachlabs.com/docs/learn-cockroachdb-sql.html). + +## Accessing the admin UI + +If you want to see information about how the cluster is doing, you can try +pulling up the CockroachDB admin UI by port-forwarding from your local machine +to one of the pods: + +```shell +kubectl port-forward cockroachdb-0 8080 +``` + +Once you’ve done that, you should be able to access the admin UI by visiting +http://localhost:8080/ in your web browser. + +## Simulating failures + +When all (or enough) nodes are up, simulate a failure like this: + +```shell +kubectl exec cockroachdb-0 -- /bin/bash -c "while true; do kill 1; done" +``` + +You can then reconnect to the database as demonstrated above and verify +that no data was lost. The example runs with three-fold replication, so +it can tolerate one failure of any given node at a time. Note also that +there is a brief period of time immediately after the creation of the +cluster during which the three-fold replication is established, and during +which killing a node may lead to unavailability. + +The [demo script](demo.sh) gives an example of killing one instance of the +database and ensuring the other replicas have all data that was written. + +## Scaling up or down + +Scale the Stateful Set by running + +```shell +kubectl scale statefulset cockroachdb --replicas=4 +``` + +Note that you may need to create a new persistent volume claim first. If you +ran `minikube.sh`, there's a spare volume so you can immediately scale up by +one. If you're running on GCE or AWS, you can scale up by as many as you want +because new volumes will automatically be created for you. Convince yourself +that the new node immediately serves reads and writes. + +## Cleaning up when you're done + +Because all of the resources in this example have been tagged with the label `app=cockroachdb`, +we can clean up everything that we created in one quick command using a selector on that label: + +```shell +kubectl delete statefulsets,persistentvolumes,persistentvolumeclaims,services,poddisruptionbudget -l app=cockroachdb +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/cockroachdb/README.md?pixel)]() + diff --git a/cockroachdb/cockroachdb-statefulset.yaml b/cockroachdb/cockroachdb-statefulset.yaml new file mode 100644 index 000000000..739d77105 --- /dev/null +++ b/cockroachdb/cockroachdb-statefulset.yaml @@ -0,0 +1,193 @@ +apiVersion: v1 +kind: Service +metadata: + # This service is meant to be used by clients of the database. It exposes a ClusterIP that will + # automatically load balance connections to the different database pods. + name: cockroachdb-public + labels: + app: cockroachdb +spec: + ports: + # The main port, served by gRPC, serves Postgres-flavor SQL, internode + # traffic and the cli. + - port: 26257 + targetPort: 26257 + name: grpc + # The secondary port serves the UI as well as health and debug endpoints. + - port: 8080 + targetPort: 8080 + name: http + selector: + app: cockroachdb +--- +apiVersion: v1 +kind: Service +metadata: + # This service only exists to create DNS entries for each pod in the stateful + # set such that they can resolve each other's IP addresses. It does not + # create a load-balanced ClusterIP and should not be used directly by clients + # in most circumstances. + name: cockroachdb + labels: + app: cockroachdb + annotations: + # This is needed to make the peer-finder work properly and to help avoid + # edge cases where instance 0 comes up after losing its data and needs to + # decide whether it should create a new cluster or try to join an existing + # one. If it creates a new cluster when it should have joined an existing + # one, we'd end up with two separate clusters listening at the same service + # endpoint, which would be very bad. + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + # Enable automatic monitoring of all instances when Prometheus is running in the cluster. + prometheus.io/scrape: "true" + prometheus.io/path: "_status/vars" + prometheus.io/port: "8080" +spec: + ports: + - port: 26257 + targetPort: 26257 + name: grpc + - port: 8080 + targetPort: 8080 + name: http + clusterIP: None + selector: + app: cockroachdb +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: cockroachdb-budget + labels: + app: cockroachdb +spec: + selector: + matchLabels: + app: cockroachdb + minAvailable: 67% +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: cockroachdb +spec: + serviceName: "cockroachdb" + replicas: 3 + template: + metadata: + labels: + app: cockroachdb + annotations: + scheduler.alpha.kubernetes.io/affinity: > + { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "weight": 100, + "labelSelector": { + "matchExpressions": [{ + "key": "app", + "operator": "In", + "values": ["cockroachdb"] + }] + }, + "topologyKey": "kubernetes.io/hostname" + }] + } + } + # Init containers are run only once in the lifetime of a pod, before + # it's started up for the first time. It has to exit successfully + # before the pod's main containers are allowed to start. + # This particular init container does a DNS lookup for other pods in + # the set to help determine whether or not a cluster already exists. + # If any other pods exist, it creates a file in the cockroach-data + # directory to pass that information along to the primary container that + # has to decide what command-line flags to use when starting CockroachDB. + # This only matters when a pod's persistent volume is empty - if it has + # data from a previous execution, that data will always be used. + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "bootstrap", + "image": "cockroachdb/cockroach-k8s-init:0.1", + "imagePullPolicy": "IfNotPresent", + "args": [ + "-on-start=/on-start.sh", + "-service=cockroachdb" + ], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "datadir", + "mountPath": "/cockroach/cockroach-data" + } + ] + } + ]' + spec: + containers: + - name: cockroachdb + # Runs the master branch. Not recommended for production, but since + # CockroachDB is in Beta, you don't want to run it in production + # anyway. See + # https://hub.docker.com/r/cockroachdb/cockroach/tags/ + # if you prefer to run a beta release. + image: cockroachdb/cockroach + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + name: grpc + - containerPort: 8080 + name: http + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + # The use of qualified `hostname -f` is crucial: + # Other nodes aren't able to look up the unqualified hostname. + CRARGS=("start" "--logtostderr" "--insecure" "--host" "$(hostname -f)" "--http-host" "0.0.0.0") + # We only want to initialize a new cluster (by omitting the join flag) + # if we're sure that we're the first node (i.e. index 0) and that + # there aren't any other nodes running as part of the cluster that + # this is supposed to be a part of (which indicates that a cluster + # already exists and we should make sure not to create a new one). + # It's fine to run without --join on a restart if there aren't any + # other nodes. + if [ ! "$(hostname)" == "cockroachdb-0" ] || \ + [ -e "/cockroach/cockroach-data/cluster_exists_marker" ] + then + # We don't join cockroachdb in order to avoid a node attempting + # to join itself, which currently doesn't work + # (https://github.com/cockroachdb/cockroach/issues/9625). + CRARGS+=("--join" "cockroachdb-public") + fi + exec /cockroach/cockroach ${CRARGS[*]} + # No pre-stop hook is required, a SIGTERM plus some time is all that's + # needed for graceful shutdown of a node. + terminationGracePeriodSeconds: 60 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: datadir + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi diff --git a/cockroachdb/demo.sh b/cockroachdb/demo.sh new file mode 100755 index 000000000..8b3031fb2 --- /dev/null +++ b/cockroachdb/demo.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +function sql() { + # TODO(knz): Why does the more idiomatic read from stdin not produce any + # output? + kubectl exec "cockroachdb-${1}" -- /cockroach/cockroach sql \ + --host "cockroachdb-${1}.cockroachdb" \ + -e "$(cat /dev/stdin)" +} + +function kill() { + ! kubectl exec -t "cockroachdb-${1}" -- /bin/bash -c "while true; do kill 1; done" &> /dev/null +} + +# Create database on second node (idempotently for convenience). +cat < Current pod descriptors use an `emptyDir` for storing data in each data node container. This is meant to be for the sake of simplicity and [should be adapted according to your storage needs](../../docs/design/persistent-storage.md). + +## Docker image + +The [pre-built image](https://github.com/pires/docker-elasticsearch-kubernetes) used in this example will not be supported. Feel free to fork to fit your own needs, but keep in mind that you will need to change Kubernetes descriptors accordingly. + +## Deploy + +Let's kickstart our cluster with 1 instance of Elasticsearch. + +``` +kubectl create -f examples/elasticsearch/service-account.yaml +kubectl create -f examples/elasticsearch/es-svc.yaml +kubectl create -f examples/elasticsearch/es-rc.yaml +``` + +Let's see if it worked: + +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +es-kfymw 1/1 Running 0 7m +kube-dns-p3v1u 3/3 Running 0 19m +``` + +``` +$ kubectl logs es-kfymw +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +[2015-08-30 10:01:31,946][INFO ][node ] [Hammerhead] version[1.7.1], pid[7], build[b88f43f/2015-07-29T09:54:16Z] +[2015-08-30 10:01:31,946][INFO ][node ] [Hammerhead] initializing ... +[2015-08-30 10:01:32,110][INFO ][plugins ] [Hammerhead] loaded [cloud-kubernetes], sites [] +[2015-08-30 10:01:32,153][INFO ][env ] [Hammerhead] using [1] data paths, mounts [[/data (/dev/sda9)]], net usable_space [14.4gb], net total_space [15.5gb], types [ext4] +[2015-08-30 10:01:37,188][INFO ][node ] [Hammerhead] initialized +[2015-08-30 10:01:37,189][INFO ][node ] [Hammerhead] starting ... +[2015-08-30 10:01:37,499][INFO ][transport ] [Hammerhead] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/10.244.48.2:9300]} +[2015-08-30 10:01:37,550][INFO ][discovery ] [Hammerhead] myesdb/n2-6uu_UT3W5XNrjyqBPiA +[2015-08-30 10:01:43,966][INFO ][cluster.service ] [Hammerhead] new_master [Hammerhead][n2-6uu_UT3W5XNrjyqBPiA][es-kfymw][inet[/10.244.48.2:9300]]{master=true}, reason: zen-disco-join (elected_as_master) +[2015-08-30 10:01:44,010][INFO ][http ] [Hammerhead] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/10.244.48.2:9200]} +[2015-08-30 10:01:44,011][INFO ][node ] [Hammerhead] started +[2015-08-30 10:01:44,042][INFO ][gateway ] [Hammerhead] recovered [0] indices into cluster_state +``` + +So we have a 1-node Elasticsearch cluster ready to handle some work. + +## Scale + +Scaling is as easy as: + +``` +kubectl scale --replicas=3 rc es +``` + +Did it work? + +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +es-78e0s 1/1 Running 0 8m +es-kfymw 1/1 Running 0 17m +es-rjmer 1/1 Running 0 8m +kube-dns-p3v1u 3/3 Running 0 30m +``` + +Let's take a look at logs: + +``` +$ kubectl logs es-kfymw +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +[2015-08-30 10:01:31,946][INFO ][node ] [Hammerhead] version[1.7.1], pid[7], build[b88f43f/2015-07-29T09:54:16Z] +[2015-08-30 10:01:31,946][INFO ][node ] [Hammerhead] initializing ... +[2015-08-30 10:01:32,110][INFO ][plugins ] [Hammerhead] loaded [cloud-kubernetes], sites [] +[2015-08-30 10:01:32,153][INFO ][env ] [Hammerhead] using [1] data paths, mounts [[/data (/dev/sda9)]], net usable_space [14.4gb], net total_space [15.5gb], types [ext4] +[2015-08-30 10:01:37,188][INFO ][node ] [Hammerhead] initialized +[2015-08-30 10:01:37,189][INFO ][node ] [Hammerhead] starting ... +[2015-08-30 10:01:37,499][INFO ][transport ] [Hammerhead] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/10.244.48.2:9300]} +[2015-08-30 10:01:37,550][INFO ][discovery ] [Hammerhead] myesdb/n2-6uu_UT3W5XNrjyqBPiA +[2015-08-30 10:01:43,966][INFO ][cluster.service ] [Hammerhead] new_master [Hammerhead][n2-6uu_UT3W5XNrjyqBPiA][es-kfymw][inet[/10.244.48.2:9300]]{master=true}, reason: zen-disco-join (elected_as_master) +[2015-08-30 10:01:44,010][INFO ][http ] [Hammerhead] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/10.244.48.2:9200]} +[2015-08-30 10:01:44,011][INFO ][node ] [Hammerhead] started +[2015-08-30 10:01:44,042][INFO ][gateway ] [Hammerhead] recovered [0] indices into cluster_state +[2015-08-30 10:08:02,517][INFO ][cluster.service ] [Hammerhead] added {[Tenpin][2gv5MiwhRiOSsrTOF3DhuA][es-78e0s][inet[/10.244.54.4:9300]]{master=true},}, reason: zen-disco-receive(join from node[[Tenpin][2gv5MiwhRiOSsrTOF3DhuA][es-78e0s][inet[/10.244.54.4:9300]]{master=true}]) +[2015-08-30 10:10:10,645][INFO ][cluster.service ] [Hammerhead] added {[Evilhawk][ziTq2PzYRJys43rNL2tbyg][es-rjmer][inet[/10.244.33.3:9300]]{master=true},}, reason: zen-disco-receive(join from node[[Evilhawk][ziTq2PzYRJys43rNL2tbyg][es-rjmer][inet[/10.244.33.3:9300]]{master=true}]) +``` + +So we have a 3-node Elasticsearch cluster ready to handle more work. + +## Access the service + +*Don't forget* that services in Kubernetes are only acessible from containers in the cluster. For different behavior you should [configure the creation of an external load-balancer](http://kubernetes.io/v1.0/docs/user-guide/services.html#type-loadbalancer). While it's supported within this example service descriptor, its usage is out of scope of this document, for now. + +``` +$ kubectl get service elasticsearch +NAME LABELS SELECTOR IP(S) PORT(S) +elasticsearch component=elasticsearch component=elasticsearch 10.100.108.94 9200/TCP + 9300/TCP +``` + +From any host on your cluster (that's running `kube-proxy`), run: + +``` +$ curl 10.100.108.94:9200 +``` + +You should see something similar to the following: + + +```json +{ + "status" : 200, + "name" : "Hammerhead", + "cluster_name" : "myesdb", + "version" : { + "number" : "1.7.1", + "build_hash" : "b88f43fc40b0bcd7f173a1f9ee2e97816de80b19", + "build_timestamp" : "2015-07-29T09:54:16Z", + "build_snapshot" : false, + "lucene_version" : "4.10.4" + }, + "tagline" : "You Know, for Search" +} +``` + +Or if you want to check cluster information: + + +``` +curl 10.100.108.94:9200/_cluster/health?pretty +``` + +You should see something similar to the following: + +```json +{ + "cluster_name" : "myesdb", + "status" : "green", + "timed_out" : false, + "number_of_nodes" : 3, + "number_of_data_nodes" : 3, + "active_primary_shards" : 0, + "active_shards" : 0, + "relocating_shards" : 0, + "initializing_shards" : 0, + "unassigned_shards" : 0, + "delayed_unassigned_shards" : 0, + "number_of_pending_tasks" : 0, + "number_of_in_flight_fetch" : 0 +} +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/elasticsearch/README.md?pixel)]() + diff --git a/elasticsearch/es-rc.yaml b/elasticsearch/es-rc.yaml new file mode 100644 index 000000000..31f38e406 --- /dev/null +++ b/elasticsearch/es-rc.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: es + labels: + component: elasticsearch +spec: + replicas: 1 + template: + metadata: + labels: + component: elasticsearch + spec: + serviceAccount: elasticsearch + containers: + - name: es + securityContext: + capabilities: + add: + - IPC_LOCK + image: quay.io/pires/docker-elasticsearch-kubernetes:1.7.1-4 + env: + - name: KUBERNETES_CA_CERTIFICATE_FILE + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "CLUSTER_NAME" + value: "myesdb" + - name: "DISCOVERY_SERVICE" + value: "elasticsearch" + - name: NODE_MASTER + value: "true" + - name: NODE_DATA + value: "true" + - name: HTTP_ENABLE + value: "true" + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - mountPath: /data + name: storage + volumes: + - name: storage + emptyDir: {} diff --git a/elasticsearch/es-svc.yaml b/elasticsearch/es-svc.yaml new file mode 100644 index 000000000..3a5dd4564 --- /dev/null +++ b/elasticsearch/es-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + component: elasticsearch +spec: + type: LoadBalancer + selector: + component: elasticsearch + ports: + - name: http + port: 9200 + protocol: TCP + - name: transport + port: 9300 + protocol: TCP diff --git a/elasticsearch/production_cluster/README.md b/elasticsearch/production_cluster/README.md new file mode 100644 index 000000000..58a2ff581 --- /dev/null +++ b/elasticsearch/production_cluster/README.md @@ -0,0 +1,189 @@ +# Elasticsearch for Kubernetes + +Kubernetes makes it trivial for anyone to easily build and scale [Elasticsearch](http://www.elasticsearch.org/) clusters. Here, you'll find how to do so. +Current Elasticsearch version is `1.7.1`. + +Before we start, one needs to know that Elasticsearch best-practices recommend to separate nodes in three roles: +* `Master` nodes - intended for clustering management only, no data, no HTTP API +* `Client` nodes - intended for client usage, no data, with HTTP API +* `Data` nodes - intended for storing and indexing your data, no HTTP API + +This is enforced throughout this document. + +WARNING Current pod descriptors use an `emptyDir` for storing data in each data node container. This is meant to be for the sake of simplicity and [should be adapted according to your storage needs](../../../docs/design/persistent-storage.md). + +## Docker image + +This example uses [this pre-built image](https://github.com/pires/docker-elasticsearch-kubernetes). Feel free to fork and update it to fit your own needs, but keep in mind that you will need to change Kubernetes descriptors accordingly. + +## Deploy + +``` +kubectl create -f examples/elasticsearch/production_cluster/service-account.yaml +kubectl create -f examples/elasticsearch/production_cluster/es-discovery-svc.yaml +kubectl create -f examples/elasticsearch/production_cluster/es-svc.yaml +kubectl create -f examples/elasticsearch/production_cluster/es-master-rc.yaml +``` + +Wait until `es-master` is provisioned, and + +``` +kubectl create -f examples/elasticsearch/production_cluster/es-client-rc.yaml +``` + +Wait until `es-client` is provisioned, and + +``` +kubectl create -f examples/elasticsearch/production_cluster/es-data-rc.yaml +``` + +Wait until `es-data` is provisioned. + +Now, I leave up to you how to validate the cluster, but a first step is to wait for containers to be in ```RUNNING``` state and check the Elasticsearch master logs: + +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +es-client-2ep9o 1/1 Running 0 2m +es-data-r9tgv 1/1 Running 0 1m +es-master-vxl6c 1/1 Running 0 6m +``` + +``` +$ kubectl logs es-master-vxl6c +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +[2015-08-21 10:58:51,324][INFO ][node ] [Arc] version[1.7.1], pid[8], build[b88f43f/2015-07-29T09:54:16Z] +[2015-08-21 10:58:51,328][INFO ][node ] [Arc] initializing ... +[2015-08-21 10:58:51,542][INFO ][plugins ] [Arc] loaded [cloud-kubernetes], sites [] +[2015-08-21 10:58:51,624][INFO ][env ] [Arc] using [1] data paths, mounts [[/data (/dev/sda9)]], net usable_space [14.4gb], net total_space [15.5gb], types [ext4] +[2015-08-21 10:58:57,439][INFO ][node ] [Arc] initialized +[2015-08-21 10:58:57,439][INFO ][node ] [Arc] starting ... +[2015-08-21 10:58:57,782][INFO ][transport ] [Arc] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/10.244.15.2:9300]} +[2015-08-21 10:58:57,847][INFO ][discovery ] [Arc] myesdb/-x16XFUzTCC8xYqWoeEOYQ +[2015-08-21 10:59:05,167][INFO ][cluster.service ] [Arc] new_master [Arc][-x16XFUzTCC8xYqWoeEOYQ][es-master-vxl6c][inet[/10.244.15.2:9300]]{data=false, master=true}, reason: zen-disco-join (elected_as_master) +[2015-08-21 10:59:05,202][INFO ][node ] [Arc] started +[2015-08-21 10:59:05,238][INFO ][gateway ] [Arc] recovered [0] indices into cluster_state +[2015-08-21 11:02:28,797][INFO ][cluster.service ] [Arc] added {[Gideon][4EfhWSqaTqikbK4tI7bODA][es-data-r9tgv][inet[/10.244.59.4:9300]]{master=false},}, reason: zen-disco-receive(join from node[[Gideon][4EfhWSqaTqikbK4tI7bODA][es-data-r9tgv][inet[/10.244.59.4:9300]]{master=false}]) +[2015-08-21 11:03:16,822][INFO ][cluster.service ] [Arc] added {[Venomm][tFYxwgqGSpOejHLG4umRqg][es-client-2ep9o][inet[/10.244.53.2:9300]]{data=false, master=false},}, reason: zen-disco-receive(join from node[[Venomm][tFYxwgqGSpOejHLG4umRqg][es-client-2ep9o][inet[/10.244.53.2:9300]]{data=false, master=false}]) +``` + +As you can assert, the cluster is up and running. Easy, wasn't it? + +## Scale + +Scaling each type of node to handle your cluster is as easy as: + +``` +kubectl scale --replicas=3 rc es-master +kubectl scale --replicas=2 rc es-client +kubectl scale --replicas=2 rc es-data +``` + +Did it work? + +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +es-client-2ep9o 1/1 Running 0 4m +es-client-ye5s1 1/1 Running 0 50s +es-data-8az22 1/1 Running 0 47s +es-data-r9tgv 1/1 Running 0 3m +es-master-57h7k 1/1 Running 0 52s +es-master-kuwse 1/1 Running 0 52s +es-master-vxl6c 1/1 Running 0 8m +``` + +Let's take another look of the Elasticsearch master logs: + +``` +$ kubectl logs es-master-vxl6c +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +log4j:WARN No such property [maxBackupIndex] in org.apache.log4j.DailyRollingFileAppender. +[2015-08-21 10:58:51,324][INFO ][node ] [Arc] version[1.7.1], pid[8], build[b88f43f/2015-07-29T09:54:16Z] +[2015-08-21 10:58:51,328][INFO ][node ] [Arc] initializing ... +[2015-08-21 10:58:51,542][INFO ][plugins ] [Arc] loaded [cloud-kubernetes], sites [] +[2015-08-21 10:58:51,624][INFO ][env ] [Arc] using [1] data paths, mounts [[/data (/dev/sda9)]], net usable_space [14.4gb], net total_space [15.5gb], types [ext4] +[2015-08-21 10:58:57,439][INFO ][node ] [Arc] initialized +[2015-08-21 10:58:57,439][INFO ][node ] [Arc] starting ... +[2015-08-21 10:58:57,782][INFO ][transport ] [Arc] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/10.244.15.2:9300]} +[2015-08-21 10:58:57,847][INFO ][discovery ] [Arc] myesdb/-x16XFUzTCC8xYqWoeEOYQ +[2015-08-21 10:59:05,167][INFO ][cluster.service ] [Arc] new_master [Arc][-x16XFUzTCC8xYqWoeEOYQ][es-master-vxl6c][inet[/10.244.15.2:9300]]{data=false, master=true}, reason: zen-disco-join (elected_as_master) +[2015-08-21 10:59:05,202][INFO ][node ] [Arc] started +[2015-08-21 10:59:05,238][INFO ][gateway ] [Arc] recovered [0] indices into cluster_state +[2015-08-21 11:02:28,797][INFO ][cluster.service ] [Arc] added {[Gideon][4EfhWSqaTqikbK4tI7bODA][es-data-r9tgv][inet[/10.244.59.4:9300]]{master=false},}, reason: zen-disco-receive(join from node[[Gideon][4EfhWSqaTqikbK4tI7bODA][es-data-r9tgv][inet[/10.244.59.4:9300]]{master=false}]) +[2015-08-21 11:03:16,822][INFO ][cluster.service ] [Arc] added {[Venomm][tFYxwgqGSpOejHLG4umRqg][es-client-2ep9o][inet[/10.244.53.2:9300]]{data=false, master=false},}, reason: zen-disco-receive(join from node[[Venomm][tFYxwgqGSpOejHLG4umRqg][es-client-2ep9o][inet[/10.244.53.2:9300]]{data=false, master=false}]) +[2015-08-21 11:04:40,781][INFO ][cluster.service ] [Arc] added {[Erik Josten][QUJlahfLTi-MsxzM6_Da0g][es-master-kuwse][inet[/10.244.59.5:9300]]{data=false, master=true},}, reason: zen-disco-receive(join from node[[Erik Josten][QUJlahfLTi-MsxzM6_Da0g][es-master-kuwse][inet[/10.244.59.5:9300]]{data=false, master=true}]) +[2015-08-21 11:04:41,076][INFO ][cluster.service ] [Arc] added {[Power Princess][V4qnR-6jQOS5ovXQsPgo7g][es-master-57h7k][inet[/10.244.53.3:9300]]{data=false, master=true},}, reason: zen-disco-receive(join from node[[Power Princess][V4qnR-6jQOS5ovXQsPgo7g][es-master-57h7k][inet[/10.244.53.3:9300]]{data=false, master=true}]) +[2015-08-21 11:04:53,966][INFO ][cluster.service ] [Arc] added {[Cagliostro][Wpfx5fkBRiG2qCEWd8laaQ][es-client-ye5s1][inet[/10.244.15.3:9300]]{data=false, master=false},}, reason: zen-disco-receive(join from node[[Cagliostro][Wpfx5fkBRiG2qCEWd8laaQ][es-client-ye5s1][inet[/10.244.15.3:9300]]{data=false, master=false}]) +[2015-08-21 11:04:56,803][INFO ][cluster.service ] [Arc] added {[Thog][vkdEtX3ESfWmhXXf-Wi0_Q][es-data-8az22][inet[/10.244.15.4:9300]]{master=false},}, reason: zen-disco-receive(join from node[[Thog][vkdEtX3ESfWmhXXf-Wi0_Q][es-data-8az22][inet[/10.244.15.4:9300]]{master=false}]) +``` + +## Access the service + +*Don't forget* that services in Kubernetes are only accessible from containers in the cluster. For different behavior you should [configure the creation of an external load-balancer](http://kubernetes.io/v1.0/docs/user-guide/services.html#type-loadbalancer). While it's supported within this example service descriptor, its usage is out of scope of this document, for now. + +``` +$ kubectl get service elasticsearch +NAME LABELS SELECTOR IP(S) PORT(S) +elasticsearch component=elasticsearch,role=client component=elasticsearch,role=client 10.100.134.2 9200/TCP +``` + +From any host on your cluster (that's running `kube-proxy`), run: + +``` +curl http://10.100.134.2:9200 +``` + +You should see something similar to the following: + + +```json +{ + "status" : 200, + "name" : "Cagliostro", + "cluster_name" : "myesdb", + "version" : { + "number" : "1.7.1", + "build_hash" : "b88f43fc40b0bcd7f173a1f9ee2e97816de80b19", + "build_timestamp" : "2015-07-29T09:54:16Z", + "build_snapshot" : false, + "lucene_version" : "4.10.4" + }, + "tagline" : "You Know, for Search" +} +``` + +Or if you want to check cluster information: + + +``` +curl http://10.100.134.2:9200/_cluster/health?pretty +``` + +You should see something similar to the following: + +```json +{ + "cluster_name" : "myesdb", + "status" : "green", + "timed_out" : false, + "number_of_nodes" : 7, + "number_of_data_nodes" : 2, + "active_primary_shards" : 0, + "active_shards" : 0, + "relocating_shards" : 0, + "initializing_shards" : 0, + "unassigned_shards" : 0, + "delayed_unassigned_shards" : 0, + "number_of_pending_tasks" : 0, + "number_of_in_flight_fetch" : 0 +} +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/elasticsearch/production_cluster/README.md?pixel)]() + diff --git a/elasticsearch/production_cluster/es-client-rc.yaml b/elasticsearch/production_cluster/es-client-rc.yaml new file mode 100644 index 000000000..a2e2f2037 --- /dev/null +++ b/elasticsearch/production_cluster/es-client-rc.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: es-client + labels: + component: elasticsearch + role: client +spec: + replicas: 1 + template: + metadata: + labels: + component: elasticsearch + role: client + spec: + serviceAccount: elasticsearch + containers: + - name: es-client + securityContext: + capabilities: + add: + - IPC_LOCK + image: quay.io/pires/docker-elasticsearch-kubernetes:1.7.1-4 + env: + - name: KUBERNETES_CA_CERTIFICATE_FILE + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "CLUSTER_NAME" + value: "myesdb" + - name: NODE_MASTER + value: "false" + - name: NODE_DATA + value: "false" + - name: HTTP_ENABLE + value: "true" + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - mountPath: /data + name: storage + volumes: + - name: storage + emptyDir: {} diff --git a/elasticsearch/production_cluster/es-data-rc.yaml b/elasticsearch/production_cluster/es-data-rc.yaml new file mode 100644 index 000000000..6fed15f01 --- /dev/null +++ b/elasticsearch/production_cluster/es-data-rc.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: es-data + labels: + component: elasticsearch + role: data +spec: + replicas: 1 + template: + metadata: + labels: + component: elasticsearch + role: data + spec: + serviceAccount: elasticsearch + containers: + - name: es-data + securityContext: + capabilities: + add: + - IPC_LOCK + image: quay.io/pires/docker-elasticsearch-kubernetes:1.7.1-4 + env: + - name: KUBERNETES_CA_CERTIFICATE_FILE + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "CLUSTER_NAME" + value: "myesdb" + - name: NODE_MASTER + value: "false" + - name: HTTP_ENABLE + value: "false" + ports: + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - mountPath: /data + name: storage + volumes: + - name: storage + emptyDir: {} diff --git a/elasticsearch/production_cluster/es-discovery-svc.yaml b/elasticsearch/production_cluster/es-discovery-svc.yaml new file mode 100644 index 000000000..cfdc5daa2 --- /dev/null +++ b/elasticsearch/production_cluster/es-discovery-svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-discovery + labels: + component: elasticsearch + role: master +spec: + selector: + component: elasticsearch + role: master + ports: + - name: transport + port: 9300 + protocol: TCP diff --git a/elasticsearch/production_cluster/es-master-rc.yaml b/elasticsearch/production_cluster/es-master-rc.yaml new file mode 100644 index 000000000..49eadbfd3 --- /dev/null +++ b/elasticsearch/production_cluster/es-master-rc.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: es-master + labels: + component: elasticsearch + role: master +spec: + replicas: 1 + template: + metadata: + labels: + component: elasticsearch + role: master + spec: + serviceAccount: elasticsearch + containers: + - name: es-master + securityContext: + capabilities: + add: + - IPC_LOCK + image: quay.io/pires/docker-elasticsearch-kubernetes:1.7.1-4 + env: + - name: KUBERNETES_CA_CERTIFICATE_FILE + value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "CLUSTER_NAME" + value: "myesdb" + - name: NODE_MASTER + value: "true" + - name: NODE_DATA + value: "false" + - name: HTTP_ENABLE + value: "false" + ports: + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - mountPath: /data + name: storage + volumes: + - name: storage + emptyDir: {} diff --git a/elasticsearch/production_cluster/es-svc.yaml b/elasticsearch/production_cluster/es-svc.yaml new file mode 100644 index 000000000..03bc4efda --- /dev/null +++ b/elasticsearch/production_cluster/es-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + component: elasticsearch + role: client +spec: + type: LoadBalancer + selector: + component: elasticsearch + role: client + ports: + - name: http + port: 9200 + protocol: TCP diff --git a/elasticsearch/production_cluster/service-account.yaml b/elasticsearch/production_cluster/service-account.yaml new file mode 100644 index 000000000..7b7b80b20 --- /dev/null +++ b/elasticsearch/production_cluster/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch diff --git a/elasticsearch/service-account.yaml b/elasticsearch/service-account.yaml new file mode 100644 index 000000000..7b7b80b20 --- /dev/null +++ b/elasticsearch/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch diff --git a/examples_test.go b/examples_test.go new file mode 100644 index 000000000..64fdc9627 --- /dev/null +++ b/examples_test.go @@ -0,0 +1,464 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package examples_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/golang/glog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/testapi" + "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/apis/apps" + appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation" + "k8s.io/kubernetes/pkg/apis/batch" + "k8s.io/kubernetes/pkg/apis/extensions" + expvalidation "k8s.io/kubernetes/pkg/apis/extensions/validation" + "k8s.io/kubernetes/pkg/capabilities" + "k8s.io/kubernetes/pkg/registry/batch/job" + schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api" + schedulerapilatest "k8s.io/kubernetes/plugin/pkg/scheduler/api/latest" +) + +func validateObject(obj runtime.Object) (errors field.ErrorList) { + switch t := obj.(type) { + case *api.ReplicationController: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateReplicationController(t) + case *api.ReplicationControllerList: + for i := range t.Items { + errors = append(errors, validateObject(&t.Items[i])...) + } + case *api.Service: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateService(t) + case *api.ServiceList: + for i := range t.Items { + errors = append(errors, validateObject(&t.Items[i])...) + } + case *api.Pod: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidatePod(t) + case *api.PodList: + for i := range t.Items { + errors = append(errors, validateObject(&t.Items[i])...) + } + case *api.PersistentVolume: + errors = validation.ValidatePersistentVolume(t) + case *api.PersistentVolumeClaim: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidatePersistentVolumeClaim(t) + case *api.PodTemplate: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidatePodTemplate(t) + case *api.Endpoints: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateEndpoints(t) + case *api.Namespace: + errors = validation.ValidateNamespace(t) + case *api.Secret: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateSecret(t) + case *api.LimitRange: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateLimitRange(t) + case *api.ResourceQuota: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = validation.ValidateResourceQuota(t) + case *extensions.Deployment: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = expvalidation.ValidateDeployment(t) + case *batch.Job: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + // Job needs generateSelector called before validation, and job.Validate does this. + // See: https://github.com/kubernetes/kubernetes/issues/20951#issuecomment-187787040 + t.ObjectMeta.UID = types.UID("fakeuid") + errors = job.Strategy.Validate(nil, t) + case *extensions.Ingress: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = expvalidation.ValidateIngress(t) + case *extensions.DaemonSet: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = expvalidation.ValidateDaemonSet(t) + case *apps.StatefulSet: + if t.Namespace == "" { + t.Namespace = metav1.NamespaceDefault + } + errors = appsvalidation.ValidateStatefulSet(t) + default: + errors = field.ErrorList{} + errors = append(errors, field.InternalError(field.NewPath(""), fmt.Errorf("no validation defined for %#v", obj))) + } + return errors +} + +func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error { + return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() && path != inDir { + return filepath.SkipDir + } + + file := filepath.Base(path) + if ext := filepath.Ext(file); ext == ".json" || ext == ".yaml" { + glog.Infof("Testing %s", path) + data, err := ioutil.ReadFile(path) + if err != nil { + return err + } + name := strings.TrimSuffix(file, ext) + + if ext == ".yaml" { + out, err := yaml.ToJSON(data) + if err != nil { + return fmt.Errorf("%s: %v", path, err) + } + data = out + } + + fn(name, path, data) + } + return nil + }) +} + +func TestExampleObjectSchemas(t *testing.T) { + cases := map[string]map[string]runtime.Object{ + "../examples/guestbook": { + "frontend-deployment": &extensions.Deployment{}, + "redis-slave-deployment": &extensions.Deployment{}, + "redis-master-deployment": &extensions.Deployment{}, + "frontend-service": &api.Service{}, + "redis-master-service": &api.Service{}, + "redis-slave-service": &api.Service{}, + }, + "../examples/guestbook/legacy": { + "frontend-controller": &api.ReplicationController{}, + "redis-slave-controller": &api.ReplicationController{}, + "redis-master-controller": &api.ReplicationController{}, + }, + "../examples/guestbook-go": { + "guestbook-controller": &api.ReplicationController{}, + "redis-slave-controller": &api.ReplicationController{}, + "redis-master-controller": &api.ReplicationController{}, + "guestbook-service": &api.Service{}, + "redis-master-service": &api.Service{}, + "redis-slave-service": &api.Service{}, + }, + "../examples/volumes/iscsi": { + "chap-secret": &api.Secret{}, + "iscsi": &api.Pod{}, + "iscsi-chap": &api.Pod{}, + }, + "../examples/volumes/glusterfs": { + "glusterfs-pod": &api.Pod{}, + "glusterfs-endpoints": &api.Endpoints{}, + "glusterfs-service": &api.Service{}, + }, + "../examples": { + "scheduler-policy-config": &schedulerapi.Policy{}, + "scheduler-policy-config-with-extender": &schedulerapi.Policy{}, + }, + "../examples/volumes/rbd/secret": { + "ceph-secret": &api.Secret{}, + }, + "../examples/volumes/rbd": { + "rbd": &api.Pod{}, + "rbd-with-secret": &api.Pod{}, + }, + "../examples/storage/cassandra": { + "cassandra-daemonset": &extensions.DaemonSet{}, + "cassandra-controller": &api.ReplicationController{}, + "cassandra-service": &api.Service{}, + "cassandra-statefulset": &apps.StatefulSet{}, + }, + "../examples/cluster-dns": { + "dns-backend-rc": &api.ReplicationController{}, + "dns-backend-service": &api.Service{}, + "dns-frontend-pod": &api.Pod{}, + "namespace-dev": &api.Namespace{}, + "namespace-prod": &api.Namespace{}, + }, + "../examples/elasticsearch": { + "es-rc": &api.ReplicationController{}, + "es-svc": &api.Service{}, + "service-account": nil, + }, + "../examples/explorer": { + "pod": &api.Pod{}, + }, + "../examples/storage/hazelcast": { + "hazelcast-deployment": &extensions.Deployment{}, + "hazelcast-service": &api.Service{}, + }, + "../examples/meteor": { + "meteor-controller": &api.ReplicationController{}, + "meteor-service": &api.Service{}, + "mongo-pod": &api.Pod{}, + "mongo-service": &api.Service{}, + }, + "../examples/mysql-wordpress-pd": { + "gce-volumes": &api.PersistentVolume{}, + "local-volumes": &api.PersistentVolume{}, + "mysql-deployment": &api.Service{}, + "wordpress-deployment": &api.Service{}, + }, + "../examples/volumes/nfs": { + "nfs-busybox-rc": &api.ReplicationController{}, + "nfs-server-rc": &api.ReplicationController{}, + "nfs-server-service": &api.Service{}, + "nfs-pv": &api.PersistentVolume{}, + "nfs-pvc": &api.PersistentVolumeClaim{}, + "nfs-web-rc": &api.ReplicationController{}, + "nfs-web-service": &api.Service{}, + }, + "../examples/openshift-origin": { + "openshift-origin-namespace": &api.Namespace{}, + "openshift-controller": &extensions.Deployment{}, + "openshift-service": &api.Service{}, + "etcd-controller": &extensions.Deployment{}, + "etcd-service": &api.Service{}, + "etcd-discovery-controller": &extensions.Deployment{}, + "etcd-discovery-service": &api.Service{}, + "secret": nil, + }, + "../examples/phabricator": { + "phabricator-controller": &api.ReplicationController{}, + "phabricator-service": &api.Service{}, + }, + "../examples/storage/redis": { + "redis-controller": &api.ReplicationController{}, + "redis-master": &api.Pod{}, + "redis-sentinel-controller": &api.ReplicationController{}, + "redis-sentinel-service": &api.Service{}, + }, + "../examples/storage/rethinkdb": { + "admin-pod": &api.Pod{}, + "admin-service": &api.Service{}, + "driver-service": &api.Service{}, + "rc": &api.ReplicationController{}, + }, + "../examples/spark": { + "namespace-spark-cluster": &api.Namespace{}, + "spark-master-controller": &api.ReplicationController{}, + "spark-master-service": &api.Service{}, + "spark-ui-proxy-controller": &api.ReplicationController{}, + "spark-ui-proxy-service": &api.Service{}, + "spark-worker-controller": &api.ReplicationController{}, + "zeppelin-controller": &api.ReplicationController{}, + "zeppelin-service": &api.Service{}, + }, + "../examples/spark/spark-gluster": { + "spark-master-service": &api.Service{}, + "spark-master-controller": &api.ReplicationController{}, + "spark-worker-controller": &api.ReplicationController{}, + "glusterfs-endpoints": &api.Endpoints{}, + }, + "../examples/storm": { + "storm-nimbus-service": &api.Service{}, + "storm-nimbus": &api.Pod{}, + "storm-worker-controller": &api.ReplicationController{}, + "zookeeper-service": &api.Service{}, + "zookeeper": &api.Pod{}, + }, + "../examples/volumes/cephfs/": { + "cephfs": &api.Pod{}, + "cephfs-with-secret": &api.Pod{}, + }, + "../examples/volumes/fibre_channel": { + "fc": &api.Pod{}, + }, + "../examples/javaweb-tomcat-sidecar": { + "javaweb": &api.Pod{}, + "javaweb-2": &api.Pod{}, + }, + "../examples/volumes/azure_file": { + "azure": &api.Pod{}, + }, + "../examples/volumes/azure_disk": { + "azure": &api.Pod{}, + }, + } + + capabilities.SetForTests(capabilities.Capabilities{ + AllowPrivileged: true, + }) + + for path, expected := range cases { + tested := 0 + err := walkJSONFiles(path, func(name, path string, data []byte) { + expectedType, found := expected[name] + if !found { + t.Errorf("%s: %s does not have a test case defined", path, name) + return + } + tested++ + if expectedType == nil { + t.Logf("skipping : %s/%s\n", path, name) + return + } + if strings.Contains(name, "scheduler-policy-config") { + if err := runtime.DecodeInto(schedulerapilatest.Codec, data, expectedType); err != nil { + t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) + return + } + //TODO: Add validate method for &schedulerapi.Policy + } else { + codec, err := testapi.GetCodecForObject(expectedType) + if err != nil { + t.Errorf("Could not get codec for %s: %s", expectedType, err) + } + if err := runtime.DecodeInto(codec, data, expectedType); err != nil { + t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data)) + return + } + if errors := validateObject(expectedType); len(errors) > 0 { + t.Errorf("%s did not validate correctly: %v", path, errors) + } + } + }) + if err != nil { + t.Errorf("Expected no error, Got %v", err) + } + if tested != len(expected) { + t.Errorf("Directory %v: Expected %d examples, Got %d", path, len(expected), tested) + } + } +} + +// This regex is tricky, but it works. For future me, here is the decode: +// +// Flags: (?ms) = multiline match, allow . to match \n +// 1) Look for a line that starts with ``` (a markdown code block) +// 2) (?: ... ) = non-capturing group +// 3) (P) = capture group as "name" +// 4) Look for #1 followed by either: +// 4a) "yaml" followed by any word-characters followed by a newline (e.g. ```yamlfoo\n) +// 4b) "any word-characters followed by a newline (e.g. ```json\n) +// 5) Look for either: +// 5a) #4a followed by one or more characters (non-greedy) +// 5b) #4b followed by { followed by one or more characters (non-greedy) followed by } +// 6) Look for #5 followed by a newline followed by ``` (end of the code block) +// +// This could probably be simplified, but is already too delicate. Before any +// real changes, we should have a testscase that just tests this regex. +var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?Pyaml)\\w*\\n(?P.+?)|\\w*\\n(?P\\{.+?\\}))\\n^```") +var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}") + +func TestReadme(t *testing.T) { + paths := []struct { + file string + expectedType []runtime.Object + }{ + {"../README.md", []runtime.Object{&api.Pod{}}}, + {"../examples/volumes/iscsi/README.md", []runtime.Object{&api.Secret{}}}, + } + + for _, path := range paths { + data, err := ioutil.ReadFile(path.file) + if err != nil { + t.Errorf("Unable to read file %s: %v", path, err) + continue + } + + matches := sampleRegexp.FindAllStringSubmatch(string(data), -1) + if matches == nil { + continue + } + ix := 0 + for _, match := range matches { + var content, subtype string + for i, name := range sampleRegexp.SubexpNames() { + if name == "type" { + subtype = match[i] + } + if name == "content" && match[i] != "" { + content = match[i] + } + } + if subtype == "yaml" && subsetRegexp.FindString(content) != "" { + t.Logf("skipping (%s): \n%s", subtype, content) + continue + } + + var expectedType runtime.Object + if len(path.expectedType) == 1 { + expectedType = path.expectedType[0] + } else { + expectedType = path.expectedType[ix] + ix++ + } + json, err := yaml.ToJSON([]byte(content)) + if err != nil { + t.Errorf("%s could not be converted to JSON: %v\n%s", path, err, string(content)) + } + if err := runtime.DecodeInto(testapi.Default.Codec(), json, expectedType); err != nil { + t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content)) + continue + } + if errors := validateObject(expectedType); len(errors) > 0 { + t.Errorf("%s did not validate correctly: %v", path, errors) + } + _, err = runtime.Encode(testapi.Default.Codec(), expectedType) + if err != nil { + t.Errorf("Could not encode object: %v", err) + continue + } + } + } +} diff --git a/explorer/BUILD b/explorer/BUILD new file mode 100644 index 000000000..562838d55 --- /dev/null +++ b/explorer/BUILD @@ -0,0 +1,35 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "explorer", + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["explorer.go"], + tags = ["automanaged"], + deps = ["//vendor/github.com/davecgh/go-spew/spew:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/explorer/Dockerfile b/explorer/Dockerfile new file mode 100644 index 000000000..71c686c56 --- /dev/null +++ b/explorer/Dockerfile @@ -0,0 +1,19 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM scratch +ADD explorer explorer +ADD README.md README.md +EXPOSE 8080 +ENTRYPOINT ["/explorer"] diff --git a/explorer/Makefile b/explorer/Makefile new file mode 100644 index 000000000..35dd5bd7e --- /dev/null +++ b/explorer/Makefile @@ -0,0 +1,30 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +# Keep this one version ahead, so no one accidentally blows away the latest published version. +TAG = 1.1 + +explorer: explorer.go + CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' ./explorer.go + +container: explorer + docker build --pull -t gcr.io/google_containers/explorer:$(TAG) . + +push: container + gcloud docker -- push gcr.io/google_containers/explorer:$(TAG) + +clean: + rm -f explorer diff --git a/explorer/README.md b/explorer/README.md new file mode 100644 index 000000000..8fb1fac95 --- /dev/null +++ b/explorer/README.md @@ -0,0 +1,133 @@ +### explorer + +Explorer is a little container for examining the runtime environment Kubernetes produces for your pods. + +The intended use is to substitute gcr.io/google_containers/explorer for your intended container, and then visit it via the proxy. + +Currently, you can look at: + * The environment variables to make sure Kubernetes is doing what you expect. + * The filesystem to make sure the mounted volumes and files are also what you expect. + * Perform DNS lookups, to see how DNS works. + +`pod.yaml` is supplied as an example. You can control the port it serves on with the -port flag. + +Example from command line (the DNS lookup looks better from a web browser): + +```console +$ kubectl create -f examples/explorer/pod.yaml +$ kubectl proxy & +Starting to serve on localhost:8001 + +$ curl localhost:8001/api/v1/proxy/namespaces/default/pods/explorer:8080/vars/ +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +HOSTNAME=explorer +KIBANA_LOGGING_PORT_5601_TCP_PORT=5601 +KUBERNETES_SERVICE_HOST=10.0.0.2 +MONITORING_GRAFANA_PORT_80_TCP_PROTO=tcp +MONITORING_INFLUXDB_UI_PORT_80_TCP_PROTO=tcp +KIBANA_LOGGING_SERVICE_PORT=5601 +MONITORING_HEAPSTER_PORT_80_TCP_PORT=80 +MONITORING_INFLUXDB_UI_PORT_80_TCP_PORT=80 +KIBANA_LOGGING_SERVICE_HOST=10.0.204.206 +KIBANA_LOGGING_PORT_5601_TCP=tcp://10.0.204.206:5601 +KUBERNETES_PORT=tcp://10.0.0.2:443 +MONITORING_INFLUXDB_PORT=tcp://10.0.2.30:80 +MONITORING_INFLUXDB_PORT_80_TCP_PROTO=tcp +MONITORING_INFLUXDB_UI_PORT=tcp://10.0.36.78:80 +KUBE_DNS_PORT_53_UDP=udp://10.0.0.10:53 +MONITORING_INFLUXDB_SERVICE_HOST=10.0.2.30 +ELASTICSEARCH_LOGGING_PORT=tcp://10.0.48.200:9200 +ELASTICSEARCH_LOGGING_PORT_9200_TCP_PORT=9200 +KUBERNETES_PORT_443_TCP=tcp://10.0.0.2:443 +ELASTICSEARCH_LOGGING_PORT_9200_TCP_PROTO=tcp +KIBANA_LOGGING_PORT_5601_TCP_ADDR=10.0.204.206 +KUBE_DNS_PORT_53_UDP_ADDR=10.0.0.10 +MONITORING_HEAPSTER_PORT_80_TCP_PROTO=tcp +MONITORING_INFLUXDB_PORT_80_TCP_ADDR=10.0.2.30 +KIBANA_LOGGING_PORT=tcp://10.0.204.206:5601 +MONITORING_GRAFANA_SERVICE_PORT=80 +MONITORING_HEAPSTER_SERVICE_PORT=80 +MONITORING_HEAPSTER_PORT_80_TCP=tcp://10.0.150.238:80 +ELASTICSEARCH_LOGGING_PORT_9200_TCP=tcp://10.0.48.200:9200 +ELASTICSEARCH_LOGGING_PORT_9200_TCP_ADDR=10.0.48.200 +MONITORING_GRAFANA_PORT_80_TCP_PORT=80 +MONITORING_HEAPSTER_PORT=tcp://10.0.150.238:80 +MONITORING_INFLUXDB_PORT_80_TCP=tcp://10.0.2.30:80 +KUBE_DNS_SERVICE_PORT=53 +KUBE_DNS_PORT_53_UDP_PORT=53 +MONITORING_GRAFANA_PORT_80_TCP_ADDR=10.0.100.174 +MONITORING_INFLUXDB_UI_SERVICE_HOST=10.0.36.78 +KIBANA_LOGGING_PORT_5601_TCP_PROTO=tcp +MONITORING_GRAFANA_PORT=tcp://10.0.100.174:80 +MONITORING_INFLUXDB_UI_PORT_80_TCP_ADDR=10.0.36.78 +KUBE_DNS_SERVICE_HOST=10.0.0.10 +KUBERNETES_PORT_443_TCP_PORT=443 +MONITORING_HEAPSTER_PORT_80_TCP_ADDR=10.0.150.238 +MONITORING_INFLUXDB_UI_SERVICE_PORT=80 +KUBE_DNS_PORT=udp://10.0.0.10:53 +ELASTICSEARCH_LOGGING_SERVICE_HOST=10.0.48.200 +KUBERNETES_SERVICE_PORT=443 +MONITORING_HEAPSTER_SERVICE_HOST=10.0.150.238 +MONITORING_INFLUXDB_SERVICE_PORT=80 +MONITORING_INFLUXDB_PORT_80_TCP_PORT=80 +KUBE_DNS_PORT_53_UDP_PROTO=udp +MONITORING_GRAFANA_PORT_80_TCP=tcp://10.0.100.174:80 +ELASTICSEARCH_LOGGING_SERVICE_PORT=9200 +MONITORING_GRAFANA_SERVICE_HOST=10.0.100.174 +MONITORING_INFLUXDB_UI_PORT_80_TCP=tcp://10.0.36.78:80 +KUBERNETES_PORT_443_TCP_PROTO=tcp +KUBERNETES_PORT_443_TCP_ADDR=10.0.0.2 +HOME=/ + +$ curl localhost:8001/api/v1/proxy/namespaces/default/pods/explorer:8080/fs/ +mount/ +var/ +.dockerenv +etc/ +dev/ +proc/ +.dockerinit +sys/ +README.md +explorer + +$ curl localhost:8001/api/v1/proxy/namespaces/default/pods/explorer:8080/dns?q=elasticsearch-logging + +
+ + +
+

LookupNS(elasticsearch-logging):
+Result: ([]*net.NS)
+Error: <*>lookup elasticsearch-logging: no such host
+
+LookupTXT(elasticsearch-logging):
+Result: ([]string)
+Error: <*>lookup elasticsearch-logging: no such host
+
+LookupSRV("", "", elasticsearch-logging):
+cname: elasticsearch-logging.default.svc.cluster.local.
+Result: ([]*net.SRV)[<*>{Target:(string)elasticsearch-logging.default.svc.cluster.local. Port:(uint16)9200 Priority:(uint16)10 Weight:(uint16)100}]
+Error: 
+
+LookupHost(elasticsearch-logging):
+Result: ([]string)[10.0.60.245]
+Error: 
+
+LookupIP(elasticsearch-logging):
+Result: ([]net.IP)[10.0.60.245]
+Error: 
+
+LookupMX(elasticsearch-logging):
+Result: ([]*net.MX)
+Error: <*>lookup elasticsearch-logging: no such host
+
+
+ + +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/explorer/README.md?pixel)]() + diff --git a/explorer/explorer.go b/explorer/explorer.go new file mode 100644 index 000000000..cdba95f54 --- /dev/null +++ b/explorer/explorer.go @@ -0,0 +1,122 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A tiny web server for viewing the environment kubernetes creates for your +// containers. It exposes the filesystem and environment variables via http +// server. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + + "github.com/davecgh/go-spew/spew" +) + +var ( + port = flag.Int("port", 8080, "Port number to serve at.") +) + +func main() { + flag.Parse() + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("Error getting hostname: %v", err) + } + + links := []struct { + link, desc string + }{ + {"/fs/", "Complete file system as seen by this container."}, + {"/vars/", "Environment variables as seen by this container."}, + {"/hostname/", "Hostname as seen by this container."}, + {"/dns?q=google.com", "Explore DNS records seen by this container."}, + {"/quit", "Cause this container to exit."}, + } + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, " Kubernetes environment explorer

") + for _, v := range links { + fmt.Fprintf(w, `%v: %v
`, v.link, v.link, v.desc) + } + }) + + http.Handle("/fs/", http.StripPrefix("/fs/", http.FileServer(http.Dir("/")))) + http.HandleFunc("/vars/", func(w http.ResponseWriter, r *http.Request) { + for _, v := range os.Environ() { + fmt.Fprintf(w, "%v\n", v) + } + }) + http.HandleFunc("/hostname/", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, hostname) + }) + http.HandleFunc("/quit", func(w http.ResponseWriter, r *http.Request) { + os.Exit(0) + }) + http.HandleFunc("/dns", dns) + + go log.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", *port), nil)) + + select {} +} + +func dns(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + // Note that the below is NOT safe from input attacks, but that's OK + // because this is just for debugging. + fmt.Fprintf(w, ` +
+ + +
+

`, q)
+	{
+		res, err := net.LookupNS(q)
+		spew.Fprintf(w, "LookupNS(%v):\nResult: %#v\nError: %v\n\n", q, res, err)
+	}
+	{
+		res, err := net.LookupTXT(q)
+		spew.Fprintf(w, "LookupTXT(%v):\nResult: %#v\nError: %v\n\n", q, res, err)
+	}
+	{
+		cname, res, err := net.LookupSRV("", "", q)
+		spew.Fprintf(w, `LookupSRV("", "", %v):
+cname: %v
+Result: %#v
+Error: %v
+
+`, q, cname, res, err)
+	}
+	{
+		res, err := net.LookupHost(q)
+		spew.Fprintf(w, "LookupHost(%v):\nResult: %#v\nError: %v\n\n", q, res, err)
+	}
+	{
+		res, err := net.LookupIP(q)
+		spew.Fprintf(w, "LookupIP(%v):\nResult: %#v\nError: %v\n\n", q, res, err)
+	}
+	{
+		res, err := net.LookupMX(q)
+		spew.Fprintf(w, "LookupMX(%v):\nResult: %#v\nError: %v\n\n", q, res, err)
+	}
+	fmt.Fprintf(w, `
+ +`) +} diff --git a/explorer/pod.yaml b/explorer/pod.yaml new file mode 100644 index 000000000..2c26c3e17 --- /dev/null +++ b/explorer/pod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: explorer +spec: + containers: + - name: explorer + image: gcr.io/google_containers/explorer:1.0 + args: ["-port=8080"] + ports: + - containerPort: 8080 + protocol: TCP + volumeMounts: + - mountPath: "/mount/test-volume" + name: test-volume + volumes: + - name: test-volume + emptyDir: {} diff --git a/guestbook-go/.gitignore b/guestbook-go/.gitignore new file mode 100644 index 000000000..a45a95c2f --- /dev/null +++ b/guestbook-go/.gitignore @@ -0,0 +1 @@ +guestbook_bin diff --git a/guestbook-go/BUILD b/guestbook-go/BUILD new file mode 100644 index 000000000..c24a6d7bb --- /dev/null +++ b/guestbook-go/BUILD @@ -0,0 +1,39 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "guestbook-go", + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["main.go"], + tags = ["automanaged"], + deps = [ + "//vendor/github.com/codegangsta/negroni:go_default_library", + "//vendor/github.com/gorilla/mux:go_default_library", + "//vendor/github.com/xyproto/simpleredis:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/guestbook-go/Dockerfile b/guestbook-go/Dockerfile new file mode 100644 index 000000000..a58ebae38 --- /dev/null +++ b/guestbook-go/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM busybox:ubuntu-14.04 + +ADD ./guestbook_bin /app/guestbook +ADD ./public/index.html /app/public/index.html +ADD ./public/script.js /app/public/script.js +ADD ./public/style.css /app/public/style.css + +WORKDIR /app +CMD ["./guestbook"] +EXPOSE 3000 diff --git a/guestbook-go/Makefile b/guestbook-go/Makefile new file mode 100644 index 000000000..9c63819eb --- /dev/null +++ b/guestbook-go/Makefile @@ -0,0 +1,41 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the guestbook-go example + +# Usage: +# [VERSION=v3] [REGISTRY="gcr.io/google_containers"] make build +VERSION?=v3 +REGISTRY?=gcr.io/google_containers + +release: clean build push clean + +# builds a docker image that builds the app and packages it into a minimal docker image +build: + @cp ../../bazel-bin/examples/guestbook-go/guestbook-go guestbook_bin + docker build --pull --rm --force-rm -t ${REGISTRY}/guestbook-builder . + docker run --rm ${REGISTRY}/guestbook-builder | docker build --pull -t "${REGISTRY}/guestbook:${VERSION}" - + +# push the image to an registry +push: + gcloud docker -- push ${REGISTRY}/guestbook:${VERSION} + +# remove previous images and containers +clean: + rm -f guestbook_bin + docker rm -f ${REGISTRY}/guestbook-builder 2> /dev/null || true + docker rmi -f ${REGISTRY}/guestbook-builder || true + docker rmi -f "${REGISTRY}/guestbook:${VERSION}" || true + +.PHONY: release clean build push diff --git a/guestbook-go/README.md b/guestbook-go/README.md new file mode 100644 index 000000000..7382c4ea7 --- /dev/null +++ b/guestbook-go/README.md @@ -0,0 +1,271 @@ +## Guestbook Example + +This example shows how to build a simple multi-tier web application using Kubernetes and Docker. The application consists of a web front-end, Redis master for storage, and replicated set of Redis slaves, all for which we will create Kubernetes replication controllers, pods, and services. + +If you are running a cluster in Google Container Engine (GKE), instead see the [Guestbook Example for Google Container Engine](https://cloud.google.com/container-engine/docs/tutorials/guestbook). + +##### Table of Contents + + * [Step Zero: Prerequisites](#step-zero) + * [Step One: Create the Redis master pod](#step-one) + * [Step Two: Create the Redis master service](#step-two) + * [Step Three: Create the Redis slave pods](#step-three) + * [Step Four: Create the Redis slave service](#step-four) + * [Step Five: Create the guestbook pods](#step-five) + * [Step Six: Create the guestbook service](#step-six) + * [Step Seven: View the guestbook](#step-seven) + * [Step Eight: Cleanup](#step-eight) + +### Step Zero: Prerequisites + +This example assumes that you have a working cluster. See the [Getting Started Guides](../../docs/getting-started-guides/) for details about creating a cluster. + +**Tip:** View all the `kubectl` commands, including their options and descriptions in the [kubectl CLI reference](../../docs/user-guide/kubectl/kubectl.md). + +### Step One: Create the Redis master pod + +Use the `examples/guestbook-go/redis-master-controller.json` file to create a [replication controller](../../docs/user-guide/replication-controller.md) and Redis master [pod](../../docs/user-guide/pods.md). The pod runs a Redis key-value server in a container. Using a replication controller is the preferred way to launch long-running pods, even for 1 replica, so that the pod benefits from the self-healing mechanism in Kubernetes (keeps the pods alive). + +1. Use the [redis-master-controller.json](redis-master-controller.json) file to create the Redis master replication controller in your Kubernetes cluster by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/redis-master-controller.json + replicationcontrollers/redis-master + ``` + +2. To verify that the redis-master controller is up, list the replication controllers you created in the cluster with the `kubectl get rc` command(if you don't specify a `--namespace`, the `default` namespace will be used. The same below): + + ```console + $ kubectl get rc + CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS + redis-master redis-master gurpartap/redis app=redis,role=master 1 + ... + ``` + + Result: The replication controller then creates the single Redis master pod. + +3. To verify that the redis-master pod is running, list the pods you created in cluster with the `kubectl get pods` command: + + ```console + $ kubectl get pods + NAME READY STATUS RESTARTS AGE + redis-master-xx4uv 1/1 Running 0 1m + ... + ``` + + Result: You'll see a single Redis master pod and the machine where the pod is running after the pod gets placed (may take up to thirty seconds). + +4. To verify what containers are running in the redis-master pod, you can SSH to that machine with `gcloud compute ssh --zone` *`zone_name`* *`host_name`* and then run `docker ps`: + + ```console + me@workstation$ gcloud compute ssh --zone us-central1-b kubernetes-node-bz1p + + me@kubernetes-node-3:~$ sudo docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS + d5c458dabe50 redis "/entrypoint.sh redis" 5 minutes ago Up 5 minutes + ``` + + Note: The initial `docker pull` can take a few minutes, depending on network conditions. + +### Step Two: Create the Redis master service + +A Kubernetes [service](../../docs/user-guide/services.md) is a named load balancer that proxies traffic to one or more pods. The services in a Kubernetes cluster are discoverable inside other pods via environment variables or DNS. + +Services find the pods to load balance based on pod labels. The pod that you created in Step One has the label `app=redis` and `role=master`. The selector field of the service determines which pods will receive the traffic sent to the service. + +1. Use the [redis-master-service.json](redis-master-service.json) file to create the service in your Kubernetes cluster by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/redis-master-service.json + services/redis-master + ``` + +2. To verify that the redis-master service is up, list the services you created in the cluster with the `kubectl get services` command: + + ```console + $ kubectl get services + NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE + redis-master 10.0.136.3 6379/TCP app=redis,role=master 1h + ... + ``` + + Result: All new pods will see the `redis-master` service running on the host (`$REDIS_MASTER_SERVICE_HOST` environment variable) at port 6379, or running on `redis-master:6379`. After the service is created, the service proxy on each node is configured to set up a proxy on the specified port (in our example, that's port 6379). + + +### Step Three: Create the Redis slave pods + +The Redis master we created earlier is a single pod (REPLICAS = 1), while the Redis read slaves we are creating here are 'replicated' pods. In Kubernetes, a replication controller is responsible for managing the multiple instances of a replicated pod. + +1. Use the file [redis-slave-controller.json](redis-slave-controller.json) to create the replication controller by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/redis-slave-controller.json + replicationcontrollers/redis-slave + ``` + +2. To verify that the redis-slave controller is running, run the `kubectl get rc` command: + + ```console + $ kubectl get rc + CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS + redis-master redis-master redis app=redis,role=master 1 + redis-slave redis-slave kubernetes/redis-slave:v2 app=redis,role=slave 2 + ... + ``` + + Result: The replication controller creates and configures the Redis slave pods through the redis-master service (name:port pair, in our example that's `redis-master:6379`). + + Example: + The Redis slaves get started by the replication controller with the following command: + + ```console + redis-server --slaveof redis-master 6379 + ``` + +3. To verify that the Redis master and slaves pods are running, run the `kubectl get pods` command: + + ```console + $ kubectl get pods + NAME READY STATUS RESTARTS AGE + redis-master-xx4uv 1/1 Running 0 18m + redis-slave-b6wj4 1/1 Running 0 1m + redis-slave-iai40 1/1 Running 0 1m + ... + ``` + + Result: You see the single Redis master and two Redis slave pods. + +### Step Four: Create the Redis slave service + +Just like the master, we want to have a service to proxy connections to the read slaves. In this case, in addition to discovery, the Redis slave service provides transparent load balancing to clients. + +1. Use the [redis-slave-service.json](redis-slave-service.json) file to create the Redis slave service by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/redis-slave-service.json + services/redis-slave + ``` + +2. To verify that the redis-slave service is up, list the services you created in the cluster with the `kubectl get services` command: + + ```console + $ kubectl get services + NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE + redis-master 10.0.136.3 6379/TCP app=redis,role=master 1h + redis-slave 10.0.21.92 6379/TCP app-redis,role=slave 1h + ... + ``` + + Result: The service is created with labels `app=redis` and `role=slave` to identify that the pods are running the Redis slaves. + +Tip: It is helpful to set labels on your services themselves--as we've done here--to make it easy to locate them later. + +### Step Five: Create the guestbook pods + +This is a simple Go `net/http` ([negroni](https://github.com/codegangsta/negroni) based) server that is configured to talk to either the slave or master services depending on whether the request is a read or a write. The pods we are creating expose a simple JSON interface and serves a jQuery-Ajax based UI. Like the Redis read slaves, these pods are also managed by a replication controller. + +1. Use the [guestbook-controller.json](guestbook-controller.json) file to create the guestbook replication controller by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/guestbook-controller.json + replicationcontrollers/guestbook + ``` + + Tip: If you want to modify the guestbook code open the `_src` of this example and read the README.md and the Makefile. If you have pushed your custom image be sure to update the `image` accordingly in the guestbook-controller.json. + +2. To verify that the guestbook replication controller is running, run the `kubectl get rc` command: + + ```console + $ kubectl get rc + CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS + guestbook guestbook gcr.io/google_containers/guestbook:v3 app=guestbook 3 + redis-master redis-master redis app=redis,role=master 1 + redis-slave redis-slave kubernetes/redis-slave:v2 app=redis,role=slave 2 + ... + ``` + +3. To verify that the guestbook pods are running (it might take up to thirty seconds to create the pods), list the pods you created in cluster with the `kubectl get pods` command: + + ```console + $ kubectl get pods + NAME READY STATUS RESTARTS AGE + guestbook-3crgn 1/1 Running 0 2m + guestbook-gv7i6 1/1 Running 0 2m + guestbook-x405a 1/1 Running 0 2m + redis-master-xx4uv 1/1 Running 0 23m + redis-slave-b6wj4 1/1 Running 0 6m + redis-slave-iai40 1/1 Running 0 6m + ... + ``` + + Result: You see a single Redis master, two Redis slaves, and three guestbook pods. + +### Step Six: Create the guestbook service + +Just like the others, we create a service to group the guestbook pods but this time, to make the guestbook front-end externally visible, we specify `"type": "LoadBalancer"`. + +1. Use the [guestbook-service.json](guestbook-service.json) file to create the guestbook service by running the `kubectl create -f` *`filename`* command: + + ```console + $ kubectl create -f examples/guestbook-go/guestbook-service.json + ``` + + +2. To verify that the guestbook service is up, list the services you created in the cluster with the `kubectl get services` command: + + ```console + $ kubectl get services + NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE + guestbook 10.0.217.218 146.148.81.8 3000/TCP app=guestbook 1h + redis-master 10.0.136.3 6379/TCP app=redis,role=master 1h + redis-slave 10.0.21.92 6379/TCP app-redis,role=slave 1h + ... + ``` + + Result: The service is created with label `app=guestbook`. + +### Step Seven: View the guestbook + +You can now play with the guestbook that you just created by opening it in a browser (it might take a few moments for the guestbook to come up). + + * **Local Host:** + If you are running Kubernetes locally, to view the guestbook, navigate to `http://localhost:3000` in your browser. + + * **Remote Host:** + 1. To view the guestbook on a remote host, locate the external IP of the load balancer in the **IP** column of the `kubectl get services` output. In our example, the internal IP address is `10.0.217.218` and the external IP address is `146.148.81.8` (*Note: you might need to scroll to see the IP column*). + + 2. Append port `3000` to the IP address (for example `http://146.148.81.8:3000`), and then navigate to that address in your browser. + + Result: The guestbook displays in your browser: + + ![Guestbook](guestbook-page.png) + + **Further Reading:** + If you're using Google Compute Engine, see the details about limiting traffic to specific sources at [Google Compute Engine firewall documentation][gce-firewall-docs]. + +[cloud-console]: https://console.developer.google.com +[gce-firewall-docs]: https://cloud.google.com/compute/docs/networking#firewalls + +### Step Eight: Cleanup + +After you're done playing with the guestbook, you can cleanup by deleting the guestbook service and removing the associated resources that were created, including load balancers, forwarding rules, target pools, and Kubernetes replication controllers and services. + +Delete all the resources by running the following `kubectl delete -f` *`filename`* command: + +```console +$ kubectl delete -f examples/guestbook-go +guestbook-controller +guestbook +redid-master-controller +redis-master +redis-slave-controller +redis-slave +``` + +Tip: To turn down your Kubernetes cluster, follow the corresponding instructions in the version of the +[Getting Started Guides](../../docs/getting-started-guides/) that you previously used to create your cluster. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/guestbook-go/README.md?pixel)]() + diff --git a/guestbook-go/guestbook-controller.json b/guestbook-go/guestbook-controller.json new file mode 100644 index 000000000..82c0e9134 --- /dev/null +++ b/guestbook-go/guestbook-controller.json @@ -0,0 +1,37 @@ +{ + "kind":"ReplicationController", + "apiVersion":"v1", + "metadata":{ + "name":"guestbook", + "labels":{ + "app":"guestbook" + } + }, + "spec":{ + "replicas":3, + "selector":{ + "app":"guestbook" + }, + "template":{ + "metadata":{ + "labels":{ + "app":"guestbook" + } + }, + "spec":{ + "containers":[ + { + "name":"guestbook", + "image":"gcr.io/google_containers/guestbook:v3", + "ports":[ + { + "name":"http-server", + "containerPort":3000 + } + ] + } + ] + } + } + } +} diff --git a/guestbook-go/guestbook-page.png b/guestbook-go/guestbook-page.png new file mode 100644 index 000000000..776835fd2 Binary files /dev/null and b/guestbook-go/guestbook-page.png differ diff --git a/guestbook-go/guestbook-service.json b/guestbook-go/guestbook-service.json new file mode 100644 index 000000000..cc7640e4c --- /dev/null +++ b/guestbook-go/guestbook-service.json @@ -0,0 +1,22 @@ +{ + "kind":"Service", + "apiVersion":"v1", + "metadata":{ + "name":"guestbook", + "labels":{ + "app":"guestbook" + } + }, + "spec":{ + "ports": [ + { + "port":3000, + "targetPort":"http-server" + } + ], + "selector":{ + "app":"guestbook" + }, + "type": "LoadBalancer" + } +} diff --git a/guestbook-go/main.go b/guestbook-go/main.go new file mode 100644 index 000000000..c6e8cf187 --- /dev/null +++ b/guestbook-go/main.go @@ -0,0 +1,91 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "net/http" + "os" + "strings" + + "github.com/codegangsta/negroni" + "github.com/gorilla/mux" + "github.com/xyproto/simpleredis" +) + +var ( + masterPool *simpleredis.ConnectionPool + slavePool *simpleredis.ConnectionPool +) + +func ListRangeHandler(rw http.ResponseWriter, req *http.Request) { + key := mux.Vars(req)["key"] + list := simpleredis.NewList(slavePool, key) + members := HandleError(list.GetAll()).([]string) + membersJSON := HandleError(json.MarshalIndent(members, "", " ")).([]byte) + rw.Write(membersJSON) +} + +func ListPushHandler(rw http.ResponseWriter, req *http.Request) { + key := mux.Vars(req)["key"] + value := mux.Vars(req)["value"] + list := simpleredis.NewList(masterPool, key) + HandleError(nil, list.Add(value)) + ListRangeHandler(rw, req) +} + +func InfoHandler(rw http.ResponseWriter, req *http.Request) { + info := HandleError(masterPool.Get(0).Do("INFO")).([]byte) + rw.Write(info) +} + +func EnvHandler(rw http.ResponseWriter, req *http.Request) { + environment := make(map[string]string) + for _, item := range os.Environ() { + splits := strings.Split(item, "=") + key := splits[0] + val := strings.Join(splits[1:], "=") + environment[key] = val + } + + envJSON := HandleError(json.MarshalIndent(environment, "", " ")).([]byte) + rw.Write(envJSON) +} + +func HandleError(result interface{}, err error) (r interface{}) { + if err != nil { + panic(err) + } + return result +} + +func main() { + masterPool = simpleredis.NewConnectionPoolHost("redis-master:6379") + defer masterPool.Close() + slavePool = simpleredis.NewConnectionPoolHost("redis-slave:6379") + defer slavePool.Close() + + r := mux.NewRouter() + r.Path("/lrange/{key}").Methods("GET").HandlerFunc(ListRangeHandler) + r.Path("/rpush/{key}/{value}").Methods("GET").HandlerFunc(ListPushHandler) + r.Path("/info").Methods("GET").HandlerFunc(InfoHandler) + r.Path("/env").Methods("GET").HandlerFunc(EnvHandler) + + n := negroni.Classic() + n.UseHandler(r) + n.Run(":3000") +} diff --git a/guestbook-go/public/index.html b/guestbook-go/public/index.html new file mode 100644 index 000000000..f525f4b76 --- /dev/null +++ b/guestbook-go/public/index.html @@ -0,0 +1,34 @@ + + + + + + + + Guestbook + + + + +
+

Waiting for database connection...

+
+ +
+
+ + Submit +
+
+ +
+

+

/env + /info

+
+ + + + diff --git a/guestbook-go/public/script.js b/guestbook-go/public/script.js new file mode 100644 index 000000000..a0a545b05 --- /dev/null +++ b/guestbook-go/public/script.js @@ -0,0 +1,46 @@ +$(document).ready(function() { + var headerTitleElement = $("#header h1"); + var entriesElement = $("#guestbook-entries"); + var formElement = $("#guestbook-form"); + var submitElement = $("#guestbook-submit"); + var entryContentElement = $("#guestbook-entry-content"); + var hostAddressElement = $("#guestbook-host-address"); + + var appendGuestbookEntries = function(data) { + entriesElement.empty(); + $.each(data, function(key, val) { + entriesElement.append("

" + val + "

"); + }); + } + + var handleSubmission = function(e) { + e.preventDefault(); + var entryValue = entryContentElement.val() + if (entryValue.length > 0) { + entriesElement.append("

...

"); + $.getJSON("rpush/guestbook/" + entryValue, appendGuestbookEntries); + } + return false; + } + + // colors = purple, blue, red, green, yellow + var colors = ["#549", "#18d", "#d31", "#2a4", "#db1"]; + var randomColor = colors[Math.floor(5 * Math.random())]; + (function setElementsColor(color) { + headerTitleElement.css("color", color); + entryContentElement.css("box-shadow", "inset 0 0 0 2px " + color); + submitElement.css("background-color", color); + })(randomColor); + + submitElement.click(handleSubmission); + formElement.submit(handleSubmission); + hostAddressElement.append(document.URL); + + // Poll every second. + (function fetchGuestbook() { + $.getJSON("lrange/guestbook").done(appendGuestbookEntries).always( + function() { + setTimeout(fetchGuestbook, 1000); + }); + })(); +}); diff --git a/guestbook-go/public/style.css b/guestbook-go/public/style.css new file mode 100644 index 000000000..fd1c393fb --- /dev/null +++ b/guestbook-go/public/style.css @@ -0,0 +1,61 @@ +body, input { + color: #123; + font-family: "Gill Sans", sans-serif; +} + +div { + overflow: hidden; + padding: 1em 0; + position: relative; + text-align: center; +} + +h1, h2, p, input, a { + font-weight: 300; + margin: 0; +} + +h1 { + color: #BDB76B; + font-size: 3.5em; +} + +h2 { + color: #999; +} + +form { + margin: 0 auto; + max-width: 50em; + text-align: center; +} + +input { + border: 0; + border-radius: 1000px; + box-shadow: inset 0 0 0 2px #BDB76B; + display: inline; + font-size: 1.5em; + margin-bottom: 1em; + outline: none; + padding: .5em 5%; + width: 55%; +} + +form a { + background: #BDB76B; + border: 0; + border-radius: 1000px; + color: #FFF; + font-size: 1.25em; + font-weight: 400; + padding: .75em 2em; + text-decoration: none; + text-transform: uppercase; + white-space: normal; +} + +p { + font-size: 1.5em; + line-height: 1.5; +} diff --git a/guestbook-go/redis-master-controller.json b/guestbook-go/redis-master-controller.json new file mode 100644 index 000000000..4ffe53a72 --- /dev/null +++ b/guestbook-go/redis-master-controller.json @@ -0,0 +1,40 @@ +{ + "kind":"ReplicationController", + "apiVersion":"v1", + "metadata":{ + "name":"redis-master", + "labels":{ + "app":"redis", + "role":"master" + } + }, + "spec":{ + "replicas":1, + "selector":{ + "app":"redis", + "role":"master" + }, + "template":{ + "metadata":{ + "labels":{ + "app":"redis", + "role":"master" + } + }, + "spec":{ + "containers":[ + { + "name":"redis-master", + "image":"redis:2.8.23", + "ports":[ + { + "name":"redis-server", + "containerPort":6379 + } + ] + } + ] + } + } + } +} diff --git a/guestbook-go/redis-master-service.json b/guestbook-go/redis-master-service.json new file mode 100644 index 000000000..3a7426ead --- /dev/null +++ b/guestbook-go/redis-master-service.json @@ -0,0 +1,23 @@ +{ + "kind":"Service", + "apiVersion":"v1", + "metadata":{ + "name":"redis-master", + "labels":{ + "app":"redis", + "role":"master" + } + }, + "spec":{ + "ports": [ + { + "port":6379, + "targetPort":"redis-server" + } + ], + "selector":{ + "app":"redis", + "role":"master" + } + } +} diff --git a/guestbook-go/redis-slave-controller.json b/guestbook-go/redis-slave-controller.json new file mode 100644 index 000000000..eec652afa --- /dev/null +++ b/guestbook-go/redis-slave-controller.json @@ -0,0 +1,40 @@ +{ + "kind":"ReplicationController", + "apiVersion":"v1", + "metadata":{ + "name":"redis-slave", + "labels":{ + "app":"redis", + "role":"slave" + } + }, + "spec":{ + "replicas":2, + "selector":{ + "app":"redis", + "role":"slave" + }, + "template":{ + "metadata":{ + "labels":{ + "app":"redis", + "role":"slave" + } + }, + "spec":{ + "containers":[ + { + "name":"redis-slave", + "image":"kubernetes/redis-slave:v2", + "ports":[ + { + "name":"redis-server", + "containerPort":6379 + } + ] + } + ] + } + } + } +} diff --git a/guestbook-go/redis-slave-service.json b/guestbook-go/redis-slave-service.json new file mode 100644 index 000000000..7e8f49a6a --- /dev/null +++ b/guestbook-go/redis-slave-service.json @@ -0,0 +1,23 @@ +{ + "kind":"Service", + "apiVersion":"v1", + "metadata":{ + "name":"redis-slave", + "labels":{ + "app":"redis", + "role":"slave" + } + }, + "spec":{ + "ports": [ + { + "port":6379, + "targetPort":"redis-server" + } + ], + "selector":{ + "app":"redis", + "role":"slave" + } + } +} diff --git a/guestbook/README.md b/guestbook/README.md new file mode 100644 index 000000000..e8be154da --- /dev/null +++ b/guestbook/README.md @@ -0,0 +1,702 @@ + +## Guestbook Example + +This example shows how to build a simple, multi-tier web application using Kubernetes and [Docker](https://www.docker.com/). + +**Table of Contents** + + + - [Guestbook Example](#guestbook-example) + - [Prerequisites](#prerequisites) + - [Quick Start](#quick-start) + - [Step One: Start up the redis master](#step-one-start-up-the-redis-master) + - [Define a Deployment](#define-a-deployment) + - [Define a Service](#define-a-service) + - [Create a Service](#create-a-service) + - [Finding a Service](#finding-a-service) + - [Environment variables](#environment-variables) + - [DNS service](#dns-service) + - [Create a Deployment](#create-a-deployment) + - [Optional Interlude](#optional-interlude) + - [Step Two: Start up the redis slave](#step-two-start-up-the-redis-slave) + - [Step Three: Start up the guestbook frontend](#step-three-start-up-the-guestbook-frontend) + - [Using 'type: LoadBalancer' for the frontend service (cloud-provider-specific)](#using-type-loadbalancer-for-the-frontend-service-cloud-provider-specific) + - [Step Four: Cleanup](#step-four-cleanup) + - [Troubleshooting](#troubleshooting) + - [Appendix: Accessing the guestbook site externally](#appendix-accessing-the-guestbook-site-externally) + - [Google Compute Engine External Load Balancer Specifics](#google-compute-engine-external-load-balancer-specifics) + + + +The example consists of: + +- A web frontend +- A [redis](http://redis.io/) master (for storage), and a replicated set of redis 'slaves'. + +The web frontend interacts with the redis master via javascript redis API calls. + +**Note**: If you are running this example on a [Google Container Engine](https://cloud.google.com/container-engine/) installation, see [this Google Container Engine guestbook walkthrough](https://cloud.google.com/container-engine/docs/tutorials/guestbook) instead. The basic concepts are the same, but the walkthrough is tailored to a Container Engine setup. + +### Prerequisites + +This example requires a running Kubernetes cluster. First, check that kubectl is properly configured by getting the cluster state: + +```console +$ kubectl cluster-info +``` + +If you see a url response, you are ready to go. If not, read the [Getting Started guides](http://kubernetes.io/docs/getting-started-guides/) for how to get started, and follow the [prerequisites](http://kubernetes.io/docs/user-guide/prereqs/) to install and configure `kubectl`. As noted above, if you have a Google Container Engine cluster set up, read [this example](https://cloud.google.com/container-engine/docs/tutorials/guestbook) instead. + +All the files referenced in this example can be downloaded in [current folder](./). + +### Quick Start + +This section shows the simplest way to get the example work. If you want to know the details, you should skip this and read [the rest of the example](#step-one-start-up-the-redis-master). + +Start the guestbook with one command: + +```console +$ kubectl create -f examples/guestbook/all-in-one/guestbook-all-in-one.yaml +service "redis-master" created +deployment "redis-master" created +service "redis-slave" created +deployment "redis-slave" created +service "frontend" created +deployment "frontend" created +``` + +Alternatively, you can start the guestbook by running: + +```console +$ kubectl create -f examples/guestbook/ +``` + +Then, list all your Services: + +```console +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +frontend 10.0.0.117 80/TCP 20s +redis-master 10.0.0.170 6379/TCP 20s +redis-slave 10.0.0.201 6379/TCP 20s +``` + +Now you can access the guestbook on each node with frontend Service's `:`, e.g. `10.0.0.117:80` in this guide. `` is a cluster-internal IP. If you want to access the guestbook from outside of the cluster, add `type: NodePort` to the frontend Service `spec` field. Then you can access the guestbook with `:NodePort` from outside of the cluster. On cloud providers which support external load balancers, adding `type: LoadBalancer` to the frontend Service `spec` field will provision a load balancer for your Service. There are several ways for you to access the guestbook. You may learn from [Accessing services running on the cluster](https://kubernetes.io/docs/concepts/cluster-administration/access-cluster/#accessing-services-running-on-the-cluster). + +Clean up the guestbook: + +```console +$ kubectl delete -f examples/guestbook/all-in-one/guestbook-all-in-one.yaml +``` + +or + +```console +$ kubectl delete -f examples/guestbook/ +``` + + +### Step One: Start up the redis master + +Before continuing to the gory details, we also recommend you to read Kubernetes [concepts and user guide](http://kubernetes.io/docs/user-guide/). +**Note**: The redis master in this example is *not* highly available. Making it highly available would be an interesting, but intricate exercise — redis doesn't actually support multi-master Deployments at this point in time, so high availability would be a somewhat tricky thing to implement, and might involve periodic serialization to disk, and so on. + +#### Define a Deployment + +To start the redis master, use the file [redis-master-deployment.yaml](redis-master-deployment.yaml), which describes a single [pod](http://kubernetes.io/docs/user-guide/pods/) running a redis key-value server in a container. + +Although we have a single instance of our redis master, we are using a [Deployment](http://kubernetes.io/docs/user-guide/deployments/) to enforce that exactly one pod keeps running. E.g., if the node were to go down, the Deployment will ensure that the redis master gets restarted on a healthy node. (In our simplified example, this could result in data loss.) + +The file [redis-master-deployment.yaml](redis-master-deployment.yaml) defines the redis master Deployment: + + + +```yaml +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-master + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: master + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 1 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: master + # tier: backend + template: + metadata: + labels: + app: redis + role: master + tier: backend + spec: + containers: + - name: master + image: gcr.io/google_containers/redis:e2e # or just image: redis + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 +``` + +[Download example](redis-master-deployment.yaml?raw=true) + + +#### Define a Service + +A Kubernetes [Service](http://kubernetes.io/docs/user-guide/services/) is a named load balancer that proxies traffic to one or more containers. This is done using the [labels](http://kubernetes.io/docs/user-guide/labels/) metadata that we defined in the `redis-master` pod above. As mentioned, we have only one redis master, but we nevertheless want to create a Service for it. Why? Because it gives us a deterministic way to route to the single master using an elastic IP. + +Services find the pods to load balance based on the pods' labels. +The selector field of the Service description determines which pods will receive the traffic sent to the Service, and the `port` and `targetPort` information defines what port the Service proxy will run at. + +The file [redis-master-service.yaml](redis-master-deployment.yaml) defines the redis master Service: + + + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: redis-master + labels: + app: redis + role: master + tier: backend +spec: + ports: + # the port that this service should serve on + - port: 6379 + targetPort: 6379 + selector: + app: redis + role: master + tier: backend +``` + +[Download example](redis-master-service.yaml?raw=true) + + +#### Create a Service + +According to the [config best practices](http://kubernetes.io/docs/user-guide/config-best-practices/), create a Service before corresponding Deployments so that the scheduler can spread the pods comprising the Service. So we first create the Service by running: + +```console +$ kubectl create -f examples/guestbook/redis-master-service.yaml +service "redis-master" created +``` + +Then check the list of services, which should include the redis-master: + +```console +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +redis-master 10.0.76.248 6379/TCP 1s +``` + +This will cause all pods to see the redis master apparently running on `:`. A Service can map an incoming port to any `targetPort` in the backend pod. Once created, the Service proxy on each node is configured to set up a proxy on the specified port (in this case port `6379`). + +`targetPort` will default to `port` if it is omitted in the configuration. `targetPort` is the port the container accepts traffic on, and `port` is the abstracted Service port, which can be any port other pods use to access the Service. For simplicity's sake, we omit it in the following configurations. + +The traffic flow from slaves to masters can be described in two steps: + + - A *redis slave* will connect to `port` on the *redis master Service* + - Traffic will be forwarded from the Service `port` (on the Service node) to the `targetPort` on the pod that the Service listens to. + +For more details, please see [Connecting applications](http://kubernetes.io/docs/user-guide/connecting-applications/). + +#### Finding a Service + +Kubernetes supports two primary modes of finding a Service — environment variables and DNS. + + +##### Environment variables + +The services in a Kubernetes cluster are discoverable inside other containers via [environment variables](https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables). + +##### DNS service + +An alternative is to use the [cluster's DNS service](https://kubernetes.io/docs/concepts/services-networking/service/#dns), if it has been enabled for the cluster. This lets all pods do name resolution of services automatically, based on the Service name. + +This example has been configured to use the DNS service by default. + +If your cluster does not have the DNS service enabled, then you can use environment variables by setting the +`GET_HOSTS_FROM` env value in both +[redis-slave-deployment.yaml](redis-slave-deployment.yaml) and [frontend-deployment.yaml](frontend-deployment.yaml) +from `dns` to `env` before you start up the app. +(However, this is unlikely to be necessary. You can check for the DNS service in the list of the cluster's services by +running `kubectl --namespace=kube-system get rc -l k8s-app=kube-dns`.) +Note that switching to env causes creation-order dependencies, since Services need to be created before their clients that require env vars. + +#### Create a Deployment + +Second, create the redis master pod in your Kubernetes cluster by running: + +```console +$ kubectl create -f examples/guestbook/redis-master-deployment.yaml +deployment "redis-master" created +``` + +You can see the Deployment for your cluster by running: + +```console +$ kubectl get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +redis-master 1 1 1 1 27s +``` + +Then, you can list the pods in the cluster, to verify that the master is running: + +```console +$ kubectl get pods +``` + +You'll see all pods in the cluster, including the redis master pod, and the status of each pod. +The name of the redis master will look similar to that in the following list: + +```console +NAME READY STATUS RESTARTS AGE +redis-master-2353460263-1ecey 1/1 Running 0 1m +... +``` + +(Note that an initial `docker pull` to grab a container image may take a few minutes, depending on network conditions. A pod will be reported as `Pending` while its image is being downloaded.) + +`kubectl get pods` will show only the pods in the default [namespace](http://kubernetes.io/docs/user-guide/namespaces/). To see pods in all namespaces, run: + +``` +kubectl get pods --all-namespaces +``` + +For more details, please see [Configuring containers](http://kubernetes.io/docs/user-guide/configuring-containers/) and [Deploying applications](http://kubernetes.io/docs/user-guide/deploying-applications/). + +#### Optional Interlude + +You can get information about a pod, including the machine that it is running on, via `kubectl describe pods/`. E.g., for the redis master, you should see something like the following (your pod name will be different): + +```console +$ kubectl describe pods redis-master-2353460263-1ecey +Name: redis-master-2353460263-1ecey +Node: kubernetes-node-m0k7/10.240.0.5 +... +Labels: app=redis,pod-template-hash=2353460263,role=master,tier=backend +Status: Running +IP: 10.244.2.3 +Controllers: ReplicaSet/redis-master-2353460263 +Containers: + master: + Container ID: docker://76cf8115485966131587958ea3cbe363e2e1dcce129e2e624883f393ce256f6c + Image: gcr.io/google_containers/redis:e2e + Image ID: docker://e5f6c5a2b5646828f51e8e0d30a2987df7e8183ab2c3ed0ca19eaa03cc5db08c + Port: 6379/TCP +... +``` + +The `Node` is the name and IP of the machine, e.g. `kubernetes-node-m0k7` in the example above. You can find more details about this node with `kubectl describe nodes kubernetes-node-m0k7`. + +If you want to view the container logs for a given pod, you can run: + +```console +$ kubectl logs +``` + +These logs will usually give you enough information to troubleshoot. + +However, if you should want to SSH to the listed host machine, you can inspect various logs there directly as well. For example, with Google Compute Engine, using `gcloud`, you can SSH like this: + +```console +me@workstation$ gcloud compute ssh +``` + +Then, you can look at the Docker containers on the remote machine. You should see something like this (the specifics of the IDs will be different): + +```console +me@kubernetes-node-krxw:~$ sudo docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +... +0ffef9649265 redis:latest "/entrypoint.sh redi" About a minute ago Up About a minute k8s_master.869d22f3_redis-master-dz33o_default_1449a58a-5ead-11e5-a104-688f84ef8ef6_d74cb2b5 +``` + +If you want to see the logs for a given container, you can run: + +```console +$ docker logs +``` + +### Step Two: Start up the redis slave + +Now that the redis master is running, we can start up its 'read slaves'. + +We'll define these as replicated pods as well, though this time — unlike for the redis master — we'll define the number of replicas to be 2. +In Kubernetes, a Deployment is responsible for managing multiple instances of a replicated pod. The Deployment will automatically launch new pods if the number of replicas falls below the specified number. +(This particular replicated pod is a great one to test this with -- you can try killing the Docker processes for your pods directly, then watch them come back online on a new node shortly thereafter.) + +Just like the master, we want to have a Service to proxy connections to the redis slaves. In this case, in addition to discovery, the slave Service will provide transparent load balancing to web app clients. + +This time we put the Service and Deployment into one [file](http://kubernetes.io/docs/user-guide/managing-deployments/#organizing-resource-configurations). Grouping related objects together in a single file is often better than having separate files. +The specification for the slaves is in [all-in-one/redis-slave.yaml](all-in-one/redis-slave.yaml): + + + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: redis + role: slave + tier: backend +spec: + ports: + # the port that this service should serve on + - port: 6379 + selector: + app: redis + role: slave + tier: backend +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: slave + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 +``` + +[Download example](all-in-one/redis-slave.yaml?raw=true) + + +This time the selector for the Service is `app=redis,role=slave,tier=backend`, because that identifies the pods running redis slaves. It is generally helpful to set labels on your Service itself as we've done here to make it easy to locate them with the `kubectl get services -l "app=redis,role=slave,tier=backend"` command. For more information on the usage of labels, see [using-labels-effectively](http://kubernetes.io/docs/user-guide/managing-deployments/#using-labels-effectively). + +Now that you have created the specification, create the Service in your cluster by running: + +```console +$ kubectl create -f examples/guestbook/all-in-one/redis-slave.yaml +service "redis-slave" created +deployment "redis-slave" created + +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +redis-master 10.0.76.248 6379/TCP 20m +redis-slave 10.0.112.188 6379/TCP 16s + +$ kubectl get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +redis-master 1 1 1 1 22m +redis-slave 2 2 2 2 2m +``` + +Once the Deployment is up, you can list the pods in the cluster, to verify that the master and slaves are running. You should see a list that includes something like the following: + +```console +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +redis-master-2353460263-1ecey 1/1 Running 0 35m +redis-slave-1691881626-dlf5f 1/1 Running 0 15m +redis-slave-1691881626-sfn8t 1/1 Running 0 15m +``` + +You should see a single redis master pod and two redis slave pods. As mentioned above, you can get more information about any pod with: `kubectl describe pods/`. And also can view the resources on [kube-ui](http://kubernetes.io/docs/user-guide/ui/). + +### Step Three: Start up the guestbook frontend + +A frontend pod is a simple PHP server that is configured to talk to either the slave or master services, depending on whether the client request is a read or a write. It exposes a simple AJAX interface, and serves an Angular-based UX. +Again we'll create a set of replicated frontend pods instantiated by a Deployment — this time, with three replicas. + +As with the other pods, we now want to create a Service to group the frontend pods. +The Deployment and Service are described in the file [all-in-one/frontend.yaml](all-in-one/frontend.yaml): + + + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: frontend + labels: + app: guestbook + tier: frontend +spec: + # if your cluster supports it, uncomment the following to automatically create + # an external load-balanced IP for the frontend service. + # type: LoadBalancer + ports: + # the port that this service should serve on + - port: 80 + selector: + app: guestbook + tier: frontend +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google-samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 +``` + +[Download example](all-in-one/frontend.yaml?raw=true) + + +#### Using 'type: LoadBalancer' for the frontend service (cloud-provider-specific) + +For supported cloud providers, such as Google Compute Engine or Google Container Engine, you can specify to use an external load balancer +in the service `spec`, to expose the service onto an external load balancer IP. +To do this, uncomment the `type: LoadBalancer` line in the [all-in-one/frontend.yaml](all-in-one/frontend.yaml) file before you start the service. + +[See the appendix below](#appendix-accessing-the-guestbook-site-externally) on accessing the guestbook site externally for more details. + +Create the service and Deployment like this: + +```console +$ kubectl create -f examples/guestbook/all-in-one/frontend.yaml +service "frontend" created +deployment "frontend" created +``` + +Then, list all your services again: + +```console +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +frontend 10.0.63.63 80/TCP 1m +redis-master 10.0.76.248 6379/TCP 39m +redis-slave 10.0.112.188 6379/TCP 19m +``` + +Also list all your Deployments: + +```console +$ kubectl get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +frontend 3 3 3 3 2m +redis-master 1 1 1 1 39m +redis-slave 2 2 2 2 20m +``` + +Once it's up, i.e. when desired replicas match current replicas (again, it may take up to thirty seconds to create the pods), you can list the pods with specified labels in the cluster, to verify that the master, slaves and frontends are all running. You should see a list containing pods with label 'tier' like the following: + +```console +$ kubectl get pods -L tier +NAME READY STATUS RESTARTS AGE TIER +frontend-1211764471-4e1j2 1/1 Running 0 4m frontend +frontend-1211764471-gkbkv 1/1 Running 0 4m frontend +frontend-1211764471-rk1cf 1/1 Running 0 4m frontend +redis-master-2353460263-1ecey 1/1 Running 0 42m backend +redis-slave-1691881626-dlf5f 1/1 Running 0 22m backend +redis-slave-1691881626-sfn8t 1/1 Running 0 22m backend +``` + +You should see a single redis master pod, two redis slaves, and three frontend pods. + +The code for the PHP server that the frontends are running is in `examples/guestbook/php-redis/guestbook.php`. It looks like this: + +```php + 'tcp', + 'host' => $host, + 'port' => 6379, + ]); + + $client->set($_GET['key'], $_GET['value']); + print('{"message": "Updated"}'); + } else { + $host = 'redis-slave'; + if (getenv('GET_HOSTS_FROM') == 'env') { + $host = getenv('REDIS_SLAVE_SERVICE_HOST'); + } + $client = new Predis\Client([ + 'scheme' => 'tcp', + 'host' => $host, + 'port' => 6379, + ]); + + $value = $client->get($_GET['key']); + print('{"data": "' . $value . '"}'); + } +} else { + phpinfo(); +} ?> +``` + +Note the use of the `redis-master` and `redis-slave` host names -- we're finding those Services via the Kubernetes cluster's DNS service, as discussed above. All the frontend replicas will write to the load-balancing redis-slaves service, which can be highly replicated as well. + +### Step Four: Cleanup + +If you are in a live Kubernetes cluster, you can just kill the pods by deleting the Deployments and Services. Using labels to select the resources to delete is an easy way to do this in one command. + +```console +$ kubectl delete deployments,services -l "app in (redis, guestbook)" +``` + +To completely tear down a Kubernetes cluster, if you ran this from source, you can use: + +```console +$ /cluster/kube-down.sh +``` + +### Troubleshooting + +If you are having trouble bringing up your guestbook app, double check that your external IP is properly defined for your frontend Service, and that the firewall for your cluster nodes is open to port 80. + +Then, see the [troubleshooting documentation](http://kubernetes.io/docs/troubleshooting/) for a further list of common issues and how you can diagnose them. + + + +### Appendix: Accessing the guestbook site externally + +You'll want to set up your guestbook Service so that it can be accessed from outside of the internal Kubernetes network. Above, we introduced one way to do that, by setting `type: LoadBalancer` to Service `spec`. + +More generally, Kubernetes supports two ways of exposing a Service onto an external IP address: `NodePort`s and `LoadBalancer`s , as described [here](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types). + +If the `LoadBalancer` specification is used, it can take a short period for an external IP to show up in `kubectl get services` output, but you should then see it listed as well, e.g. like this: + +```console +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +frontend 10.0.63.63 23.236.59.54 80/TCP 1m +redis-master 10.0.76.248 6379/TCP 39m +redis-slave 10.0.112.188 6379/TCP 19m +``` + +Once you've exposed the service to an external IP, visit the IP to see your guestbook in action, i.e. `http://:`. + +You should see a web page that looks something like this (without the messages). Try adding some entries to it! + + + +If you are more advanced in the ops arena, you can also manually get the service IP from looking at the output of `kubectl get pods,services`, and modify your firewall using standard tools and services (firewalld, iptables, selinux) which you are already familiar with. + +#### Google Compute Engine External Load Balancer Specifics + +In Google Compute Engine, Kubernetes automatically creates forwarding rules for services with `LoadBalancer`. + +You can list the forwarding rules like this (the forwarding rule also indicates the external IP): + +```console +$ gcloud compute forwarding-rules list +NAME REGION IP_ADDRESS IP_PROTOCOL TARGET +frontend us-central1 130.211.188.51 TCP us-central1/targetPools/frontend +``` + +In Google Compute Engine, you also may need to open the firewall for port 80 using the [console][cloud-console] or the `gcloud` tool. The following command will allow traffic from any source to instances tagged `kubernetes-node` (replace with your tags as appropriate): + +```console +$ gcloud compute firewall-rules create --allow=tcp:80 --target-tags=kubernetes-node kubernetes-node-80 +``` + +For GCE Kubernetes startup details, see the [Getting started on Google Compute Engine](http://kubernetes.io/docs/getting-started-guides/gce/) + +For Google Compute Engine details about limiting traffic to specific sources, see the [Google Compute Engine firewall documentation][gce-firewall-docs]. + +[cloud-console]: https://console.developer.google.com +[gce-firewall-docs]: https://cloud.google.com/compute/docs/networking#firewalls + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/guestbook/README.md?pixel)]() + diff --git a/guestbook/all-in-one/frontend.yaml b/guestbook/all-in-one/frontend.yaml new file mode 100644 index 000000000..780b5c0d7 --- /dev/null +++ b/guestbook/all-in-one/frontend.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Service +metadata: + name: frontend + labels: + app: guestbook + tier: frontend +spec: + # if your cluster supports it, uncomment the following to automatically create + # an external load-balanced IP for the frontend service. + # type: LoadBalancer + ports: + # the port that this service should serve on + - port: 80 + selector: + app: guestbook + tier: frontend +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google-samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 diff --git a/guestbook/all-in-one/guestbook-all-in-one.yaml b/guestbook/all-in-one/guestbook-all-in-one.yaml new file mode 100644 index 000000000..c6675e0eb --- /dev/null +++ b/guestbook/all-in-one/guestbook-all-in-one.yaml @@ -0,0 +1,179 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-master + labels: + app: redis + tier: backend + role: master +spec: + ports: + # the port that this service should serve on + - port: 6379 + targetPort: 6379 + selector: + app: redis + tier: backend + role: master +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-master + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: master + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 1 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: master + # tier: backend + template: + metadata: + labels: + app: redis + role: master + tier: backend + spec: + containers: + - name: master + image: gcr.io/google_containers/redis:e2e # or just image: redis + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: redis + tier: backend + role: slave +spec: + ports: + # the port that this service should serve on + - port: 6379 + selector: + app: redis + tier: backend + role: slave +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: slave + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend + labels: + app: guestbook + tier: frontend +spec: + # if your cluster supports it, uncomment the following to automatically create + # an external load-balanced IP for the frontend service. + # type: LoadBalancer + ports: + # the port that this service should serve on + - port: 80 + selector: + app: guestbook + tier: frontend +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google-samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 diff --git a/guestbook/all-in-one/redis-slave.yaml b/guestbook/all-in-one/redis-slave.yaml new file mode 100644 index 000000000..88c3a9955 --- /dev/null +++ b/guestbook/all-in-one/redis-slave.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: redis + role: slave + tier: backend +spec: + ports: + # the port that this service should serve on + - port: 6379 + selector: + app: redis + role: slave + tier: backend +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: slave + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 diff --git a/guestbook/frontend-deployment.yaml b/guestbook/frontend-deployment.yaml new file mode 100644 index 000000000..1d67946f4 --- /dev/null +++ b/guestbook/frontend-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google-samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 diff --git a/guestbook/frontend-service.yaml b/guestbook/frontend-service.yaml new file mode 100644 index 000000000..72ea61327 --- /dev/null +++ b/guestbook/frontend-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: frontend + labels: + app: guestbook + tier: frontend +spec: + # if your cluster supports it, uncomment the following to automatically create + # an external load-balanced IP for the frontend service. + # type: LoadBalancer + ports: + # the port that this service should serve on + - port: 80 + selector: + app: guestbook + tier: frontend diff --git a/guestbook/legacy/frontend-controller.yaml b/guestbook/legacy/frontend-controller.yaml new file mode 100644 index 000000000..b9b8c40f1 --- /dev/null +++ b/guestbook/legacy/frontend-controller.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: frontend + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: guestbook + # tier: frontend +spec: + # this replicas value is default + # modify it according to your case + replicas: 3 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # app: guestbook + # tier: frontend + template: + metadata: + labels: + app: guestbook + tier: frontend + spec: + containers: + - name: php-redis + image: gcr.io/google_samples/gb-frontend:v4 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access environment variables to find service host + # info, comment out the 'value: dns' line above, and uncomment the + # line below. + # value: env + ports: + - containerPort: 80 diff --git a/guestbook/legacy/redis-master-controller.yaml b/guestbook/legacy/redis-master-controller.yaml new file mode 100644 index 000000000..2b9b8ec7a --- /dev/null +++ b/guestbook/legacy/redis-master-controller.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: redis-master + # these labels can be applied automatically + # from the labels in the pod template if not set + labels: + app: redis + role: master + tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 1 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # app: guestbook + # role: master + # tier: backend + template: + metadata: + labels: + app: redis + role: master + tier: backend + spec: + containers: + - name: master + image: gcr.io/google_containers/redis:e2e # or just image: redis + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 diff --git a/guestbook/legacy/redis-slave-controller.yaml b/guestbook/legacy/redis-slave-controller.yaml new file mode 100644 index 000000000..1d54a8280 --- /dev/null +++ b/guestbook/legacy/redis-slave-controller.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + labels: + app: redis + role: slave + tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 diff --git a/guestbook/php-redis/Dockerfile b/guestbook/php-redis/Dockerfile new file mode 100644 index 000000000..e6f5a2f84 --- /dev/null +++ b/guestbook/php-redis/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM php:5-apache + +RUN apt-get update +RUN apt-get install -y php-pear +RUN pear channel-discover pear.nrk.io +RUN pear install nrk/Predis + +# If the container's stdio is connected to systemd-journald, +# /proc/self/fd/{1,2} are Unix sockets and apache will not be able to open() +# them. Use "cat" to write directly to the already opened fds without opening +# them again. +RUN sed -i 's#ErrorLog /proc/self/fd/2#ErrorLog "|$/bin/cat 1>\&2"#' /etc/apache2/apache2.conf +RUN sed -i 's#CustomLog /proc/self/fd/1 combined#CustomLog "|/bin/cat" combined#' /etc/apache2/apache2.conf + +ADD guestbook.php /var/www/html/guestbook.php +ADD controllers.js /var/www/html/controllers.js +ADD index.html /var/www/html/index.html diff --git a/guestbook/php-redis/controllers.js b/guestbook/php-redis/controllers.js new file mode 100644 index 000000000..1e4b55042 --- /dev/null +++ b/guestbook/php-redis/controllers.js @@ -0,0 +1,29 @@ +var redisApp = angular.module('redis', ['ui.bootstrap']); + +/** + * Constructor + */ +function RedisController() {} + +RedisController.prototype.onRedis = function() { + this.scope_.messages.push(this.scope_.msg); + this.scope_.msg = ""; + var value = this.scope_.messages.join(); + this.http_.get("guestbook.php?cmd=set&key=messages&value=" + value) + .success(angular.bind(this, function(data) { + this.scope_.redisResponse = "Updated."; + })); +}; + +redisApp.controller('RedisCtrl', function ($scope, $http, $location) { + $scope.controller = new RedisController(); + $scope.controller.scope_ = $scope; + $scope.controller.location_ = $location; + $scope.controller.http_ = $http; + + $scope.controller.http_.get("guestbook.php?cmd=get&key=messages") + .success(function(data) { + console.log(data); + $scope.messages = data.data.split(","); + }); +}); diff --git a/guestbook/php-redis/guestbook.php b/guestbook/php-redis/guestbook.php new file mode 100644 index 000000000..ee0670ee6 --- /dev/null +++ b/guestbook/php-redis/guestbook.php @@ -0,0 +1,41 @@ + 'tcp', + 'host' => $host, + 'port' => 6379, + ]); + + $client->set($_GET['key'], $_GET['value']); + print('{"message": "Updated"}'); + } else { + $host = 'redis-slave'; + if (getenv('GET_HOSTS_FROM') == 'env') { + $host = getenv('REDIS_SLAVE_SERVICE_HOST'); + } + $client = new Predis\Client([ + 'scheme' => 'tcp', + 'host' => $host, + 'port' => 6379, + ]); + + $value = $client->get($_GET['key']); + print('{"data": "' . $value . '"}'); + } +} else { + phpinfo(); +} ?> diff --git a/guestbook/php-redis/index.html b/guestbook/php-redis/index.html new file mode 100644 index 000000000..4ffb4ed2a --- /dev/null +++ b/guestbook/php-redis/index.html @@ -0,0 +1,25 @@ + + + Guestbook + + + + + + +
+

Guestbook

+
+
+
+ +
+
+
+
+ {{msg}} +
+
+
+ + diff --git a/guestbook/redis-master-deployment.yaml b/guestbook/redis-master-deployment.yaml new file mode 100644 index 000000000..2ef40abd6 --- /dev/null +++ b/guestbook/redis-master-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-master + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: master + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 1 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: master + # tier: backend + template: + metadata: + labels: + app: redis + role: master + tier: backend + spec: + containers: + - name: master + image: gcr.io/google_containers/redis:e2e # or just image: redis + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 diff --git a/guestbook/redis-master-service.yaml b/guestbook/redis-master-service.yaml new file mode 100644 index 000000000..a9db70312 --- /dev/null +++ b/guestbook/redis-master-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-master + labels: + app: redis + role: master + tier: backend +spec: + ports: + # the port that this service should serve on + - port: 6379 + targetPort: 6379 + selector: + app: redis + role: master + tier: backend diff --git a/guestbook/redis-slave-deployment.yaml b/guestbook/redis-slave-deployment.yaml new file mode 100644 index 000000000..5686961d1 --- /dev/null +++ b/guestbook/redis-slave-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: redis-slave + # these labels can be applied automatically + # from the labels in the pod template if not set + # labels: + # app: redis + # role: slave + # tier: backend +spec: + # this replicas value is default + # modify it according to your case + replicas: 2 + # selector can be applied automatically + # from the labels in the pod template if not set + # selector: + # matchLabels: + # app: guestbook + # role: slave + # tier: backend + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # If your cluster config does not include a dns service, then to + # instead access an environment variable to find the master + # service's host, comment out the 'value: dns' line above, and + # uncomment the line below. + # value: env + ports: + - containerPort: 6379 diff --git a/guestbook/redis-slave-service.yaml b/guestbook/redis-slave-service.yaml new file mode 100644 index 000000000..e1f040171 --- /dev/null +++ b/guestbook/redis-slave-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: redis + role: slave + tier: backend +spec: + ports: + # the port that this service should serve on + - port: 6379 + selector: + app: redis + role: slave + tier: backend diff --git a/guestbook/redis-slave/Dockerfile b/guestbook/redis-slave/Dockerfile new file mode 100644 index 000000000..e90b22588 --- /dev/null +++ b/guestbook/redis-slave/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM redis + +ADD run.sh /run.sh + +RUN chmod a+x /run.sh + +CMD /run.sh diff --git a/guestbook/redis-slave/run.sh b/guestbook/redis-slave/run.sh new file mode 100755 index 000000000..d9037d48c --- /dev/null +++ b/guestbook/redis-slave/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ ${GET_HOSTS_FROM:-dns} == "env" ]]; then + redis-server --slaveof ${REDIS_MASTER_SERVICE_HOST} 6379 +else + redis-server --slaveof redis-master 6379 +fi diff --git a/guidelines.md b/guidelines.md new file mode 100644 index 000000000..a7b798157 --- /dev/null +++ b/guidelines.md @@ -0,0 +1,88 @@ +# Example Guidelines + +## An Example Is + +An example demonstrates running an application/framework/workload on +Kubernetes in a meaningful way. It is educational and informative. + +Examples are not: + +* Full app deployments, ready to use, with no explanation. These + belong to [Helm charts](https://github.com/helm/charts). +* Simple toys to show how to use a Kubernetes feature. These belong in + the [user guide](../docs/user-guide/). +* Demos that follow a script to show a Kubernetes feature in + action. Example: killing a node to demonstrate controller + self-healing. +* A tutorial which guides the user through multiple progressively more + complex deployments to arrive at the final solution. An example + should just demonstrate how to setup the correct deployment + +## An Example Includes + +### Up front + +* Has a "this is what you'll learn" section. +* Has a Table of Contents. +* Has a section that brings up the app in the fewest number of + commands (TL;DR / quickstart), without cloning the repo (kubectl + apply -f http://...). +* Points to documentation of prerequisites. + * [Create a cluster](../docs/getting-started-guides/) (e.g., single-node docker). + * [Setup kubectl](../docs/user-guide/prereqs.md). + * etc. +* Should specify which release of Kubernetes is required and any other + prerequisites, such as DNS, a cloudprovider with PV provisioning, a + cloudprovider with external load balancers, etc. + * Point to general documentation about alternatives for those + mechanisms rather than present the alternatives in each example. + * Tries to balance between using using new features, and being + compatible across environments. + +### Throughout + +* Should point to documentation on first mention: + [kubectl](../docs/user-guide/kubectl-overview.md), + [pods](../docs/user-guide/pods.md), + [services](../docs/user-guide/services.md), + [deployments](../docs/user-guide/deployments.md), + [replication controllers](../docs/user-guide/replication-controller.md), + [jobs](../docs/user-guide/jobs.md), + [labels](../docs/user-guide/labels.md), + [persistent volumes](../docs/user-guide/persistent-volumes.md), + etc. +* Most examples should be cloudprovider-independent (e.g., using PVCs, not PDs). + * Other examples with cloudprovider-specific bits could be somewhere else. +* Actually show the app working -- console output, and or screenshots. + * Ascii animations and screencasts are recommended. +* Follows [config best practices](../docs/user-guide/config-best-practices.md). +* Shouldn't duplicate the [thorough walk-through](../docs/user-guide/#thorough-walkthrough). +* Docker images are pre-built, and source is contained in a subfolder. + * Source is the Dockerfile and any custom files needed beyond the + upstream app being packaged. + * Images are pushed to `gcr.io/google-samples`. Contact @jeffmendoza + to have an image pushed + * Images are tagged with a version (not latest) that is referenced + in the example config. +* Only use the code highlighting types + [supported by Rouge](https://github.com/jneen/rouge/wiki/list-of-supported-languages-and-lexers), + as this is what GitHub Pages uses. +* Commands to be copied use the `shell` syntax highlighting type, and + do not include any kind of prompt. +* Example output is in a separate block quote to distinguish it from + the command (which doesn't have a prompt). +* When providing an example command or config for which the user is + expected to substitute text with something specific to them, use + angle brackets: `` for the text to be substituted. +* Use `kubectl` instead of `cluster\kubectl.sh` for example cli + commands. + +### At the end + +* Should have a section suggesting what to look at next, both in terms + of "additional resources" and "what example to look at next". + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/guidelines.md?pixel)]() + diff --git a/https-nginx/BUILD b/https-nginx/BUILD new file mode 100644 index 000000000..b8cd0357b --- /dev/null +++ b/https-nginx/BUILD @@ -0,0 +1,40 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "https-nginx", + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["make_secret.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/install:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/https-nginx/Dockerfile b/https-nginx/Dockerfile new file mode 100644 index 000000000..f58408861 --- /dev/null +++ b/https-nginx/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nginx + + +COPY index2.html /usr/share/nginx/html/index2.html +RUN chmod +r /usr/share/nginx/html/index2.html +COPY auto-reload-nginx.sh /home/auto-reload-nginx.sh +RUN chmod +x /home/auto-reload-nginx.sh + +# install inotify +RUN apt-get update && apt-get install -y inotify-tools diff --git a/https-nginx/Makefile b/https-nginx/Makefile new file mode 100644 index 000000000..f8203dcac --- /dev/null +++ b/https-nginx/Makefile @@ -0,0 +1,38 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: + +TAG = 1.0 +PREFIX = bprashanth/nginxhttps +KEY = /tmp/nginx.key +CERT = /tmp/nginx.crt +SECRET = /tmp/secret.json + +keys: + # The CName used here is specific to the service specified in nginx-app.yaml. + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout $(KEY) -out $(CERT) -subj "/CN=nginxsvc/O=nginxsvc" + +secret: + go run make_secret.go -crt $(CERT) -key $(KEY) > $(SECRET) + +container: + docker build --pull -t $(PREFIX):$(TAG) . + +push: container + docker push $(PREFIX):$(TAG) + +clean: + rm $(KEY) + rm $(CERT) diff --git a/https-nginx/README.md b/https-nginx/README.md new file mode 100644 index 000000000..a64b04abd --- /dev/null +++ b/https-nginx/README.md @@ -0,0 +1,129 @@ + +# Nginx https service + +This example creates a basic nginx https service useful in verifying proof of concept, keys, secrets, configmap, and end-to-end https service creation in kubernetes. +It uses an [nginx server block](http://wiki.nginx.org/ServerBlockExample) to serve the index page over both http and https. It will detect changes to nginx's configuration file, default.conf, mounted as a configmap volume and reload nginx automatically. + +### Generate certificates + +First generate a self signed rsa key and certificate that the server can use for TLS. This step invokes the make_secret.go script in the same directory, which uses the kubernetes api to generate a secret json config in /tmp/secret.json. + +```sh +$ make keys secret KEY=/tmp/nginx.key CERT=/tmp/nginx.crt SECRET=/tmp/secret.json +``` + +### Create a https nginx application running in a kubernetes cluster + +You need a [running kubernetes cluster](../../docs/getting-started-guides/) for this to work. + +Create a secret and a configmap. + +```sh +$ kubectl create -f /tmp/secret.json +secret "nginxsecret" created + +$ kubectl create configmap nginxconfigmap --from-file=examples/https-nginx/default.conf +configmap "nginxconfigmap" created +``` + +Create a service and a replication controller using the configuration in nginx-app.yaml. + +```sh +$ kubectl create -f examples/https-nginx/nginx-app.yaml +You have exposed your service on an external port on all nodes in your +cluster. If you want to expose this service to the external internet, you may +need to set up firewall rules for the service port(s) (tcp:32211,tcp:30028) to serve traffic. +... +service "nginxsvc" created +replicationcontroller "my-nginx" created +``` + +Then, find the node port that Kubernetes is using for http and https traffic. + +```sh +$ kubectl get service nginxsvc -o json +... + { + "name": "http", + "protocol": "TCP", + "port": 80, + "targetPort": 80, + "nodePort": 32211 + }, + { + "name": "https", + "protocol": "TCP", + "port": 443, + "targetPort": 443, + "nodePort": 30028 + } +... +``` + +If you are using Kubernetes on a cloud provider, you may need to create cloud firewall rules to serve traffic. +If you are using GCE or GKE, you can use the following commands to add firewall rules. + +```sh +$ gcloud compute firewall-rules create allow-nginx-http --allow tcp:32211 --description "Incoming http allowed." +Created [https://www.googleapis.com/compute/v1/projects/hello-world-job/global/firewalls/allow-nginx-http]. +NAME NETWORK SRC_RANGES RULES SRC_TAGS TARGET_TAGS +allow-nginx-http default 0.0.0.0/0 tcp:32211 + +$ gcloud compute firewall-rules create allow-nginx-https --allow tcp:30028 --description "Incoming https allowed." +Created [https://www.googleapis.com/compute/v1/projects/hello-world-job/global/firewalls/allow-nginx-https]. +NAME NETWORK SRC_RANGES RULES SRC_TAGS TARGET_TAGS +allow-nginx-https default 0.0.0.0/0 tcp:30028 +``` + +Find your nodes' IPs. + +```sh +$ kubectl get nodes -o json | grep ExternalIP -A 2 + "type": "ExternalIP", + "address": "104.198.1.26" + } +-- + "type": "ExternalIP", + "address": "104.198.12.158" + } +-- + "type": "ExternalIP", + "address": "104.198.11.137" + } +``` + +Now your service is up. You can either use your browser or type the following commands. + +```sh +$ curl https://: -k + +$ curl https://104.198.1.26:30028 -k +... +Welcome to nginx! +... +``` + +Then we will update the configmap by changing `index.html` to `index2.html`. + +```sh +kubectl create configmap nginxconfigmap --from-file=examples/https-nginx/default.conf -o yaml --dry-run\ +| sed 's/index.html/index2.html/g' | kubectl apply -f - +configmap "nginxconfigmap" configured +``` + +Wait a few seconds to let the change propagate. Now you should be able to either use your browser or type the following commands to verify Nginx has been reloaded with new configuration. + +```sh +$ curl https://: -k + +$ curl https://104.198.1.26:30028 -k +... +Nginx reloaded! +... +``` + +For more information on how to run this in a kubernetes cluster, please see the [user-guide](../../docs/user-guide/connecting-applications.md). + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/https-nginx/README.md?pixel)]() + diff --git a/https-nginx/auto-reload-nginx.sh b/https-nginx/auto-reload-nginx.sh new file mode 100755 index 000000000..78144b059 --- /dev/null +++ b/https-nginx/auto-reload-nginx.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nginx "$@" +oldcksum=`cksum /etc/nginx/conf.d/default.conf` + +inotifywait -e modify,move,create,delete -mr --timefmt '%d/%m/%y %H:%M' --format '%T' \ +/etc/nginx/conf.d/ | while read date time; do + + newcksum=`cksum /etc/nginx/conf.d/default.conf` + if [ "$newcksum" != "$oldcksum" ]; then + echo "At ${time} on ${date}, config file update detected." + oldcksum=$newcksum + nginx -s reload + fi + +done diff --git a/https-nginx/default.conf b/https-nginx/default.conf new file mode 100644 index 000000000..d91a5ba6d --- /dev/null +++ b/https-nginx/default.conf @@ -0,0 +1,17 @@ +server { + listen 80 default_server; + listen [::]:80 default_server ipv6only=on; + + listen 443 ssl; + + root /usr/share/nginx/html; + index index.html; + + server_name localhost; + ssl_certificate /etc/nginx/ssl/nginx.crt; + ssl_certificate_key /etc/nginx/ssl/nginx.key; + + location / { + try_files $uri $uri/ =404; + } +} diff --git a/https-nginx/index2.html b/https-nginx/index2.html new file mode 100644 index 000000000..86280c986 --- /dev/null +++ b/https-nginx/index2.html @@ -0,0 +1,28 @@ + + + +Nginx reloaded! + + + +

Nginx has been reloaded!

+

If you see this page, the nginx web server has been automaticly reloaded, since the config file has been updated using Kubernetes.

+ + +

For online documentation and support please refer to +kubernetes.io.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ + diff --git a/https-nginx/make_secret.go b/https-nginx/make_secret.go new file mode 100644 index 000000000..8299b5042 --- /dev/null +++ b/https-nginx/make_secret.go @@ -0,0 +1,70 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A small script that converts the given open ssl public/private keys to +// a secret that it writes to stdout as json. Most common use case is to +// create a secret from self signed certificates used to authenticate with +// a devserver. Usage: go run make_secret.go -crt ca.crt -key priv.key > secret.json +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api" + + // This installs the legacy v1 API + _ "k8s.io/kubernetes/pkg/api/install" +) + +// TODO: +// Add a -o flag that writes to the specified destination file. +// Teach the script to create crt and key if -crt and -key aren't specified. +var ( + crt = flag.String("crt", "", "path to nginx certificates.") + key = flag.String("key", "", "path to nginx private key.") +) + +func read(file string) []byte { + b, err := ioutil.ReadFile(file) + if err != nil { + log.Fatalf("Cannot read file %v, %v", file, err) + } + return b +} + +func main() { + flag.Parse() + if *crt == "" || *key == "" { + log.Fatalf("Need to specify -crt -key and -template") + } + nginxCrt := read(*crt) + nginxKey := read(*key) + secret := &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginxsecret", + }, + Data: map[string][]byte{ + "nginx.crt": nginxCrt, + "nginx.key": nginxKey, + }, + } + fmt.Printf(runtime.EncodeOrDie(api.Codecs.LegacyCodec(api.Registry.EnabledVersions()...), secret)) +} diff --git a/https-nginx/nginx-app.yaml b/https-nginx/nginx-app.yaml new file mode 100644 index 000000000..a5ce0bd37 --- /dev/null +++ b/https-nginx/nginx-app.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginxsvc + labels: + app: nginx +spec: + type: NodePort + ports: + - port: 80 + protocol: TCP + name: http + - port: 443 + protocol: TCP + name: https + selector: + app: nginx +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: my-nginx +spec: + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + volumes: + - name: secret-volume + secret: + secretName: nginxsecret + - name: configmap-volume + configMap: + name: nginxconfigmap + containers: + - name: nginxhttps + image: ymqytw/nginxhttps:1.5 + command: ["/home/auto-reload-nginx.sh"] + ports: + - containerPort: 443 + - containerPort: 80 + livenessProbe: + httpGet: + path: /index.html + port: 80 + initialDelaySeconds: 30 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/nginx/ssl + name: secret-volume + - mountPath: /etc/nginx/conf.d + name: configmap-volume diff --git a/javaee/README.md b/javaee/README.md new file mode 100644 index 000000000..7fc8b1b87 --- /dev/null +++ b/javaee/README.md @@ -0,0 +1,134 @@ +## Java EE Application using WildFly and MySQL + +The following document describes the deployment of a Java EE application using [WildFly](http://wildfly.org) application server and MySQL database server on Kubernetes. The sample application source code is at: https://github.com/javaee-samples/javaee7-simple-sample. + +### Prerequisites + +https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/prereqs.md + +### Start MySQL Pod + +In Kubernetes a [_Pod_](../../docs/user-guide/pods.md) is the smallest deployable unit that can be created, scheduled, and managed. It's a collocated group of containers that share an IP and storage volume. + +Here is the config for MySQL pod: [mysql-pod.yaml](mysql-pod.yaml) + + + + +Create the MySQL pod: + +```sh +kubectl create -f examples/javaee/mysql-pod.yaml +``` + +Check status of the pod: + +```sh +kubectl get -w po +NAME READY STATUS RESTARTS AGE +mysql-pod 0/1 Pending 0 4s +NAME READY STATUS RESTARTS AGE +mysql-pod 0/1 Running 0 44s +mysql-pod 1/1 Running 0 44s +``` + +Wait for the status to `1/1` and `Running`. + +### Start MySQL Service + +We are creating a [_Service_](../../docs/user-guide/services.md) to expose the TCP port of the MySQL server. A Service distributes traffic across a set of Pods. The order of Service and the targeted Pods does not matter. However Service needs to be started before any other Pods consuming the Service are started. + +In this application, we will use a Kubernetes Service to provide a discoverable endpoints for the MySQL endpoint in the cluster. MySQL service target pods with the labels `name: mysql-pod` and `context: docker-k8s-lab`. + +Here is definition of the MySQL service: [mysql-service.yaml](mysql-service.yaml) + + + + +Create this service: + +```sh +kubectl create -f examples/javaee/mysql-service.yaml +``` + +Get status of the service: + +```sh +kubectl get -w svc +NAME LABELS SELECTOR IP(S) PORT(S) +kubernetes component=apiserver,provider=kubernetes 10.247.0.1 443/TCP +mysql-service context=docker-k8s-lab,name=mysql-pod context=docker-k8s-lab,name=mysql-pod 10.247.63.43 3306/TCP +``` + +If multiple services are running, then it can be narrowed by specifying labels: + +```sh +kubectl get -w po -l context=docker-k8s-lab,name=mysql-pod +NAME READY STATUS RESTARTS AGE +mysql-pod 1/1 Running 0 4m +``` + +This is also the selector label used by service to target pods. + +When a Service is run on a node, the kubelet adds a set of environment variables for each active Service. It supports both Docker links compatible variables and simpler `{SVCNAME}_SERVICE_HOST` and `{SVCNAME}_SERVICE_PORT` variables, where the Service name is upper-cased and dashes are converted to underscores. + +Our service name is ``mysql-service'' and so ``MYSQL_SERVICE_SERVICE_HOST'' and ``MYSQL_SERVICE_SERVICE_PORT'' variables are available to other pods. This host and port variables are then used to create the JDBC resource in WildFly. + +### Start WildFly Replication Controller + +WildFly is a lightweight Java EE 7 compliant application server. It is wrapped in a Replication Controller and used as the Java EE runtime. + +In Kubernetes a [_Replication Controller_](../../docs/user-guide/replication-controller.md) is responsible for replicating sets of identical pods. Like a _Service_ it has a selector query which identifies the members of it's set. Unlike a service it also has a desired number of replicas, and it will create or delete pods to ensure that the number of pods matches up with it's desired state. + +Here is definition of the MySQL service: [wildfly-rc.yaml](wildfly-rc.yaml). + + + + +Create this controller: + +```sh +kubectl create -f examples/javaee/wildfly-rc.yaml +``` + +Check status of the pod inside replication controller: + +```sh +kubectl get po +NAME READY STATUS RESTARTS AGE +mysql-pod 1/1 Running 0 1h +wildfly-rc-w2kk5 1/1 Running 0 6m +``` + +### Access the application + +Get IP address of the pod: + +```sh +kubectl get -o template po wildfly-rc-w2kk5 --template={{.status.podIP}} +10.246.1.23 +``` + +Log in to node and access the application: + +```sh +vagrant ssh node-1 +Last login: Thu Jul 16 00:24:36 2015 from 10.0.2.2 +[vagrant@kubernetes-node-1 ~]$ curl http://10.246.1.23:8080/employees/resources/employees/ +1Penny2Sheldon3Amy4Leonard5Bernadette6Raj7Howard8Priya +``` + +### Delete resources + +All resources created in this application can be deleted: + +```sh +kubectl delete -f examples/javaee/mysql-pod.yaml +kubectl delete -f examples/javaee/mysql-service.yaml +kubectl delete -f examples/javaee/wildfly-rc.yaml +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/javaee/README.md?pixel)]() + diff --git a/javaee/mysql-pod.yaml b/javaee/mysql-pod.yaml new file mode 100644 index 000000000..b8884f386 --- /dev/null +++ b/javaee/mysql-pod.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mysql-pod + labels: + name: mysql-pod + context: docker-k8s-lab +spec: + containers: + - + name: mysql + image: mysql:latest + env: + - + name: "MYSQL_USER" + value: "mysql" + - + name: "MYSQL_PASSWORD" + value: "mysql" + - + name: "MYSQL_DATABASE" + value: "sample" + - + name: "MYSQL_ROOT_PASSWORD" + value: "supersecret" + ports: + - + containerPort: 3306 diff --git a/javaee/mysql-service.yaml b/javaee/mysql-service.yaml new file mode 100644 index 000000000..0cbb329a8 --- /dev/null +++ b/javaee/mysql-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql-service + labels: + name: mysql-pod + context: docker-k8s-lab +spec: + ports: + # the port that this service should serve on + - port: 3306 + # label keys and values that must match in order to receive traffic for this service + selector: + name: mysql-pod + context: docker-k8s-lab diff --git a/javaee/wildfly-rc.yaml b/javaee/wildfly-rc.yaml new file mode 100644 index 000000000..303b63f8d --- /dev/null +++ b/javaee/wildfly-rc.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: wildfly-rc + labels: + name: wildfly + context: docker-k8s-lab +spec: + replicas: 1 + template: + metadata: + labels: + name: wildfly + spec: + containers: + - name: wildfly-rc-pod + image: arungupta/wildfly-mysql-javaee7:k8s + ports: + - containerPort: 8080 \ No newline at end of file diff --git a/javaweb-tomcat-sidecar/README.md b/javaweb-tomcat-sidecar/README.md new file mode 100644 index 000000000..7f715f53d --- /dev/null +++ b/javaweb-tomcat-sidecar/README.md @@ -0,0 +1,185 @@ +## Java Web Application with Tomcat and Sidecar Container + +The following document describes the deployment of a Java Web application using Tomcat. Instead of packaging `war` file inside the Tomcat image or mount the `war` as a volume, we use a sidecar container as `war` file provider. + +### Prerequisites + +https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/prereqs.md + +### Overview + +This sidecar mode brings a new workflow for Java users: + +![](workflow.png?raw=true "Workflow") + +As you can see, user can create a `sample:v2` container as sidecar to "provide" war file to Tomcat by copying it to the shared `emptyDir` volume. And Pod will make sure the two containers compose an "atomic" scheduling unit, which is perfect for this case. Thus, your application version management will be totally separated from web server management. + +For example, if you are going to change the configurations of your Tomcat: + +```console +$ docker exec -it /bin/bash +# make some change, and then commit it to a new image +$ docker commit mytomcat:7.0-dev +``` + +Done! The new Tomcat image **will not** mess up with your `sample.war` file. You can re-use your tomcat image with lots of different war container images for lots of different apps without having to build lots of different images. + +Also this means that rolling out a new Tomcat to patch security or whatever else, doesn't require rebuilding N different images. + +**Why not put my `sample.war` in a host dir and mount it to tomcat container?** + +You have to **manage the volumes** in this case, for example, when you restart or scale the pod on another node, your contents is not ready on that host. + +Generally, we have to set up a distributed file system (NFS at least) volume to solve this (if we do not have GCE PD volume). But this is generally unnecessary. + +### How To Set this Up + +In Kubernetes a [_Pod_](../../docs/user-guide/pods.md) is the smallest deployable unit that can be created, scheduled, and managed. It's a collocated group of containers that share an IP and storage volume. + +Here is the config [javaweb.yaml](javaweb.yaml) for Java Web pod: + +NOTE: you should define `war` container **first** as it is the "provider". + + + +``` +apiVersion: v1 +kind: Pod +metadata: + name: javaweb +spec: + containers: + - image: resouer/sample:v1 + name: war + volumeMounts: + - mountPath: /app + name: app-volume + - image: resouer/mytomcat:7.0 + name: tomcat + command: ["sh","-c","/root/apache-tomcat-7.0.42-v2/bin/start.sh"] + volumeMounts: + - mountPath: /root/apache-tomcat-7.0.42-v2/webapps + name: app-volume + ports: + - containerPort: 8080 + hostPort: 8001 + volumes: + - name: app-volume + emptyDir: {} +``` + + + +The only magic here is the `resouer/sample:v1` image: + +``` +FROM busybox:latest +ADD sample.war sample.war +CMD "sh" "mv.sh" +``` + +And the contents of `mv.sh` is: + +```sh +cp /sample.war /app +tail -f /dev/null +``` + +#### Explanation + +1. 'war' container only contains the `war` file of your app +2. 'war' container's CMD tries to copy `sample.war` to the `emptyDir` volume path +3. The last line of `tail -f` is just used to hold the container, as Replication Controller does not support one-off task +4. 'tomcat' container will load the `sample.war` from volume path + +What's more, if you don't want to enclose a build-in `mv.sh` script in the `war` container, you can use Pod lifecycle handler to do the copy work, here's a example [javaweb-2.yaml](javaweb-2.yaml): + + + + +``` +apiVersion: v1 +kind: Pod +metadata: + name: javaweb-2 +spec: + containers: + - image: resouer/sample:v2 + name: war + lifecycle: + postStart: + exec: + command: + - "cp" + - "/sample.war" + - "/app" + volumeMounts: + - mountPath: /app + name: app-volume + - image: resouer/mytomcat:7.0 + name: tomcat + command: ["sh","-c","/root/apache-tomcat-7.0.42-v2/bin/start.sh"] + volumeMounts: + - mountPath: /root/apache-tomcat-7.0.42-v2/webapps + name: app-volume + ports: + - containerPort: 8080 + hostPort: 8001 + volumes: + - name: app-volume + emptyDir: {} +``` + + + +And the `resouer/sample:v2` Dockerfile is quite simple: + +``` +FROM busybox:latest +ADD sample.war sample.war +CMD "tail" "-f" "/dev/null" +``` + +#### Explanation + +1. 'war' container only contains the `war` file of your app +2. 'war' container's CMD uses `tail -f` to hold the container, nothing more +3. The `postStart` lifecycle handler will do `cp` after the `war` container is started +4. Again 'tomcat' container will load the `sample.war` from volume path + +Done! Now your `war` container contains nothing except `sample.war`, clean enough. + +### Test It Out + +Create the Java web pod: + +```console +$ kubectl create -f examples/javaweb-tomcat-sidecar/javaweb-2.yaml +``` + +Check status of the pod: + +```console +$ kubectl get -w po +NAME READY STATUS RESTARTS AGE +javaweb-2 2/2 Running 0 7s +``` + +Wait for the status to `2/2` and `Running`. Then you can visit "Hello, World" page on `http://localhost:8001/sample/index.html` + +You can also test `javaweb.yaml` in the same way. + +### Delete Resources + +All resources created in this application can be deleted: + +```console +$ kubectl delete -f examples/javaweb-tomcat-sidecar/javaweb-2.yaml +``` + + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/javaweb-tomcat-sidecar/README.md?pixel)]() + diff --git a/javaweb-tomcat-sidecar/javaweb-2.yaml b/javaweb-tomcat-sidecar/javaweb-2.yaml new file mode 100644 index 000000000..b34d5ab6e --- /dev/null +++ b/javaweb-tomcat-sidecar/javaweb-2.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: javaweb-2 +spec: + containers: + - image: resouer/sample:v2 + name: war + lifecycle: + postStart: + exec: + command: + - "cp" + - "/sample.war" + - "/app" + volumeMounts: + - mountPath: /app + name: app-volume + - image: resouer/mytomcat:7.0 + name: tomcat + command: ["sh","-c","/root/apache-tomcat-7.0.42-v2/bin/start.sh"] + volumeMounts: + - mountPath: /root/apache-tomcat-7.0.42-v2/webapps + name: app-volume + ports: + - containerPort: 8080 + hostPort: 8001 + volumes: + - name: app-volume + emptyDir: {} + diff --git a/javaweb-tomcat-sidecar/javaweb.yaml b/javaweb-tomcat-sidecar/javaweb.yaml new file mode 100644 index 000000000..d77f6a727 --- /dev/null +++ b/javaweb-tomcat-sidecar/javaweb.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: javaweb +spec: + containers: + - image: resouer/sample:v1 + name: war + volumeMounts: + - mountPath: /app + name: app-volume + - image: resouer/mytomcat:7.0 + name: tomcat + command: ["sh","-c","/root/apache-tomcat-7.0.42-v2/bin/start.sh"] + volumeMounts: + - mountPath: /root/apache-tomcat-7.0.42-v2/webapps + name: app-volume + ports: + - containerPort: 8080 + hostPort: 8001 + volumes: + - name: app-volume + emptyDir: {} + diff --git a/javaweb-tomcat-sidecar/workflow.png b/javaweb-tomcat-sidecar/workflow.png new file mode 100644 index 000000000..7be56d082 Binary files /dev/null and b/javaweb-tomcat-sidecar/workflow.png differ diff --git a/job/expansions/README.md b/job/expansions/README.md new file mode 100644 index 000000000..ea72d5663 --- /dev/null +++ b/job/expansions/README.md @@ -0,0 +1,7 @@ + +This file has moved to: http://kubernetes.io/docs/user-guide/jobs/ + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/job/expansions/README.md?pixel)]() + diff --git a/job/work-queue-1/README.md b/job/work-queue-1/README.md new file mode 100644 index 000000000..d32d130f6 --- /dev/null +++ b/job/work-queue-1/README.md @@ -0,0 +1,7 @@ + +This file has moved to: http://kubernetes.io/docs/user-guide/jobs/ + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/job/work-queue-1/README.md?pixel)]() + diff --git a/job/work-queue-2/README.md b/job/work-queue-2/README.md new file mode 100644 index 000000000..94355c0f6 --- /dev/null +++ b/job/work-queue-2/README.md @@ -0,0 +1,7 @@ + +This file has moved to: http://kubernetes.io/docs/user-guide/jobs/ + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/job/work-queue-2/README.md?pixel)]() + diff --git a/kubectl-container/.gitignore b/kubectl-container/.gitignore new file mode 100644 index 000000000..50a4a06fd --- /dev/null +++ b/kubectl-container/.gitignore @@ -0,0 +1,2 @@ +kubectl +.tag diff --git a/kubectl-container/Dockerfile b/kubectl-container/Dockerfile new file mode 100644 index 000000000..add0569c0 --- /dev/null +++ b/kubectl-container/Dockerfile @@ -0,0 +1,17 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM scratch +ADD kubectl kubectl +ENTRYPOINT ["/kubectl"] diff --git a/kubectl-container/Makefile b/kubectl-container/Makefile new file mode 100644 index 000000000..ea127f151 --- /dev/null +++ b/kubectl-container/Makefile @@ -0,0 +1,48 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use: +# +# `make kubectl` will build kubectl. +# `make tag` will suggest a tag. +# `make container` will build a container-- you must supply a tag. +# `make push` will push the container-- you must supply a tag. + +GOARCH?=$(shell go env GOARCH) +GOOS?=$(shell go env GOOS) + +kubectl: + make -C ../../ WHAT=cmd/kubectl KUBE_STATIC_OVERRIDES="kubectl"; \ + cp ../../_output/local/bin/$(GOOS)/$(GOARCH)/kubectl . + +.tag: kubectl + ./kubectl version --client | grep -o 'GitVersion:"[^"]*"' | sed 's/[^"]*"\([^"+]*\).*/\1/' > .tag + +tag: .tag + @echo "Suggest using TAG=$(shell cat .tag)" + @echo "$$ make container TAG=$(shell cat .tag)" + @echo "or" + @echo "$$ make push TAG=$(shell cat .tag)" + +container: + $(if $(TAG),,$(error TAG is not defined. Use 'make tag' to see a suggestion)) + docker build --pull -t gcr.io/google_containers/kubectl:$(TAG) . + +push: container + $(if $(TAG),,$(error TAG is not defined. Use 'make tag' to see a suggestion)) + gcloud docker -- push gcr.io/google_containers/kubectl:$(TAG) + +clean: + rm -f kubectl + rm -f .tag diff --git a/kubectl-container/README.md b/kubectl-container/README.md new file mode 100644 index 000000000..9f43e2cf8 --- /dev/null +++ b/kubectl-container/README.md @@ -0,0 +1,31 @@ +To access the Kubernetes API [from a Pod](../../docs/user-guide/accessing-the-cluster.md#accessing-the-api-from-a-pod) one of the solution is to run `kubectl proxy` in a so-called sidecar container within the Pod. To do this, you need to package `kubectl` in a container. It is useful when service accounts are being used for accessing the API and the old no-auth KUBERNETES_RO service is not available. Since all containers in a Pod share the same network namespace, containers will be able to reach the API on localhost. + +This example contains a [Dockerfile](Dockerfile) and [Makefile](Makefile) for packaging up `kubectl` into +a container and pushing the resulting container image on the Google Container Registry. You can modify the Makefile to push to a different registry if needed. + +Assuming that you have checked out the Kubernetes source code and setup your environment to be able to build it. The typical build step of this kubectl container will be: + + $ cd examples/kubectl-container + $ make kubectl + $ make tag + $ make container + $ make push + +It is not currently automated as part of a release process, so for the moment +this is an example of what to do if you want to package `kubectl` into a +container and use it within a pod. + +In the future, we may release consistently versioned groups of containers when +we cut a release, in which case the source of gcr.io/google_containers/kubectl +would become that automated process. + +[```pod.json```](pod.json) is provided as an example of running `kubectl` as a sidecar +container in a Pod, and to help you verify that `kubectl` works correctly in +this configuration. To launch this Pod, you will need a configured Kubernetes endpoint and `kubectl` installed locally, then simply create the Pod: + + $ kubectl create -f pod.json + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/kubectl-container/README.md?pixel)]() + diff --git a/kubectl-container/pod.json b/kubectl-container/pod.json new file mode 100644 index 000000000..ed0ec6599 --- /dev/null +++ b/kubectl-container/pod.json @@ -0,0 +1,53 @@ +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "kubectl-tester" + }, + "spec": { + "containers": [ + { + "name": "bb", + "image": "gcr.io/google_containers/busybox", + "command": [ + "sh", "-c", "sleep 5; wget -O - ${KUBERNETES_RO_SERVICE_HOST}:${KUBERNETES_RO_SERVICE_PORT}/api/v1/pods/; sleep 10000" + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "env": [ + { + "name": "KUBERNETES_RO_SERVICE_HOST", + "value": "127.0.0.1" + }, + { + "name": "KUBERNETES_RO_SERVICE_PORT", + "value": "8001" + } + ], + "volumeMounts": [ + { + "name": "test-volume", + "mountPath": "/mount/test-volume" + } + ] + }, + { + "name": "kubectl", + "image": "gcr.io/google_containers/kubectl:v0.18.0-120-gaeb4ac55ad12b1-dirty", + "imagePullPolicy": "Always", + "args": [ + "proxy", "-p", "8001" + ] + } + ], + "volumes": [ + { + "name": "test-volume", + "emptyDir": {} + } + ] + } +} diff --git a/meteor/README.md b/meteor/README.md new file mode 100644 index 000000000..f0beb51d1 --- /dev/null +++ b/meteor/README.md @@ -0,0 +1,215 @@ +Meteor on Kubernetes +==================== + +This example shows you how to package and run a +[Meteor](https://www.meteor.com/) app on Kubernetes. + +Get started on Google Compute Engine +------------------------------------ + +Meteor uses MongoDB, and we will use the `GCEPersistentDisk` type of +volume for persistent storage. Therefore, this example is only +applicable to [Google Compute +Engine](https://cloud.google.com/compute/). Take a look at the +[volumes documentation](../../docs/user-guide/volumes.md) for other options. + +First, if you have not already done so: + +1. [Create](https://cloud.google.com/compute/docs/quickstart) a +[Google Cloud Platform](https://cloud.google.com/) project. +2. [Enable +billing](https://developers.google.com/console/help/new/#billing). +3. Install the [gcloud SDK](https://cloud.google.com/sdk/). + +Authenticate with gcloud and set the gcloud default project name to +point to the project you want to use for your Kubernetes cluster: + +```sh +gcloud auth login +gcloud config set project +``` + +Next, start up a Kubernetes cluster: + +```sh +wget -q -O - https://get.k8s.io | bash +``` + +Please see the [Google Compute Engine getting started +guide](../../docs/getting-started-guides/gce.md) for full +details and other options for starting a cluster. + +Build a container for your Meteor app +------------------------------------- + +To be able to run your Meteor app on Kubernetes you need to build a +Docker container for it first. To do that you need to install +[Docker](https://www.docker.com) Once you have that you need to add 2 +files to your existing Meteor project `Dockerfile` and +`.dockerignore`. + +`Dockerfile` should contain the below lines. You should replace the +`ROOT_URL` with the actual hostname of your app. + +``` +FROM chees/meteor-kubernetes +ENV ROOT_URL http://myawesomeapp.com +``` + +The `.dockerignore` file should contain the below lines. This tells +Docker to ignore the files on those directories when it's building +your container. + +``` +.meteor/local +packages/*/.build* +``` + +You can see an example meteor project already set up at: +[meteor-gke-example](https://github.com/Q42/meteor-gke-example). Feel +free to use this app for this example. + +> Note: The next step will not work if you have added mobile platforms +> to your meteor project. Check with `meteor list-platforms` + +Now you can build your container by running this in +your Meteor project directory: + +``` +docker build -t my-meteor . +``` + +Pushing to a registry +--------------------- + +For the [Docker Hub](https://hub.docker.com/), tag your app image with +your username and push to the Hub with the below commands. Replace +`` with your Hub username. + +``` +docker tag my-meteor /my-meteor +docker push /my-meteor +``` + +For [Google Container +Registry](https://cloud.google.com/tools/container-registry/), tag +your app image with your project ID, and push to GCR. Replace +`` with your project ID. + +``` +docker tag my-meteor gcr.io//my-meteor +gcloud docker -- push gcr.io//my-meteor +``` + +Running +------- + +Now that you have containerized your Meteor app it's time to set up +your cluster. Edit [`meteor-controller.json`](meteor-controller.json) +and make sure the `image:` points to the container you just pushed to +the Docker Hub or GCR. + +We will need to provide MongoDB a persistent Kubernetes volume to +store its data. See the [volumes documentation](../../docs/user-guide/volumes.md) for +options. We're going to use Google Compute Engine persistent +disks. Create the MongoDB disk by running: + +``` +gcloud compute disks create --size=200GB mongo-disk +``` + +Now you can start Mongo using that disk: + +``` +kubectl create -f examples/meteor/mongo-pod.json +kubectl create -f examples/meteor/mongo-service.json +``` + +Wait until Mongo is started completely and then start up your Meteor app: + +``` +kubectl create -f examples/meteor/meteor-service.json +kubectl create -f examples/meteor/meteor-controller.json +``` + +Note that [`meteor-service.json`](meteor-service.json) creates a load balancer, so +your app should be available through the IP of that load balancer once +the Meteor pods are started. We also created the service before creating the rc to +aid the scheduler in placing pods, as the scheduler ranks pod placement according to +service anti-affinity (among other things). You can find the IP of your load balancer +by running: + +``` +kubectl get service meteor --template="{{range .status.loadBalancer.ingress}} {{.ip}} {{end}}" +``` + +You will have to open up port 80 if it's not open yet in your +environment. On Google Compute Engine, you may run the below command. + +``` +gcloud compute firewall-rules create meteor-80 --allow=tcp:80 --target-tags kubernetes-node +``` + +What is going on? +----------------- + +Firstly, the `FROM chees/meteor-kubernetes` line in your `Dockerfile` +specifies the base image for your Meteor app. The code for that image +is located in the `dockerbase/` subdirectory. Open up the `Dockerfile` +to get an insight of what happens during the `docker build` step. The +image is based on the Node.js official image. It then installs Meteor +and copies in your apps' code. The last line specifies what happens +when your app container is run. + +```sh +ENTRYPOINT MONGO_URL=mongodb://$MONGO_SERVICE_HOST:$MONGO_SERVICE_PORT /usr/local/bin/node main.js +``` + +Here we can see the MongoDB host and port information being passed +into the Meteor app. The `MONGO_SERVICE...` environment variables are +set by Kubernetes, and point to the service named `mongo` specified in +[`mongo-service.json`](mongo-service.json). See the [environment +documentation](../../docs/user-guide/container-environment.md) for more details. + +As you may know, Meteor uses long lasting connections, and requires +_sticky sessions_. With Kubernetes you can scale out your app easily +with session affinity. The +[`meteor-service.json`](meteor-service.json) file contains +`"sessionAffinity": "ClientIP"`, which provides this for us. See the +[service +documentation](../../docs/user-guide/services.md#virtual-ips-and-service-proxies) for +more information. + +As mentioned above, the mongo container uses a volume which is mapped +to a persistent disk by Kubernetes. In [`mongo-pod.json`](mongo-pod.json) the container +section specifies the volume: + +```json +{ + "volumeMounts": [ + { + "name": "mongo-disk", + "mountPath": "/data/db" + } +``` + +The name `mongo-disk` refers to the volume specified outside the +container section: + +```json +{ + "volumes": [ + { + "name": "mongo-disk", + "gcePersistentDisk": { + "pdName": "mongo-disk", + "fsType": "ext4" + } + } + ], +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/meteor/README.md?pixel)]() + diff --git a/meteor/dockerbase/Dockerfile b/meteor/dockerbase/Dockerfile new file mode 100644 index 000000000..708e2cb43 --- /dev/null +++ b/meteor/dockerbase/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM node:0.10 + +ONBUILD WORKDIR /appsrc +ONBUILD COPY . /appsrc + +ONBUILD RUN curl https://install.meteor.com/ | sh && \ + meteor build ../app --directory --architecture os.linux.x86_64 && \ + rm -rf /appsrc +# TODO rm meteor so it doesn't take space in the image? + +ONBUILD WORKDIR /app/bundle + +ONBUILD RUN (cd programs/server && npm install) +EXPOSE 8080 +CMD [] +ENV PORT 8080 +ENTRYPOINT MONGO_URL=mongodb://$MONGO_SERVICE_HOST:$MONGO_SERVICE_PORT /usr/local/bin/node main.js diff --git a/meteor/dockerbase/README.md b/meteor/dockerbase/README.md new file mode 100644 index 000000000..2c04dff78 --- /dev/null +++ b/meteor/dockerbase/README.md @@ -0,0 +1,14 @@ +Building the meteor-kubernetes base image +----------------------------------------- + +As a normal user you don't need to do this since the image is already built and pushed to Docker Hub. You can just use it as a base image. See [this example](https://github.com/Q42/meteor-gke-example/blob/master/Dockerfile). + +To build and push the base meteor-kubernetes image: + + docker build -t chees/meteor-kubernetes . + docker push chees/meteor-kubernetes + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/meteor/dockerbase/README.md?pixel)]() + diff --git a/meteor/meteor-controller.json b/meteor/meteor-controller.json new file mode 100644 index 000000000..fa85afdcb --- /dev/null +++ b/meteor/meteor-controller.json @@ -0,0 +1,34 @@ +{ + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "meteor-controller", + "labels": { + "name": "meteor" + } + }, + "spec": { + "replicas": 2, + "template": { + "metadata": { + "labels": { + "name": "meteor" + } + }, + "spec": { + "containers": [ + { + "name": "meteor", + "image": "chees/meteor-gke-example:latest", + "ports": [ + { + "name": "http-server", + "containerPort": 8080 + } + ] + } + ] + } + } + } +} diff --git a/meteor/meteor-service.json b/meteor/meteor-service.json new file mode 100644 index 000000000..2dc55a041 --- /dev/null +++ b/meteor/meteor-service.json @@ -0,0 +1,20 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "meteor" + }, + "spec": { + "ports": [ + { + "port": 80, + "targetPort": "http-server" + } + ], + "selector": { + "name": "meteor" + }, + "sessionAffinity": "ClientIP", + "type": "LoadBalancer" + } +} diff --git a/meteor/mongo-pod.json b/meteor/mongo-pod.json new file mode 100644 index 000000000..a5b80ac5d --- /dev/null +++ b/meteor/mongo-pod.json @@ -0,0 +1,40 @@ +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "mongo", + "labels": { + "name": "mongo", + "role": "mongo" + } + }, + "spec": { + "volumes": [ + { + "name": "mongo-disk", + "gcePersistentDisk": { + "pdName": "mongo-disk", + "fsType": "ext4" + } + } + ], + "containers": [ + { + "name": "mongo", + "image": "mongo:latest", + "ports": [ + { + "name": "mongo", + "containerPort": 27017 + } + ], + "volumeMounts": [ + { + "name": "mongo-disk", + "mountPath": "/data/db" + } + ] + } + ] + } +} diff --git a/meteor/mongo-service.json b/meteor/mongo-service.json new file mode 100644 index 000000000..bec687e99 --- /dev/null +++ b/meteor/mongo-service.json @@ -0,0 +1,22 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "mongo", + "labels": { + "name": "mongo" + } + }, + "spec": { + "ports": [ + { + "port": 27017, + "targetPort": "mongo" + } + ], + "selector": { + "name": "mongo", + "role": "mongo" + } + } +} diff --git a/mysql-cinder-pd/README.md b/mysql-cinder-pd/README.md new file mode 100644 index 000000000..fb089a01c --- /dev/null +++ b/mysql-cinder-pd/README.md @@ -0,0 +1,51 @@ +# MySQL installation with cinder volume plugin + +Cinder is a Block Storage service for OpenStack. This example shows how it can be used as an attachment mounted to a pod in Kubernets. + +### Prerequisites + +Start kubelet with cloud provider as openstack with a valid cloud config +Sample cloud_config: + +``` +[Global] +auth-url=https://os-identity.vip.foo.bar.com:5443/v2.0 +username=user +password=pass +region=region1 +tenant-id=0c331a1df18571594d49fe68asa4e +``` + +Currently the cinder volume plugin is designed to work only on linux hosts and offers ext4 and ext3 as supported fs types +Make sure that kubelet host machine has the following executables + +``` +/bin/lsblk -- To Find out the fstype of the volume +/sbin/mkfs.ext3 and /sbin/mkfs.ext4 -- To format the volume if required +/usr/bin/udevadm -- To probe the volume attached so that a symlink is created under /dev/disk/by-id/ with a virtio- prefix +``` + +Ensure cinder is installed and configured properly in the region in which kubelet is spun up + +### Example + +Create a cinder volume Ex: + +`cinder create --display-name=test-repo 2` + +Use the id of the cinder volume created to create a pod [definition](mysql.yaml) +Create a new pod with the definition + +`cluster/kubectl.sh create -f examples/mysql-cinder-pd/mysql.yaml` + +This should now + +1. Attach the specified volume to the kubelet's host machine +2. Format the volume if required (only if the volume specified is not already formatted to the fstype specified) +3. Mount it on the kubelet's host machine +4. Spin up a container with this volume mounted to the path specified in the pod definition + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/mysql-cinder-pd/README.md?pixel)]() + diff --git a/mysql-cinder-pd/mysql-service.yaml b/mysql-cinder-pd/mysql-service.yaml new file mode 100644 index 000000000..6e2c019ac --- /dev/null +++ b/mysql-cinder-pd/mysql-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: mysql + name: mysql +spec: + ports: + # the port that this service should serve on + - port: 3306 + # label keys and values that must match in order to receive traffic for this service + selector: + name: mysql \ No newline at end of file diff --git a/mysql-cinder-pd/mysql.yaml b/mysql-cinder-pd/mysql.yaml new file mode 100644 index 000000000..e224d0afe --- /dev/null +++ b/mysql-cinder-pd/mysql.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mysql + labels: + name: mysql +spec: + containers: + - resources: + limits : + cpu: 0.5 + image: mysql + name: mysql + args: + - "--ignore-db-dir" + - "lost+found" + env: + - name: MYSQL_ROOT_PASSWORD + # change this + value: yourpassword + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + # name must match the volume name below + - name: mysql-persistent-storage + # mount path within the container + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + cinder: + volumeID: bd82f7e2-wece-4c01-a505-4acf60b07f4a + fsType: ext4 diff --git a/mysql-wordpress-pd/OWNERS b/mysql-wordpress-pd/OWNERS new file mode 100644 index 000000000..d349c8eeb --- /dev/null +++ b/mysql-wordpress-pd/OWNERS @@ -0,0 +1,20 @@ +approvers: +- jeffmendoza +reviewers: +- thockin +- lavalamp +- brendandburns +- caesarxuchao +- mikedanese +- davidopp +- pmorie +- dchen1107 +- janetkuo +- roberthbailey +- eparis +- mwielgus +- jlowdermilk +- david-mcmahon +- jeffvance +- jeffmendoza +- RichieEscarez diff --git a/mysql-wordpress-pd/README.md b/mysql-wordpress-pd/README.md new file mode 100644 index 000000000..a4a8a2353 --- /dev/null +++ b/mysql-wordpress-pd/README.md @@ -0,0 +1,364 @@ +# Persistent Installation of MySQL and WordPress on Kubernetes + +This example describes how to run a persistent installation of +[WordPress](https://wordpress.org/) and +[MySQL](https://www.mysql.com/) on Kubernetes. We'll use the +[mysql](https://registry.hub.docker.com/_/mysql/) and +[wordpress](https://registry.hub.docker.com/_/wordpress/) official +[Docker](https://www.docker.com/) images for this installation. (The +WordPress image includes an Apache server). + +Demonstrated Kubernetes Concepts: + +* [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) to + define persistent disks (disk lifecycle not tied to the Pods). +* [Services](https://kubernetes.io/docs/concepts/services-networking/service/) to enable Pods to + locate one another. +* [External Load Balancers](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer) + to expose Services externally. +* [Deployments](http://kubernetes.io/docs/user-guide/deployments/) to ensure Pods + stay up and running. +* [Secrets](http://kubernetes.io/docs/user-guide/secrets/) to store sensitive + passwords. + +## Quickstart + +Put your desired MySQL password in a file called `password.txt` with +no trailing newline. The first `tr` command will remove the newline if +your editor added one. + +**Note:** if your cluster enforces **_selinux_** and you will be using [Host Path](#host-path) for storage, then please follow this [extra step](#selinux). + +```shell +tr --delete '\n' .strippedpassword.txt && mv .strippedpassword.txt password.txt +kubectl create -f https://raw.githubusercontent.com/kubernetes/kubernetes/master/examples/mysql-wordpress-pd/local-volumes.yaml +kubectl create secret generic mysql-pass --from-file=password.txt +kubectl create -f https://raw.githubusercontent.com/kubernetes/kubernetes/master/examples/mysql-wordpress-pd/mysql-deployment.yaml +kubectl create -f https://raw.githubusercontent.com/kubernetes/kubernetes/master/examples/mysql-wordpress-pd/wordpress-deployment.yaml +``` + +## Table of Contents + + + +- [Persistent Installation of MySQL and WordPress on Kubernetes](#persistent-installation-of-mysql-and-wordpress-on-kubernetes) + - [Quickstart](#quickstart) + - [Table of Contents](#table-of-contents) + - [Cluster Requirements](#cluster-requirements) + - [Decide where you will store your data](#decide-where-you-will-store-your-data) + - [Host Path](#host-path) + - [SELinux](#selinux) + - [GCE Persistent Disk](#gce-persistent-disk) + - [Create the MySQL Password Secret](#create-the-mysql-password-secret) + - [Deploy MySQL](#deploy-mysql) + - [Deploy WordPress](#deploy-wordpress) + - [Visit your new WordPress blog](#visit-your-new-wordpress-blog) + - [Take down and restart your blog](#take-down-and-restart-your-blog) + - [Next Steps](#next-steps) + + + +## Cluster Requirements + +Kubernetes runs in a variety of environments and is inherently +modular. Not all clusters are the same. These are the requirements for +this example. + +* Kubernetes version 1.2 is required due to using newer features, such + at PV Claims and Deployments. Run `kubectl version` to see your + cluster version. +* [Cluster DNS](https://github.com/kubernetes/dns) will be used for service discovery. +* An [external load balancer](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer) + will be used to access WordPress. +* [Persistent Volume Claims](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) + are used. You must create Persistent Volumes in your cluster to be + claimed. This example demonstrates how to create two types of + volumes, but any volume is sufficient. + +Consult a +[Getting Started Guide](http://kubernetes.io/docs/getting-started-guides/) +to set up a cluster and the +[kubectl](http://kubernetes.io/docs/user-guide/prereqs/) command-line client. + +## Decide where you will store your data + +MySQL and WordPress will each use a +[Persistent Volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +to store their data. We will use a Persistent Volume Claim to claim an +available persistent volume. This example covers HostPath and +GCEPersistentDisk volumes. Choose one of the two, or see +[Types of Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes) +for more options. + +### Host Path + +Host paths are volumes mapped to directories on the host. **These +should be used for testing or single-node clusters only**. The data +will not be moved between nodes if the pod is recreated on a new +node. If the pod is deleted and recreated on a new node, data will be +lost. + +##### SELinux + +On systems supporting selinux it is preferred to leave it enabled/enforcing. +However, docker containers mount the host path with the "_svirt_sandbox_file_t_" +label type, which is incompatible with the default label type for /tmp ("_tmp_t_"), +resulting in a permissions error when the mysql container attempts to `chown` +_/var/lib/mysql_. +Therefore, on selinx systems using host path, you should pre-create the host path +directory (/tmp/data/) and change it's selinux label type to "_svirt_sandbox_file_t_", +as follows: + +```shell +## on every node: +mkdir -p /tmp/data +chmod a+rwt /tmp/data # match /tmp permissions +chcon -Rt svirt_sandbox_file_t /tmp/data +``` + +Continuing with host path, create the persistent volume objects in Kubernetes using +[local-volumes.yaml](local-volumes.yaml): + +```shell +export KUBE_REPO=https://raw.githubusercontent.com/kubernetes/kubernetes/master +kubectl create -f $KUBE_REPO/examples/mysql-wordpress-pd/local-volumes.yaml +``` + + +### GCE Persistent Disk + +This storage option is applicable if you are running on +[Google Compute Engine](http://kubernetes.io/docs/getting-started-guides/gce/). + +Create two persistent disks. You will need to create the disks in the +same [GCE zone](https://cloud.google.com/compute/docs/zones) as the +Kubernetes cluster. The default setup script will create the cluster +in the `us-central1-b` zone, as seen in the +[config-default.sh](../../cluster/gce/config-default.sh) file. Replace +`` below with the appropriate zone. The names `wordpress-1` and +`wordpress-2` must match the `pdName` fields we have specified in +[gce-volumes.yaml](gce-volumes.yaml). + +```shell +gcloud compute disks create --size=20GB --zone= wordpress-1 +gcloud compute disks create --size=20GB --zone= wordpress-2 +``` + +Create the persistent volume objects in Kubernetes for those disks: + +```shell +export KUBE_REPO=https://raw.githubusercontent.com/kubernetes/kubernetes/master +kubectl create -f $KUBE_REPO/examples/mysql-wordpress-pd/gce-volumes.yaml +``` + +## Create the MySQL Password Secret + +Use a [Secret](http://kubernetes.io/docs/user-guide/secrets/) object +to store the MySQL password. First create a file (in the same directory +as the wordpress sample files) called +`password.txt` and save your password in it. Make sure to not have a +trailing newline at the end of the password. The first `tr` command +will remove the newline if your editor added one. Then, create the +Secret object. + +```shell +tr --delete '\n' .strippedpassword.txt && mv .strippedpassword.txt password.txt +kubectl create secret generic mysql-pass --from-file=password.txt +``` + +This secret is referenced by the MySQL and WordPress pod configuration +so that those pods will have access to it. The MySQL pod will set the +database password, and the WordPress pod will use the password to +access the database. + +## Deploy MySQL + +Now that the persistent disks and secrets are defined, the Kubernetes +pods can be launched. Start MySQL using +[mysql-deployment.yaml](mysql-deployment.yaml). + +```shell +kubectl create -f $KUBE_REPO/examples/mysql-wordpress-pd/mysql-deployment.yaml +``` + +Take a look at [mysql-deployment.yaml](mysql-deployment.yaml), and +note that we've defined a volume mount for `/var/lib/mysql`, and then +created a Persistent Volume Claim that looks for a 20G volume. This +claim is satisfied by any volume that meets the requirements, in our +case one of the volumes we created above. + +Also look at the `env` section and see that we specified the password +by referencing the secret `mysql-pass` that we created above. Secrets +can have multiple key:value pairs. Ours has only one key +`password.txt` which was the name of the file we used to create the +secret. The [MySQL image](https://hub.docker.com/_/mysql/) sets the +database password using the `MYSQL_ROOT_PASSWORD` environment +variable. + +It may take a short period before the new pod reaches the `Running` +state. List all pods to see the status of this new pod. + +```shell +kubectl get pods +``` + +``` +NAME READY STATUS RESTARTS AGE +wordpress-mysql-cqcf4-9q8lo 1/1 Running 0 1m +``` + +Kubernetes logs the stderr and stdout for each pod. Take a look at the +logs for a pod by using `kubectl log`. Copy the pod name from the +`get pods` command, and then: + +```shell +kubectl logs +``` + +``` +... +2016-02-19 16:58:05 1 [Note] InnoDB: 128 rollback segment(s) are active. +2016-02-19 16:58:05 1 [Note] InnoDB: Waiting for purge to start +2016-02-19 16:58:05 1 [Note] InnoDB: 5.6.29 started; log sequence number 1626007 +2016-02-19 16:58:05 1 [Note] Server hostname (bind-address): '*'; port: 3306 +2016-02-19 16:58:05 1 [Note] IPv6 is available. +2016-02-19 16:58:05 1 [Note] - '::' resolves to '::'; +2016-02-19 16:58:05 1 [Note] Server socket created on IP: '::'. +2016-02-19 16:58:05 1 [Warning] 'proxies_priv' entry '@ root@wordpress-mysql-cqcf4-9q8lo' ignored in --skip-name-resolve mode. +2016-02-19 16:58:05 1 [Note] Event Scheduler: Loaded 0 events +2016-02-19 16:58:05 1 [Note] mysqld: ready for connections. +Version: '5.6.29' socket: '/var/run/mysqld/mysqld.sock' port: 3306 MySQL Community Server (GPL) +``` + +Also in [mysql-deployment.yaml](mysql-deployment.yaml) we created a +service to allow other pods to reach this mysql instance. The name is +`wordpress-mysql` which resolves to the pod IP. + +Up to this point one Deployment, one Pod, one PVC, one Service, one Endpoint, +two PVs, and one Secret have been created, shown below: + +```shell +kubectl get deployment,pod,svc,endpoints,pvc -l app=wordpress -o wide && \ + kubectl get secret mysql-pass && \ + kubectl get pv +``` + +```shell +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +deploy/wordpress-mysql 1 1 1 1 3m +NAME READY STATUS RESTARTS AGE IP NODE +po/wordpress-mysql-3040864217-40soc 1/1 Running 0 3m 172.17.0.2 127.0.0.1 +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +svc/wordpress-mysql None 3306/TCP 3m app=wordpress,tier=mysql +NAME ENDPOINTS AGE +ep/wordpress-mysql 172.17.0.2:3306 3m +NAME STATUS VOLUME CAPACITY ACCESSMODES AGE +pvc/mysql-pv-claim Bound local-pv-2 20Gi RWO 3m +NAME TYPE DATA AGE +mysql-pass Opaque 1 3m +NAME CAPACITY ACCESSMODES STATUS CLAIM REASON AGE +local-pv-1 20Gi RWO Available 3m +local-pv-2 20Gi RWO Bound default/mysql-pv-claim 3m +``` + +## Deploy WordPress + +Next deploy WordPress using +[wordpress-deployment.yaml](wordpress-deployment.yaml): + +```shell +kubectl create -f $KUBE_REPO/examples/mysql-wordpress-pd/wordpress-deployment.yaml +``` + +Here we are using many of the same features, such as a volume claim +for persistent storage and a secret for the password. + +The [WordPress image](https://hub.docker.com/_/wordpress/) accepts the +database hostname through the environment variable +`WORDPRESS_DB_HOST`. We set the env value to the name of the MySQL +service we created: `wordpress-mysql`. + +The WordPress service has the setting `type: LoadBalancer`. This will +set up the wordpress service behind an external IP. + +Find the external IP for your WordPress service. **It may take a minute +to have an external IP assigned to the service, depending on your +cluster environment.** + +```shell +kubectl get services wordpress +``` + +``` +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +wordpress 10.0.0.5 1.2.3.4 80/TCP 19h +``` + +## Visit your new WordPress blog + +Now, we can visit the running WordPress app. Use the external IP of +the service that you obtained above. + +``` +http:// +``` + +You should see the familiar WordPress init page. + +![WordPress init page](WordPress.png "WordPress init page") + +> Warning: Do not leave your WordPress installation on this page. If +> it is found by another user, they can set up a website on your +> instance and use it to serve potentially malicious content. You +> should either continue with the installation past the point at which +> you create your username and password, delete your instance, or set +> up a firewall to restrict access. + +## Take down and restart your blog + +Set up your WordPress blog and play around with it a bit. Then, take +down its pods and bring them back up again. Because you used +persistent disks, your blog state will be preserved. + +All of the resources are labeled with `app=wordpress`, so you can +easily bring them down using a label selector: + +```shell +kubectl delete deployment,service -l app=wordpress +kubectl delete secret mysql-pass +``` + +Later, re-creating the resources with the original commands will pick +up the original disks with all your data intact. Because we did not +delete the PV Claims, no other pods in the cluster could claim them +after we deleted our pods. Keeping the PV Claims also ensured +recreating the Pods did not cause the PD to switch Pods. + +If you are ready to release your persistent volumes and the data on them, run: + +```shell +kubectl delete pvc -l app=wordpress +``` + +And then delete the volume objects themselves: + +```shell +kubectl delete pv local-pv-1 local-pv-2 +``` + +or + +```shell +kubectl delete pv wordpress-pv-1 wordpress-pv-2 +``` + +## Next Steps + +* [Introspection and Debugging](http://kubernetes.io/docs/user-guide/introspection-and-debugging/) +* [Jobs](http://kubernetes.io/docs/user-guide/jobs/) may be useful to run SQL queries. +* [Exec](http://kubernetes.io/docs/user-guide/getting-into-containers/) +* [Port Forwarding](http://kubernetes.io/docs/user-guide/connecting-to-applications-port-forward/) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/mysql-wordpress-pd/README.md?pixel)]() + diff --git a/mysql-wordpress-pd/WordPress.png b/mysql-wordpress-pd/WordPress.png new file mode 100644 index 000000000..cabcd09a6 Binary files /dev/null and b/mysql-wordpress-pd/WordPress.png differ diff --git a/mysql-wordpress-pd/gce-volumes.yaml b/mysql-wordpress-pd/gce-volumes.yaml new file mode 100644 index 000000000..17aeb0596 --- /dev/null +++ b/mysql-wordpress-pd/gce-volumes.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: wordpress-pv-1 +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + gcePersistentDisk: + pdName: wordpress-1 + fsType: ext4 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: wordpress-pv-2 +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + gcePersistentDisk: + pdName: wordpress-2 + fsType: ext4 diff --git a/mysql-wordpress-pd/local-volumes.yaml b/mysql-wordpress-pd/local-volumes.yaml new file mode 100644 index 000000000..896411333 --- /dev/null +++ b/mysql-wordpress-pd/local-volumes.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: local-pv-1 + labels: + type: local +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp/data/pv-1 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: local-pv-2 + labels: + type: local +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp/data/pv-2 diff --git a/mysql-wordpress-pd/mysql-deployment.yaml b/mysql-wordpress-pd/mysql-deployment.yaml new file mode 100644 index 000000000..2dafc08f0 --- /dev/null +++ b/mysql-wordpress-pd/mysql-deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + name: wordpress-mysql + labels: + app: wordpress +spec: + ports: + - port: 3306 + selector: + app: wordpress + tier: mysql + clusterIP: None +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim + labels: + app: wordpress +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: wordpress-mysql + labels: + app: wordpress +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app: wordpress + tier: mysql + spec: + containers: + - image: mysql:5.6 + name: mysql + env: + # $ kubectl create secret generic mysql-pass --from-file=password.txt + # make sure password.txt does not have a trailing newline + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-pass + key: password.txt + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim diff --git a/mysql-wordpress-pd/wordpress-deployment.yaml b/mysql-wordpress-pd/wordpress-deployment.yaml new file mode 100644 index 000000000..83f5f4c5d --- /dev/null +++ b/mysql-wordpress-pd/wordpress-deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + name: wordpress + labels: + app: wordpress +spec: + ports: + - port: 80 + selector: + app: wordpress + tier: frontend + type: LoadBalancer +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: wp-pv-claim + labels: + app: wordpress +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: wordpress + labels: + app: wordpress +spec: + strategy: + type: Recreate + template: + metadata: + labels: + app: wordpress + tier: frontend + spec: + containers: + - image: wordpress:4.7.3-apache + name: wordpress + env: + - name: WORDPRESS_DB_HOST + value: wordpress-mysql + - name: WORDPRESS_DB_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-pass + key: password.txt + ports: + - containerPort: 80 + name: wordpress + volumeMounts: + - name: wordpress-persistent-storage + mountPath: /var/www/html + volumes: + - name: wordpress-persistent-storage + persistentVolumeClaim: + claimName: wp-pv-claim diff --git a/newrelic/README.md b/newrelic/README.md new file mode 100644 index 000000000..ed7d22824 --- /dev/null +++ b/newrelic/README.md @@ -0,0 +1,157 @@ +## New Relic Server Monitoring Agent Example + +This example shows how to run a New Relic server monitoring agent as a pod in a DaemonSet on an existing Kubernetes cluster. + +This example will create a DaemonSet which places the New Relic monitoring agent on every node in the cluster. It's also fairly trivial to exclude specific Kubernetes nodes from the DaemonSet to just monitor specific servers. + +### Step 0: Prerequisites + +This process will create privileged containers which have full access to the host system for logging. Beware of the security implications of this. + +If you are using a Salt based KUBERNETES\_PROVIDER (**gce**, **vagrant**, **aws**), you should make sure the creation of privileged containers via the API is enabled. Check `cluster/saltbase/pillar/privilege.sls`. + +DaemonSets must be enabled on your cluster. Instructions for enabling DaemonSet can be found [here](../../docs/api.md#enabling-the-extensions-group). + +### Step 1: Configure New Relic Agent + +The New Relic agent is configured via environment variables. We will configure these environment variables in a sourced bash script, encode the environment file data, and store it in a secret which will be loaded at container runtime. + +The [New Relic Linux Server configuration page] +(https://docs.newrelic.com/docs/servers/new-relic-servers-linux/installation-configuration/configuring-servers-linux) lists all the other settings for nrsysmond. + +To create an environment variable for a setting, prepend NRSYSMOND_ to its name. For example, + +```console +loglevel=debug +``` + +translates to + +```console +NRSYSMOND_loglevel=debug +``` + +Edit examples/newrelic/nrconfig.env and set up the environment variables for your NewRelic agent. Be sure to edit the license key field and fill in your own New Relic license key. + +Now, let's vendor the config into a secret. + +```console +$ cd examples/newrelic/ +$ ./config-to-secret.sh +``` + + + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: newrelic-config +type: Opaque +data: + config: {{config_data}} +``` + +[Download example](newrelic-config-template.yaml?raw=true) + + +The script will encode the config file and write it to `newrelic-config.yaml`. + +Finally, submit the config to the cluster: + +```console +$ kubectl create -f examples/newrelic/newrelic-config.yaml +``` + +### Step 2: Create the DaemonSet definition. + +The DaemonSet definition instructs Kubernetes to place a newrelic sysmond agent on each Kubernetes node. + + + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: newrelic-agent + labels: + tier: monitoring + app: newrelic-agent + version: v1 +spec: + template: + metadata: + labels: + name: newrelic + spec: + # Filter to specific nodes: + # nodeSelector: + # app: newrelic + hostPID: true + hostIPC: true + hostNetwork: true + containers: + - resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + env: + - name: NRSYSMOND_logfile + value: "/var/log/nrsysmond.log" + image: newrelic/nrsysmond + name: newrelic + command: [ "bash", "-c", "source /etc/kube-newrelic/config && /usr/sbin/nrsysmond -E -F" ] + volumeMounts: + - name: newrelic-config + mountPath: /etc/kube-newrelic + readOnly: true + - name: dev + mountPath: /dev + - name: run + mountPath: /var/run/docker.sock + - name: sys + mountPath: /sys + - name: log + mountPath: /var/log + volumes: + - name: newrelic-config + secret: + secretName: newrelic-config + - name: dev + hostPath: + path: /dev + - name: run + hostPath: + path: /var/run/docker.sock + - name: sys + hostPath: + path: /sys + - name: log + hostPath: + path: /var/log +``` + +[Download example](newrelic-daemonset.yaml?raw=true) + + +The daemonset instructs Kubernetes to spawn pods on each node, mapping /dev/, /run/, /sys/, and /var/log to the container. It also maps the secrets we set up earlier to /etc/kube-newrelic/config, and sources them in the startup script, configuring the agent properly. + +#### DaemonSet customization + +- To include a custom hostname prefix (or other per-container environment variables that can be generated at run-time), you can modify the DaemonSet `command` value: + +``` +command: [ "bash", "-c", "source /etc/kube-newrelic/config && export NRSYSMOND_hostname=mycluster-$(hostname) && /usr/sbin/nrsysmond -E -F" ] +``` + +When the New Relic agent starts, `NRSYSMOND_hostname` is set using the output of `hostname` with `mycluster` prepended. + + +### Known issues + +It's a bit cludgy to define the environment variables like we do here in these config files. There is [another issue](https://github.com/kubernetes/kubernetes/issues/4710) to discuss adding mapping secrets to environment variables in Kubernetes. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/newrelic/README.md?pixel)]() + diff --git a/newrelic/config-to-secret.sh b/newrelic/config-to-secret.sh new file mode 100755 index 000000000..520c71990 --- /dev/null +++ b/newrelic/config-to-secret.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Encodes the environment variables into a Kubernetes secret. + +BASE64_ENC=$(cat nrconfig.env | base64 | tr -d '\n') +sed -e "s#{{config_data}}#${BASE64_ENC}#g" ./newrelic-config-template.yaml > newrelic-config.yaml diff --git a/newrelic/newrelic-config-template.yaml b/newrelic/newrelic-config-template.yaml new file mode 100644 index 000000000..361a30792 --- /dev/null +++ b/newrelic/newrelic-config-template.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: newrelic-config +type: Opaque +data: + config: {{config_data}} diff --git a/newrelic/newrelic-config.yaml b/newrelic/newrelic-config.yaml new file mode 100644 index 000000000..d2b551aa6 --- /dev/null +++ b/newrelic/newrelic-config.yaml @@ -0,0 +1,10 @@ +# This file should be overwritten by ./config-to-secret.sh +# This file is in place to satisfy the kubernetes documentation tests. + +# apiVersion: v1 +# kind: Secret +# metadata: +# name: newrelic-config +# type: Opaque +# data: +# config: base64 encoded diff --git a/newrelic/newrelic-daemonset.yaml b/newrelic/newrelic-daemonset.yaml new file mode 100644 index 000000000..24adb3823 --- /dev/null +++ b/newrelic/newrelic-daemonset.yaml @@ -0,0 +1,60 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: newrelic-agent + labels: + tier: monitoring + app: newrelic-agent + version: v1 +spec: + template: + metadata: + labels: + name: newrelic + spec: + # Filter to specific nodes: + # nodeSelector: + # app: newrelic + hostPID: true + hostIPC: true + hostNetwork: true + containers: + - resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + env: + - name: NRSYSMOND_logfile + value: "/var/log/nrsysmond.log" + image: newrelic/nrsysmond + name: newrelic + command: [ "bash", "-c", "source /etc/kube-newrelic/config && /usr/sbin/nrsysmond -E -F" ] + volumeMounts: + - name: newrelic-config + mountPath: /etc/kube-newrelic + readOnly: true + - name: dev + mountPath: /dev + - name: run + mountPath: /var/run/docker.sock + - name: sys + mountPath: /sys + - name: log + mountPath: /var/log + volumes: + - name: newrelic-config + secret: + secretName: newrelic-config + - name: dev + hostPath: + path: /dev + - name: run + hostPath: + path: /var/run/docker.sock + - name: sys + hostPath: + path: /sys + - name: log + hostPath: + path: /var/log diff --git a/newrelic/nrconfig.env b/newrelic/nrconfig.env new file mode 100644 index 000000000..ddce85294 --- /dev/null +++ b/newrelic/nrconfig.env @@ -0,0 +1,2 @@ +export NRSYSMOND_loglevel=debug +export NRSYSMOND_license_key=REPLACE_LICENSE_KEY_HERE diff --git a/nodesjs-mongodb/README.md b/nodesjs-mongodb/README.md new file mode 100644 index 000000000..b8d486c50 --- /dev/null +++ b/nodesjs-mongodb/README.md @@ -0,0 +1,282 @@ +## Node.js and MongoDB on Kubernetes + +The following document describes the deployment of a basic Node.js and MongoDB web stack on Kubernetes. Currently this example does not use replica sets for MongoDB. + +For more a in-depth explanation of this example, please [read this post.](https://medium.com/google-cloud-platform-developer-advocates/running-a-mean-stack-on-google-cloud-platform-with-kubernetes-149ca81c2b5d) + +### Prerequisites + +This example assumes that you have a basic understanding of Kubernetes conecepts (Pods, Services, Replication Controllers), a Kubernetes cluster up and running, and that you have installed the ```kubectl``` command line tool somewhere in your path. Please see the [getting started](../../docs/getting-started-guides/) for installation instructions for your platform. + +Note: This example was tested on [Google Container Engine](https://cloud.google.com/container-engine/docs/). Some optional commands require the [Google Cloud SDK](https://cloud.google.com/sdk/). + +### Creating the MongoDB Service + +The first thing to do is create the MongoDB Service. This service is used by the other Pods in the cluster to find and connect to the MongoDB instance. + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + name: mongo + name: mongo +spec: + ports: + - port: 27017 + targetPort: 27017 + selector: + name: mongo +``` + +[Download file](mongo-service.yaml) + +This service looks for all pods with the "mongo" tag, and creates a Service on port 27017 that targets port 27017 on the MongoDB pods. Port 27017 is the standard MongoDB port. + +To start the service, run: + +```sh +kubectl create -f examples/nodesjs-mongodb/mongo-service.yaml +``` + +### Creating the MongoDB Controller + +Next, create the MongoDB instance that runs the Database. Databases also need persistent storage, which will be different for each platform. + +```yaml +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: mongo + name: mongo-controller +spec: + replicas: 1 + template: + metadata: + labels: + name: mongo + spec: + containers: + - image: mongo + name: mongo + ports: + - name: mongo + containerPort: 27017 + hostPort: 27017 + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + gcePersistentDisk: + pdName: mongo-disk + fsType: ext4 +``` + +[Download file](mongo-controller.yaml) + +Looking at this file from the bottom up: + +First, it creates a volume called "mongo-persistent-storage." + +In the above example, it is using a "gcePersistentDisk" to back the storage. This is only applicable if you are running your Kubernetes cluster in Google Cloud Platform. + +If you don't already have a [Google Persistent Disk](https://cloud.google.com/compute/docs/disks) created in the same zone as your cluster, create a new disk in the same Google Compute Engine / Container Engine zone as your cluster with this command: + +```sh +gcloud compute disks create --size=200GB --zone=$ZONE mongo-disk +``` + +If you are using AWS, replace the "volumes" section with this (untested): + +```yaml + volumes: + - name: mongo-persistent-storage + awsElasticBlockStore: + volumeID: aws://{region}/{volume ID} + fsType: ext4 +``` + +If you don't have a EBS volume in the same region as your cluster, create a new EBS volume in the same region with this command (untested): + +```sh +ec2-create-volume --size 200 --region $REGION --availability-zone $ZONE +``` + +This command will return a volume ID to use. + +For other storage options (iSCSI, NFS, OpenStack), please follow the documentation. + +Now that the volume is created and usable by Kubernetes, the next step is to create the Pod. + +Looking at the container section: It uses the official MongoDB container, names itself "mongo", opens up port 27017, and mounts the disk to "/data/db" (where the mongo container expects the data to be). + +Now looking at the rest of the file, it is creating a Replication Controller with one replica, called mongo-controller. It is important to use a Replication Controller and not just a Pod, as a Replication Controller will restart the instance in case it crashes. + +Create this controller with this command: + +```sh +kubectl create -f examples/nodesjs-mongodb/mongo-controller.yaml +``` + +At this point, MongoDB is up and running. + +Note: There is no password protection or auth running on the database by default. Please keep this in mind! + +### Creating the Node.js Service + +The next step is to create the Node.js service. This service is what will be the endpoint for the web site, and will load balance requests to the Node.js instances. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + name: web +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 3000 + protocol: TCP + selector: + name: web +``` + +[Download file](web-service.yaml) + +This service is called "web," and it uses a [LoadBalancer](../../docs/user-guide/services.md#type-loadbalancer) to distribute traffic on port 80 to port 3000 running on Pods with the "web" tag. Port 80 is the standard HTTP port, and port 3000 is the standard Node.js port. + +On Google Container Engine, a [network load balancer](https://cloud.google.com/compute/docs/load-balancing/network/) and [firewall rule](https://cloud.google.com/compute/docs/networking#addingafirewall) to allow traffic are automatically created. + +To start the service, run: + +```sh +kubectl create -f examples/nodesjs-mongodb/web-service.yaml +``` + +If you are running on a platform that does not support LoadBalancer (i.e Bare Metal), you need to use a [NodePort](../../docs/user-guide/services.md#type-nodeport) with your own load balancer. + +You may also need to open appropriate Firewall ports to allow traffic. + +### Creating the Node.js Controller + +The final step is deploying the Node.js container that will run the application code. This container can easily by replaced by any other web serving frontend, such as Rails, LAMP, Java, Go, etc. + +The most important thing to keep in mind is how to access the MongoDB service. + +If you were running MongoDB and Node.js on the same server, you would access MongoDB like so: + +```javascript +MongoClient.connect('mongodb://localhost:27017/database-name', function(err, db) { console.log(db); }); +``` + +With this Kubernetes setup, that line of code would become: + +```javascript +MongoClient.connect('mongodb://mongo:27017/database-name', function(err, db) { console.log(db); }); +``` + +The MongoDB Service previously created tells Kubernetes to configure the cluster so 'mongo' points to the MongoDB instance created earlier. + +#### Custom Container + +You should have your own container that runs your Node.js code hosted in a container registry. + +See [this example](https://medium.com/google-cloud-platform-developer-advocates/running-a-mean-stack-on-google-cloud-platform-with-kubernetes-149ca81c2b5d#8edc) to see how to make your own Node.js container. + +Once you have created your container, create the web controller. + +```yaml +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: web + name: web-controller +spec: + replicas: 2 + selector: + name: web + template: + metadata: + labels: + name: web + spec: + containers: + - image: + name: web + ports: + - containerPort: 3000 + name: http-server +``` + +[Download file](web-controller.yaml) + +Replace with the url of your container. + +This Controller will create two replicas of the Node.js container, and each Node.js container will have the tag "web" and expose port 3000. The Service LoadBalancer will forward port 80 traffic to port 3000 automatically, along with load balancing traffic between the two instances. + +To start the Controller, run: + +```sh +kubectl create -f examples/nodesjs-mongodb/web-controller.yaml +``` + +#### Demo Container + +If you DON'T want to create a custom container, you can use the following YAML file: + +Note: You cannot run both Controllers at the same time, as they both try to control the same Pods. + +```yaml +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: web + name: web-controller +spec: + replicas: 2 + selector: + name: web + template: + metadata: + labels: + name: web + spec: + containers: + - image: node:0.10.40 + command: ['/bin/sh', '-c'] + args: ['cd /home && git clone https://github.com/ijason/NodeJS-Sample-App.git demo && cd demo/EmployeeDB/ && npm install && sed -i -- ''s/localhost/mongo/g'' app.js && node app.js'] + name: web + ports: + - containerPort: 3000 + name: http-server +``` + +[Download file](web-controller-demo.yaml) + +This will use the default Node.js container, and will pull and execute code at run time. This is not recommended; typically, your code should be part of the container. + +To start the Controller, run: + +```sh +kubectl create -f examples/nodesjs-mongodb/web-controller-demo.yaml +``` + +### Testing it out + +Now that all the components are running, visit the IP address of the load balancer to access the website. + +With Google Cloud Platform, get the IP address of all load balancers with the following command: + +```sh +gcloud compute forwarding-rules list +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/nodesjs-mongodb/README.md?pixel)]() + diff --git a/nodesjs-mongodb/mongo-controller.yaml b/nodesjs-mongodb/mongo-controller.yaml new file mode 100644 index 000000000..e9288151d --- /dev/null +++ b/nodesjs-mongodb/mongo-controller.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: mongo + name: mongo-controller +spec: + replicas: 1 + template: + metadata: + labels: + name: mongo + spec: + containers: + - image: mongo + name: mongo + ports: + - name: mongo + containerPort: 27017 + hostPort: 27017 + volumeMounts: + - name: mongo-persistent-storage + mountPath: /data/db + volumes: + - name: mongo-persistent-storage + gcePersistentDisk: + pdName: mongo-disk + fsType: ext4 \ No newline at end of file diff --git a/nodesjs-mongodb/mongo-service.yaml b/nodesjs-mongodb/mongo-service.yaml new file mode 100644 index 000000000..81785d5b8 --- /dev/null +++ b/nodesjs-mongodb/mongo-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: mongo + name: mongo +spec: + ports: + - port: 27017 + targetPort: 27017 + selector: + name: mongo \ No newline at end of file diff --git a/nodesjs-mongodb/web-controller-demo.yaml b/nodesjs-mongodb/web-controller-demo.yaml new file mode 100644 index 000000000..5fa8ef84e --- /dev/null +++ b/nodesjs-mongodb/web-controller-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: web + name: web-controller +spec: + replicas: 2 + selector: + name: web + template: + metadata: + labels: + name: web + spec: + containers: + - image: node:0.10.40 + command: ['/bin/sh', '-c'] + args: ['cd /home && git clone https://github.com/ijason/NodeJS-Sample-App.git demo && cd demo/EmployeeDB/ && npm install && sed -i -- ''s/localhost/mongo/g'' app.js && node app.js'] + name: web + ports: + - containerPort: 3000 + name: http-server \ No newline at end of file diff --git a/nodesjs-mongodb/web-controller.yaml b/nodesjs-mongodb/web-controller.yaml new file mode 100644 index 000000000..654138099 --- /dev/null +++ b/nodesjs-mongodb/web-controller.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + name: web + name: web-controller +spec: + replicas: 2 + selector: + name: web + template: + metadata: + labels: + name: web + spec: + containers: + - image: + name: web + ports: + - containerPort: 3000 + name: http-server \ No newline at end of file diff --git a/nodesjs-mongodb/web-service.yaml b/nodesjs-mongodb/web-service.yaml new file mode 100644 index 000000000..4cef05b33 --- /dev/null +++ b/nodesjs-mongodb/web-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + name: web +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 3000 + protocol: TCP + selector: + name: web \ No newline at end of file diff --git a/oms/README.md b/oms/README.md new file mode 100644 index 000000000..71b30e323 --- /dev/null +++ b/oms/README.md @@ -0,0 +1,61 @@ +# Microsoft Operations Management Suite (OMS) Container Monitoring Example + +The [Microsoft Operations Management Suite (OMS)](https://www.microsoft.com/en-us/cloud-platform/operations-management-suite) is a software-as-a-service offering from Microsoft that allows Enterprise IT to manage any hybrid cloud. + +This example will create a DeamonSet to deploy the OMS Linux agents running as containers to every node in the Kubernetes cluster. + +### Supported Linux Operating Systems & Docker +- Docker 1.10 thru 1.12.1 + +- An x64 version of the following: + - Ubuntu 14.04 LTS, 16.04 LTS + - CoreOS (stable) + - Amazon Linux 2016.09.0 + - openSUSE 13.2 + - CentOS 7 + - SLES 12 + - RHEL 7.2 + +## Step 1 + +If you already have a Microsoft Azure account, you can quickly create a free OMS account by following the steps [here](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-get-started#sign-up-quickly-using-microsoft-azure). + +If you don't have a Microsoft Azure account, you can create a free OMS account by following the guide [here](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-get-started#sign-up-in-3-steps-using-oms). + +## Step 2 + +You will need to edit the [omsagent-daemonset.yaml](./omsagent-daemonset.yaml) file to add your Workspace ID and Primary Key of your OMS account. + +``` +- env: + - name: WSID + value: + - name: KEY + value: +``` + +The Workspace ID and Primary Key can be found inside the OMS Portal under Settings in the connected sources tab (see below screenshot). +![connected-resources](./images/connected-resources.png) + +## Step 3 + +Run the following command to deploy the OMS agent to your Kubernetes nodes: + +``` +kubectl -f omsagent-daemonset.yaml +``` + +## Step 4 + +Add the Container solution to your OMS workspace: + +1. Log in to the OMS portal. +2. Click the Solutions Gallery tile. +3. On the OMS Solutions Gallery page, click on Containers. +4. On the page for the Containers solution, detailed information about the solution is displayed. Click Add. + +A new tile for the Container solution that you added appears on the Overview page in OMS. It would take 5 minutes for your data to appear in OMS. + +![oms-portal](./images/oms-portal.png) + +![coms-container-solution](./images/oms-container-solution.png) \ No newline at end of file diff --git a/oms/images/connected-resources.png b/oms/images/connected-resources.png new file mode 100644 index 000000000..32bd05791 Binary files /dev/null and b/oms/images/connected-resources.png differ diff --git a/oms/images/oms-container-solution.png b/oms/images/oms-container-solution.png new file mode 100644 index 000000000..1bc90d11d Binary files /dev/null and b/oms/images/oms-container-solution.png differ diff --git a/oms/images/oms-portal.png b/oms/images/oms-portal.png new file mode 100644 index 000000000..300114e1f Binary files /dev/null and b/oms/images/oms-portal.png differ diff --git a/oms/omsagent-daemonset.yaml b/oms/omsagent-daemonset.yaml new file mode 100644 index 000000000..bf59b370e --- /dev/null +++ b/oms/omsagent-daemonset.yaml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: omsagent +spec: + template: + metadata: + labels: + app: omsagent + spec: + containers: + - env: + - name: WSID + value: + - name: KEY + value: + image: microsoft/oms + name: omsagent + ports: + - containerPort: 25225 + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock \ No newline at end of file diff --git a/openshift-origin/.gitignore b/openshift-origin/.gitignore new file mode 100644 index 000000000..f733c4b5f --- /dev/null +++ b/openshift-origin/.gitignore @@ -0,0 +1 @@ +config/ diff --git a/openshift-origin/README.md b/openshift-origin/README.md new file mode 100644 index 000000000..34b243518 --- /dev/null +++ b/openshift-origin/README.md @@ -0,0 +1,211 @@ +## OpenShift Origin example + +This example shows how to run OpenShift Origin as a pod on an existing Kubernetes cluster. + +OpenShift Origin runs with a rich set of role based policy rules out of the box that requires authentication from users via certificates. When run as a pod on an existing Kubernetes cluster, it proxies access to the underlying Kubernetes services to provide security. + +As a result, this example is a complex end-to-end configuration that shows how to configure certificates for a service that runs on Kubernetes, and requires a number of configuration files to be injected dynamically via a secret volume to the pod. + +This example will create a pod running the OpenShift Origin master. In addition, it will run a three-pod etcd setup to hold OpenShift content. OpenShift embeds Kubernetes in the stand-alone setup, so the configuration for OpenShift when it is running against an external Kubernetes cluster is different: content specific to Kubernetes will be stored in the Kubernetes etcd repository (i.e. pods, services, replication controllers, etc.), but OpenShift specific content (builds, images, users, policies, etc.) are stored in its etcd setup. + +### Step 0: Prerequisites + +This example assumes that you have an understanding of Kubernetes and that you have forked the repository. + +OpenShift Origin creates privileged containers when running Docker builds during the source-to-image process. + +If you are using a Salt based KUBERNETES_PROVIDER (**gce**, **vagrant**, **aws**), you should enable the +ability to create privileged containers via the API. + +```sh +$ cd kubernetes +$ vi cluster/saltbase/pillar/privilege.sls + +# If true, allow privileged containers to be created by API +allow_privileged: true +``` + +Now spin up a cluster using your preferred KUBERNETES_PROVIDER. Remember that `kube-up.sh` may start other pods on your nodes, so ensure that you have enough resources to run the five pods for this example. + + +```sh +$ export KUBERNETES_PROVIDER=${YOUR_PROVIDER} +$ cluster/kube-up.sh +``` + +Next, let's setup some variables, and create a local folder that will hold generated configuration files. + +```sh +$ export OPENSHIFT_EXAMPLE=$(pwd)/examples/openshift-origin +$ export OPENSHIFT_CONFIG=${OPENSHIFT_EXAMPLE}/config +$ mkdir ${OPENSHIFT_CONFIG} + +$ export ETCD_INITIAL_CLUSTER_TOKEN=$(python -c "import string; import random; print(''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(40)))") +$ export ETCD_DISCOVERY_TOKEN=$(python -c "import string; import random; print(\"etcd-cluster-\" + ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(5)))") +$ sed -i.bak -e "s/INSERT_ETCD_INITIAL_CLUSTER_TOKEN/\"${ETCD_INITIAL_CLUSTER_TOKEN}\"/g" -e "s/INSERT_ETCD_DISCOVERY_TOKEN/\"${ETCD_DISCOVERY_TOKEN}\"/g" ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml +``` + +This will have created a `etcd-controller.yaml.bak` file in your directory, which you should remember to restore when doing cleanup (or use the given `cleanup.sh`). Finally, let's start up the external etcd pods and the discovery service necessary for their initialization: + +```sh +$ kubectl create -f examples/openshift-origin/openshift-origin-namespace.yaml +$ kubectl create -f examples/openshift-origin/etcd-discovery-controller.yaml --namespace="openshift-origin" +$ kubectl create -f examples/openshift-origin/etcd-discovery-service.yaml --namespace="openshift-origin" +$ kubectl create -f examples/openshift-origin/etcd-controller.yaml --namespace="openshift-origin" +$ kubectl create -f examples/openshift-origin/etcd-service.yaml --namespace="openshift-origin" +``` + +### Step 1: Export your Kubernetes configuration file for use by OpenShift pod + +OpenShift Origin uses a configuration file to know how to access your Kubernetes cluster with administrative authority. + +``` +$ cluster/kubectl.sh config view --output=yaml --flatten=true --minify=true > ${OPENSHIFT_CONFIG}/kubeconfig +``` + +The output from this command will contain a single file that has all the required information needed to connect to your Kubernetes cluster that you previously provisioned. This file should be considered sensitive, so do not share this file with untrusted parties. + +We will later use this file to tell OpenShift how to bootstrap its own configuration. + +### Step 2: Create an External Load Balancer to Route Traffic to OpenShift + +An external load balancer is needed to route traffic to our OpenShift master service that will run as a pod on your Kubernetes cluster. + + +```sh +$ cluster/kubectl.sh create -f $OPENSHIFT_EXAMPLE/openshift-service.yaml --namespace="openshift-origin" +``` + +### Step 3: Generate configuration file for your OpenShift master pod + +The OpenShift master requires a configuration file as input to know how to bootstrap the system. + +In order to build this configuration file, we need to know the public IP address of our external load balancer in order to build default certificates. + +Grab the public IP address of the service we previously created: the two-line script below will attempt to do so, but make sure to check that the IP was set as a result - if it was not, try again after a couple seconds. + + +```sh +$ export PUBLIC_OPENSHIFT_IP=$(kubectl get services openshift --namespace="openshift-origin" --template="{{ index .status.loadBalancer.ingress 0 \"ip\" }}") +$ echo ${PUBLIC_OPENSHIFT_IP} +``` + +You can automate the process with the following script, as it might take more than a minute for the IP to be set and discoverable. + +```shell +$ while [ ${#PUBLIC_OPENSHIFT_IP} -lt 1 ]; do + echo -n . + sleep 1 + { + export PUBLIC_OPENSHIFT_IP=$(kubectl get services openshift --namespace="openshift-origin" --template="{{ index .status.loadBalancer.ingress 0 \"ip\" }}") + } 2> ${OPENSHIFT_EXAMPLE}/openshift-startup.log + if [[ ! ${PUBLIC_OPENSHIFT_IP} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + export PUBLIC_OPENSHIFT_IP="" + fi + done +$ echo +$ echo "Public OpenShift IP set to: ${PUBLIC_OPENSHIFT_IP}" +``` + +Ensure you have a valid PUBLIC_IP address before continuing in the example. + +We now need to run a command on your host to generate a proper OpenShift configuration. To do this, we will volume mount the configuration directory that holds your Kubernetes kubeconfig file from the prior step. + + +```sh +$ docker run --privileged -v ${OPENSHIFT_CONFIG}:/config openshift/origin start master --write-config=/config --kubeconfig=/config/kubeconfig --master=https://localhost:8443 --public-master=https://${PUBLIC_OPENSHIFT_IP}:8443 --etcd=http://etcd:2379 +``` + +You should now see a number of certificates minted in your configuration directory, as well as a master-config.yaml file that tells the OpenShift master how to execute. We need to make some adjustments to this configuration directory in order to allow the OpenShift cluster to use Kubernetes serviceaccounts. First, write the Kubernetes service account key to the `${OPENSHIFT_CONFIG}` directory. The following script assumes you are using GCE. If you are not, use `scp` or `ssh` to get the key from the master node running Kubernetes. It is usually located at `/srv/kubernetes/server.key`. + +```shell +$ export ZONE=$(gcloud compute instances list | grep "${KUBE_GCE_INSTANCE_PREFIX}\-master" | awk '{print $2}' | head -1) +$ echo "sudo cat /srv/kubernetes/server.key; exit;" | gcloud compute ssh ${KUBE_GCE_INSTANCE_PREFIX}-master --zone ${ZONE} | grep -Ex "(^\-.*\-$|^\S+$)" > ${OPENSHIFT_CONFIG}/serviceaccounts.private.key + +``` + +Although we are retrieving the private key from the Kubernetes master, OpenShift will take care of the conversion for us so that serviceaccounts are created with the public key. Edit your `master-config.yaml` file in the `${OPENSHIFT_CONFIG}` directory to add `serviceaccounts.private.key` to the list of `publicKeyFiles`: + +```shell +$ sed -i -e 's/publicKeyFiles:.*$/publicKeyFiles:/g' -e '/publicKeyFiles:/a \ \ - serviceaccounts.private.key' ${OPENSHIFT_CONFIG}/master-config.yaml +``` + +Now, the configuration files are complete. In the next step, we will bundle the resulting configuration into a Kubernetes Secret that our OpenShift master pod will consume. + +### Step 4: Bundle the configuration into a Secret + +We now need to bundle the contents of our configuration into a secret for use by our OpenShift master pod. + +OpenShift includes an experimental command to make this easier. + +First, update the ownership for the files previously generated: + +``` +$ sudo -E chown -R ${USER} ${OPENSHIFT_CONFIG} +``` + +Then run the following command to collapse them into a Kubernetes secret. + +```sh +$ docker run -it --privileged -e="KUBECONFIG=/config/admin.kubeconfig" -v ${OPENSHIFT_CONFIG}:/config openshift/origin cli secrets new openshift-config /config -o json &> examples/openshift-origin/secret.json +``` + +Now, lets create the secret in your Kubernetes cluster. + +```sh +$ cluster/kubectl.sh create -f examples/openshift-origin/secret.json --namespace="openshift-origin" +``` + +**NOTE: This secret is secret and should not be shared with untrusted parties.** + +### Step 5: Deploy OpenShift Master + +We are now ready to deploy OpenShift. + +We will deploy a pod that runs the OpenShift master. The OpenShift master will delegate to the underlying Kubernetes +system to manage Kubernetes specific resources. For the sake of simplicity, the OpenShift master will run with an embedded etcd to hold OpenShift specific content. This demonstration will evolve in the future to show how to run etcd in a pod so that content is not destroyed if the OpenShift master fails. + +```sh +$ cluster/kubectl.sh create -f ${OPENSHIFT_EXAMPLE}/openshift-controller.yaml --namespace="openshift-origin" +``` + +You should now get a pod provisioned whose name begins with openshift. + +```sh +$ cluster/kubectl.sh get pods | grep openshift +$ cluster/kubectl.sh log openshift-t7147 origin +Running: cluster/../cluster/gce/../../cluster/../_output/dockerized/bin/linux/amd64/kubectl logs openshift-t7t47 origin +2015-04-30T15:26:00.454146869Z I0430 15:26:00.454005 1 start_master.go:296] Starting an OpenShift master, reachable at 0.0.0.0:8443 (etcd: [https://10.0.27.2:4001]) +2015-04-30T15:26:00.454231211Z I0430 15:26:00.454223 1 start_master.go:297] OpenShift master public address is https://104.197.73.241:8443 +``` + +Depending upon your cloud provider, you may need to open up an external firewall rule for tcp:8443. For GCE, you can run the following: + +```sh +$ gcloud compute --project "your-project" firewall-rules create "origin" --allow tcp:8443 --network "your-network" --source-ranges "0.0.0.0/0" +``` + +Consult your cloud provider's documentation for more information. + +Open a browser and visit the OpenShift master public address reported in your log. + +You can use the CLI commands by running the following: + +```sh +$ docker run --privileged --entrypoint="/usr/bin/bash" -it -e="OPENSHIFTCONFIG=/config/admin.kubeconfig" -v ${OPENSHIFT_CONFIG}:/config openshift/origin +$ osc config use-context public-default +$ osc --help +``` + +## Cleanup + +Clean up your cluster from resources created with this example: + +```sh +$ ${OPENSHIFT_EXAMPLE}/cleanup.sh +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/openshift-origin/README.md?pixel)]() + diff --git a/openshift-origin/cleanup.sh b/openshift-origin/cleanup.sh new file mode 100755 index 000000000..a2c931571 --- /dev/null +++ b/openshift-origin/cleanup.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleans up resources from the example, assumed to be run from Kubernetes repo root +echo +echo +export OPENSHIFT_EXAMPLE=$(pwd)/examples/openshift-origin +export OPENSHIFT_CONFIG=${OPENSHIFT_EXAMPLE}/config + +echo "===> Removing the OpenShift namespace:" +kubectl delete namespace openshift-origin +echo + +echo "===> Removing local files:" +rm -rf ${OPENSHIFT_CONFIG} +rm ${OPENSHIFT_EXAMPLE}/openshift-startup.log +rm ${OPENSHIFT_EXAMPLE}/secret.json +touch ${OPENSHIFT_EXAMPLE}/secret.json +echo + +echo "===> Restoring changed YAML specifcations:" +if [ -f "${OPENSHIFT_EXAMPLE}/etcd-controller.yaml.bak" ]; then + rm ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml + mv -v ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml.bak ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml +else + echo "No changed specifications found." +fi +echo + +echo Done. diff --git a/openshift-origin/create.sh b/openshift-origin/create.sh new file mode 100755 index 000000000..717df61b7 --- /dev/null +++ b/openshift-origin/create.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Creates resources from the example, assumed to be run from Kubernetes repo root +echo +echo "===> Initializing:" +if [ ! $(which python) ] +then + echo "Python is a prerequisite for running this script. Please install Python and try running again." + exit 1 +fi + +if [ ! $(which gcloud) ] +then + echo "gcloud is a prerequisite for running this script. Please install gcloud and try running again." + exit 1 +fi + +gcloud_instances=$(gcloud compute instances list | grep "\-master") +if [ -z "$gcloud_instances" ] || [ -z "${KUBE_GCE_INSTANCE_PREFIX}" ] +then + echo "This script is only able to supply the necessary serviceaccount key if you are running on Google" + echo "Compute Engine using a cluster/kube-up.sh script with KUBE_GCE_INSTANCE_PREFIX set. If this is not" + echo "the case, be ready to supply a path to the serviceaccount public key." + if [ -z "${KUBE_GCE_INSTANCE_PREFIX}" ] + then + echo "Please provide your KUBE_GCE_INSTANCE_PREFIX now:" + read KUBE_GCE_INSTANCE_PREFIX + fi +fi + +export OPENSHIFT_EXAMPLE=$(pwd)/examples/openshift-origin +echo Set OPENSHIFT_EXAMPLE=${OPENSHIFT_EXAMPLE} +export OPENSHIFT_CONFIG=${OPENSHIFT_EXAMPLE}/config +echo Set OPENSHIFT_CONFIG=${OPENSHIFT_CONFIG} +mkdir ${OPENSHIFT_CONFIG} +echo Made dir ${OPENSHIFT_CONFIG} +echo + +echo "===> Setting up OpenShift-Origin namespace:" +kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-origin-namespace.yaml +echo + +echo "===> Setting up etcd-discovery:" +# A token etcd uses to generate unique cluster ID and member ID. Conforms to [a-z0-9]{40} +export ETCD_INITIAL_CLUSTER_TOKEN=$(python -c "import string; import random; print(''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(40)))") + +# A unique token used by the discovery service. Conforms to etcd-cluster-[a-z0-9]{5} +export ETCD_DISCOVERY_TOKEN=$(python -c "import string; import random; print(\"etcd-cluster-\" + ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(5)))") +sed -i.bak -e "s/INSERT_ETCD_INITIAL_CLUSTER_TOKEN/\"${ETCD_INITIAL_CLUSTER_TOKEN}\"/g" -e "s/INSERT_ETCD_DISCOVERY_TOKEN/\"${ETCD_DISCOVERY_TOKEN}\"/g" ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml + +kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-discovery-controller.yaml --namespace='openshift-origin' +kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-discovery-service.yaml --namespace='openshift-origin' +echo + +echo "===> Setting up etcd:" +kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-controller.yaml --namespace='openshift-origin' +kubectl create -f ${OPENSHIFT_EXAMPLE}/etcd-service.yaml --namespace='openshift-origin' +echo + +echo "===> Setting up openshift-origin:" +kubectl config view --output=yaml --flatten=true --minify=true > ${OPENSHIFT_CONFIG}/kubeconfig +kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-service.yaml --namespace='openshift-origin' +echo + +export PUBLIC_OPENSHIFT_IP="" +echo "===> Waiting for public IP to be set for the OpenShift Service." +echo "Mistakes in service setup can cause this to loop infinitely if an" +echo "external IP is never set. Ensure that the OpenShift service" +echo "is set to use an external load balancer. This process may take" +echo "a few minutes. Errors can be found in the log file found at:" +echo ${OPENSHIFT_EXAMPLE}/openshift-startup.log +echo "" > ${OPENSHIFT_EXAMPLE}/openshift-startup.log +while [ ${#PUBLIC_OPENSHIFT_IP} -lt 1 ]; do + echo -n . + sleep 1 + { + export PUBLIC_OPENSHIFT_IP=$(kubectl get services openshift --namespace="openshift-origin" --template="{{ index .status.loadBalancer.ingress 0 \"ip\" }}") + } >> ${OPENSHIFT_EXAMPLE}/openshift-startup.log 2>&1 + if [[ ! ${PUBLIC_OPENSHIFT_IP} =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + export PUBLIC_OPENSHIFT_IP="" + fi +done +echo +echo "Public OpenShift IP set to: ${PUBLIC_OPENSHIFT_IP}" +echo + +echo "===> Configuring OpenShift:" +docker run --privileged -v ${OPENSHIFT_CONFIG}:/config openshift/origin start master --write-config=/config --kubeconfig=/config/kubeconfig --master=https://localhost:8443 --public-master=https://${PUBLIC_OPENSHIFT_IP}:8443 --etcd=http://etcd:2379 +sudo -E chown -R ${USER} ${OPENSHIFT_CONFIG} + +# The following assumes GCE and that KUBE_GCE_INSTANCE_PREFIX is set +export ZONE=$(gcloud compute instances list | grep "${KUBE_GCE_INSTANCE_PREFIX}\-master" | awk '{print $2}' | head -1) +echo "sudo cat /srv/kubernetes/server.key; exit;" | gcloud compute ssh ${KUBE_GCE_INSTANCE_PREFIX}-master --zone ${ZONE} | grep -Ex "(^\-.*\-$|^\S+$)" > ${OPENSHIFT_CONFIG}/serviceaccounts.private.key +# The following insertion will fail if indentation changes +sed -i -e 's/publicKeyFiles:.*$/publicKeyFiles:/g' -e '/publicKeyFiles:/a \ \ - serviceaccounts.private.key' ${OPENSHIFT_CONFIG}/master-config.yaml + +docker run -it --privileged -e="KUBECONFIG=/config/admin.kubeconfig" -v ${OPENSHIFT_CONFIG}:/config openshift/origin cli secrets new openshift-config /config -o json &> ${OPENSHIFT_EXAMPLE}/secret.json +kubectl create -f ${OPENSHIFT_EXAMPLE}/secret.json --namespace='openshift-origin' +echo + +echo "===> Running OpenShift Master:" +kubectl create -f ${OPENSHIFT_EXAMPLE}/openshift-controller.yaml --namespace='openshift-origin' +echo + +echo Done. diff --git a/openshift-origin/etcd-controller.yaml b/openshift-origin/etcd-controller.yaml new file mode 100644 index 000000000..419c57dbd --- /dev/null +++ b/openshift-origin/etcd-controller.yaml @@ -0,0 +1,52 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: etcd +spec: + strategy: + type: Recreate + replicas: 3 + selector: + matchLabels: + name: etcd + template: + metadata: + labels: + name: etcd + spec: + containers: + - name: member + image: openshift/etcd-20-centos7 + ports: + - containerPort: 2379 + protocol: TCP + - containerPort: 2380 + protocol: TCP + env: + # ETCD_NUM_MEMBERS is the maximum number of members to launch (have to match with # of replicas) + - name: ETCD_NUM_MEMBERS + value: "3" + - name: ETCD_INITIAL_CLUSTER_STATE + value: "new" + # ETCD_INITIAL_CLUSTER_TOKEN is a token etcd uses to generate unique cluster ID and member ID. Conforms to [a-z0-9]{40} + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: INSERT_ETCD_INITIAL_CLUSTER_TOKEN + # ETCD_DISCOVERY_TOKEN is a unique token used by the discovery service. Conforms to etcd-cluster-[a-z0-9]{5} + - name: ETCD_DISCOVERY_TOKEN + value: INSERT_ETCD_DISCOVERY_TOKEN + # ETCD_DISCOVERY_URL connects etcd instances together by storing a list of peer addresses, + # metadata and the initial size of the cluster under a unique address + - name: ETCD_DISCOVERY_URL + value: "http://etcd-discovery:2379" + - name: ETCDCTL_PEERS + value: "http://etcd:2379" + resources: {} + terminationMessagePath: "/dev/termination-log" + imagePullPolicy: IfNotPresent + securityContext: + capabilities: {} + privileged: false + restartPolicy: Always + dnsPolicy: ClusterFirst + serviceAccount: '' +status: {} diff --git a/openshift-origin/etcd-discovery-controller.yaml b/openshift-origin/etcd-discovery-controller.yaml new file mode 100644 index 000000000..33f593ac7 --- /dev/null +++ b/openshift-origin/etcd-discovery-controller.yaml @@ -0,0 +1,34 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: etcd-discovery +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + name: etcd-discovery + template: + metadata: + labels: + name: etcd-discovery + spec: + containers: + - name: discovery + image: openshift/etcd-20-centos7 + args: + - etcd-discovery.sh + ports: + - containerPort: 2379 + protocol: TCP + resources: {} + terminationMessagePath: "/dev/termination-log" + imagePullPolicy: IfNotPresent + securityContext: + capabilities: {} + privileged: false + restartPolicy: Always + dnsPolicy: ClusterFirst + serviceAccount: '' +status: {} diff --git a/openshift-origin/etcd-discovery-service.yaml b/openshift-origin/etcd-discovery-service.yaml new file mode 100644 index 000000000..99f464716 --- /dev/null +++ b/openshift-origin/etcd-discovery-service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: etcd-discovery + labels: + name: etcd-discovery +spec: + ports: + - protocol: TCP + port: 2379 + targetPort: 2379 + nodePort: 0 + selector: + name: etcd-discovery + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/openshift-origin/etcd-service.yaml b/openshift-origin/etcd-service.yaml new file mode 100644 index 000000000..00bc56ef0 --- /dev/null +++ b/openshift-origin/etcd-service.yaml @@ -0,0 +1,24 @@ +kind: Service +apiVersion: v1 +metadata: + name: etcd + labels: + name: etcd +spec: + ports: + - name: client + protocol: TCP + port: 2379 + targetPort: 2379 + nodePort: 0 + - name: server + protocol: TCP + port: 2380 + targetPort: 2380 + nodePort: 0 + selector: + name: etcd + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/openshift-origin/openshift-controller.yaml b/openshift-origin/openshift-controller.yaml new file mode 100644 index 000000000..1844a70cc --- /dev/null +++ b/openshift-origin/openshift-controller.yaml @@ -0,0 +1,34 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + name: openshift + name: openshift +spec: + replicas: 1 + selector: + matchLabels: + name: openshift + template: + metadata: + labels: + name: openshift + spec: + containers: + - args: + - start + - master + - --config=/config/master-config.yaml + image: "openshift/origin" + name: origin + ports: + - containerPort: 8443 + name: openshift + volumeMounts: + - mountPath: /config + name: config + readOnly: true + volumes: + - name: config + secret: + secretName: openshift-config \ No newline at end of file diff --git a/openshift-origin/openshift-origin-namespace.yaml b/openshift-origin/openshift-origin-namespace.yaml new file mode 100644 index 000000000..1596417f6 --- /dev/null +++ b/openshift-origin/openshift-origin-namespace.yaml @@ -0,0 +1,6 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: "openshift-origin" + labels: + name: "openshift-origin" \ No newline at end of file diff --git a/openshift-origin/openshift-service.yaml b/openshift-origin/openshift-service.yaml new file mode 100644 index 000000000..a8d599da6 --- /dev/null +++ b/openshift-origin/openshift-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: openshift +spec: + ports: + - name: openshift + port: 8443 + targetPort: 8443 + selector: + name: openshift + type: LoadBalancer diff --git a/openshift-origin/secret.json b/openshift-origin/secret.json new file mode 100644 index 000000000..e69de29bb diff --git a/persistent-volume-provisioning/README.md b/persistent-volume-provisioning/README.md new file mode 100644 index 000000000..f00319176 --- /dev/null +++ b/persistent-volume-provisioning/README.md @@ -0,0 +1,480 @@ +## Persistent Volume Provisioning + +This example shows how to use dynamic persistent volume provisioning. + +### Prerequisites + +This example assumes that you have an understanding of Kubernetes administration and can modify the +scripts that launch kube-controller-manager. + +### Admin Configuration + +The admin must define `StorageClass` objects that describe named "classes" of storage offered in a cluster. Different classes might map to arbitrary levels or policies determined by the admin. When configuring a `StorageClass` object for persistent volume provisioning, the admin will need to describe the type of provisioner to use and the parameters that will be used by the provisioner when it provisions a `PersistentVolume` belonging to the class. + +The name of a StorageClass object is significant, and is how users can request a particular class, by specifying the name in their `PersistentVolumeClaim`. The `provisioner` field must be specified as it determines what volume plugin is used for provisioning PVs. The `parameters` field contains the parameters that describe volumes belonging to the storage class. Different parameters may be accepted depending on the `provisioner`. For example, the value `io1`, for the parameter `type`, and the parameter `iopsPerGB` are specific to EBS . When a parameter is omitted, some default is used. + +See [Kubernetes StorageClass documentation](https://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses) for complete reference of all supported parameters. + +#### AWS + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/aws-ebs +parameters: + type: io1 + zone: us-east-1d + iopsPerGB: "10" +``` + +* `type`: `io1`, `gp2`, `sc1`, `st1`. See AWS docs for details. Default: `gp2`. +* `zone`: AWS zone. If not specified, a random zone from those where Kubernetes cluster has a node is chosen. +* `iopsPerGB`: only for `io1` volumes. I/O operations per second per GiB. AWS volume plugin multiplies this with size of requested volume to compute IOPS of the volume and caps it at 20 000 IOPS (maximum supported by AWS, see AWS docs). +* `encrypted`: denotes whether the EBS volume should be encrypted or not. Valid values are `true` or `false`. +* `kmsKeyId`: optional. The full Amazon Resource Name of the key to use when encrypting the volume. If none is supplied but `encrypted` is true, a key is generated by AWS. See AWS docs for valid ARN value. + +#### GCE + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard + zone: us-central1-a +``` + +* `type`: `pd-standard` or `pd-ssd`. Default: `pd-ssd` +* `zone`: GCE zone. If not specified, a random zone in the same region as controller-manager will be chosen. + +#### vSphere + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: eagerzeroedthick + fstype: ext3 +``` + +* `diskformat`: `thin`, `zeroedthick` and `eagerzeroedthick`. See vSphere docs for details. Default: `"thin"`. +* `fstype`: fstype that are supported by kubernetes. Default: `"ext4"`. + +#### Portworx Volume + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: portworx-io-priority-high +provisioner: kubernetes.io/portworx-volume +parameters: + repl: "1" + snap_interval: "70" + io_priority: "high" + +``` + +* `fs`: filesystem to be laid out: [none/xfs/ext4] (default: `ext4`) +* `block_size`: block size in Kbytes (default: `32`) +* `repl`: replication factor [1..3] (default: `1`) +* `io_priority`: IO Priority: [high/medium/low] (default: `low`) +* `snap_interval`: snapshot interval in minutes, 0 disables snaps (default: `0`) +* `aggregation_level`: specifies the number of chunks the volume would be distributed into, 0 indicates a non-aggregated volume (default: `0`) +* `ephemeral`: ephemeral storage [true/false] (default `false`) + +For a complete example refer ([Portworx Volume docs](../volumes/portworx/README.md)) + +#### GLUSTERFS + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://127.0.0.1:8081" + clusterid: "630372ccdc720a92c681fb928f27b53f" + restuser: "admin" + secretNamespace: "default" + secretName: "heketi-secret" + gidMin: "40000" + gidMax: "50000" + volumetype: "replicate:3" +``` + +* `resturl` : Gluster REST service/Heketi service url which provision gluster volumes on demand. The general format should be `IPaddress:Port` and this is a mandatory parameter for GlusterFS dynamic provisioner. If Heketi service is exposed as a routable service in openshift/kubernetes setup, this can have a format similar to +`http://heketi-storage-project.cloudapps.mystorage.com` where the fqdn is a resolvable heketi service url. +* `restauthenabled` : Gluster REST service authentication boolean that enables authentication to the REST server. If this value is 'true', `restuser` and `restuserkey` or `secretNamespace` + `secretName` have to be filled. This option is deprecated, authentication is enabled when any of `restuser`, `restuserkey`, `secretName` or `secretNamespace` is specified. +* `restuser` : Gluster REST service/Heketi user who has access to create volumes in the Gluster Trusted Pool. +* `restuserkey` : Gluster REST service/Heketi user's password which will be used for authentication to the REST server. This parameter is deprecated in favor of `secretNamespace` + `secretName`. +* `secretNamespace` + `secretName` : Identification of Secret instance that contains user password to use when talking to Gluster REST service. These parameters are optional, empty password will be used when both `secretNamespace` and `secretName` are omitted. The provided secret must have type "kubernetes.io/glusterfs". +When both `restuserkey` and `secretNamespace` + `secretName` is specified, the secret will be used. +* `clusterid`: `630372ccdc720a92c681fb928f27b53f` is the ID of the cluster which will be used by Heketi when provisioning the volume. It can also be a list of clusterids, for ex: +"8452344e2becec931ece4e33c4674e4e,42982310de6c63381718ccfa6d8cf397". This is an optional parameter. + +Example of a secret can be found in [glusterfs-secret.yaml](glusterfs/glusterfs-secret.yaml). + +* `gidMin` + `gidMax` : The minimum and maximum value of GID range for the storage class. A unique value (GID) in this range ( gidMin-gidMax ) will be used for dynamically provisioned volumes. These are optional values. If not specified, the volume will be provisioned with a value between 2000-2147483647 which are defaults for gidMin and gidMax respectively. + +* `volumetype` : The volume type and its parameters can be configured with this optional value. If the volume type is not mentioned, it's up to the provisioner to decide the volume type. +For example: + 'Replica volume': + `volumetype: replicate:3` where '3' is replica count. + 'Disperse/EC volume': + `volumetype: disperse:4:2` where '4' is data and '2' is the redundancy count. + 'Distribute volume': + `volumetype: none` + +For available volume types and its administration options refer: ([Administration Guide](https://access.redhat.com/documentation/en-US/Red_Hat_Storage/3.1/html/Administration_Guide/part-Overview.html)) + +Reference : ([How to configure Heketi](https://github.com/heketi/heketi/wiki/Setting-up-the-topology)) + +When the persistent volumes are dynamically provisioned, the Gluster plugin automatically create an endpoint and a headless service in the name `gluster-dynamic-`. This dynamic endpoint and service will be deleted automatically when the persistent volume claim is deleted. + + +#### OpenStack Cinder + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: gold +provisioner: kubernetes.io/cinder +parameters: + type: fast + availability: nova +``` + +* `type`: [VolumeType](http://docs.openstack.org/admin-guide/dashboard-manage-volumes.html) created in Cinder. Default is empty. +* `availability`: Availability Zone. Default is empty. + +#### Ceph RBD + +```yaml + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: fast + provisioner: kubernetes.io/rbd + parameters: + monitors: 10.16.153.105:6789 + adminId: kube + adminSecretName: ceph-secret + adminSecretNamespace: kube-system + pool: kube + userId: kube + userSecretName: ceph-secret-user +``` + +* `monitors`: Ceph monitors, comma delimited. It is required. +* `adminId`: Ceph client ID that is capable of creating images in the pool. Default is "admin". +* `adminSecret`: Secret Name for `adminId`. It is required. The provided secret must have type "kubernetes.io/rbd". +* `adminSecretNamespace`: The namespace for `adminSecret`. Default is "default". +* `pool`: Ceph RBD pool. Default is "rbd". +* `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`. +* `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required. + +#### Quobyte + + + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/quobyte +parameters: + quobyteAPIServer: "http://138.68.74.142:7860" + registry: "138.68.74.142:7861" + adminSecretName: "quobyte-admin-secret" + adminSecretNamespace: "kube-system" + user: "root" + group: "root" + quobyteConfig: "BASE" + quobyteTenant: "DEFAULT" +``` + +[Download example](quobyte/quobyte-storage-class.yaml?raw=true) + + +* **quobyteAPIServer** API Server of Quobyte in the format http(s)://api-server:7860 +* **registry** Quobyte registry to use to mount the volume. You can specify the registry as : pair or if you want to specify multiple registries you just have to put a comma between them e.q. :,:,:. The host can be an IP address or if you have a working DNS you can also provide the DNS names. +* **adminSecretName** secret that holds information about the Quobyte user and the password to authenticate against the API server. The provided secret must have type "kubernetes.io/quobyte". +* **adminSecretNamespace** The namespace for **adminSecretName**. Default is `default`. +* **user** maps all access to this user. Default is `root`. +* **group** maps all access to this group. Default is `nfsnobody`. +* **quobyteConfig** use the specified configuration to create the volume. You can create a new configuration or modify an existing one with the Web console or the quobyte CLI. Default is `BASE` +* **quobyteTenant** use the specified tenant ID to create/delete the volume. This Quobyte tenant has to be already present in Quobyte. Default is `DEFAULT` + +First create Quobyte admin's Secret in the system namespace. Here the Secret is created in `kube-system`: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml --namespace=kube-system +``` + +Then create the Quobyte storage class: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml +``` + +Now create a PVC + +``` +$ kubectl create -f examples/persistent-volume-provisioning/claim1.json +``` + +Check the created PVC: + +``` +$ kubectl describe pvc +Name: claim1 +Namespace: default +Status: Bound +Volume: pvc-bdb82652-694a-11e6-b811-080027242396 +Labels: +Capacity: 3Gi +Access Modes: RWO +No events. + +$ kubectl describe pv +Name: pvc-bdb82652-694a-11e6-b811-080027242396 +Labels: +Status: Bound +Claim: default/claim1 +Reclaim Policy: Delete +Access Modes: RWO +Capacity: 3Gi +Message: +Source: + Type: Quobyte (a Quobyte mount on the host that shares a pod's lifetime) + Registry: 138.68.79.14:7861 + Volume: kubernetes-dynamic-pvc-bdb97c58-694a-11e6-91b6-080027242396 + ReadOnly: false +No events. +``` + +Create a Pod to use the PVC: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/quobyte/example-pod.yaml +``` + +#### Azure Disk + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/azure-disk +parameters: + skuName: Standard_LRS + location: eastus + storageAccount: azure_storage_account_name +``` + +* `skuName`: Azure storage account Sku tier. Default is empty. +* `location`: Azure storage account location. Default is empty. +* `storageAccount`: Azure storage account name. If storage account is not provided, all storage accounts associated with the resource group are searched to find one that matches `skuName` and `location`. If storage account is provided, it must reside in the same resource group as the cluster, and `skuName` and `location` are ignored. + +#### Azure File + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: slow +provisioner: kubernetes.io/azure-file +parameters: + skuName: Standard_LRS + location: eastus + storageAccount: azure_storage_account_name +``` + +The parameters are the same as those used by [Azure Disk](#azure-disk) + +### User provisioning requests + +Users request dynamically provisioned storage by including a storage class in their `PersistentVolumeClaim` using `spec.storageClassName` attribute. +It is required that this value matches the name of a `StorageClass` configured by the administrator. + +``` +{ + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "claim1" + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "3Gi" + } + }, + "storageClassName": "slow" + } +} +``` + +### Sample output + +#### GCE + +This example uses GCE but any provisioner would follow the same flow. + +First we note there are no Persistent Volumes in the cluster. After creating a storage class and a claim including that storage class, we see a new PV is created +and automatically bound to the claim requesting storage. + + +``` +$ kubectl get pv + +$ kubectl create -f examples/persistent-volume-provisioning/gce-pd.yaml +storageclass "slow" created + +$ kubectl create -f examples/persistent-volume-provisioning/claim1.json +persistentvolumeclaim "claim1" created + +$ kubectl get pv +NAME CAPACITY ACCESSMODES STATUS CLAIM REASON AGE +pvc-bb6d2f0c-534c-11e6-9348-42010af00002 3Gi RWO Bound default/claim1 4s + +$ kubectl get pvc +NAME LABELS STATUS VOLUME CAPACITY ACCESSMODES AGE +claim1 Bound pvc-bb6d2f0c-534c-11e6-9348-42010af00002 3Gi RWO 7s + +# delete the claim to release the volume +$ kubectl delete pvc claim1 +persistentvolumeclaim "claim1" deleted + +# the volume is deleted in response to being release of its claim +$ kubectl get pv + +``` + + +#### Ceph RBD + +This section will guide you on how to configure and use the Ceph RBD provisioner. + +##### Pre-requisites + +For this to work you must have a functional Ceph cluster, and the `rbd` command line utility must be installed on any host/container that `kube-controller-manager` or `kubelet` is running on. + +##### Configuration + +First we must identify the Ceph client admin key. This is usually found in `/etc/ceph/ceph.client.admin.keyring` on your Ceph cluster nodes. The file will look something like this: + +``` +[client.admin] + key = AQBfxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" +``` + +From the key value, we will create a secret. We must create the Ceph admin Secret in the namespace defined in our `StorageClass`. In this example we've set the namespace to `kube-system`. + +``` +$ kubectl create secret generic ceph-secret-admin --from-literal=key='AQBfxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==' --namespace=kube-system --type=kubernetes.io/rbd +``` + +Now modify `examples/persistent-volume-provisioning/rbd/rbd-storage-class.yaml` to reflect your environment, particularly the `monitors` field. We are now ready to create our RBD Storage Class: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/rbd/rbd-storage-class.yaml +``` + +The kube-controller-manager is now able to provision storage, however we still need to be able to map the RBD volume to a node. Mapping should be done with a non-privileged key, if you have existing users you can get all keys by running `ceph auth list` on your Ceph cluster with the admin key. For this example we will create a new user and pool. + +``` +$ ceph osd pool create kube 512 +$ ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube' +[client.kube] + key = AQBQyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy== +``` + +This key will be made into a secret, just like the admin secret. However this user secret will need to be created in every namespace where you intend to consume RBD volumes provisioned in our example storage class. Let's create a namespace called `myns`, and create the user secret in that namespace. + +``` +kubectl create namespace myns +kubectl create secret generic ceph-secret-user --from-literal=key='AQBQyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy==' --namespace=myns --type=kubernetes.io/rbd +``` + +You are now ready to provision and use RBD storage. + +##### Usage + +With the storageclass configured, let's create a PVC in our example namespace, `myns`: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/claim1.json --namespace=myns +``` + +Eventually the PVC creation will result in a PV and RBD volume to match: + +``` +$ kubectl describe pvc --namespace=myns +Name: claim1 +Namespace: myns +Status: Bound +Volume: pvc-1cfa23b3-664b-11e6-9eb9-90b11c09520d +Labels: +Capacity: 3Gi +Access Modes: RWO +No events. + +$ kubectl describe pv +Name: pvc-1cfa23b3-664b-11e6-9eb9-90b11c09520d +Labels: +Status: Bound +Claim: myns/claim1 +Reclaim Policy: Delete +Access Modes: RWO +Capacity: 3Gi +Message: +Source: + Type: RBD (a Rados Block Device mount on the host that shares a pod's lifetime) + CephMonitors: [127.0.0.1:6789] + RBDImage: kubernetes-dynamic-pvc-1cfb1862-664b-11e6-9a5d-90b11c09520d + FSType: + RBDPool: kube + RadosUser: kube + Keyring: /etc/ceph/keyring + SecretRef: &{ceph-secret-user} + ReadOnly: false +No events. +``` + +With our storage provisioned, we can now create a Pod to use the PVC: + +``` +$ kubectl create -f examples/persistent-volume-provisioning/rbd/pod.yaml --namespace=myns +``` + +Now our pod has an RBD mount! + +``` +$ export PODNAME=`kubectl get pod --selector='role=server' --namespace=myns --output=template --template="{{with index .items 0}}{{.metadata.name}}{{end}}"` +$ kubectl exec -it $PODNAME --namespace=myns -- df -h | grep rbd +/dev/rbd1 2.9G 4.5M 2.8G 1% /var/lib/www/html +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/persistent-volume-provisioning/README.md?pixel)]() + diff --git a/persistent-volume-provisioning/aws-ebs.yaml b/persistent-volume-provisioning/aws-ebs.yaml new file mode 100644 index 000000000..4fc5ba924 --- /dev/null +++ b/persistent-volume-provisioning/aws-ebs.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/aws-ebs +parameters: + type: io1 + zone: us-east-1d + iopsPerGB: "10" diff --git a/persistent-volume-provisioning/claim1.json b/persistent-volume-provisioning/claim1.json new file mode 100644 index 000000000..75c71a669 --- /dev/null +++ b/persistent-volume-provisioning/claim1.json @@ -0,0 +1,18 @@ +{ + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "claim1" + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "3Gi" + } + }, + "storageClassName": "slow" + } +} diff --git a/persistent-volume-provisioning/gce-pd.yaml b/persistent-volume-provisioning/gce-pd.yaml new file mode 100644 index 000000000..3d8e134ba --- /dev/null +++ b/persistent-volume-provisioning/gce-pd.yaml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard + zone: us-central1-a diff --git a/persistent-volume-provisioning/glusterfs/glusterfs-dp.yaml b/persistent-volume-provisioning/glusterfs/glusterfs-dp.yaml new file mode 100644 index 000000000..ad2e76597 --- /dev/null +++ b/persistent-volume-provisioning/glusterfs/glusterfs-dp.yaml @@ -0,0 +1,14 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://127.0.0.1:8081" + clusterid: "630372ccdc720a92c681fb928f27b53f" + restuser: "admin" + secretNamespace: "default" + secretName: "heketi-secret" + gidMin: "40000" + gidMax: "50000" + volumetype: "replicate:3" diff --git a/persistent-volume-provisioning/glusterfs/glusterfs-secret.yaml b/persistent-volume-provisioning/glusterfs/glusterfs-secret.yaml new file mode 100644 index 000000000..bb9c806a2 --- /dev/null +++ b/persistent-volume-provisioning/glusterfs/glusterfs-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: heketi-secret + namespace: default +data: + # base64 encoded password. E.g.: echo -n "mypassword" | base64 + key: bXlwYXNzd29yZA== +type: kubernetes.io/glusterfs diff --git a/persistent-volume-provisioning/quobyte/example-pod.yaml b/persistent-volume-provisioning/quobyte/example-pod.yaml new file mode 100644 index 000000000..eb814f552 --- /dev/null +++ b/persistent-volume-provisioning/quobyte/example-pod.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: server +spec: + replicas: 1 + selector: + role: server + template: + metadata: + labels: + role: server + spec: + containers: + - name: server + image: nginx + volumeMounts: + - mountPath: /var/lib/www/html + name: quobytepvc + volumes: + - name: quobytepvc + persistentVolumeClaim: + claimName: claim1 diff --git a/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml b/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml new file mode 100644 index 000000000..9e395a89b --- /dev/null +++ b/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: quobyte-admin-secret +data: + password: cXVvYnl0ZQ== + user: YWRtaW4= +type: kubernetes.io/quobyte diff --git a/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml b/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml new file mode 100644 index 000000000..739b94fe9 --- /dev/null +++ b/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml @@ -0,0 +1,14 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/quobyte +parameters: + quobyteAPIServer: "http://138.68.74.142:7860" + registry: "138.68.74.142:7861" + adminSecretName: "quobyte-admin-secret" + adminSecretNamespace: "kube-system" + user: "root" + group: "root" + quobyteConfig: "BASE" + quobyteTenant: "DEFAULT" diff --git a/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml b/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml new file mode 100644 index 000000000..f86d975f7 --- /dev/null +++ b/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ceph-secret-admin +type: "kubernetes.io/rbd" +data: +#Please note this value is base64 encoded. + key: QVFEQ1pMdFhPUnQrSmhBQUFYaERWNHJsZ3BsMmNjcDR6RFZST0E9PQ== +type: kubernetes.io/rbd diff --git a/persistent-volume-provisioning/rbd/ceph-secret-user.yaml b/persistent-volume-provisioning/rbd/ceph-secret-user.yaml new file mode 100644 index 000000000..e538dcafb --- /dev/null +++ b/persistent-volume-provisioning/rbd/ceph-secret-user.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ceph-secret-user +type: "kubernetes.io/rbd" +data: +#Please note this value is base64 encoded. + key: QVFBTWdYaFZ3QkNlRGhBQTlubFBhRnlmVVNhdEdENGRyRldEdlE9PQ== diff --git a/persistent-volume-provisioning/rbd/pod.yaml b/persistent-volume-provisioning/rbd/pod.yaml new file mode 100644 index 000000000..6eea26f94 --- /dev/null +++ b/persistent-volume-provisioning/rbd/pod.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: server +spec: + replicas: 1 + selector: + role: server + template: + metadata: + labels: + role: server + spec: + containers: + - name: server + image: nginx + volumeMounts: + - mountPath: /var/lib/www/html + name: mypvc + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: claim1 diff --git a/persistent-volume-provisioning/rbd/rbd-storage-class.yaml b/persistent-volume-provisioning/rbd/rbd-storage-class.yaml new file mode 100644 index 000000000..e2d7d67d9 --- /dev/null +++ b/persistent-volume-provisioning/rbd/rbd-storage-class.yaml @@ -0,0 +1,14 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/rbd +parameters: + monitors: 127.0.0.1:6789 + adminId: admin + adminSecretName: ceph-secret-admin + adminSecretNamespace: "kube-system" + pool: kube + userId: kube + userSecretName: ceph-secret-user + diff --git a/phabricator/README.md b/phabricator/README.md new file mode 100644 index 000000000..6e3c2adcc --- /dev/null +++ b/phabricator/README.md @@ -0,0 +1,216 @@ +## Phabricator example + +This example shows how to build a simple multi-tier web application using Kubernetes and Docker. + +The example combines a web frontend and an external service that provides MySQL database. We use CloudSQL on Google Cloud Platform in this example, but in principle any approach to running MySQL should work. + +### Step Zero: Prerequisites + +This example assumes that you have a basic understanding of kubernetes [services](../../docs/user-guide/services.md) and that you have forked the repository and [turned up a Kubernetes cluster](../../docs/getting-started-guides/): + +```sh +$ cd kubernetes +$ cluster/kube-up.sh +``` + +### Step One: Set up Cloud SQL instance + +Follow the [official instructions](https://cloud.google.com/sql/docs/getting-started) to set up Cloud SQL instance. + +In the remaining part of this example we will assume that your instance is named "phabricator-db", has IP 1.2.3.4, is listening on port 3306 and the password is "1234". + +### Step Two: Authenticate phabricator in Cloud SQL + +In order to allow phabricator to connect to your Cloud SQL instance you need to run the following command to authorize all your nodes within a cluster: + +```bash +NODE_NAMES=`kubectl get nodes | cut -d" " -f1 | tail -n+2` +NODE_IPS=`gcloud compute instances list $NODE_NAMES | tr -s " " | cut -d" " -f 5 | tail -n+2` +gcloud sql instances patch phabricator-db --authorized-networks $NODE_IPS +``` + +Otherwise you will see the following logs: + +```bash +$ kubectl logs phabricator-controller-02qp4 +[...] +Raw MySQL Error: Attempt to connect to root@1.2.3.4 failed with error +#2013: Lost connection to MySQL server at 'reading initial communication packet', system error: 0. + +``` + +### Step Three: Turn up the phabricator + +To start Phabricator server use the file [`examples/phabricator/phabricator-controller.json`](phabricator-controller.json) which describes a [replication controller](../../docs/user-guide/replication-controller.md) with a single [pod](../../docs/user-guide/pods.md) running an Apache server with Phabricator PHP source: + + + +```json +{ + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "phabricator-controller", + "labels": { + "name": "phabricator" + } + }, + "spec": { + "replicas": 1, + "selector": { + "name": "phabricator" + }, + "template": { + "metadata": { + "labels": { + "name": "phabricator" + } + }, + "spec": { + "containers": [ + { + "name": "phabricator", + "image": "fgrzadkowski/example-php-phabricator", + "ports": [ + { + "name": "http-server", + "containerPort": 80 + } + ], + "env": [ + { + "name": "MYSQL_SERVICE_IP", + "value": "1.2.3.4" + }, + { + "name": "MYSQL_SERVICE_PORT", + "value": "3306" + }, + { + "name": "MYSQL_PASSWORD", + "value": "1234" + } + ] + } + ] + } + } + } +} +``` + +[Download example](phabricator-controller.json?raw=true) + + +Create the phabricator pod in your Kubernetes cluster by running: + +```sh +$ kubectl create -f examples/phabricator/phabricator-controller.json +``` + +**Note:** Remember to substitute environment variable values in json file before create replication controller. + +Once that's up you can list the pods in the cluster, to verify that it is running: + +```sh +kubectl get pods +``` + +You'll see a single phabricator pod. It will also display the machine that the pod is running on once it gets placed (may take up to thirty seconds): + +``` +NAME READY STATUS RESTARTS AGE +phabricator-controller-9vy68 1/1 Running 0 1m +``` + +If you ssh to that machine, you can run `docker ps` to see the actual pod: + +```sh +me@workstation$ gcloud compute ssh --zone us-central1-b kubernetes-node-2 + +$ sudo docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +54983bc33494 fgrzadkowski/phabricator:latest "/run.sh" 2 hours ago Up 2 hours k8s_phabricator.d6b45054_phabricator-controller-02qp4.default.api_eafb1e53-b6a9-11e4-b1ae-42010af05ea6_01c2c4ca +``` + +(Note that initial `docker pull` may take a few minutes, depending on network conditions. During this time, the `get pods` command will return `Pending` because the container has not yet started ) + +### Step Four: Turn up the phabricator service + +A Kubernetes 'service' is a named load balancer that proxies traffic to one or more containers. The services in a Kubernetes cluster are discoverable inside other containers via *environment variables*. Services find the containers to load balance based on pod labels. These environment variables are typically referenced in application code, shell scripts, or other places where one node needs to talk to another in a distributed system. You should catch up on [kubernetes services](../../docs/user-guide/services.md) before proceeding. + +The pod that you created in Step Three has the label `name=phabricator`. The selector field of the service determines which pods will receive the traffic sent to the service. + +Use the file [`examples/phabricator/phabricator-service.json`](phabricator-service.json): + + + +```json +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "phabricator" + }, + "spec": { + "ports": [ + { + "port": 80, + "targetPort": "http-server" + } + ], + "selector": { + "name": "phabricator" + }, + "type": "LoadBalancer" + } +} +``` + +[Download example](phabricator-service.json?raw=true) + + +To create the service run: + +```sh +$ kubectl create -f examples/phabricator/phabricator-service.json +phabricator +``` + +To play with the service itself, find the external IP of the load balancer: + +```console +$ kubectl get services +NAME LABELS SELECTOR IP(S) PORT(S) +kubernetes component=apiserver,provider=kubernetes 10.0.0.1 443/TCP +phabricator name=phabricator 10.0.31.173 80/TCP +$ kubectl get services phabricator -o json | grep ingress -A 4 + "ingress": [ + { + "ip": "104.197.13.125" + } + ] +``` + +and then visit port 80 of that IP address. + +**Note**: Provisioning of the external IP address may take few minutes. + +**Note**: You may need to open the firewall for port 80 using the [console][cloud-console] or the `gcloud` tool. The following command will allow traffic from any source to instances tagged `kubernetes-node`: + +```sh +$ gcloud compute firewall-rules create phabricator-node-80 --allow=tcp:80 --target-tags kubernetes-node +``` + +### Step Six: Cleanup + +To turn down a Kubernetes cluster: + +```sh +$ cluster/kube-down.sh +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/phabricator/README.md?pixel)]() + diff --git a/phabricator/phabricator-controller.json b/phabricator/phabricator-controller.json new file mode 100644 index 000000000..b13103e11 --- /dev/null +++ b/phabricator/phabricator-controller.json @@ -0,0 +1,51 @@ +{ + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "phabricator-controller", + "labels": { + "name": "phabricator" + } + }, + "spec": { + "replicas": 1, + "selector": { + "name": "phabricator" + }, + "template": { + "metadata": { + "labels": { + "name": "phabricator" + } + }, + "spec": { + "containers": [ + { + "name": "phabricator", + "image": "fgrzadkowski/example-php-phabricator", + "ports": [ + { + "name": "http-server", + "containerPort": 80 + } + ], + "env": [ + { + "name": "MYSQL_SERVICE_IP", + "value": "1.2.3.4" + }, + { + "name": "MYSQL_SERVICE_PORT", + "value": "3306" + }, + { + "name": "MYSQL_PASSWORD", + "value": "1234" + } + ] + } + ] + } + } + } +} diff --git a/phabricator/phabricator-service.json b/phabricator/phabricator-service.json new file mode 100644 index 000000000..b2ec74cf4 --- /dev/null +++ b/phabricator/phabricator-service.json @@ -0,0 +1,19 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "phabricator" + }, + "spec": { + "ports": [ + { + "port": 80, + "targetPort": "http-server" + } + ], + "selector": { + "name": "phabricator" + }, + "type": "LoadBalancer" + } +} diff --git a/phabricator/php-phabricator/000-default.conf b/phabricator/php-phabricator/000-default.conf new file mode 100644 index 000000000..2ec64d687 --- /dev/null +++ b/phabricator/php-phabricator/000-default.conf @@ -0,0 +1,12 @@ + + Require all granted + + + + DocumentRoot /home/www-data/phabricator/webroot + + RewriteEngine on + RewriteRule ^/rsrc/(.*) - [L,QSA] + RewriteRule ^/favicon.ico - [L,QSA] + RewriteRule ^(.*)$ /index.php?__path__=$1 [B,L,QSA] + diff --git a/phabricator/php-phabricator/Dockerfile b/phabricator/php-phabricator/Dockerfile new file mode 100644 index 000000000..f39b9421a --- /dev/null +++ b/phabricator/php-phabricator/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:14.04 + +# Install all the required packages. +RUN apt-get update +RUN apt-get -y install \ + git apache2 dpkg-dev python-pygments \ + php5 php5-mysql php5-gd php5-dev php5-curl php-apc php5-cli php5-json php5-xhprof +RUN a2enmod rewrite +RUN apt-get source php5 +RUN (cd `ls -1F | grep '^php5-.*/$'`/ext/pcntl && phpize && ./configure && make && sudo make install) + +# Load code source. +RUN mkdir /home/www-data +RUN cd /home/www-data && git clone https://github.com/phacility/libphutil.git +RUN cd /home/www-data && git clone https://github.com/phacility/arcanist.git +RUN cd /home/www-data && git clone https://github.com/phacility/phabricator.git +RUN chown -R www-data /home/www-data +RUN chgrp -R www-data /home/www-data + +ADD 000-default.conf /etc/apache2/sites-available/000-default.conf +ADD run.sh /run.sh +RUN chmod a+x /*.sh + +# Run Apache2. +EXPOSE 80 +CMD ["/run.sh"] diff --git a/phabricator/php-phabricator/run.sh b/phabricator/php-phabricator/run.sh new file mode 100755 index 000000000..1f2b8387f --- /dev/null +++ b/phabricator/php-phabricator/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "MySQL host IP ${MYSQL_SERVICE_IP} port ${MYSQL_SERVICE_PORT}." +/home/www-data/phabricator/bin/config set mysql.host $MYSQL_SERVICE_IP +/home/www-data/phabricator/bin/config set mysql.port $MYSQL_SERVICE_PORT +/home/www-data/phabricator/bin/config set mysql.pass $MYSQL_PASSWORD + +echo "Running storage upgrade" +/home/www-data/phabricator/bin/storage --force upgrade || exit 1 + +source /etc/apache2/envvars +echo "Starting Apache2" +apache2 -D FOREGROUND + diff --git a/phabricator/setup.sh b/phabricator/setup.sh new file mode 100755 index 000000000..588b1f5f9 --- /dev/null +++ b/phabricator/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Create Phabricator replication controller" && kubectl create -f phabricator-controller.json +echo "Create Phabricator service" && kubectl create -f phabricator-service.json +echo "Create firewall rule" && gcloud compute firewall-rules create phabricator-node-80 --allow=tcp:80 --target-tags kubernetes-node + diff --git a/phabricator/teardown.sh b/phabricator/teardown.sh new file mode 100755 index 000000000..266313912 --- /dev/null +++ b/phabricator/teardown.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Deleting Phabricator service" && kubectl delete -f phabricator-service.json +echo "Deleting Phabricator replication controller" && kubectl delete rc phabricator-controller +echo "Delete firewall rule" && gcloud compute firewall-rules delete -q phabricator-node-80 + diff --git a/pod b/pod new file mode 100644 index 000000000..363381d55 --- /dev/null +++ b/pod @@ -0,0 +1,13 @@ +# Copy of pod.yaml without file extension for test +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/podsecuritypolicy/rbac/README.md b/podsecuritypolicy/rbac/README.md new file mode 100644 index 000000000..e606d23de --- /dev/null +++ b/podsecuritypolicy/rbac/README.md @@ -0,0 +1,196 @@ +## PSP RBAC Example + +This example demonstrates the usage of *PodSecurityPolicy* to control access to privileged containers +based on role and groups. + +### Prerequisites + +The server must be started to enable the appropriate APIs and flags + +1. allow privileged containers +1. allow security contexts +1. enable RBAC and accept any token +1. enable PodSecurityPolicies +1. use the PodSecurityPolicy admission controller + +If you are using the `local-up-cluster.sh` script you may enable these settings with the following syntax + +``` +PSP_ADMISSION=true ALLOW_PRIVILEGED=true ALLOW_SECURITY_CONTEXT=true ALLOW_ANY_TOKEN=true ENABLE_RBAC=true RUNTIME_CONFIG="extensions/v1beta1=true,extensions/v1beta1/podsecuritypolicy=true" hack/local-up-cluster.sh +``` + +### Using the protected port + +It is important to note that this example uses the following syntax to test with RBAC + +1. `--server=https://127.0.0.1:6443`: when performing requests this ensures that the protected port is used so +that RBAC will be enforced +1. `--token={user}/{group(s)}`: this syntax allows a request to specify the username and groups to use for +testing. It relies on the `ALLOW_ANY_TOKEN` setting. + +## Creating the policies, roles, and bindings + +### Policies + +The first step to enforcing cluster constraints via PSP is to create your policies. In this +example we will use two policies, `restricted` and `privileged`. For simplicity, the only difference +between these policies is the ability to run a privileged container. + +```yaml +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged +spec: + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted +spec: + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' + +``` + +To create these policies run + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/system:masters create -f examples/podsecuritypolicy/rbac/policies.yaml +podsecuritypolicy "privileged" created +podsecuritypolicy "restricted" created +``` + +### Roles and bindings + +In order to create a pod, either the creating user or the service account +specified by the pod must be authorized to use a `PodSecurityPolicy` object +that allows the pod. That authorization is determined by the ability to perform +the `use` verb on a particular `podsecuritypolicies` resource. The `use` verb +is a special verb that grants access to use a policy while not permitting any +other access. For this example, we'll first create RBAC `ClusterRoles` that +enable access to `use` specific policies. + +1. `restricted-psp-user`: this role allows the `use` verb on the `restricted` policy only +2. `privileged-psp-user`: this role allows the `use` verb on the `privileged` policy only + + +We can then create `ClusterRoleBindings` to grant groups of users the +"restricted" and/or "privileged" `ClusterRoles`. In this example, the bindings +grant the following roles to groups. + +1. `privileged`: this group is bound to the `privilegedPSP` role and `restrictedPSP` role which gives users +in this group access to both policies. +1. `restricted`: this group is bound to the `restrictedPSP` role. +1. `system:authenticated`: this is a system group for any authenticated user. It is bound to the `edit` +role which is already provided by the cluster. + +To create these roles and bindings run + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/system:masters create -f examples/podsecuritypolicy/rbac/roles.yaml +clusterrole "restricted-psp-user" created +clusterrole "privileged-psp-user" created + +$ kubectl --server=https://127.0.0.1:6443 --token=foo/system:masters create -f examples/podsecuritypolicy/rbac/bindings.yaml +clusterrolebinding "privileged-psp-users" created +clusterrolebinding "restricted-psp-users" created +clusterrolebinding "edit" created +``` + +## Testing access + +### Restricted user can create non-privileged pods + +Create the pod + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/restricted-psp-users create -f examples/podsecuritypolicy/rbac/pod.yaml +pod "nginx" created +``` + +Check the PSP that allowed the pod + +``` +$ kubectl get pod nginx -o yaml | grep psp + kubernetes.io/psp: restricted +``` + +### Restricted user cannot create privileged pods + +Delete the existing pod + +``` +$ kubectl delete pod nginx +pod "nginx" deleted +``` + +Create the privileged pod + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/restricted-psp-users create -f examples/podsecuritypolicy/rbac/pod_priv.yaml +Error from server (Forbidden): error when creating "examples/podsecuritypolicy/rbac/pod_priv.yaml": pods "nginx" is forbidden: unable to validate against any pod security policy: [spec.containers[0].securityContext.privileged: Invalid value: true: Privileged containers are not allowed] +``` + +### Privileged user can create non-privileged pods + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/privileged-psp-users create -f examples/podsecuritypolicy/rbac/pod.yaml +pod "nginx" created +``` + +Check the PSP that allowed the pod. Note, this could be the `restricted` or `privileged` PSP since both allow +for the creation of non-privileged pods. + +``` +$ kubectl get pod nginx -o yaml | egrep "psp|privileged" + kubernetes.io/psp: privileged + privileged: false +``` + +### Privileged user can create privileged pods + +Delete the existing pod + +``` +$ kubectl delete pod nginx +pod "nginx" deleted +``` + +Create the privileged pod + +``` +$ kubectl --server=https://127.0.0.1:6443 --token=foo/privileged-psp-users create -f examples/podsecuritypolicy/rbac/pod_priv.yaml +pod "nginx" created +``` + +Check the PSP that allowed the pod. + +``` +$ kubectl get pod nginx -o yaml | egrep "psp|privileged" + kubernetes.io/psp: privileged + privileged: true +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/podsecuritypolicy/rbac/README.md?pixel)]() + diff --git a/podsecuritypolicy/rbac/bindings.yaml b/podsecuritypolicy/rbac/bindings.yaml new file mode 100644 index 000000000..b07f99ee2 --- /dev/null +++ b/podsecuritypolicy/rbac/bindings.yaml @@ -0,0 +1,49 @@ +# privilegedPSP gives the privilegedPSP role +# to the group privileged. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: privileged-psp-users +subjects: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: privileged-psp-users +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: privileged-psp-user +--- +# restrictedPSP grants the restrictedPSP role to +# the groups restricted and privileged. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: restricted-psp-users +subjects: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: restricted-psp-users +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: privileged-psp-users +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: restricted-psp-user +--- +# edit grants edit role to system:authenticated. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: edit +subjects: +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: privileged-psp-users +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: restricted-psp-users +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit diff --git a/podsecuritypolicy/rbac/pod.yaml b/podsecuritypolicy/rbac/pod.yaml new file mode 100644 index 000000000..5b7b1efdc --- /dev/null +++ b/podsecuritypolicy/rbac/pod.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/podsecuritypolicy/rbac/pod_priv.yaml b/podsecuritypolicy/rbac/pod_priv.yaml new file mode 100644 index 000000000..6c638c449 --- /dev/null +++ b/podsecuritypolicy/rbac/pod_priv.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + labels: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + securityContext: + privileged: true diff --git a/podsecuritypolicy/rbac/policies.yaml b/podsecuritypolicy/rbac/policies.yaml new file mode 100644 index 000000000..7519091b8 --- /dev/null +++ b/podsecuritypolicy/rbac/policies.yaml @@ -0,0 +1,38 @@ +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged +spec: + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted +spec: + privileged: false + fsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'emptyDir' + - 'secret' + - 'downwardAPI' + - 'configMap' + - 'persistentVolumeClaim' + diff --git a/podsecuritypolicy/rbac/roles.yaml b/podsecuritypolicy/rbac/roles.yaml new file mode 100644 index 000000000..43aecf2a0 --- /dev/null +++ b/podsecuritypolicy/rbac/roles.yaml @@ -0,0 +1,33 @@ +# restrictedPSP grants access to use +# the restricted PSP. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: restricted-psp-user +rules: +- apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + - restricted + verbs: + - use +--- +# privilegedPSP grants access to use the privileged +# PSP. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: privileged-psp-user +rules: +- apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + - privileged + verbs: + - use + + diff --git a/runtime-constraints/README.md b/runtime-constraints/README.md new file mode 100644 index 000000000..82f6c0ee1 --- /dev/null +++ b/runtime-constraints/README.md @@ -0,0 +1,285 @@ +## Runtime Constraints example + +This example demonstrates how Kubernetes enforces runtime constraints for compute resources. + +### Prerequisites + +For the purpose of this example, we will spin up a 1 node cluster using the Vagrant provider that +is not running with any additional add-ons that consume node resources. This keeps our demonstration +of compute resources easier to follow by starting with an empty cluster. + +``` +$ export KUBERNETES_PROVIDER=vagrant +$ export NUM_NODES=1 +$ export KUBE_ENABLE_CLUSTER_MONITORING=none +$ export KUBE_ENABLE_CLUSTER_DNS=false +$ export KUBE_ENABLE_CLUSTER_UI=false +$ cluster/kube-up.sh +``` + +We should now have a single node cluster running 0 pods. + +``` +$ cluster/kubectl.sh get nodes +NAME LABELS STATUS AGE +10.245.1.3 kubernetes.io/hostname=10.245.1.3 Ready 17m +$ cluster/kubectl.sh get pods --all-namespaces +``` + +When demonstrating runtime constraints, it's useful to show what happens when a node is under heavy load. For +this scenario, we have a single node with 2 cpus and 1GB of memory to demonstrate behavior under load, but the +results extend to multi-node scenarios. + +### CPU requests + +Each container in a pod may specify the amount of CPU it requests on a node. CPU requests are used at schedule time, and represent a minimum amount of CPU that should be reserved for your container to run. + +When executing your container, the Kubelet maps your containers CPU requests to CFS shares in the Linux kernel. CFS CPU shares do not impose a ceiling on the actual amount of CPU the container can use. Instead, it defines a relative weight across all containers on the system for how much CPU time the container should get if there is CPU contention. + +Let's demonstrate this concept using a simple container that will consume as much CPU as possible. + +``` +$ cluster/kubectl.sh run cpuhog \ + --image=busybox \ + --requests=cpu=100m \ + -- md5sum /dev/urandom +``` + +This will create a single pod on your node that requests 1/10 of a CPU, but it has no limit on how much CPU it may actually consume +on the node. + +To demonstrate this, if you SSH into your machine, you will see it is consuming as much CPU as possible on the node. + +``` +$ vagrant ssh node-1 +$ sudo docker stats $(sudo docker ps -q) +CONTAINER CPU % MEM USAGE/LIMIT MEM % NET I/O +6b593b1a9658 0.00% 1.425 MB/1.042 GB 0.14% 1.038 kB/738 B +ae8ae4ffcfe4 150.06% 831.5 kB/1.042 GB 0.08% 0 B/0 B +``` + +As you can see, its consuming 150% of the total CPU. + +If we scale our replication controller to 20 pods, we should see that each container is given an equal proportion of CPU time. + +``` +$ cluster/kubectl.sh scale rc/cpuhog --replicas=20 +``` + +Once all the pods are running, you will see on your node that each container is getting approximately an equal proportion of CPU time. + +``` +$ sudo docker stats $(sudo docker ps -q) +CONTAINER CPU % MEM USAGE/LIMIT MEM % NET I/O +089e2d061dee 9.24% 786.4 kB/1.042 GB 0.08% 0 B/0 B +0be33d6e8ddb 10.48% 823.3 kB/1.042 GB 0.08% 0 B/0 B +0f4e3c4a93e0 10.43% 786.4 kB/1.042 GB 0.08% 0 B/0 B +``` + +Each container is getting 10% of the CPU time per their scheduling request, and we are unable to schedule more. + +As you can see CPU requests are used to schedule pods to the node in a manner that provides weighted distribution of CPU time +when under contention. If the node is not being actively consumed by other containers, a container is able to burst up to as much +available CPU time as possible. If there is contention for CPU, CPU time is shared based on the requested value. + +Let's delete all existing resources in preparation for the next scenario. Verify all the pods are deleted and terminated. + +``` +$ cluster/kubectl.sh delete rc --all +$ cluster/kubectl.sh get pods +NAME READY STATUS RESTARTS AGE +``` + +### CPU limits + +So what do you do if you want to control the maximum amount of CPU that your container can burst to use in order provide a consistent +level of service independent of CPU contention on the node? You can specify an upper limit on the total amount of CPU that a pod's +container may consume. + +To enforce this feature, your node must run a docker version >= 1.7, and your operating system kernel must +have support for CFS quota enabled. Finally, your the Kubelet must be started with the following flag: + +``` +kubelet --cpu-cfs-quota=true +``` + +To demonstrate, let's create the same pod again, but this time set an upper limit to use 50% of a single CPU. + +``` +$ cluster/kubectl.sh run cpuhog \ + --image=busybox \ + --requests=cpu=100m \ + --limits=cpu=500m \ + -- md5sum /dev/urandom +``` + +Let's SSH into the node, and look at usage stats. + +``` +$ vagrant ssh node-1 +$ sudo su +$ docker stats $(docker ps -q) +CONTAINER CPU % MEM USAGE/LIMIT MEM % NET I/O +2a196edf7de2 47.38% 835.6 kB/1.042 GB 0.08% 0 B/0 B +... +``` + +As you can see, the container is no longer allowed to consume all available CPU on the node. Instead, it is being limited to use +50% of a CPU over every 100ms period. As a result, the reported value will be in the range of 50% but may oscillate above and below. + +Let's delete all existing resources in preparation for the next scenario. Verify all the pods are deleted and terminated. + +``` +$ cluster/kubectl.sh delete rc --all +$ cluster/kubectl.sh get pods +NAME READY STATUS RESTARTS AGE +``` + +### Memory requests + +By default, a container is able to consume as much memory on the node as possible. In order to improve placement of your +pods in the cluster, it is recommended to specify the amount of memory your container will require to run. The scheduler +will then take available node memory capacity into account prior to binding your pod to a node. + +Let's demonstrate this by creating a pod that runs a single container which requests 100Mi of memory. The container will +allocate and write to 200MB of memory every 2 seconds. + +``` +$ cluster/kubectl.sh run memhog \ + --image=derekwaynecarr/memhog \ + --requests=memory=100Mi \ + --command \ + -- /bin/sh -c "while true; do memhog -r100 200m; sleep 1; done" +``` + +If you look at output of docker stats on the node: + +``` +$ docker stats $(docker ps -q) +CONTAINER CPU % MEM USAGE/LIMIT MEM % NET I/O +2badf74ae782 0.00% 1.425 MB/1.042 GB 0.14% 816 B/348 B +a320182967fa 105.81% 214.2 MB/1.042 GB 20.56% 0 B/0 B + +``` + +As you can see, the container is using approximately 200MB of memory, and is only limited to the 1GB of memory on the node. + +We scheduled against 100Mi, but have burst our memory usage to a greater value. + +We refer to this as memory having __Burstable__ quality of service for this container. + +Let's delete all existing resources in preparation for the next scenario. Verify all the pods are deleted and terminated. + +``` +$ cluster/kubectl.sh delete rc --all +$ cluster/kubectl.sh get pods +NAME READY STATUS RESTARTS AGE +``` + +### Memory limits + +If you specify a memory limit, you can constrain the amount of memory your container can use. + +For example, let's limit our container to 200Mi of memory, and just consume 100MB. + +``` +$ cluster/kubectl.sh run memhog \ + --image=derekwaynecarr/memhog \ + --limits=memory=200Mi \ + --command -- /bin/sh -c "while true; do memhog -r100 100m; sleep 1; done" +``` + +If you look at output of docker stats on the node: + +``` +$ docker stats $(docker ps -q) +CONTAINER CPU % MEM USAGE/LIMIT MEM % NET I/O +5a7c22ae1837 125.23% 109.4 MB/209.7 MB 52.14% 0 B/0 B +c1d7579c9291 0.00% 1.421 MB/1.042 GB 0.14% 1.038 kB/816 B +``` + +As you can see, we are limited to 200Mi memory, and are only consuming 109.4MB on the node. + +Let's demonstrate what happens if you exceed your allowed memory usage by creating a replication controller +whose pod will keep being OOM killed because it attempts to allocate 300MB of memory, but is limited to 200Mi. + +``` +$ cluster/kubectl.sh run memhog-oom --image=derekwaynecarr/memhog --limits=memory=200Mi --command -- memhog -r100 300m +``` + +If we describe the created pod, you will see that it keeps restarting until it ultimately goes into a CrashLoopBackOff. + +The reason it is killed and restarts is because it is OOMKilled as it attempts to exceed its memory limit. + +``` +$ cluster/kubectl.sh get pods +NAME READY STATUS RESTARTS AGE +memhog-oom-gj9hw 0/1 CrashLoopBackOff 2 26s +$ cluster/kubectl.sh describe pods/memhog-oom-gj9hw | grep -C 3 "Terminated" + memory: 200Mi + State: Waiting + Reason: CrashLoopBackOff + Last Termination State: Terminated + Reason: OOMKilled + Exit Code: 137 + Started: Wed, 23 Sep 2015 15:23:58 -0400 +``` + +Let's clean-up before proceeding further. + +``` +$ cluster/kubectl.sh delete rc --all +``` + +### What if my node runs out of memory? + +If you only schedule __Guaranteed__ memory containers, where the request is equal to the limit, then you are not in major danger of +causing an OOM event on your node. If any individual container consumes more than their specified limit, it will be killed. + +If you schedule __BestEffort__ memory containers, where the request and limit is not specified, or __Burstable__ memory containers, where +the request is less than any specified limit, then it is possible that a container will request more memory than what is actually available on the node. + +If this occurs, the system will attempt to prioritize the containers that are killed based on their quality of service. This is done +by using the OOMScoreAdjust feature in the Linux kernel which provides a heuristic to rank a process between -1000 and 1000. Processes +with lower values are preserved in favor of processes with higher values. The system daemons (kubelet, kube-proxy, docker) all run with +low OOMScoreAdjust values. + +In simplest terms, containers with __Guaranteed__ memory containers are given a lower value than __Burstable__ containers which has +a lower value than __BestEffort__ containers. As a consequence, containers with __BestEffort__ should be killed before the other tiers. + +To demonstrate this, let's spin up a set of different replication controllers that will over commit the node. + +``` +$ cluster/kubectl.sh run mem-guaranteed --image=derekwaynecarr/memhog --replicas=2 --requests=cpu=10m --limits=memory=600Mi --command -- memhog -r100000 500m +$ cluster/kubectl.sh run mem-burstable --image=derekwaynecarr/memhog --replicas=2 --requests=cpu=10m,memory=600Mi --command -- memhog -r100000 100m +$ cluster/kubectl.sh run mem-besteffort --replicas=10 --image=derekwaynecarr/memhog --requests=cpu=10m --command -- memhog -r10000 500m +``` + +This will induce a SystemOOM + +``` +$ cluster/kubectl.sh get events | grep OOM +43m 8m 178 10.245.1.3 Node SystemOOM {kubelet 10.245.1.3} System OOM encountered +``` + +If you look at the pods: + +``` +$ cluster/kubectl.sh get pods +NAME READY STATUS RESTARTS AGE +... +mem-besteffort-zpnpm 0/1 CrashLoopBackOff 4 3m +mem-burstable-n0yz1 1/1 Running 0 4m +mem-burstable-q3dts 1/1 Running 0 4m +mem-guaranteed-fqsw8 1/1 Running 0 4m +mem-guaranteed-rkqso 1/1 Running 0 4m +``` + +You see that our BestEffort pod goes in a restart cycle, but the pods with greater levels of quality of service continue to function. + +As you can see, we rely on the Kernel to react to system OOM events. Depending on how your host operating +system was configured, and which process the Kernel ultimately decides to kill on your Node, you may experience unstable results. In addition, during an OOM event, while the kernel is cleaning up processes, the system may experience significant periods of slow down or appear unresponsive. As a result, while the system allows you to overcommit on memory, we recommend to not induce a Kernel sys OOM. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/runtime-constraints/README.md?pixel)]() + diff --git a/scheduler-policy-config-with-extender.json b/scheduler-policy-config-with-extender.json new file mode 100644 index 000000000..eea38f5c4 --- /dev/null +++ b/scheduler-policy-config-with-extender.json @@ -0,0 +1,28 @@ +{ +"kind" : "Policy", +"apiVersion" : "v1", +"predicates" : [ + {"name" : "PodFitsHostPorts"}, + {"name" : "PodFitsResources"}, + {"name" : "NoDiskConflict"}, + {"name" : "MatchNodeSelector"}, + {"name" : "HostName"} + ], +"priorities" : [ + {"name" : "LeastRequestedPriority", "weight" : 1}, + {"name" : "BalancedResourceAllocation", "weight" : 1}, + {"name" : "ServiceSpreadingPriority", "weight" : 1}, + {"name" : "EqualPriority", "weight" : 1} + ], +"extenders":[ + { + "urlPrefix": "http://127.0.0.1:12346/scheduler", + "apiVersion": "v1beta1", + "filterVerb": "filter", + "prioritizeVerb": "prioritize", + "weight": 5, + "enableHttps": false, + "nodeCacheCapable": false + } + ] +} diff --git a/scheduler-policy-config.json b/scheduler-policy-config.json new file mode 100644 index 000000000..f93c39d13 --- /dev/null +++ b/scheduler-policy-config.json @@ -0,0 +1,18 @@ +{ +"kind" : "Policy", +"apiVersion" : "v1", +"predicates" : [ + {"name" : "PodFitsHostPorts"}, + {"name" : "PodFitsResources"}, + {"name" : "NoDiskConflict"}, + {"name" : "NoVolumeZoneConflict"}, + {"name" : "MatchNodeSelector"}, + {"name" : "HostName"} + ], +"priorities" : [ + {"name" : "LeastRequestedPriority", "weight" : 1}, + {"name" : "BalancedResourceAllocation", "weight" : 1}, + {"name" : "ServiceSpreadingPriority", "weight" : 1}, + {"name" : "EqualPriority", "weight" : 1} + ] +} diff --git a/selenium/README.md b/selenium/README.md new file mode 100644 index 000000000..55c441715 --- /dev/null +++ b/selenium/README.md @@ -0,0 +1,199 @@ +## Selenium on Kubernetes + +Selenium is a browser automation tool used primarily for testing web applications. However when Selenium is used in a CI pipeline to test applications, there is often contention around the use of Selenium resources. This example shows you how to deploy Selenium to Kubernetes in a scalable fashion. + +### Prerequisites + +This example assumes you have a working Kubernetes cluster and a properly configured kubectl client. See the [Getting Started Guides](../../docs/getting-started-guides/) for details. + +Google Container Engine is also a quick way to get Kubernetes up and running: https://cloud.google.com/container-engine/ + +Your cluster must have 4 CPU and 6 GB of RAM to complete the example up to the scaling portion. + +### Deploy Selenium Grid Hub: + +We will be using Selenium Grid Hub to make our Selenium install scalable via a master/worker model. The Selenium Hub is the master, and the Selenium Nodes are the workers(not to be confused with Kubernetes nodes). We only need one hub, but we're using a replication controller to ensure that the hub is always running: + +```console +kubectl create --filename=examples/selenium/selenium-hub-rc.yaml +``` + +The Selenium Nodes will need to know how to get to the Hub, let's create a service for the nodes to connect to. + +```console +kubectl create --filename=examples/selenium/selenium-hub-svc.yaml +``` + +### Verify Selenium Hub Deployment + +Let's verify our deployment of Selenium hub by connecting to the web console. + +#### Kubernetes Nodes Reachable + +If your Kubernetes nodes are reachable from your network, you can verify the hub by hitting it on the nodeport. You can retrieve the nodeport by typing `kubectl describe svc selenium-hub`, however the snippet below automates that by using kubectl's template functionality: + +```console +export NODEPORT=`kubectl get svc --selector='app=selenium-hub' --output=template --template="{{ with index .items 0}}{{with index .spec.ports 0 }}{{.nodePort}}{{end}}{{end}}"` +export NODE=`kubectl get nodes --output=template --template="{{with index .items 0 }}{{.metadata.name}}{{end}}"` + +curl http://$NODE:$NODEPORT +``` + +#### Kubernetes Nodes Unreachable + +If you cannot reach your Kubernetes nodes from your network, you can proxy via kubectl. + +```console +export PODNAME=`kubectl get pods --selector="app=selenium-hub" --output=template --template="{{with index .items 0}}{{.metadata.name}}{{end}}"` +kubectl port-forward $PODNAME 4444:4444 +``` + +In a separate terminal, you can now check the status. + +```console +curl http://localhost:4444 +``` + +#### Using Google Container Engine + +If you are using Google Container Engine, you can expose your hub via the internet. This is a bad idea for many reasons, but you can do it as follows: + +```console +kubectl expose rc selenium-hub --name=selenium-hub-external --labels="app=selenium-hub,external=true" --type=LoadBalancer +``` + +Then wait a few minutes, eventually your new `selenium-hub-external` service will be assigned a load balanced IP from gcloud. Once `kubectl get svc selenium-hub-external` shows two IPs, run this snippet. + +```console +export INTERNET_IP=`kubectl get svc --selector="app=selenium-hub,external=true" --output=template --template="{{with index .items 0}}{{with index .status.loadBalancer.ingress 0}}{{.ip}}{{end}}{{end}}"` + +curl http://$INTERNET_IP:4444/ +``` + +You should now be able to hit `$INTERNET_IP` via your web browser, and so can everyone else on the Internet! + +### Deploy Firefox and Chrome Nodes: + +Now that the Hub is up, we can deploy workers. + +This will deploy 2 Chrome nodes. + +```console +kubectl create --filename=examples/selenium/selenium-node-chrome-rc.yaml +``` + +And 2 Firefox nodes to match. + +```console +kubectl create --filename=examples/selenium/selenium-node-firefox-rc.yaml +``` + +Once the pods start, you will see them show up in the Selenium Hub interface. + +### Run a Selenium Job + +Let's run a quick Selenium job to validate our setup. + +#### Setup Python Environment + +First, we need to start a python container that we can attach to. + +```console +kubectl run selenium-python --image=google/python-hello +``` + +Next, we need to get inside this container. + +```console +export PODNAME=`kubectl get pods --selector="run=selenium-python" --output=template --template="{{with index .items 0}}{{.metadata.name}}{{end}}"` +kubectl exec --stdin=true --tty=true $PODNAME bash +``` + +Once inside, we need to install the Selenium library + +```console +pip install selenium +``` + +#### Run Selenium Job with Python + +We're all set up, start the python interpreter. + +```console +python +``` + +And paste in the contents of selenium-test.py. + +```python +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities + +def check_browser(browser): + driver = webdriver.Remote( + command_executor='http://selenium-hub:4444/wd/hub', + desired_capabilities=getattr(DesiredCapabilities, browser) + ) + driver.get("http://google.com") + assert "google" in driver.page_source + driver.close() + print("Browser %s checks out!" % browser) + + +check_browser("FIREFOX") +check_browser("CHROME") +``` + +You should get + +``` +>>> check_browser("FIREFOX") +Browser FIREFOX checks out! +>>> check_browser("CHROME") +Browser CHROME checks out! +``` + +Congratulations, your Selenium Hub is up, with Firefox and Chrome nodes! + +### Scale your Firefox and Chrome nodes. + +If you need more Firefox or Chrome nodes, your hardware is the limit: + +```console +kubectl scale rc selenium-node-firefox --replicas=10 +kubectl scale rc selenium-node-chrome --replicas=10 +``` + +You now have 10 Firefox and 10 Chrome nodes, happy Seleniuming! + +### Debugging + +Sometimes it is necessary to check on a hung test. Each pod is running VNC. To check on one of the browser nodes via VNC, it's recommended that you proxy, since we don't want to expose a service for every pod, and the containers have a weak VNC password. Replace POD_NAME with the name of the pod you want to connect to. + +```console +kubectl port-forward $POD_NAME 5900:5900 +``` + +Then connect to localhost:5900 with your VNC client using the password "secret" + +Enjoy your scalable Selenium Grid! + +Adapted from: https://github.com/SeleniumHQ/docker-selenium + +### Teardown + +To remove all created resources, run the following: + +```console +kubectl delete rc selenium-hub +kubectl delete rc selenium-node-chrome +kubectl delete rc selenium-node-firefox +kubectl delete deployment selenium-python +kubectl delete svc selenium-hub +kubectl delete svc selenium-hub-external +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/selenium/README.md?pixel)]() + diff --git a/selenium/selenium-hub-rc.yaml b/selenium/selenium-hub-rc.yaml new file mode 100644 index 000000000..f48510e1c --- /dev/null +++ b/selenium/selenium-hub-rc.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: selenium-hub + labels: + app: selenium-hub +spec: + replicas: 1 + selector: + app: selenium-hub + template: + metadata: + labels: + app: selenium-hub + spec: + containers: + - name: selenium-hub + image: selenium/hub:2.53.0 + ports: + - containerPort: 4444 + resources: + limits: + memory: "1000Mi" + cpu: ".5" + livenessProbe: + httpGet: + path: /grid/console + port: 4444 + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /grid/console + port: 4444 + initialDelaySeconds: 30 + timeoutSeconds: 5 diff --git a/selenium/selenium-hub-svc.yaml b/selenium/selenium-hub-svc.yaml new file mode 100644 index 000000000..0b252ede3 --- /dev/null +++ b/selenium/selenium-hub-svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: selenium-hub + labels: + app: selenium-hub +spec: + ports: + - port: 4444 + targetPort: 4444 + name: port0 + selector: + app: selenium-hub + type: NodePort + sessionAffinity: None diff --git a/selenium/selenium-node-chrome-rc.yaml b/selenium/selenium-node-chrome-rc.yaml new file mode 100644 index 000000000..3c49c1b43 --- /dev/null +++ b/selenium/selenium-node-chrome-rc.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: selenium-node-chrome + labels: + app: selenium-node-chrome +spec: + replicas: 2 + selector: + app: selenium-node-chrome + template: + metadata: + labels: + app: selenium-node-chrome + spec: + containers: + - name: selenium-node-chrome + image: selenium/node-chrome-debug:2.53.0 + ports: + - containerPort: 5900 + env: + - name: HUB_PORT_4444_TCP_ADDR + value: "selenium-hub" + - name: HUB_PORT_4444_TCP_PORT + value: "4444" + resources: + limits: + memory: "1000Mi" + cpu: ".5" diff --git a/selenium/selenium-node-firefox-rc.yaml b/selenium/selenium-node-firefox-rc.yaml new file mode 100644 index 000000000..d6f665785 --- /dev/null +++ b/selenium/selenium-node-firefox-rc.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: selenium-node-firefox + labels: + app: selenium-node-firefox +spec: + replicas: 2 + selector: + app: selenium-node-firefox + template: + metadata: + labels: + app: selenium-node-firefox + spec: + containers: + - name: selenium-node-firefox + image: selenium/node-firefox-debug:2.53.0 + ports: + - containerPort: 5900 + env: + - name: HUB_PORT_4444_TCP_ADDR + value: "selenium-hub" + - name: HUB_PORT_4444_TCP_PORT + value: "4444" + resources: + limits: + memory: "1000Mi" + cpu: ".5" diff --git a/selenium/selenium-test.py b/selenium/selenium-test.py new file mode 100644 index 000000000..80d598a3b --- /dev/null +++ b/selenium/selenium-test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities + +def check_browser(browser): + driver = webdriver.Remote( + command_executor='http://selenium-hub:4444/wd/hub', + desired_capabilities=getattr(DesiredCapabilities, browser) + ) + driver.get("http://google.com") + assert "google" in driver.page_source + driver.close() + print("Browser %s checks out!" % browser) + + +check_browser("FIREFOX") +check_browser("CHROME") + diff --git a/sharing-clusters/BUILD b/sharing-clusters/BUILD new file mode 100644 index 000000000..4405a0ab4 --- /dev/null +++ b/sharing-clusters/BUILD @@ -0,0 +1,39 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_binary( + name = "sharing-clusters", + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["make_secret.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/sharing-clusters/README.md b/sharing-clusters/README.md new file mode 100644 index 000000000..38f46fa0e --- /dev/null +++ b/sharing-clusters/README.md @@ -0,0 +1,187 @@ +# Sharing Clusters + +This example demonstrates how to access one kubernetes cluster from another. It only works if both clusters are running on the same network, on a cloud provider that provides a private ip range per network (eg: GCE, GKE, AWS). + +## Setup + +Create a cluster in US (you don't need to do this if you already have a running kubernetes cluster) + +```shell +$ cluster/kube-up.sh +``` + +Before creating our second cluster, lets have a look at the kubectl config: + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: REDACTED + server: https://104.197.84.16 + name: +... +current-context: +... +``` + +Now spin up the second cluster in Europe + +```shell +$ ./cluster/kube-up.sh +$ KUBE_GCE_ZONE=europe-west1-b KUBE_GCE_INSTANCE_PREFIX=eu ./cluster/kube-up.sh +``` + +Your kubectl config should contain both clusters: + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: REDACTED + server: https://146.148.25.221 + name: +- cluster: + certificate-authority-data: REDACTED + server: https://104.197.84.16 + name: +... +current-context: kubernetesdev_eu +... +``` + +And kubectl get nodes should agree: + +``` +$ kubectl get nodes +NAME LABELS STATUS +eu-node-0n61 kubernetes.io/hostname=eu-node-0n61 Ready +eu-node-79ua kubernetes.io/hostname=eu-node-79ua Ready +eu-node-7wz7 kubernetes.io/hostname=eu-node-7wz7 Ready +eu-node-loh2 kubernetes.io/hostname=eu-node-loh2 Ready + +$ kubectl config use-context +$ kubectl get nodes +NAME LABELS STATUS +kubernetes-node-5jtd kubernetes.io/hostname=kubernetes-node-5jtd Ready +kubernetes-node-lqfc kubernetes.io/hostname=kubernetes-node-lqfc Ready +kubernetes-node-sjra kubernetes.io/hostname=kubernetes-node-sjra Ready +kubernetes-node-wul8 kubernetes.io/hostname=kubernetes-node-wul8 Ready +``` + +## Testing reachability + +For this test to work we'll need to create a service in europe: + +``` +$ kubectl config use-context +$ kubectl create -f /tmp/secret.json +$ kubectl create -f examples/https-nginx/nginx-app.yaml +$ kubectl exec -it my-nginx-luiln -- echo "Europe nginx" >> /usr/share/nginx/html/index.html +$ kubectl get ep +NAME ENDPOINTS +kubernetes 10.240.249.92:443 +nginxsvc 10.244.0.4:80,10.244.0.4:443 +``` + +Just to test reachability, we'll try hitting the Europe nginx from our initial US central cluster. Create a basic curl pod in the US cluster: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: curlpod +spec: + containers: + - image: radial/busyboxplus:curl + command: + - sleep + - "360000000" + imagePullPolicy: IfNotPresent + name: curlcontainer + restartPolicy: Always +``` + +And test that you can actually reach the test nginx service across continents + +``` +$ kubectl config use-context +$ kubectl -it exec curlpod -- /bin/sh +[ root@curlpod:/ ]$ curl http://10.244.0.4:80 +Europe nginx +``` + +## Granting access to the remote cluster + +We will grant the US cluster access to the Europe cluster. Basically we're going to setup a secret that allows kubectl to function in a pod running in the US cluster, just like it did on our local machine in the previous step. First create a secret with the contents of the current .kube/config: + +```shell +$ kubectl config use-context +$ go run ./make_secret.go --kubeconfig=$HOME/.kube/config > /tmp/secret.json +$ kubectl config use-context +$ kubectl create -f /tmp/secret.json +``` + +Create a kubectl pod that uses the secret, in the US cluster. + +```json +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "kubectl-tester" + }, + "spec": { + "volumes": [ + { + "name": "secret-volume", + "secret": { + "secretName": "kubeconfig" + } + } + ], + "containers": [ + { + "name": "kubectl", + "image": "bprashanth/kubectl:0.0", + "imagePullPolicy": "Always", + "env": [ + { + "name": "KUBECONFIG", + "value": "/.kube/config" + } + ], + "args": [ + "proxy", "-p", "8001" + ], + "volumeMounts": [ + { + "name": "secret-volume", + "mountPath": "/.kube" + } + ] + } + ] + } +} +``` + +And check that you can access the remote cluster + +```shell +$ kubectl config use-context +$ kubectl exec -it kubectl-tester bash + +kubectl-tester $ kubectl get nodes +NAME LABELS STATUS +eu-node-0n61 kubernetes.io/hostname=eu-node-0n61 Ready +eu-node-79ua kubernetes.io/hostname=eu-node-79ua Ready +eu-node-7wz7 kubernetes.io/hostname=eu-node-7wz7 Ready +eu-node-loh2 kubernetes.io/hostname=eu-node-loh2 Ready +``` + +For a more advanced example of sharing clusters, see the [service-loadbalancer](https://github.com/kubernetes/contrib/tree/master/service-loadbalancer/README.md) + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/sharing-clusters/README.md?pixel)]() + diff --git a/sharing-clusters/make_secret.go b/sharing-clusters/make_secret.go new file mode 100644 index 000000000..4721c1167 --- /dev/null +++ b/sharing-clusters/make_secret.go @@ -0,0 +1,63 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A tiny script to help conver a given kubeconfig into a secret. +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api" +) + +// TODO: +// Add a -o flag that writes to the specified destination file. +var ( + kubeconfig = flag.String("kubeconfig", "", "path to kubeconfig file.") + name = flag.String("name", "kubeconfig", "name to use in the metadata of the secret.") + ns = flag.String("ns", "default", "namespace of the secret.") +) + +func read(file string) []byte { + b, err := ioutil.ReadFile(file) + if err != nil { + log.Fatalf("Cannot read file %v, %v", file, err) + } + return b +} + +func main() { + flag.Parse() + if *kubeconfig == "" { + log.Fatalf("Need to specify --kubeconfig") + } + cfg := read(*kubeconfig) + secret := &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: *name, + Namespace: *ns, + }, + Data: map[string][]byte{ + "config": cfg, + }, + } + fmt.Printf(runtime.EncodeOrDie(api.Codecs.LegacyCodec(api.Registry.EnabledVersions()...), secret)) +} diff --git a/simple-nginx.md b/simple-nginx.md new file mode 100644 index 000000000..b61801a4d --- /dev/null +++ b/simple-nginx.md @@ -0,0 +1,62 @@ +## Running your first containers in Kubernetes + +Ok, you've run one of the [getting started guides](../docs/getting-started-guides/) and you have +successfully turned up a Kubernetes cluster. Now what? This guide will help you get oriented +to Kubernetes and running your first containers on the cluster. + +### Running a container (simple version) + +From this point onwards, it is assumed that `kubectl` is on your path from one of the getting started guides. + +The [`kubectl run`](../docs/user-guide/kubectl/kubectl_run.md) line below will create two [nginx](https://registry.hub.docker.com/_/nginx/) [pods](../docs/user-guide/pods.md) listening on port 80. It will also create a [deployment](../docs/user-guide/deployments.md) named `my-nginx` to ensure that there are always two pods running. + +```bash +kubectl run my-nginx --image=nginx --replicas=2 --port=80 +``` + +Once the pods are created, you can list them to see what is up and running: + +```bash +kubectl get pods +``` + +You can also see the deployment that was created: + +```bash +kubectl get deployment +``` + +### Exposing your pods to the internet. + +On some platforms (for example Google Compute Engine) the kubectl command can integrate with your cloud provider to add a [public IP address](../docs/user-guide/services.md#publishing-services---service-types) for the pods, +to do this run: + +```bash +kubectl expose deployment my-nginx --port=80 --type=LoadBalancer +``` + +This should print the service that has been created, and map an external IP address to the service. Where to find this external IP address will depend on the environment you run in. For instance, for Google Compute Engine the external IP address is listed as part of the newly created service and can be retrieved by running + +```bash +kubectl get services +``` + +In order to access your nginx landing page, you also have to make sure that traffic from external IPs is allowed. Do this by opening a firewall to allow traffic on port 80. + +### Cleanup + +To delete the two replicated containers, delete the deployment: + +```bash +kubectl delete deployment my-nginx +``` + +### Next: Configuration files + +Most people will eventually want to use declarative configuration files for creating/modifying their applications. A [simplified introduction](../docs/user-guide/deploying-applications.md) +is given in a different document. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/simple-nginx.md?pixel)]() + diff --git a/spark/README.md b/spark/README.md new file mode 100644 index 000000000..8ea4f6a4f --- /dev/null +++ b/spark/README.md @@ -0,0 +1,373 @@ +# Spark example + +Following this example, you will create a functional [Apache +Spark](http://spark.apache.org/) cluster using Kubernetes and +[Docker](http://docker.io). + +You will setup a Spark master service and a set of Spark workers using Spark's [standalone mode](http://spark.apache.org/docs/latest/spark-standalone.html). + +For the impatient expert, jump straight to the [tl;dr](#tldr) +section. + +### Sources + +The Docker images are heavily based on https://github.com/mattf/docker-spark. +And are curated in https://github.com/kubernetes/application-images/tree/master/spark + +The Spark UI Proxy is taken from https://github.com/aseigneurin/spark-ui-proxy. + +The PySpark examples are taken from http://stackoverflow.com/questions/4114167/checking-if-a-number-is-a-prime-number-in-python/27946768#27946768 + +## Step Zero: Prerequisites + +This example assumes + +- You have a Kubernetes cluster installed and running. +- That you have installed the ```kubectl``` command line tool installed in your path and configured to talk to your Kubernetes cluster +- That your Kubernetes cluster is running [kube-dns](https://github.com/kubernetes/dns) or an equivalent integration. + +Optionally, your Kubernetes cluster should be configured with a Loadbalancer integration (automatically configured via kube-up or GKE) + +## Step One: Create namespace + +```sh +$ kubectl create -f examples/spark/namespace-spark-cluster.yaml +``` + +Now list all namespaces: + +```sh +$ kubectl get namespaces +NAME LABELS STATUS +default Active +spark-cluster name=spark-cluster Active +``` + +To configure kubectl to work with our namespace, we will create a new context using our current context as a base: + +```sh +$ CURRENT_CONTEXT=$(kubectl config view -o jsonpath='{.current-context}') +$ USER_NAME=$(kubectl config view -o jsonpath='{.contexts[?(@.name == "'"${CURRENT_CONTEXT}"'")].context.user}') +$ CLUSTER_NAME=$(kubectl config view -o jsonpath='{.contexts[?(@.name == "'"${CURRENT_CONTEXT}"'")].context.cluster}') +$ kubectl config set-context spark --namespace=spark-cluster --cluster=${CLUSTER_NAME} --user=${USER_NAME} +$ kubectl config use-context spark +``` + +## Step Two: Start your Master service + +The Master [service](../../docs/user-guide/services.md) is the master service +for a Spark cluster. + +Use the +[`examples/spark/spark-master-controller.yaml`](spark-master-controller.yaml) +file to create a +[replication controller](../../docs/user-guide/replication-controller.md) +running the Spark Master service. + +```console +$ kubectl create -f examples/spark/spark-master-controller.yaml +replicationcontroller "spark-master-controller" created +``` + +Then, use the +[`examples/spark/spark-master-service.yaml`](spark-master-service.yaml) file to +create a logical service endpoint that Spark workers can use to access the +Master pod: + +```console +$ kubectl create -f examples/spark/spark-master-service.yaml +service "spark-master" created +``` + +### Check to see if Master is running and accessible + +```console +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +spark-master-controller-5u0q5 1/1 Running 0 8m +``` + +Check logs to see the status of the master. (Use the pod retrieved from the previous output.) + +```sh +$ kubectl logs spark-master-controller-5u0q5 +starting org.apache.spark.deploy.master.Master, logging to /opt/spark-1.5.1-bin-hadoop2.6/sbin/../logs/spark--org.apache.spark.deploy.master.Master-1-spark-master-controller-g0oao.out +Spark Command: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -cp /opt/spark-1.5.1-bin-hadoop2.6/sbin/../conf/:/opt/spark-1.5.1-bin-hadoop2.6/lib/spark-assembly-1.5.1-hadoop2.6.0.jar:/opt/spark-1.5.1-bin-hadoop2.6/lib/datanucleus-rdbms-3.2.9.jar:/opt/spark-1.5.1-bin-hadoop2.6/lib/datanucleus-core-3.2.10.jar:/opt/spark-1.5.1-bin-hadoop2.6/lib/datanucleus-api-jdo-3.2.6.jar -Xms1g -Xmx1g org.apache.spark.deploy.master.Master --ip spark-master --port 7077 --webui-port 8080 +======================================== +15/10/27 21:25:05 INFO Master: Registered signal handlers for [TERM, HUP, INT] +15/10/27 21:25:05 INFO SecurityManager: Changing view acls to: root +15/10/27 21:25:05 INFO SecurityManager: Changing modify acls to: root +15/10/27 21:25:05 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) +15/10/27 21:25:06 INFO Slf4jLogger: Slf4jLogger started +15/10/27 21:25:06 INFO Remoting: Starting remoting +15/10/27 21:25:06 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkMaster@spark-master:7077] +15/10/27 21:25:06 INFO Utils: Successfully started service 'sparkMaster' on port 7077. +15/10/27 21:25:07 INFO Master: Starting Spark master at spark://spark-master:7077 +15/10/27 21:25:07 INFO Master: Running Spark version 1.5.1 +15/10/27 21:25:07 INFO Utils: Successfully started service 'MasterUI' on port 8080. +15/10/27 21:25:07 INFO MasterWebUI: Started MasterWebUI at http://spark-master:8080 +15/10/27 21:25:07 INFO Utils: Successfully started service on port 6066. +15/10/27 21:25:07 INFO StandaloneRestServer: Started REST server for submitting applications on port 6066 +15/10/27 21:25:07 INFO Master: I have been elected leader! New state: ALIVE +``` + +Once the master is started, we'll want to check the Spark WebUI. In order to access the Spark WebUI, we will deploy a [specialized proxy](https://github.com/aseigneurin/spark-ui-proxy). This proxy is neccessary to access worker logs from the Spark UI. + +Deploy the proxy controller with [`examples/spark/spark-ui-proxy-controller.yaml`](spark-ui-proxy-controller.yaml): + +```console +$ kubectl create -f examples/spark/spark-ui-proxy-controller.yaml +replicationcontroller "spark-ui-proxy-controller" created +``` + +We'll also need a corresponding Loadbalanced service for our Spark Proxy [`examples/spark/spark-ui-proxy-service.yaml`](spark-ui-proxy-service.yaml): + +```console +$ kubectl create -f examples/spark/spark-ui-proxy-service.yaml +service "spark-ui-proxy" created +``` + +After creating the service, you should eventually get a loadbalanced endpoint: + +```console +$ kubectl get svc spark-ui-proxy -o wide + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +spark-ui-proxy 10.0.51.107 aad59283284d611e6839606c214502b5-833417581.us-east-1.elb.amazonaws.com 80/TCP 9m component=spark-ui-proxy +``` + +The Spark UI in the above example output will be available at http://aad59283284d611e6839606c214502b5-833417581.us-east-1.elb.amazonaws.com + +If your Kubernetes cluster is not equipped with a Loadbalancer integration, you will need to use the [kubectl proxy](../../docs/user-guide/accessing-the-cluster.md#using-kubectl-proxy) to +connect to the Spark WebUI: + +```console +kubectl proxy --port=8001 +``` + +At which point the UI will be available at +[http://localhost:8001/api/v1/proxy/namespaces/spark-cluster/services/spark-master:8080/](http://localhost:8001/api/v1/proxy/namespaces/spark-cluster/services/spark-master:8080/). + +## Step Three: Start your Spark workers + +The Spark workers do the heavy lifting in a Spark cluster. They +provide execution resources and data cache capabilities for your +program. + +The Spark workers need the Master service to be running. + +Use the [`examples/spark/spark-worker-controller.yaml`](spark-worker-controller.yaml) file to create a +[replication controller](../../docs/user-guide/replication-controller.md) that manages the worker pods. + +```console +$ kubectl create -f examples/spark/spark-worker-controller.yaml +replicationcontroller "spark-worker-controller" created +``` + +### Check to see if the workers are running + +If you launched the Spark WebUI, your workers should just appear in the UI when +they're ready. (It may take a little bit to pull the images and launch the +pods.) You can also interrogate the status in the following way: + +```console +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +spark-master-controller-5u0q5 1/1 Running 0 25m +spark-worker-controller-e8otp 1/1 Running 0 6m +spark-worker-controller-fiivl 1/1 Running 0 6m +spark-worker-controller-ytc7o 1/1 Running 0 6m + +$ kubectl logs spark-master-controller-5u0q5 +[...] +15/10/26 18:20:14 INFO Master: Registering worker 10.244.1.13:53567 with 2 cores, 6.3 GB RAM +15/10/26 18:20:14 INFO Master: Registering worker 10.244.2.7:46195 with 2 cores, 6.3 GB RAM +15/10/26 18:20:14 INFO Master: Registering worker 10.244.3.8:39926 with 2 cores, 6.3 GB RAM +``` + +## Step Four: Start the Zeppelin UI to launch jobs on your Spark cluster + +The Zeppelin UI pod can be used to launch jobs into the Spark cluster either via +a web notebook frontend or the traditional Spark command line. See +[Zeppelin](https://zeppelin.incubator.apache.org/) and +[Spark architecture](https://spark.apache.org/docs/latest/cluster-overview.html) +for more details. + +Deploy Zeppelin: + +```console +$ kubectl create -f examples/spark/zeppelin-controller.yaml +replicationcontroller "zeppelin-controller" created +``` + +And the corresponding service: + +```console +$ kubectl create -f examples/spark/zeppelin-service.yaml +service "zeppelin" created +``` + +Zeppelin needs the spark-master service to be running. + +### Check to see if Zeppelin is running + +```console +$ kubectl get pods -l component=zeppelin +NAME READY STATUS RESTARTS AGE +zeppelin-controller-ja09s 1/1 Running 0 53s +``` + +## Step Five: Do something with the cluster + +Now you have two choices, depending on your predilections. You can do something +graphical with the Spark cluster, or you can stay in the CLI. + +For both choices, we will be working with this Python snippet: + +```python +from math import sqrt; from itertools import count, islice + +def isprime(n): + return n > 1 and all(n%i for i in islice(count(2), int(sqrt(n)-1))) + +nums = sc.parallelize(xrange(10000000)) +print nums.filter(isprime).count() +``` + +### Do something fast with pyspark! + +Simply copy and paste the python snippet into pyspark from within the zeppelin pod: + +```console +$ kubectl exec zeppelin-controller-ja09s -it pyspark +Python 2.7.9 (default, Mar 1 2015, 12:57:24) +[GCC 4.9.2] on linux2 +Type "help", "copyright", "credits" or "license" for more information. +Welcome to + ____ __ + / __/__ ___ _____/ /__ + _\ \/ _ \/ _ `/ __/ '_/ + /__ / .__/\_,_/_/ /_/\_\ version 1.5.1 + /_/ + +Using Python version 2.7.9 (default, Mar 1 2015 12:57:24) +SparkContext available as sc, HiveContext available as sqlContext. +>>> from math import sqrt; from itertools import count, islice +>>> +>>> def isprime(n): +... return n > 1 and all(n%i for i in islice(count(2), int(sqrt(n)-1))) +... +>>> nums = sc.parallelize(xrange(10000000)) + +>>> print nums.filter(isprime).count() +664579 +``` + +Congratulations, you now know how many prime numbers there are within the first 10 million numbers! + +### Do something graphical and shiny! + +Creating the Zeppelin service should have yielded you a Loadbalancer endpoint: + +```console +$ kubectl get svc zeppelin -o wide + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +zeppelin 10.0.154.1 a596f143884da11e6839506c114532b5-121893930.us-east-1.elb.amazonaws.com 80/TCP 3m component=zeppelin +``` + +If your Kubernetes cluster does not have a Loadbalancer integration, then we will have to use port forwarding. + +Take the Zeppelin pod from before and port-forward the WebUI port: + +```console +$ kubectl port-forward zeppelin-controller-ja09s 8080:8080 +``` + +This forwards `localhost` 8080 to container port 8080. You can then find +Zeppelin at [http://localhost:8080/](http://localhost:8080/). + +Once you've loaded up the Zeppelin UI, create a "New Notebook". In there we will paste our python snippet, but we need to add a `%pyspark` hint for Zeppelin to understand it: + +``` +%pyspark +from math import sqrt; from itertools import count, islice + +def isprime(n): + return n > 1 and all(n%i for i in islice(count(2), int(sqrt(n)-1))) + +nums = sc.parallelize(xrange(10000000)) +print nums.filter(isprime).count() +``` + +After pasting in our code, press shift+enter or click the play icon to the right of our snippet. The Spark job will run and once again we'll have our result! + +## Result + +You now have services and replication controllers for the Spark master, Spark +workers and Spark driver. You can take this example to the next step and start +using the Apache Spark cluster you just created, see +[Spark documentation](https://spark.apache.org/documentation.html) for more +information. + +## tl;dr + +```console +kubectl create -f examples/spark +``` + +After it's setup: + +```console +kubectl get pods # Make sure everything is running +kubectl get svc -o wide # Get the Loadbalancer endpoints for spark-ui-proxy and zeppelin +``` + +At which point the Master UI and Zeppelin will be available at the URLs under the `EXTERNAL-IP` field. + +You can also interact with the Spark cluster using the traditional `spark-shell` / +`spark-subsubmit` / `pyspark` commands by using `kubectl exec` against the +`zeppelin-controller` pod. + +If your Kubernetes cluster does not have a Loadbalancer integration, use `kubectl proxy` and `kubectl port-forward` to access the Spark UI and Zeppelin. + +For Spark UI: + +```console +kubectl proxy --port=8001 +``` + +Then visit [http://localhost:8001/api/v1/proxy/namespaces/spark-cluster/services/spark-ui-proxy/](http://localhost:8001/api/v1/proxy/namespaces/spark-cluster/services/spark-ui-proxy/). + +For Zeppelin: + +```console +kubectl port-forward zeppelin-controller-abc123 8080:8080 & +``` + +Then visit [http://localhost:8080/](http://localhost:8080/). + +## Known Issues With Spark + +* This provides a Spark configuration that is restricted to the cluster network, + meaning the Spark master is only available as a cluster service. If you need + to submit jobs using external client other than Zeppelin or `spark-submit` on + the `zeppelin` pod, you will need to provide a way for your clients to get to + the + [`examples/spark/spark-master-service.yaml`](spark-master-service.yaml). See + [Services](../../docs/user-guide/services.md) for more information. + +## Known Issues With Zeppelin + +* The Zeppelin pod is large, so it may take a while to pull depending on your + network. The size of the Zeppelin pod is something we're working on, see issue #17231. + +* Zeppelin may take some time (about a minute) on this pipeline the first time + you run it. It seems to take considerable time to load. + +* On GKE, `kubectl port-forward` may not be stable over long periods of time. If + you see Zeppelin go into `Disconnected` state (there will be a red dot on the + top right as well), the `port-forward` probably failed and needs to be + restarted. See #12179. + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/spark/README.md?pixel)]() + diff --git a/spark/namespace-spark-cluster.yaml b/spark/namespace-spark-cluster.yaml new file mode 100644 index 000000000..1f3dce83c --- /dev/null +++ b/spark/namespace-spark-cluster.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "spark-cluster" + labels: + name: "spark-cluster" diff --git a/spark/spark-gluster/README.md b/spark/spark-gluster/README.md new file mode 100644 index 000000000..348a64eb8 --- /dev/null +++ b/spark/spark-gluster/README.md @@ -0,0 +1,123 @@ +# Spark on GlusterFS example + +This guide is an extension of the standard [Spark on Kubernetes Guide](../../../examples/spark/) and describes how to run Spark on GlusterFS using the [Kubernetes Volume Plugin for GlusterFS](../../../examples/volumes/glusterfs/) + +The setup is the same in that you will setup a Spark Master Service in the same way you do with the standard Spark guide but you will deploy a modified Spark Master and a Modified Spark Worker ReplicationController, as they will be modified to use the GlusterFS volume plugin to mount a GlusterFS volume into the Spark Master and Spark Workers containers. Note that this example can be used as a guide for implementing any of the Kubernetes Volume Plugins with the Spark Example. + +[There is also a video available that provides a walkthrough for how to set this solution up](https://youtu.be/xyIaoM0-gM0) + +## Step Zero: Prerequisites + +This example assumes that you have been able to successfully get the standard Spark Example working in Kubernetes and that you have a GlusterFS cluster that is accessible from your Kubernetes cluster. It is also recommended that you are familiar with the GlusterFS Volume Plugin and how to configure it. + +## Step One: Define the endpoints for your GlusterFS Cluster + +Modify the `examples/spark/spark-gluster/glusterfs-endpoints.yaml` file to list the IP addresses of some of the servers in your GlusterFS cluster. The GlusterFS Volume Plugin uses these IP addresses to perform a Fuse Mount of the GlusterFS Volume into the Spark Worker Containers that are launched by the ReplicationController in the next section. + +Register your endpoints by running the following command: + +```console +$ kubectl create -f examples/spark/spark-gluster/glusterfs-endpoints.yaml +``` + +## Step Two: Modify and Submit your Spark Master ReplicationController + +Modify the `examples/spark/spark-gluster/spark-master-controller.yaml` file to reflect the GlusterFS Volume that you wish to use in the PATH parameter of the volumes subsection. + +Submit the Spark Master Pod + +```console +$ kubectl create -f examples/spark/spark-gluster/spark-master-controller.yaml +``` + +Verify that the Spark Master Pod deployed successfully. + +```console +$ kubectl get pods +``` + +Submit the Spark Master Service + +```console +$ kubectl create -f examples/spark/spark-gluster/spark-master-service.yaml +``` + +Verify that the Spark Master Service deployed successfully. + +```console +$ kubectl get services +``` + +## Step Three: Start your Spark workers + +Modify the `examples/spark/spark-gluster/spark-worker-controller.yaml` file to reflect the GlusterFS Volume that you wish to use in the PATH parameter of the Volumes subsection. + +Make sure that the replication factor for the pods is not greater than the amount of Kubernetes nodes available in your Kubernetes cluster. + +Submit your Spark Worker ReplicationController by running the following command: + +```console +$ kubectl create -f examples/spark/spark-gluster/spark-worker-controller.yaml +``` + +Verify that the Spark Worker ReplicationController deployed its pods successfully. + +```console +$ kubectl get pods +``` + +Follow the steps from the standard example to verify the Spark Worker pods have registered successfully with the Spark Master. + +## Step Four: Submit a Spark Job + +All the Spark Workers and the Spark Master in your cluster have a mount to GlusterFS. This means that any of them can be used as the Spark Client to submit a job. For simplicity, lets use the Spark Master as an example. + + +The Spark Worker and Spark Master containers include a setup_client utility script that takes two parameters, the Service IP of the Spark Master and the port that it is running on. This must be to setup the container as a Spark client prior to submitting any Spark Jobs. + +Obtain the Service IP (listed as IP:) and Full Pod Name by running + +```console +$ kubectl describe pod spark-master-controller +``` + +Now we will shell into the Spark Master Container and run a Spark Job. In the example below, we are running the Spark Wordcount example and specifying the input and output directory at the location where GlusterFS is mounted in the Spark Master Container. This will submit the job to the Spark Master who will distribute the work to all the Spark Worker Containers. + +All the Spark Worker containers will be able to access the data as they all have the same GlusterFS volume mounted at /mnt/glusterfs. The reason we are submitting the job from a Spark Worker and not an additional Spark Base container (as in the standard Spark Example) is due to the fact that the Spark instance submitting the job must be able to access the data. Only the Spark Master and Spark Worker containers have GlusterFS mounted. + +The Spark Worker and Spark Master containers include a setup_client utility script that takes two parameters, the Service IP of the Spark Master and the port that it is running on. This must be done to setup the container as a Spark client prior to submitting any Spark Jobs. + +Shell into the Master Spark Node (spark-master-controller) by running + +```console +kubectl exec spark-master-controller- -i -t -- bash -i + +root@spark-master-controller-c1sqd:/# . /setup_client.sh 7077 +root@spark-master-controller-c1sqd:/# pyspark + +Python 2.7.9 (default, Mar 1 2015, 12:57:24) +[GCC 4.9.2] on linux2 +Type "help", "copyright", "credits" or "license" for more information. +15/06/26 14:25:28 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable +Welcome to + ____ __ + / __/__ ___ _____/ /__ + _\ \/ _ \/ _ `/ __/ '_/ + /__ / .__/\_,_/_/ /_/\_\ version 1.4.0 + /_/ +Using Python version 2.7.9 (default, Mar 1 2015 12:57:24) +SparkContext available as sc, HiveContext available as sqlContext. +>>> file = sc.textFile("/mnt/glusterfs/somefile.txt") +>>> counts = file.flatMap(lambda line: line.split(" ")).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b) +>>> counts.saveAsTextFile("/mnt/glusterfs/output") +``` + +While still in the container, you can see the output of your Spark Job in the Distributed File System by running the following: + +```console +root@spark-master-controller-c1sqd:/# ls -l /mnt/glusterfs/output +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/spark/spark-gluster/README.md?pixel)]() + diff --git a/spark/spark-gluster/glusterfs-endpoints.yaml b/spark/spark-gluster/glusterfs-endpoints.yaml new file mode 100644 index 000000000..357fdb766 --- /dev/null +++ b/spark/spark-gluster/glusterfs-endpoints.yaml @@ -0,0 +1,14 @@ +kind: Endpoints +apiVersion: v1 +metadata: + name: glusterfs-cluster + namespace: spark-cluster +subsets: + - addresses: + - ip: 192.168.30.104 + ports: + - port: 1 + - addresses: + - ip: 192.168.30.105 + ports: + - port: 1 diff --git a/spark/spark-gluster/spark-master-controller.yaml b/spark/spark-gluster/spark-master-controller.yaml new file mode 100644 index 000000000..d0b365b71 --- /dev/null +++ b/spark/spark-gluster/spark-master-controller.yaml @@ -0,0 +1,34 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: spark-master-controller + namespace: spark-cluster + labels: + component: spark-master +spec: + replicas: 1 + selector: + component: spark-master + template: + metadata: + labels: + component: spark-master + spec: + containers: + - name: spark-master + image: gcr.io/google_containers/spark:1.5.2_v1 + command: ["/start-master"] + ports: + - containerPort: 7077 + volumeMounts: + - mountPath: /mnt/glusterfs + name: glusterfsvol + resources: + requests: + cpu: 100m + volumes: + - name: glusterfsvol + glusterfs: + endpoints: glusterfs-cluster + path: MyVolume + readOnly: false diff --git a/spark/spark-gluster/spark-master-service.yaml b/spark/spark-gluster/spark-master-service.yaml new file mode 100644 index 000000000..2f5bdb15d --- /dev/null +++ b/spark/spark-gluster/spark-master-service.yaml @@ -0,0 +1,13 @@ +kind: Service +apiVersion: v1 +metadata: + name: spark-master + namespace: spark-cluster + labels: + component: spark-master-service +spec: + ports: + - port: 7077 + targetPort: 7077 + selector: + component: spark-master diff --git a/spark/spark-gluster/spark-worker-controller.yaml b/spark/spark-gluster/spark-worker-controller.yaml new file mode 100644 index 000000000..69cc3cec9 --- /dev/null +++ b/spark/spark-gluster/spark-worker-controller.yaml @@ -0,0 +1,35 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: spark-gluster-worker-controller + namespace: spark-cluster + labels: + component: spark-worker +spec: + replicas: 2 + selector: + component: spark-worker + template: + metadata: + labels: + component: spark-worker + uses: spark-master + spec: + containers: + - name: spark-worker + image: gcr.io/google_containers/spark:1.5.2_v1 + command: ["/start-worker"] + ports: + - containerPort: 8888 + volumeMounts: + - mountPath: /mnt/glusterfs + name: glusterfsvol + resources: + requests: + cpu: 100m + volumes: + - name: glusterfsvol + glusterfs: + endpoints: glusterfs-cluster + path: MyVolume + readOnly: false diff --git a/spark/spark-master-controller.yaml b/spark/spark-master-controller.yaml new file mode 100644 index 000000000..60fb7ba8a --- /dev/null +++ b/spark/spark-master-controller.yaml @@ -0,0 +1,23 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: spark-master-controller +spec: + replicas: 1 + selector: + component: spark-master + template: + metadata: + labels: + component: spark-master + spec: + containers: + - name: spark-master + image: gcr.io/google_containers/spark:1.5.2_v1 + command: ["/start-master"] + ports: + - containerPort: 7077 + - containerPort: 8080 + resources: + requests: + cpu: 100m diff --git a/spark/spark-master-service.yaml b/spark/spark-master-service.yaml new file mode 100644 index 000000000..794147d28 --- /dev/null +++ b/spark/spark-master-service.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: spark-master +spec: + ports: + - port: 7077 + targetPort: 7077 + name: spark + - port: 8080 + targetPort: 8080 + name: http + selector: + component: spark-master diff --git a/spark/spark-ui-proxy-controller.yaml b/spark/spark-ui-proxy-controller.yaml new file mode 100644 index 000000000..3bd0566cd --- /dev/null +++ b/spark/spark-ui-proxy-controller.yaml @@ -0,0 +1,29 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: spark-ui-proxy-controller +spec: + replicas: 1 + selector: + component: spark-ui-proxy + template: + metadata: + labels: + component: spark-ui-proxy + spec: + containers: + - name: spark-ui-proxy + image: elsonrodriguez/spark-ui-proxy:1.0 + ports: + - containerPort: 80 + resources: + requests: + cpu: 100m + args: + - spark-master:8080 + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 120 + timeoutSeconds: 5 diff --git a/spark/spark-ui-proxy-service.yaml b/spark/spark-ui-proxy-service.yaml new file mode 100644 index 000000000..ebd6b7c94 --- /dev/null +++ b/spark/spark-ui-proxy-service.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + name: spark-ui-proxy +spec: + ports: + - port: 80 + targetPort: 80 + selector: + component: spark-ui-proxy + type: LoadBalancer diff --git a/spark/spark-worker-controller.yaml b/spark/spark-worker-controller.yaml new file mode 100644 index 000000000..9c748b3e0 --- /dev/null +++ b/spark/spark-worker-controller.yaml @@ -0,0 +1,23 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: spark-worker-controller +spec: + replicas: 2 + selector: + component: spark-worker + template: + metadata: + labels: + component: spark-worker + spec: + containers: + - name: spark-worker + image: gcr.io/google_containers/spark:1.5.2_v1 + command: ["/start-worker"] + ports: + - containerPort: 8081 + resources: + requests: + cpu: 100m + diff --git a/spark/zeppelin-controller.yaml b/spark/zeppelin-controller.yaml new file mode 100644 index 000000000..56bb90d42 --- /dev/null +++ b/spark/zeppelin-controller.yaml @@ -0,0 +1,21 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: zeppelin-controller +spec: + replicas: 1 + selector: + component: zeppelin + template: + metadata: + labels: + component: zeppelin + spec: + containers: + - name: zeppelin + image: gcr.io/google_containers/zeppelin:v0.5.6_v1 + ports: + - containerPort: 8080 + resources: + requests: + cpu: 100m diff --git a/spark/zeppelin-service.yaml b/spark/zeppelin-service.yaml new file mode 100644 index 000000000..f2c7838ad --- /dev/null +++ b/spark/zeppelin-service.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + name: zeppelin +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + component: zeppelin + type: LoadBalancer diff --git a/storage/cassandra/README.md b/storage/cassandra/README.md new file mode 100644 index 000000000..80b68e8c9 --- /dev/null +++ b/storage/cassandra/README.md @@ -0,0 +1,854 @@ + +# Cloud Native Deployments of Cassandra using Kubernetes + +## Table of Contents + + - [Prerequisites](#prerequisites) + - [Cassandra Docker](#cassandra-docker) + - [Quickstart](#quickstart) + - [Step 1: Create a Cassandra Headless Service](#step-1-create-a-cassandra-headless-service) + - [Step 2: Use a StatefulSet to create Cassandra Ring](#step-2-use-a-statefulset-to-create-cassandra-ring) + - [Step 3: Validate and Modify The Cassandra StatefulSet](#step-3-validate-and-modify-the-cassandra-statefulset) + - [Step 4: Delete Cassandra StatefulSet](#step-4-delete-cassandra-statefulset) + - [Step 5: Use a Replication Controller to create Cassandra node pods](#step-5-use-a-replication-controller-to-create-cassandra-node-pods) + - [Step 6: Scale up the Cassandra cluster](#step-6-scale-up-the-cassandra-cluster) + - [Step 7: Delete the Replication Controller](#step-7-delete-the-replication-controller) + - [Step 8: Use a DaemonSet instead of a Replication Controller](#step-8-use-a-daemonset-instead-of-a-replication-controller) + - [Step 9: Resource Cleanup](#step-9-resource-cleanup) + - [Seed Provider Source](#seed-provider-source) + +The following document describes the development of a _cloud native_ +[Cassandra](http://cassandra.apache.org/) deployment on Kubernetes. When we say +_cloud native_, we mean an application which understands that it is running +within a cluster manager, and uses this cluster management infrastructure to +help implement the application. In particular, in this instance, a custom +Cassandra `SeedProvider` is used to enable Cassandra to dynamically discover +new Cassandra nodes as they join the cluster. + +This example also uses some of the core components of Kubernetes: + +- [_Pods_](../../../docs/user-guide/pods.md) +- [ _Services_](../../../docs/user-guide/services.md) +- [_Replication Controllers_](../../../docs/user-guide/replication-controller.md) +- [_Stateful Sets_](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) +- [_Daemon Sets_](../../../docs/admin/daemons.md) + +## Prerequisites + +This example assumes that you have a Kubernetes version >=1.2 cluster installed and running, +and that you have installed the [`kubectl`](../../../docs/user-guide/kubectl/kubectl.md) +command line tool somewhere in your path. Please see the +[getting started guides](../../../docs/getting-started-guides/) +for installation instructions for your platform. + +This example also has a few code and configuration files needed. To avoid +typing these out, you can `git clone` the Kubernetes repository to your local +computer. + +## Cassandra Docker + +The pods use the [```gcr.io/google-samples/cassandra:v12```](image/Dockerfile) +image from Google's [container registry](https://cloud.google.com/container-registry/docs/). +The docker is based on `debian:jessie` and includes OpenJDK 8. This image +includes a standard Cassandra installation from the Apache Debian repo. Through the use of environment variables you are able to change values that are inserted into the `cassandra.yaml`. + +| ENV VAR | DEFAULT VALUE | +| ------------- |:-------------: | +| CASSANDRA_CLUSTER_NAME | 'Test Cluster' | +| CASSANDRA_NUM_TOKENS | 32 | +| CASSANDRA_RPC_ADDRESS | 0.0.0.0 | + +## Quickstart + +If you want to jump straight to the commands we will run, +here are the steps: + +```sh +# +# StatefulSet +# + +# create a service to track all cassandra statefulset nodes +kubectl create -f examples/storage/cassandra/cassandra-service.yaml + +# create a statefulset +kubectl create -f examples/storage/cassandra/cassandra-statefulset.yaml + +# validate the Cassandra cluster. Substitute the name of one of your pods. +kubectl exec -ti cassandra-0 -- nodetool status + +# cleanup +grace=$(kubectl get po cassandra-0 --template '{{.spec.terminationGracePeriodSeconds}}') \ + && kubectl delete statefulset,po -l app=cassandra \ + && echo "Sleeping $grace" \ + && sleep $grace \ + && kubectl delete pvc -l app=cassandra + +# +# Resource Controller Example +# + +# create a replication controller to replicate cassandra nodes +kubectl create -f examples/storage/cassandra/cassandra-controller.yaml + +# validate the Cassandra cluster. Substitute the name of one of your pods. +kubectl exec -ti cassandra-xxxxx -- nodetool status + +# scale up the Cassandra cluster +kubectl scale rc cassandra --replicas=4 + +# delete the replication controller +kubectl delete rc cassandra + +# +# Create a DaemonSet to place a cassandra node on each kubernetes node +# + +kubectl create -f examples/storage/cassandra/cassandra-daemonset.yaml --validate=false + +# resource cleanup +kubectl delete service -l app=cassandra +kubectl delete daemonset cassandra +``` + +## Step 1: Create a Cassandra Headless Service + +A Kubernetes _[Service](../../../docs/user-guide/services.md)_ describes a set of +[_Pods_](../../../docs/user-guide/pods.md) that perform the same task. In +Kubernetes, the atomic unit of an application is a Pod: one or more containers +that _must_ be scheduled onto the same host. + +The Service is used for DNS lookups between Cassandra Pods, and Cassandra clients +within the Kubernetes Cluster. + +Here is the service description: + + + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: cassandra + name: cassandra +spec: + clusterIP: None + ports: + - port: 9042 + selector: + app: cassandra +``` + +[Download example](cassandra-service.yaml?raw=true) + + +Create the service for the StatefulSet: + + +```console +$ kubectl create -f examples/storage/cassandra/cassandra-service.yaml +``` + +The following command shows if the service has been created. + +```console +$ kubectl get svc cassandra +``` + +The response should be like: + +```console +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cassandra None 9042/TCP 45s +``` + +If an error is returned the service create failed. + +## Step 2: Use a StatefulSet to create Cassandra Ring + +StatefulSets (previously PetSets) are a feature that was upgraded to a Beta component in +Kubernetes 1.5. Deploying stateful distributed applications, like Cassandra, within a clustered +environment can be challenging. We implemented StatefulSet to greatly simplify this +process. Multiple StatefulSet features are used within this example, but is out of +scope of this documentation. [Please refer to the Stateful Set documentation.](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + +The StatefulSet manifest that is included below, creates a Cassandra ring that consists +of three pods. + +This example includes using a GCE Storage Class, please update appropriately depending +on the cloud you are working with. + + + +```yaml +apiVersion: "apps/v1beta1" +kind: StatefulSet +metadata: + name: cassandra +spec: + serviceName: cassandra + replicas: 3 + template: + metadata: + labels: + app: cassandra + spec: + containers: + - name: cassandra + image: gcr.io/google-samples/cassandra:v12 + imagePullPolicy: Always + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + resources: + limits: + cpu: "500m" + memory: 1Gi + requests: + cpu: "500m" + memory: 1Gi + securityContext: + capabilities: + add: + - IPC_LOCK + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEEDS + value: "cassandra-0.cassandra.default.svc.cluster.local" + - name: CASSANDRA_CLUSTER_NAME + value: "K8Demo" + - name: CASSANDRA_DC + value: "DC1-K8Demo" + - name: CASSANDRA_RACK + value: "Rack1-K8Demo" + - name: CASSANDRA_AUTO_BOOTSTRAP + value: "false" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + readinessProbe: + exec: + command: + - /bin/bash + - -c + - /ready-probe.sh + initialDelaySeconds: 15 + timeoutSeconds: 5 + # These volume mounts are persistent. They are like inline claims, + # but not exactly because the names need to match exactly one of + # the stateful pod volumes. + volumeMounts: + - name: cassandra-data + mountPath: /cassandra_data + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + # do not use these in production until ssd GCEPersistentDisk or other ssd pd + volumeClaimTemplates: + - metadata: + name: cassandra-data + annotations: + volume.beta.kubernetes.io/storage-class: fast + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd +``` + +[Download example](cassandra-statefulset.yaml?raw=true) + + +Create the Cassandra StatefulSet as follows: + +```console +$ kubectl create -f examples/storage/cassandra/cassandra-statefulset.yaml +``` + +## Step 3: Validate and Modify The Cassandra StatefulSet + +Deploying this StatefulSet shows off two of the new features that StatefulSets provides. + +1. The pod names are known +2. The pods deploy in incremental order + +First validate that the StatefulSet has deployed, by running `kubectl` command below. + +```console +$ kubectl get statefulset cassandra +``` + +The command should respond like: + +```console +NAME DESIRED CURRENT AGE +cassandra 3 3 13s +``` + +Next watch the Cassandra pods deploy, one after another. The StatefulSet resource +deploys pods in a number fashion: 1, 2, 3, etc. If you execute the following +command before the pods deploy you are able to see the ordered creation. + +```console +$ kubectl get pods -l="app=cassandra" +NAME READY STATUS RESTARTS AGE +cassandra-0 1/1 Running 0 1m +cassandra-1 0/1 ContainerCreating 0 8s +``` + +The above example shows two of the three pods in the Cassandra StatefulSet deployed. +Once all of the pods are deployed the same command will respond with the full +StatefulSet. + +```console +$ kubectl get pods -l="app=cassandra" +NAME READY STATUS RESTARTS AGE +cassandra-0 1/1 Running 0 10m +cassandra-1 1/1 Running 0 9m +cassandra-2 1/1 Running 0 8m +``` + +Running the Cassandra utility `nodetool` will display the status of the ring. + +```console +$ kubectl exec cassandra-0 -- nodetool status +Datacenter: DC1-K8Demo +====================== +Status=Up/Down +|/ State=Normal/Leaving/Joining/Moving +-- Address Load Tokens Owns (effective) Host ID Rack +UN 10.4.2.4 65.26 KiB 32 63.7% a9d27f81-6783-461d-8583-87de2589133e Rack1-K8Demo +UN 10.4.0.4 102.04 KiB 32 66.7% 5559a58c-8b03-47ad-bc32-c621708dc2e4 Rack1-K8Demo +UN 10.4.1.4 83.06 KiB 32 69.6% 9dce943c-581d-4c0e-9543-f519969cc805 Rack1-K8Demo +``` + +You can also run `cqlsh` to describe the keyspaces in the cluster. + +```console +$ kubectl exec cassandra-0 -- cqlsh -e 'desc keyspaces' + +system_traces system_schema system_auth system system_distributed +``` + +In order to increase or decrease the size of the Cassandra StatefulSet, you must use +`kubectl edit`. You can find more information about the edit command in the [documentation](../../../docs/user-guide/kubectl/kubectl_edit.md). + +Use the following command to edit the StatefulSet. + +```console +$ kubectl edit statefulset cassandra +``` + +This will create an editor in your terminal. The line you are looking to change is +`replicas`. The example does on contain the entire contents of the terminal window, and +the last line of the example below is the replicas line that you want to change. + +```console +# Please edit the object below. Lines beginning with a '#' will be ignored, +# and an empty file will abort the edit. If an error occurs while saving this file will be +# reopened with the relevant failures. +# +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + creationTimestamp: 2016-08-13T18:40:58Z + generation: 1 + labels: + app: cassandra + name: cassandra + namespace: default + resourceVersion: "323" + selfLink: /apis/apps/v1beta1/namespaces/default/statefulsets/cassandra + uid: 7a219483-6185-11e6-a910-42010a8a0fc0 +spec: + replicas: 3 +``` + +Modify the manifest to the following, and save the manifest. + +```console +spec: + replicas: 4 +``` + +The StatefulSet will now contain four pods. + +```console +$ kubectl get statefulset cassandra +``` + +The command should respond like: + +```console +NAME DESIRED CURRENT AGE +cassandra 4 4 36m +``` + +For the Kubernetes 1.5 release, the beta StatefulSet resource does not have `kubectl scale` +functionality, like a Deployment, ReplicaSet, Replication Controller, or Job. + +## Step 4: Delete Cassandra StatefulSet + +Deleting and/or scaling a StatefulSet down will not delete the volumes associated with the StatefulSet. This is done to ensure safety first, your data is more valuable than an auto purge of all related StatefulSet resources. Deleting the Persistent Volume Claims may result in a deletion of the associated volumes, depending on the storage class and reclaim policy. You should never assume ability to access a volume after claim deletion. + +Use the following commands to delete the StatefulSet. + +```console +$ grace=$(kubectl get po cassandra-0 --template '{{.spec.terminationGracePeriodSeconds}}') \ + && kubectl delete statefulset -l app=cassandra \ + && echo "Sleeping $grace" \ + && sleep $grace \ + && kubectl delete pvc -l app=cassandra +``` + +## Step 5: Use a Replication Controller to create Cassandra node pods + +A Kubernetes +_[Replication Controller](../../../docs/user-guide/replication-controller.md)_ +is responsible for replicating sets of identical pods. Like a +Service, it has a selector query which identifies the members of its set. +Unlike a Service, it also has a desired number of replicas, and it will create +or delete Pods to ensure that the number of Pods matches up with its +desired state. + +The Replication Controller, in conjunction with the Service we just defined, +will let us easily build a replicated, scalable Cassandra cluster. + +Let's create a replication controller with two initial replicas. + + + +```yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: cassandra + # The labels will be applied automatically + # from the labels in the pod template, if not set + # labels: + # app: cassandra +spec: + replicas: 2 + # The selector will be applied automatically + # from the labels in the pod template, if not set. + # selector: + # app: cassandra + template: + metadata: + labels: + app: cassandra + spec: + containers: + - command: + - /run.sh + resources: + limits: + cpu: 0.5 + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEED_PROVIDER + value: "io.k8s.cassandra.KubernetesSeedProvider" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/google-samples/cassandra:v12 + name: cassandra + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + volumeMounts: + - mountPath: /cassandra_data + name: data + volumes: + - name: data + emptyDir: {} +``` + +[Download example](cassandra-controller.yaml?raw=true) + + +There are a few things to note in this description. + +The `selector` attribute contains the controller's selector query. It can be +explicitly specified, or applied automatically from the labels in the pod +template if not set, as is done here. + +The pod template's label, `app:cassandra`, matches the Service selector +from Step 1. This is how pods created by this replication controller are picked up +by the Service." + +The `replicas` attribute specifies the desired number of replicas, in this +case 2 initially. We'll scale up to more shortly. + +Create the Replication Controller: + +```console + +$ kubectl create -f examples/storage/cassandra/cassandra-controller.yaml + +``` + +You can list the new controller: + +```console + +$ kubectl get rc -o wide +NAME DESIRED CURRENT AGE CONTAINER(S) IMAGE(S) SELECTOR +cassandra 2 2 11s cassandra gcr.io/google-samples/cassandra:v12 app=cassandra + +``` + +Now if you list the pods in your cluster, and filter to the label +`app=cassandra`, you should see two Cassandra pods. (The `wide` argument lets +you see which Kubernetes nodes the pods were scheduled onto.) + +```console + +$ kubectl get pods -l="app=cassandra" -o wide +NAME READY STATUS RESTARTS AGE NODE +cassandra-21qyy 1/1 Running 0 1m kubernetes-minion-b286 +cassandra-q6sz7 1/1 Running 0 1m kubernetes-minion-9ye5 + +``` + +Because these pods have the label `app=cassandra`, they map to the service we +defined in Step 1. + +You can check that the Pods are visible to the Service using the following service endpoints query: + +```console + +$ kubectl get endpoints cassandra -o yaml +apiVersion: v1 +kind: Endpoints +metadata: + creationTimestamp: 2015-06-21T22:34:12Z + labels: + app: cassandra + name: cassandra + namespace: default + resourceVersion: "944373" + selfLink: /api/v1/namespaces/default/endpoints/cassandra + uid: a3d6c25f-1865-11e5-a34e-42010af01bcc +subsets: +- addresses: + - ip: 10.244.3.15 + targetRef: + kind: Pod + name: cassandra + namespace: default + resourceVersion: "944372" + uid: 9ef9895d-1865-11e5-a34e-42010af01bcc + ports: + - port: 9042 + protocol: TCP + +``` + +To show that the `SeedProvider` logic is working as intended, you can use the +`nodetool` command to examine the status of the Cassandra cluster. To do this, +use the `kubectl exec` command, which lets you run `nodetool` in one of your +Cassandra pods. Again, substitute `cassandra-xxxxx` with the actual name of one +of your pods. + +```console + +$ kubectl exec -ti cassandra-xxxxx -- nodetool status +Datacenter: datacenter1 +======================= +Status=Up/Down +|/ State=Normal/Leaving/Joining/Moving +-- Address Load Tokens Owns (effective) Host ID Rack +UN 10.244.0.5 74.09 KB 256 100.0% 86feda0f-f070-4a5b-bda1-2eeb0ad08b77 rack1 +UN 10.244.3.3 51.28 KB 256 100.0% dafe3154-1d67-42e1-ac1d-78e7e80dce2b rack1 + +``` + +## Step 6: Scale up the Cassandra cluster + +Now let's scale our Cassandra cluster to 4 pods. We do this by telling the +Replication Controller that we now want 4 replicas. + +```sh + +$ kubectl scale rc cassandra --replicas=4 + +``` + +You can see the new pods listed: + +```console + +$ kubectl get pods -l="app=cassandra" -o wide +NAME READY STATUS RESTARTS AGE NODE +cassandra-21qyy 1/1 Running 0 6m kubernetes-minion-b286 +cassandra-81m2l 1/1 Running 0 47s kubernetes-minion-b286 +cassandra-8qoyp 1/1 Running 0 47s kubernetes-minion-9ye5 +cassandra-q6sz7 1/1 Running 0 6m kubernetes-minion-9ye5 + +``` + +In a few moments, you can examine the Cassandra cluster status again, and see +that the new pods have been detected by the custom `SeedProvider`: + +```console + +$ kubectl exec -ti cassandra-xxxxx -- nodetool status +Datacenter: datacenter1 +======================= +Status=Up/Down +|/ State=Normal/Leaving/Joining/Moving +-- Address Load Tokens Owns (effective) Host ID Rack +UN 10.244.0.6 51.67 KB 256 48.9% d07b23a5-56a1-4b0b-952d-68ab95869163 rack1 +UN 10.244.1.5 84.71 KB 256 50.7% e060df1f-faa2-470c-923d-ca049b0f3f38 rack1 +UN 10.244.1.6 84.71 KB 256 47.0% 83ca1580-4f3c-4ec5-9b38-75036b7a297f rack1 +UN 10.244.0.5 68.2 KB 256 53.4% 72ca27e2-c72c-402a-9313-1e4b61c2f839 rack1 + +``` + +## Step 7: Delete the Replication Controller + +Before you start Step 5, __delete the replication controller__ you created above: + +```sh + +$ kubectl delete rc cassandra + +``` + +## Step 8: Use a DaemonSet instead of a Replication Controller + +In Kubernetes, a [_Daemon Set_](../../../docs/admin/daemons.md) can distribute pods +onto Kubernetes nodes, one-to-one. Like a _ReplicationController_, it has a +selector query which identifies the members of its set. Unlike a +_ReplicationController_, it has a node selector to limit which nodes are +scheduled with the templated pods, and replicates not based on a set target +number of pods, but rather assigns a single pod to each targeted node. + +An example use case: when deploying to the cloud, the expectation is that +instances are ephemeral and might die at any time. Cassandra is built to +replicate data across the cluster to facilitate data redundancy, so that in the +case that an instance dies, the data stored on the instance does not, and the +cluster can react by re-replicating the data to other running nodes. + +`DaemonSet` is designed to place a single pod on each node in the Kubernetes +cluster. That will give us data redundancy. Let's create a +DaemonSet to start our storage cluster: + + + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + name: cassandra + name: cassandra +spec: + template: + metadata: + labels: + app: cassandra + spec: + # Filter to specific nodes: + # nodeSelector: + # app: cassandra + containers: + - command: + - /run.sh + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEED_PROVIDER + value: "io.k8s.cassandra.KubernetesSeedProvider" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/google-samples/cassandra:v12 + name: cassandra + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + # If you need it it is going away in C* 4.0 + #- containerPort: 9160 + # name: thrift + resources: + requests: + cpu: 0.5 + volumeMounts: + - mountPath: /cassandra_data + name: data + volumes: + - name: data + emptyDir: {} +``` + +[Download example](cassandra-daemonset.yaml?raw=true) + + +Most of this DaemonSet definition is identical to the ReplicationController +definition above; it simply gives the daemon set a recipe to use when it creates +new Cassandra pods, and targets all Cassandra nodes in the cluster. + +Differentiating aspects are the `nodeSelector` attribute, which allows the +DaemonSet to target a specific subset of nodes (you can label nodes just like +other resources), and the lack of a `replicas` attribute due to the 1-to-1 node- +pod relationship. + +Create this DaemonSet: + +```console + +$ kubectl create -f examples/storage/cassandra/cassandra-daemonset.yaml + +``` + +You may need to disable config file validation, like so: + +```console + +$ kubectl create -f examples/storage/cassandra/cassandra-daemonset.yaml --validate=false + +``` + +You can see the DaemonSet running: + +```console + +$ kubectl get daemonset +NAME DESIRED CURRENT NODE-SELECTOR +cassandra 3 3 + +``` + +Now, if you list the pods in your cluster, and filter to the label +`app=cassandra`, you should see one (and only one) new cassandra pod for each +node in your network. + +```console + +$ kubectl get pods -l="app=cassandra" -o wide +NAME READY STATUS RESTARTS AGE NODE +cassandra-ico4r 1/1 Running 0 4s kubernetes-minion-rpo1 +cassandra-kitfh 1/1 Running 0 1s kubernetes-minion-9ye5 +cassandra-tzw89 1/1 Running 0 2s kubernetes-minion-b286 + +``` + +To prove that this all worked as intended, you can again use the `nodetool` +command to examine the status of the cluster. To do this, use the `kubectl +exec` command to run `nodetool` in one of your newly-launched cassandra pods. + +```console + +$ kubectl exec -ti cassandra-xxxxx -- nodetool status +Datacenter: datacenter1 +======================= +Status=Up/Down +|/ State=Normal/Leaving/Joining/Moving +-- Address Load Tokens Owns (effective) Host ID Rack +UN 10.244.0.5 74.09 KB 256 100.0% 86feda0f-f070-4a5b-bda1-2eeb0ad08b77 rack1 +UN 10.244.4.2 32.45 KB 256 100.0% 0b1be71a-6ffb-4895-ac3e-b9791299c141 rack1 +UN 10.244.3.3 51.28 KB 256 100.0% dafe3154-1d67-42e1-ac1d-78e7e80dce2b rack1 + +``` + +**Note**: This example had you delete the cassandra Replication Controller before +you created the DaemonSet. This is because – to keep this example simple – the +RC and the DaemonSet are using the same `app=cassandra` label (so that their pods map to the +service we created, and so that the SeedProvider can identify them). + +If we didn't delete the RC first, the two resources would conflict with +respect to how many pods they wanted to have running. If we wanted, we could support running +both together by using additional labels and selectors. + +## Step 9: Resource Cleanup + +When you are ready to take down your resources, do the following: + +```console + +$ kubectl delete service -l app=cassandra +$ kubectl delete daemonset cassandra + +``` + +### Custom Seed Provider + +A custom [`SeedProvider`](https://svn.apache.org/repos/asf/cassandra/trunk/src/java/org/apache/cassandra/locator/SeedProvider.java) +is included for running Cassandra on top of Kubernetes. Only when you deploy Cassandra +via a replication control or a daemonset, you will need to use the custom seed provider. +In Cassandra, a `SeedProvider` bootstraps the gossip protocol that Cassandra uses to find other +Cassandra nodes. Seed addresses are hosts deemed as contact points. Cassandra +instances use the seed list to find each other and learn the topology of the +ring. The [`KubernetesSeedProvider`](java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java) +discovers Cassandra seeds IP addresses via the Kubernetes API, those Cassandra +instances are defined within the Cassandra Service. + +Refer to the custom seed provider [README](java/README.md) for further +`KubernetesSeedProvider` configurations. For this example you should not need +to customize the Seed Provider configurations. + +See the [image](image/) directory of this example for specifics on +how the container docker image was built and what it contains. + +You may also note that we are setting some Cassandra parameters (`MAX_HEAP_SIZE` +and `HEAP_NEWSIZE`), and adding information about the +[namespace](../../../docs/user-guide/namespaces.md). +We also tell Kubernetes that the container exposes +both the `CQL` and `Thrift` API ports. Finally, we tell the cluster +manager that we need 0.1 cpu (0.1 core). + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/cassandra/README.md?pixel)]() + diff --git a/storage/cassandra/cassandra-controller.yaml b/storage/cassandra/cassandra-controller.yaml new file mode 100644 index 000000000..f2344df12 --- /dev/null +++ b/storage/cassandra/cassandra-controller.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: cassandra + # The labels will be applied automatically + # from the labels in the pod template, if not set + # labels: + # app: cassandra +spec: + replicas: 2 + # The selector will be applied automatically + # from the labels in the pod template, if not set. + # selector: + # app: cassandra + template: + metadata: + labels: + app: cassandra + spec: + containers: + - command: + - /run.sh + resources: + limits: + cpu: 0.5 + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEED_PROVIDER + value: "io.k8s.cassandra.KubernetesSeedProvider" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/google-samples/cassandra:v12 + name: cassandra + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + volumeMounts: + - mountPath: /cassandra_data + name: data + volumes: + - name: data + emptyDir: {} diff --git a/storage/cassandra/cassandra-daemonset.yaml b/storage/cassandra/cassandra-daemonset.yaml new file mode 100644 index 000000000..8305ebf88 --- /dev/null +++ b/storage/cassandra/cassandra-daemonset.yaml @@ -0,0 +1,56 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + name: cassandra + name: cassandra +spec: + template: + metadata: + labels: + app: cassandra + spec: + # Filter to specific nodes: + # nodeSelector: + # app: cassandra + containers: + - command: + - /run.sh + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEED_PROVIDER + value: "io.k8s.cassandra.KubernetesSeedProvider" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/google-samples/cassandra:v12 + name: cassandra + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + # If you need it it is going away in C* 4.0 + #- containerPort: 9160 + # name: thrift + resources: + requests: + cpu: 0.5 + volumeMounts: + - mountPath: /cassandra_data + name: data + volumes: + - name: data + emptyDir: {} diff --git a/storage/cassandra/cassandra-service.yaml b/storage/cassandra/cassandra-service.yaml new file mode 100644 index 000000000..35b07733b --- /dev/null +++ b/storage/cassandra/cassandra-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: cassandra + name: cassandra +spec: + clusterIP: None + ports: + - port: 9042 + selector: + app: cassandra diff --git a/storage/cassandra/cassandra-statefulset.yaml b/storage/cassandra/cassandra-statefulset.yaml new file mode 100644 index 000000000..452a5d262 --- /dev/null +++ b/storage/cassandra/cassandra-statefulset.yaml @@ -0,0 +1,98 @@ +apiVersion: "apps/v1beta1" +kind: StatefulSet +metadata: + name: cassandra +spec: + serviceName: cassandra + replicas: 3 + template: + metadata: + labels: + app: cassandra + spec: + containers: + - name: cassandra + image: gcr.io/google-samples/cassandra:v12 + imagePullPolicy: Always + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + resources: + limits: + cpu: "500m" + memory: 1Gi + requests: + cpu: "500m" + memory: 1Gi + securityContext: + capabilities: + add: + - IPC_LOCK + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEEDS + value: "cassandra-0.cassandra.default.svc.cluster.local" + - name: CASSANDRA_CLUSTER_NAME + value: "K8Demo" + - name: CASSANDRA_DC + value: "DC1-K8Demo" + - name: CASSANDRA_RACK + value: "Rack1-K8Demo" + - name: CASSANDRA_AUTO_BOOTSTRAP + value: "false" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + readinessProbe: + exec: + command: + - /bin/bash + - -c + - /ready-probe.sh + initialDelaySeconds: 15 + timeoutSeconds: 5 + # These volume mounts are persistent. They are like inline claims, + # but not exactly because the names need to match exactly one of + # the stateful pod volumes. + volumeMounts: + - name: cassandra-data + mountPath: /cassandra_data + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + # do not use these in production until ssd GCEPersistentDisk or other ssd pd + volumeClaimTemplates: + - metadata: + name: cassandra-data + annotations: + volume.beta.kubernetes.io/storage-class: fast + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd diff --git a/storage/cassandra/image/Dockerfile b/storage/cassandra/image/Dockerfile new file mode 100644 index 000000000..d14401b8f --- /dev/null +++ b/storage/cassandra/image/Dockerfile @@ -0,0 +1,130 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/google_containers/ubuntu-slim:0.6 + +ARG BUILD_DATE +ARG VCS_REF +ARG CASSANDRA_VERSION +ARG DEV_CONTAINER + +LABEL \ + org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.docker.dockerfile="/Dockerfile" \ + org.label-schema.license="Apache License 2.0" \ + org.label-schema.name="k8s-for-greeks/docker-cassandra-k8s" \ + org.label-schema.url="https://github.com/k8s-for-greeks/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-type="Git" \ + org.label-schema.vcs-url="https://github.com/k8s-for-greeks/docker-cassandra-k8s" + +ENV CASSANDRA_HOME=/usr/local/apache-cassandra-${CASSANDRA_VERSION} \ + CASSANDRA_CONF=/etc/cassandra \ + CASSANDRA_DATA=/cassandra_data \ + CASSANDRA_LOGS=/var/log/cassandra \ + JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + PATH=${PATH}:/usr/lib/jvm/java-8-openjdk-amd64/bin:/usr/local/apache-cassandra-${CASSANDRA_VERSION}/bin \ + DI_VERSION=1.2.0 \ + DI_SHA=81231da1cd074fdc81af62789fead8641ef3f24b6b07366a1c34e5b059faf363 + +ADD files / + +RUN set -e && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ + && apt-get update && apt-get -qq -y --force-yes install --no-install-recommends \ + openjdk-8-jre-headless \ + libjemalloc1 \ + localepurge \ + wget && \ + mirror_url=$( wget -q -O - http://www.apache.org/dyn/closer.cgi/cassandra/ \ + | sed -n 's#.*href="\(http://ftp.[^"]*\)".*#\1#p' \ + | head -n 1 \ + ) \ + && wget -q -O - ${mirror_url}/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz \ + | tar -xzf - -C /usr/local \ + && wget -q -O - https://github.com/Yelp/dumb-init/releases/download/v${DI_VERSION}/dumb-init_${DI_VERSION}_amd64 > /sbin/dumb-init \ + && echo "$DI_SHA /sbin/dumb-init" | sha256sum -c - \ + && chmod +x /sbin/dumb-init \ + && chmod +x /ready-probe.sh \ + && mkdir -p /cassandra_data/data \ + && mkdir -p /etc/cassandra \ + && mv /logback.xml /cassandra.yaml /jvm.options /etc/cassandra/ \ + && mv /usr/local/apache-cassandra-${CASSANDRA_VERSION}/conf/cassandra-env.sh /etc/cassandra/ \ + && adduser --disabled-password --no-create-home --gecos '' --disabled-login cassandra \ + && chown cassandra: /ready-probe.sh \ + && if [ -n "$DEV_CONTAINER" ]; then apt-get -y --no-install-recommends install python; else rm -rf $CASSANDRA_HOME/pylib; fi \ + && apt-get -y purge wget localepurge \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf \ + $CASSANDRA_HOME/*.txt \ + $CASSANDRA_HOME/doc \ + $CASSANDRA_HOME/javadoc \ + $CASSANDRA_HOME/tools/*.yaml \ + $CASSANDRA_HOME/tools/bin/*.bat \ + $CASSANDRA_HOME/bin/*.bat \ + doc \ + man \ + info \ + locale \ + common-licenses \ + ~/.bashrc \ + /var/lib/apt/lists/* \ + /var/log/* \ + /var/cache/debconf/* \ + /etc/systemd \ + /lib/lsb \ + /lib/udev \ + /usr/share/doc/ \ + /usr/share/doc-base/ \ + /usr/share/man/ \ + /tmp/* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/plugin \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javaws \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/jjs \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/orbd \ + /usr/lib/jvm/java-8-openjdk-amd64/bin/pack200 \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/policytool \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmid \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmiregistry \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/servertool \ + /usr/lib/jvm/java-8-openjdk-amd64/bin/tnameserv \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/unpack200 \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/javaws.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/deploy* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/desktop \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*javafx* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*jfx* \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libdecora_sse.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libprism_*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libfxplugins.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libglass.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libgstreamer-lite.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjavafx*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjfx*.so \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/jfxrt.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/nashorn.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/oblique-fonts \ + /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/plugin.jar \ + /usr/lib/jvm/java-8-openjdk-amd64/man + +VOLUME ["/$CASSANDRA_DATA"] + +# 7000: intra-node communication +# 7001: TLS intra-node communication +# 7199: JMX +# 9042: CQL +# 9160: thrift service +EXPOSE 7000 7001 7199 9042 9160 + +CMD ["/sbin/dumb-init", "/bin/bash", "/run.sh"] diff --git a/storage/cassandra/image/Makefile b/storage/cassandra/image/Makefile new file mode 100644 index 000000000..3143d5ce2 --- /dev/null +++ b/storage/cassandra/image/Makefile @@ -0,0 +1,40 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build the cassandra image. +VERSION=v12 +PROJECT_ID?=google_samples +PROJECT=gcr.io/${PROJECT_ID} +CASSANDRA_VERSION=3.9 + +all: kubernetes-cassandra.jar build + +kubernetes-cassandra.jar: ../java/* ../java/src/main/java/io/k8s/cassandra/*.java + cd ../java && mvn clean && mvn package + mv ../java/target/kubernetes-cassandra*.jar files/kubernetes-cassandra.jar + cd ../java && mvn clean + +container: + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" -t ${PROJECT}/cassandra:${VERSION} . + +container-dev: + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" --build-arg "DEV_CONTAINER=true" -t ${PROJECT}/cassandra:${VERSION}-dev . + +build: container container-dev + +push: build + gcloud docker -- push ${PROJECT}/cassandra:${VERSION} + gcloud docker -- push ${PROJECT}/cassandra:${VERSION}-dev + +.PHONY: all build push diff --git a/storage/cassandra/image/files/cassandra.yaml b/storage/cassandra/image/files/cassandra.yaml new file mode 100644 index 000000000..7df40351e --- /dev/null +++ b/storage/cassandra/image/files/cassandra.yaml @@ -0,0 +1,990 @@ +# Cassandra storage config YAML + +# NOTE: +# See http://wiki.apache.org/cassandra/StorageConfiguration for +# full explanations of configuration directives +# /NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +# +# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, +# and will use the initial_token as described below. +# +# Specifying initial_token will override this setting on the node's initial start, +# on subsequent starts, this setting will apply even if initial token is set. +# +# If you already have a cluster with 1 token per node, and wish to migrate to +# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations +num_tokens: 256 + +# Triggers automatic allocation of num_tokens tokens for this node. The allocation +# algorithm attempts to choose tokens in a way that optimizes replicated load over +# the nodes in the datacenter for the replication strategy used by the specified +# keyspace. +# +# The load assigned to each node will be close to proportional to its number of +# vnodes. +# +# Only supported with the Murmur3Partitioner. +# allocate_tokens_for_keyspace: KEYSPACE + +# initial_token allows you to specify tokens manually. While you can use # it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes # to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# See http://wiki.apache.org/cassandra/HintedHandoff +# May either be "true" or "false" to enable globally +hinted_handoff_enabled: true +# When hinted_handoff_enabled is true, a black list of data centers that will not +# perform hinted handoff +# hinted_handoff_disabled_datacenters: +# - DC1 +# - DC2 +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +max_hint_window_in_ms: 10800000 # 3 hours + +# Maximum throttle in KBs per second, per delivery thread. This will be +# reduced proportionally to the number of nodes in the cluster. (If there +# are two nodes in the cluster, each delivery thread will use the maximum +# rate; if there are three, each will throttle to half of the maximum, +# since we expect two nodes to be delivering hints simultaneously.) +hinted_handoff_throttle_in_kb: 1024 + +# Number of threads with which to deliver hints; +# Consider increasing this number when you have multi-dc deployments, since +# cross-dc handoff tends to be slower +max_hints_delivery_threads: 2 + +# Directory where Cassandra should store hints. +# If not set, the default directory is $CASSANDRA_HOME/data/hints. +hints_directory: /cassandra_data/hints + +# How often hints should be flushed from the internal buffers to disk. +# Will *not* trigger fsync. +hints_flush_period_in_ms: 10000 + +# Maximum size for a single hints file, in megabytes. +max_hints_file_size_in_mb: 128 + +# Compression to apply to the hint files. If omitted, hints files +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +#hints_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# Maximum throttle in KBs per second, total. This will be +# reduced proportionally to the number of nodes in the cluster. +batchlog_replay_throttle_in_kb: 1024 + +# Authentication backend, implementing IAuthenticator; used to identify users +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.credentials table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) +authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +authorizer: AllowAllAuthorizer + +# Part of the Authentication & Authorization backend, implementing IRoleManager; used +# to maintain grants and memberships between roles. +# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager, +# which stores role information in the system_auth keyspace. Most functions of the +# IRoleManager require an authenticated login, so unless the configured IAuthenticator +# actually implements authentication, most of this functionality will be unavailable. +# +# - CassandraRoleManager stores role data in the system_auth keyspace. Please +# increase system_auth keyspace replication factor if you use this role manager. +role_manager: CassandraRoleManager + +# Validity period for roles cache (fetching granted roles can be an expensive +# operation depending on the role manager, CassandraRoleManager is one example) +# Granted roles are cached for authenticated sessions in AuthenticatedUser and +# after the period specified here, become eligible for (async) reload. +# Defaults to 2000, set to 0 to disable caching entirely. +# Will be disabled automatically for AllowAllAuthenticator. +roles_validity_in_ms: 2000 + +# Refresh interval for roles cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If roles_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as roles_validity_in_ms. +# roles_update_interval_in_ms: 2000 + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 2000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +permissions_validity_in_ms: 2000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# Validity period for credentials cache. This cache is tightly coupled to +# the provided PasswordAuthenticator implementation of IAuthenticator. If +# another IAuthenticator implementation is configured, this cache will not +# be automatically used and so the following settings will have no effect. +# Please note, credentials are cached in their encrypted form, so while +# activating this cache may reduce the number of queries made to the +# underlying table, it may not bring a significant reduction in the +# latency of individual authentication attempts. +# Defaults to 2000, set to 0 to disable credentials caching. +credentials_validity_in_ms: 2000 + +# Refresh interval for credentials cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If credentials_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as credentials_validity_in_ms. +# credentials_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Besides Murmur3Partitioner, partitioners included for backwards +# compatibility include RandomPartitioner, ByteOrderedPartitioner, and +# OrderPreservingPartitioner. +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Directories where Cassandra should store data on disk. Cassandra +# will spread data evenly across them, subject to the granularity of +# the configured compaction strategy. +# If not set, the default directory is $CASSANDRA_HOME/data/data. +data_file_directories: + - /cassandra_data/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# If not set, the default directory is $CASSANDRA_HOME/data/commitlog. +commitlog_directory: /cassandra_data/commitlog + +# policy for data disk failures: +# die: shut down gossip and client transports and kill the JVM for any fs errors or +# single-sstable errors, so the node can be replaced. +# stop_paranoid: shut down gossip and client transports even for single-sstable errors, +# kill the JVM for errors during startup. +# stop: shut down gossip and client transports, leaving the node effectively dead, but +# can still be inspected via JMX, kill the JVM for errors during startup. +# best_effort: stop using the failed disk and respond to requests based on +# remaining available sstables. This means you WILL see obsolete +# data at CL.ONE! +# ignore: ignore fatal errors and let requests fail, as in pre-1.2 Cassandra +disk_failure_policy: stop + +# policy for commit disk failures: +# die: shut down gossip and Thrift and kill the JVM, so the node can be replaced. +# stop: shut down gossip and Thrift, leaving the node effectively dead, but +# can still be inspected via JMX. +# stop_commit: shutdown the commit log, letting writes collect but +# continuing to service reads, as in pre-2.0.5 Cassandra +# ignore: ignore fatal errors and let the batches fail +commit_failure_policy: stop + +# Maximum size of the key cache in memory. +# +# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the +# minimum, sometimes more. The key cache is fairly tiny for the amount of +# time it saves, so it's worthwhile to use it at large numbers. +# The row cache saves even more time, but must contain the entire row, +# so it is extremely space-intensive. It's best to only use the +# row cache if you have hot rows or static rows. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. +key_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the key cache. Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 14400 or 4 hours. +key_cache_save_period: 14400 + +# Number of keys from the key cache to save +# Disabled by default, meaning all keys are going to be saved +# key_cache_keys_to_save: 100 + +# Row cache implementation class name. +# Available implementations: +# org.apache.cassandra.cache.OHCProvider Fully off-heap row cache implementation (default). +# org.apache.cassandra.cache.SerializingCacheProvider This is the row cache implementation availabile +# in previous releases of Cassandra. +# row_cache_class_name: org.apache.cassandra.cache.OHCProvider + +# Maximum size of the row cache in memory. +# Please note that OHC cache implementation requires some additional off-heap memory to manage +# the map structures and some in-flight memory during operations before/after cache entries can be +# accounted against the cache capacity. This overhead is usually small compared to the whole capacity. +# Do not specify more memory that the system can afford in the worst usual situation and leave some +# headroom for OS block level cache. Do never allow your system to swap. +# +# Default value is 0, to disable row caching. +row_cache_size_in_mb: 0 + +# Duration in seconds after which Cassandra should save the row cache. +# Caches are saved to saved_caches_directory as specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 0 to disable saving the row cache. +row_cache_save_period: 0 + +# Number of keys from the row cache to save. +# Specify 0 (which is the default), meaning all keys are going to be saved +# row_cache_keys_to_save: 100 + +# Maximum size of the counter cache in memory. +# +# Counter cache helps to reduce counter locks' contention for hot counter cells. +# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before +# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration +# of the lock hold, helping with hot counter cell updates, but will not allow skipping +# the read entirely. Only the local (clock, count) tuple of a counter cell is kept +# in memory, not the whole counter, so it's relatively cheap. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. +# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. +counter_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the counter cache (keys only). Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Default is 7200 or 2 hours. +counter_cache_save_period: 7200 + +# Number of keys from the counter cache to save +# Disabled by default, meaning all keys are going to be saved +# counter_cache_keys_to_save: 100 + +# saved caches +# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. +saved_caches_directory: /cassandra_data/saved_caches + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Cassandra won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +# Max mutation size is also configurable via max_mutation_size_in_kb setting in +# cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024. +# +# NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must +# be set to at least twice the size of max_mutation_size_in_kb / 1024 +# +commitlog_segment_size_in_mb: 32 + +# Compression to apply to the commit log. If omitted, the commit log +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +#commitlog_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# any class that implements the SeedProvider interface and has a +# constructor that takes a Map of parameters will do. +seed_provider: + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + #- class_name: io.k8s.cassandra.KubernetesSeedProvider + - class_name: SEED_PROVIDER + parameters: + # seeds is actually a comma-delimited list of addresses. + # Ex: ",," + - seeds: "127.0.0.1" + +# For workloads with more data than can fit in memory, Cassandra's +# bottleneck will be reads that need to fetch data from +# disk. "concurrent_reads" should be set to (16 * number_of_drives) in +# order to allow the operations to enqueue low enough in the stack +# that the OS and drives can reorder them. Same applies to +# "concurrent_counter_writes", since counter writes read the current +# values before incrementing and writing them back. +# +# On the other hand, since writes are almost never IO bound, the ideal +# number of "concurrent_writes" is dependent on the number of cores in +# your system; (8 * number_of_cores) is a good rule of thumb. +concurrent_reads: 32 +concurrent_writes: 32 +concurrent_counter_writes: 32 + +# For materialized view writes, as there is a read involved, so this should +# be limited by the less of concurrent reads or concurrent writes. +concurrent_materialized_view_writes: 32 + +# Maximum memory to use for pooling sstable buffers. Defaults to the smaller +# of 1/4 of heap or 512MB. This pool is allocated off-heap, so is in addition +# to the memory allocated for heap. Memory is only allocated as needed. +# file_cache_size_in_mb: 512 + +# Flag indicating whether to allocate on or off heap when the sstable buffer +# pool is exhausted, that is when it has exceeded the maximum memory +# file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request. + +# buffer_pool_use_heap_if_exhausted: true + +# The strategy for optimizing disk read +# Possible values are: +# ssd (for solid state disks, the default) +# spinning (for spinning disks) +# disk_optimization_strategy: ssd + +# Total permitted memory to use for memtables. Cassandra will stop +# accepting writes when the limit is exceeded until a flush completes, +# and will trigger a flush based on memtable_cleanup_threshold +# If omitted, Cassandra will set both to 1/4 the size of the heap. +# memtable_heap_space_in_mb: 2048 +# memtable_offheap_space_in_mb: 2048 + +# Ratio of occupied non-flushing memtable size to total permitted size +# that will trigger a flush of the largest memtable. Larger mct will +# mean larger flushes and hence less compaction, but also less concurrent +# flush activity which can make it difficult to keep your disks fed +# under heavy write load. +# +# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) +# memtable_cleanup_threshold: 0.11 + +# Specify the way Cassandra allocates and manages memtable memory. +# Options are: +# heap_buffers: on heap nio buffers +# offheap_buffers: off heap (direct) nio buffers +# offheap_objects: off heap objects +memtable_allocation_type: heap_buffers + +# Total space to use for commit logs on disk. +# +# If space gets above this value, Cassandra will flush every dirty CF +# in the oldest segment and remove it. So a small total commitlog space +# will tend to cause more flush activity on less-active columnfamilies. +# +# The default value is the smaller of 8192, and 1/4 of the total space +# of the commitlog volume. +# +# commitlog_total_space_in_mb: 8192 + +# This sets the amount of memtable flush writer threads. These will +# be blocked by disk io, and each one will hold a memtable in memory +# while blocked. +# +# memtable_flush_writers defaults to one per data_file_directory. +# +# If your data directories are backed by SSD, you can increase this, but +# avoid having memtable_flush_writers * data_file_directories > number of cores +#memtable_flush_writers: 1 + +# A fixed memory pool size in MB for for SSTable index summaries. If left +# empty, this will default to 5% of the heap size. If the memory usage of +# all index summaries exceeds this limit, SSTables with low read rates will +# shrink their index summaries in order to meet this limit. However, this +# is a best-effort process. In extreme conditions Cassandra may need to use +# more than this amount of memory. +index_summary_capacity_in_mb: + +# How frequently index summaries should be resampled. This is done +# periodically to redistribute memory from the fixed-size pool to sstables +# proportional their recent read rates. Setting to -1 will disable this +# process, leaving existing index summaries at their current sampling level. +index_summary_resize_interval_in_minutes: 60 + +# Whether to, when doing sequential writing, fsync() at intervals in +# order to force the operating system to flush the dirty +# buffers. Enable this to avoid sudden dirty buffer flushing from +# impacting read latencies. Almost always a good idea on SSDs; not +# necessarily on platters. +trickle_fsync: false +trickle_fsync_interval_in_kb: 10240 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +ssl_storage_port: 7001 + +# Address or interface to bind to and tell other Cassandra nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# Set listen_address OR listen_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing _if_ the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). +# +# Setting listen_address to 0.0.0.0 is always wrong. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +listen_address: localhost +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Address to broadcast to other Cassandra nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this +# to true to listen on broadcast_address in addition to +# the listen_address, allowing nodes to communicate in both +# interfaces. +# Ignore this property if the network configuration automatically +# routes between the public and private networks such as EC2. +# listen_on_broadcast_address: false + +# Internode authentication backend, implementing IInternodeAuthenticator; +# used to allow/disallow connections from peer nodes. +# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +start_native_transport: true +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +native_transport_port: 9042 +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +# native_transport_port_ssl: 9142 +# The maximum threads for handling requests when the native transport is used. +# This is similar to rpc_max_threads though the default differs slightly (and +# there is no native_transport_min_threads, idle threads will always be stopped +# after 30 seconds). +# native_transport_max_threads: 128 +# +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# The maximum number of concurrent client connections. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections: -1 + +# The maximum number of concurrent client connections per source ip. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections_per_ip: -1 + +# Whether to start the thrift rpc server. +start_rpc: false + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# RPC address to broadcast to drivers and other Cassandra nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +# broadcast_rpc_address: 1.2.3.4 + +# enable or disable keepalive on rpc/native connections +rpc_keepalive: true + +# Cassandra provides two out-of-the-box options for the RPC Server: +# +# sync -> One thread per thrift connection. For a very large number of clients, memory +# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size +# per thread, and that will correspond to your use of virtual memory (but physical memory +# may be limited depending on use of stack space). +# +# hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled +# asynchronously using a small number of threads that does not vary with the amount +# of thrift clients (and thus scales well to many clients). The rpc requests are still +# synchronous (one thread per active request). If hsha is selected then it is essential +# that rpc_max_threads is changed from the default value of unlimited. +# +# The default is sync because on Windows hsha is about 30% slower. On Linux, +# sync/hsha performance is about the same, with hsha of course using less memory. +# +# Alternatively, can provide your own RPC server by providing the fully-qualified class name +# of an o.a.c.t.TServerFactory that can create an instance of it. +rpc_server_type: sync + +# Uncomment rpc_min|max_thread to set request pool size limits. +# +# Regardless of your choice of RPC server (see above), the number of maximum requests in the +# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync +# RPC server, it also dictates the number of clients that can be connected at all). +# +# The default is unlimited and thus provides no protection against clients overwhelming the server. You are +# encouraged to set a maximum that makes sense for you in production, but do keep in mind that +# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. +# +# rpc_min_threads: 16 +# rpc_max_threads: 2048 + +# uncomment to set socket buffer sizes on rpc connections +# rpc_send_buff_size_in_bytes: +# rpc_recv_buff_size_in_bytes: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# See: +# /proc/sys/net/core/wmem_max +# /proc/sys/net/core/rmem_max +# /proc/sys/net/ipv4/tcp_wmem +# /proc/sys/net/ipv4/tcp_wmem +# and: man tcp +# internode_send_buff_size_in_bytes: +# internode_recv_buff_size_in_bytes: + +# Frame size for thrift (maximum message length). +thrift_framed_transport_size_in_mb: 15 + +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +auto_snapshot: true + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +tombstone_warn_threshold: 1000 +tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Cassandra will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +column_index_size_in_kb: 64 + + +# Log WARN on any batch size exceeding this value. 5kb per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 5 + +# Fail any batch exceeding this value. 50kb (10x warn threshold) by default. +batch_size_fail_threshold_in_kb: 50 + +# Number of simultaneous compactions to allow, NOT including +# validation "compactions" for anti-entropy repair. Simultaneous +# compactions can help preserve read performance in a mixed read/write +# workload, by mitigating the tendency of small sstables to accumulate +# during a single long running compactions. The default is usually +# fine and if you experience problems with compaction running too +# slowly or too fast, you should look at +# compaction_throughput_mb_per_sec first. +# +# concurrent_compactors defaults to the smaller of (number of disks, +# number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. +#concurrent_compactors: 1 + +# Throttles compaction to the given total throughput across the entire +# system. The faster you insert data, the faster you need to compact in +# order to keep the sstable count down, but in general, setting this to +# 16 to 32 times the rate you are inserting data is more than sufficient. +# Setting this to 0 disables throttling. Note that this account for all types +# of compaction, including validation compaction. +compaction_throughput_mb_per_sec: 16 + +# Log a warning when compacting partitions larger than this value +compaction_large_partition_warning_threshold_mb: 100 + +# When compacting, the replacement sstable(s) can be opened before they +# are completely written, and used in place of the prior sstables for +# any range that has been written. This helps to smoothly transfer reads +# between the sstables, reducing page cache churn and keeping hot rows hot +sstable_preemptive_open_interval_in_mb: 50 + +# Throttles all outbound streaming file transfers on this node to the +# given total throughput in Mbps. This is necessary because Cassandra does +# mostly sequential IO when streaming data during bootstrap or repair, which +# can lead to saturating the network connection and degrading rpc performance. +# When unset, the default is 200 Mbps or 25 MB/s. +# stream_throughput_outbound_megabits_per_sec: 200 + +# Throttles all streaming file transfer between the datacenters, +# this setting allows users to throttle inter dc stream throughput in addition +# to throttling all network stream traffic as configured with +# stream_throughput_outbound_megabits_per_sec +# When unset, the default is 200 Mbps or 25 MB/s +# inter_dc_stream_throughput_outbound_megabits_per_sec: 200 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 +# How long the coordinator should wait for seq or index scans to complete +range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# How long the coordinator should wait for counter writes to complete +counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +request_timeout_in_ms: 10000 + +# Enable operation timeout information exchange between nodes to accurately +# measure request timeouts. If disabled, replicas will assume that requests +# were forwarded to them instantly by the coordinator, which means that +# under overload conditions we will waste that much extra time processing +# already-timed-out requests. +# +# Warning: before enabling this property make sure to ntp is installed +# and the times are synchronized between the nodes. +cross_node_timeout: false + +# Set socket timeout for streaming operation. +# The stream session is failed if no data is received by any of the +# participants within that period. +# Default value is 3600000, which means streams timeout after an hour. +# streaming_socket_timeout_in_ms: 3600000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch. The snitch has two functions: +# - it teaches Cassandra enough about your network topology to route +# requests efficiently +# - it allows Cassandra to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Cassandra will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# IF THE RACK A REPLICA IS PLACED IN CHANGES AFTER THE REPLICA HAS BEEN +# ADDED TO A RING, THE NODE MUST BE DECOMMISSIONED AND REBOOTSTRAPPED. +# +# Out of the box, Cassandra provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Cassandra will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# controls how often to perform the more expensive part of host score +# calculation +dynamic_snitch_update_interval_in_ms: 100 +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +dynamic_snitch_reset_interval_in_ms: 600000 +# if set greater than zero and read_repair_chance is < 1.0, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. Thus, a value of +# 0.2 means Cassandra would continue to prefer the static snitch values +# until the pinned host was 20% worse than the fastest. +dynamic_snitch_badness_threshold: 0.1 + +# request_scheduler -- Set this to a class that implements +# RequestScheduler, which will schedule incoming client requests +# according to the specific policy. This is useful for multi-tenancy +# with a single Cassandra cluster. +# NOTE: This is specifically for requests from the client and does +# not affect inter node communication. +# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place +# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of +# client requests to a node with a separate queue for each +# request_scheduler_id. The scheduler is further customized by +# request_scheduler_options as described below. +request_scheduler: org.apache.cassandra.scheduler.NoScheduler + +# Scheduler Options vary based on the type of scheduler +# NoScheduler - Has no options +# RoundRobin +# - throttle_limit -- The throttle_limit is the number of in-flight +# requests per client. Requests beyond +# that limit are queued up until +# running requests can complete. +# The value of 80 here is twice the number of +# concurrent_reads + concurrent_writes. +# - default_weight -- default_weight is optional and allows for +# overriding the default which is 1. +# - weights -- Weights are optional and will default to 1 or the +# overridden default_weight. The weight translates into how +# many requests are handled during each turn of the +# RoundRobin, based on the scheduler id. +# +# request_scheduler_options: +# throttle_limit: 80 +# default_weight: 5 +# weights: +# Keyspace1: 1 +# Keyspace2: 5 + +# request_scheduler_id -- An identifier based on which to perform +# the request scheduling. Currently the only valid option is keyspace. +# request_scheduler_id: keyspace + +# Enable or disable inter-node encryption +# Default settings are TLS v1, RSA 1024-bit keys (it is imperative that +# users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher +# suite for authentication, key exchange and encryption of the actual data transfers. +# Use the DHE/ECDHE ciphers if running in FIPS 140 compliant mode. +# NOTE: No custom encryption options are enabled at the moment +# The available internode options are : all, none, dc, rack +# +# If set to dc cassandra will encrypt the traffic between the DCs +# If set to rack cassandra will encrypt the traffic between the racks +# +# The passwords used in these options must match the passwords used when generating +# the keystore and truststore. For instructions on generating these files, see: +# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +# +server_encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra + # More advanced defaults below: + # protocol: TLS + # algorithm: SunX509 + # store_type: JKS + # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + # require_client_auth: false + +# enable or disable client/server encryption. +client_encryption_options: + enabled: false + # If enabled and optional is set to true encrypted and unencrypted connections are handled. + optional: false + keystore: conf/.keystore + keystore_password: cassandra + # require_client_auth: false + # Set trustore and truststore_password if require_client_auth is true + # truststore: conf/.truststore + # truststore_password: cassandra + # More advanced defaults below: + # protocol: TLS + # algorithm: SunX509 + # store_type: JKS + # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +internode_compression: all + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +inter_dc_tcp_nodelay: false + +# TTL for different trace types used during logging of the repair process. +tracetype_query_ttl: 86400 +tracetype_repair_ttl: 604800 + +# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level +# Adjust the threshold based on your application throughput requirement +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +gc_warn_threshold_in_ms: 1000 + +# UDFs (user defined functions) are disabled by default. +# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. +enable_user_defined_functions: false + +# Enables scripted UDFs (JavaScript UDFs). +# Java UDFs are always enabled, if enable_user_defined_functions is true. +# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. +# This option has no effect, if enable_user_defined_functions is false. +enable_scripted_user_defined_functions: false + +# The default Windows kernel timer and scheduling resolution is 15.6ms for power conservation. +# Lowering this value on Windows can provide much tighter latency and better throughput, however +# some virtualized environments may see a negative performance impact from changing this setting +# below their system default. The sysinternals 'clockres' tool can confirm your system's default +# setting. +windows_timer_interval: 1 + + +# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from +# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by +# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys +# can still (and should!) be in the keystore and will be used on decrypt operations +# (to handle the case of key rotation). +# +# It is strongly recommended to download and install Java Cryptography Extension (JCE) +# Unlimited Strength Jurisdiction Policy Files for your version of the JDK. +# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) +# +# Currently, only the following file types are supported for transparent data encryption, although +# more are coming in future cassandra releases: commitlog, hints +transparent_data_encryption_options: + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + # CBC IV length for AES needs to be 16 bytes (which is also the default size) + # iv_length: 16 + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + keystore_password: cassandra + store_type: JCEKS + key_password: cassandra + diff --git a/storage/cassandra/image/files/jvm.options b/storage/cassandra/image/files/jvm.options new file mode 100644 index 000000000..6568dc0d0 --- /dev/null +++ b/storage/cassandra/image/files/jvm.options @@ -0,0 +1,240 @@ +########################################################################### +# jvm.options # +# # +# - all flags defined here will be used by cassandra to startup the JVM # +# - one flag should be specified per line # +# - lines that do not start with '-' will be ignored # +# - only static flags are accepted (no variables or parameters) # +# - dynamic flags will be appended to these on cassandra-env # +########################################################################### + +###################### +# STARTUP PARAMETERS # +###################### + +# Uncomment any of the following properties to enable specific startup parameters + +# In a multi-instance deployment, multiple Cassandra instances will independently assume that all +# CPU processors are available to it. This setting allows you to specify a smaller set of processors +# and perhaps have affinity. +#-Dcassandra.available_processors=number_of_processors + +# The directory location of the cassandra.yaml file. +#-Dcassandra.config=directory + +# Sets the initial partitioner token for a node the first time the node is started. +#-Dcassandra.initial_token=token + +# Set to false to start Cassandra on a node but not have the node join the cluster. +#-Dcassandra.join_ring=true|false + +# Set to false to clear all gossip state for the node on restart. Use when you have changed node +# information in cassandra.yaml (such as listen_address). +#-Dcassandra.load_ring_state=true|false + +# Enable pluggable metrics reporter. See Pluggable metrics reporting in Cassandra 2.0.2. +#-Dcassandra.metricsReporterConfigFile=file + +# Set the port on which the CQL native transport listens for clients. (Default: 9042) +#-Dcassandra.native_transport_port=port + +# Overrides the partitioner. (Default: org.apache.cassandra.dht.Murmur3Partitioner) +#-Dcassandra.partitioner=partitioner + +# To replace a node that has died, restart a new node in its place specifying the address of the +# dead node. The new node must not have any data in its data directory, that is, it must be in the +# same state as before bootstrapping. +#-Dcassandra.replace_address=listen_address or broadcast_address of dead node + +# Allow restoring specific tables from an archived commit log. +#-Dcassandra.replayList=table + +# Allows overriding of the default RING_DELAY (1000ms), which is the amount of time a node waits +# before joining the ring. +#-Dcassandra.ring_delay_ms=ms + +# Set the port for the Thrift RPC service, which is used for client connections. (Default: 9160) +#-Dcassandra.rpc_port=port + +# Set the SSL port for encrypted communication. (Default: 7001) +#-Dcassandra.ssl_storage_port=port + +# Enable or disable the native transport server. See start_native_transport in cassandra.yaml. +# cassandra.start_native_transport=true|false + +# Enable or disable the Thrift RPC server. (Default: true) +#-Dcassandra.start_rpc=true/false + +# Set the port for inter-node communication. (Default: 7000) +#-Dcassandra.storage_port=port + +# Set the default location for the trigger JARs. (Default: conf/triggers) +#-Dcassandra.triggers_dir=directory + +# For testing new compaction and compression strategies. It allows you to experiment with different +# strategies and benchmark write performance differences without affecting the production workload. +#-Dcassandra.write_survey=true + +# To disable configuration via JMX of auth caches (such as those for credentials, permissions and +# roles). This will mean those config options can only be set (persistently) in cassandra.yaml +# and will require a restart for new values to take effect. +#-Dcassandra.disable_auth_caches_remote_configuration=true + +######################## +# GENERAL JVM SETTINGS # +######################## + +# enable assertions. disabling this in production will give a modest +# performance benefit (around 5%). +-ea + +# enable thread priorities, primarily so we can give periodic tasks +# a lower priority to avoid interfering with client workload +-XX:+UseThreadPriorities + +# allows lowering thread priority without being root on linux - probably +# not necessary on Windows but doesn't harm anything. +# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar +-XX:ThreadPriorityPolicy=42 + +# Enable heap-dump if there's an OOM +-XX:+HeapDumpOnOutOfMemoryError + +# Per-thread stack size. +-Xss256k + +# Larger interned string table, for gossip's benefit (CASSANDRA-6410) +-XX:StringTableSize=1000003 + +# Make sure all memory is faulted and zeroed on startup. +# This helps prevent soft faults in containers and makes +# transparent hugepage allocation more effective. +-XX:+AlwaysPreTouch + +# Disable biased locking as it does not benefit Cassandra. +-XX:-UseBiasedLocking + +# Enable thread-local allocation blocks and allow the JVM to automatically +# resize them at runtime. +-XX:+UseTLAB +-XX:+ResizeTLAB + +# http://www.evanjones.ca/jvm-mmap-pause.html +-XX:+PerfDisableSharedMem + +# Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See +# http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version: +# comment out this entry to enable IPv6 support). +-Djava.net.preferIPv4Stack=true + +### Debug options + +# uncomment to enable flight recorder +#-XX:+UnlockCommercialFeatures +#-XX:+FlightRecorder + +# uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414 + +# uncomment to have Cassandra JVM log internal method compilation (developers only) +#-XX:+UnlockDiagnosticVMOptions +#-XX:+LogCompilation + +################# +# HEAP SETTINGS # +################# + +# Heap size is automatically calculated by cassandra-env based on this +# formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB)) +# That is: +# - calculate 1/2 ram and cap to 1024MB +# - calculate 1/4 ram and cap to 8192MB +# - pick the max +# +# For production use you may wish to adjust this for your environment. +# If that's the case, uncomment the -Xmx and Xms options below to override the +# automatic calculation of JVM heap memory. +# +# It is recommended to set min (-Xms) and max (-Xmx) heap sizes to +# the same value to avoid stop-the-world GC pauses during resize, and +# so that we can lock the heap in memory on startup to prevent any +# of it from being swapped out. +#-Xms4G +#-Xmx4G + +# Young generation size is automatically calculated by cassandra-env +# based on this formula: min(100 * num_cores, 1/4 * heap size) +# +# The main trade-off for the young generation is that the larger it +# is, the longer GC pause times will be. The shorter it is, the more +# expensive GC will be (usually). +# +# It is not recommended to set the young generation size if using the +# G1 GC, since that will override the target pause-time goal. +# More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html +# +# The example below assumes a modern 8-core+ machine for decent +# times. If in doubt, and if you do not particularly want to tweak, go +# 100 MB per physical CPU core. +#-Xmn800M + +################# +# GC SETTINGS # +################# + +### CMS Settings + +#-XX:+UseParNewGC +#-XX:+UseConcMarkSweepGC +#-XX:+CMSParallelRemarkEnabled +#-XX:SurvivorRatio=8 +#-XX:MaxTenuringThreshold=1 +#-XX:CMSInitiatingOccupancyFraction=75 +#-XX:+UseCMSInitiatingOccupancyOnly +#-XX:CMSWaitDuration=10000 +#-XX:+CMSParallelInitialMarkEnabled +#-XX:+CMSEdenChunksRecordAlways +# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 +#-XX:+CMSClassUnloadingEnabled + +### G1 Settings (experimental, comment previous section and uncomment section below to enable) + +## Use the Hotspot garbage-first collector. +-XX:+UseG1GC +# +## Have the JVM do less remembered set work during STW, instead +## preferring concurrent GC. Reduces p99.9 latency. +-XX:G1RSetUpdatingPauseTimePercent=5 +# +## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. +## 200ms is the JVM default and lowest viable setting +## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. +#-XX:MaxGCPauseMillis=500 + +## Optional G1 Settings + +# Save CPU time on large (>= 16GB) heaps by delaying region scanning +# until the heap is 70% full. The default in Hotspot 8u40 is 40%. +#-XX:InitiatingHeapOccupancyPercent=70 + +# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. +# Otherwise equal to the number of cores when 8 or less. +# Machines with > 10 cores should try setting these to <= full cores. +#-XX:ParallelGCThreads=16 +# By default, ConcGCThreads is 1/4 of ParallelGCThreads. +# Setting both to the same value can reduce STW durations. +#-XX:ConcGCThreads=16 + +### GC logging options -- uncomment to enable + +-XX:+PrintGCDetails +-XX:+PrintGCDateStamps +-XX:+PrintHeapAtGC +-XX:+PrintTenuringDistribution +-XX:+PrintGCApplicationStoppedTime +-XX:+PrintPromotionFailure +#-XX:PrintFLSStatistics=1 +#-Xloggc:/var/log/cassandra/gc.log +-XX:+UseGCLogFileRotation +-XX:NumberOfGCLogFiles=10 +-XX:GCLogFileSize=10M diff --git a/storage/cassandra/image/files/kubernetes-cassandra.jar b/storage/cassandra/image/files/kubernetes-cassandra.jar new file mode 100644 index 000000000..c639d56d0 Binary files /dev/null and b/storage/cassandra/image/files/kubernetes-cassandra.jar differ diff --git a/storage/cassandra/image/files/logback.xml b/storage/cassandra/image/files/logback.xml new file mode 100644 index 000000000..2bb893e54 --- /dev/null +++ b/storage/cassandra/image/files/logback.xml @@ -0,0 +1,13 @@ + + + + + + %-5level %date{HH:mm:ss,SSS} %msg%n + + + + + + + \ No newline at end of file diff --git a/storage/cassandra/image/files/ready-probe.sh b/storage/cassandra/image/files/ready-probe.sh new file mode 100644 index 000000000..6601b1a49 --- /dev/null +++ b/storage/cassandra/image/files/ready-probe.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ $(nodetool status | grep $POD_IP) == *"UN"* ]]; then + if [[ $DEBUG ]]; then + echo "UN"; + fi + exit 0; +else + if [[ $DEBUG ]]; then + echo "Not Up"; + fi + exit 1; +fi diff --git a/storage/cassandra/image/files/run.sh b/storage/cassandra/image/files/run.sh new file mode 100755 index 000000000..4f7ae651d --- /dev/null +++ b/storage/cassandra/image/files/run.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +CASSANDRA_CONF_DIR=/etc/cassandra +CASSANDRA_CFG=$CASSANDRA_CONF_DIR/cassandra.yaml + +# we are doing StatefulSet or just setting our seeds +if [ -z "$CASSANDRA_SEEDS" ]; then + HOSTNAME=$(hostname -f) + CASSANDRA_SEEDS=$(hostname -f) +fi + +# The following vars relate to there counter parts in $CASSANDRA_CFG +# for instance rpc_address +CASSANDRA_RPC_ADDRESS="${CASSANDRA_RPC_ADDRESS:-0.0.0.0}" +CASSANDRA_NUM_TOKENS="${CASSANDRA_NUM_TOKENS:-32}" +CASSANDRA_CLUSTER_NAME="${CASSANDRA_CLUSTER_NAME:='Test Cluster'}" +CASSANDRA_LISTEN_ADDRESS=${POD_IP:-$HOSTNAME} +CASSANDRA_BROADCAST_ADDRESS=${POD_IP:-$HOSTNAME} +CASSANDRA_BROADCAST_RPC_ADDRESS=${POD_IP:-$HOSTNAME} +CASSANDRA_DISK_OPTIMIZATION_STRATEGY="${CASSANDRA_DISK_OPTIMIZATION_STRATEGY:-ssd}" +CASSANDRA_MIGRATION_WAIT="${CASSANDRA_MIGRATION_WAIT:-1}" +CASSANDRA_ENDPOINT_SNITCH="${CASSANDRA_ENDPOINT_SNITCH:-SimpleSnitch}" +CASSANDRA_DC="${CASSANDRA_DC}" +CASSANDRA_RACK="${CASSANDRA_RACK}" +CASSANDRA_RING_DELAY="${CASSANDRA_RING_DELAY:-30000}" +CASSANDRA_AUTO_BOOTSTRAP="${CASSANDRA_AUTO_BOOTSTRAP:-true}" +CASSANDRA_SEEDS="${CASSANDRA_SEEDS:false}" +CASSANDRA_SEED_PROVIDER="${CASSANDRA_SEED_PROVIDER:-org.apache.cassandra.locator.SimpleSeedProvider}" +CASSANDRA_AUTO_BOOTSTRAP="${CASSANDRA_AUTO_BOOTSTRAP:false}" + +# Turn off JMX auth +CASSANDRA_OPEN_JMX="${CASSANDRA_OPEN_JMX:-false}" +# send GC to STDOUT +CASSANDRA_GC_STDOUT="${CASSANDRA_GC_STDOUT:-false}" + +echo Starting Cassandra on ${CASSANDRA_LISTEN_ADDRESS} +echo CASSANDRA_CONF_DIR ${CASSANDRA_CONF_DIR} +echo CASSANDRA_CFG ${CASSANDRA_CFG} +echo CASSANDRA_AUTO_BOOTSTRAP ${CASSANDRA_AUTO_BOOTSTRAP} +echo CASSANDRA_BROADCAST_ADDRESS ${CASSANDRA_BROADCAST_ADDRESS} +echo CASSANDRA_BROADCAST_RPC_ADDRESS ${CASSANDRA_BROADCAST_RPC_ADDRESS} +echo CASSANDRA_CLUSTER_NAME ${CASSANDRA_CLUSTER_NAME} +echo CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC ${CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC} +echo CASSANDRA_CONCURRENT_COMPACTORS ${CASSANDRA_CONCURRENT_COMPACTORS} +echo CASSANDRA_CONCURRENT_READS ${CASSANDRA_CONCURRENT_READS} +echo CASSANDRA_CONCURRENT_WRITES ${CASSANDRA_CONCURRENT_WRITES} +echo CASSANDRA_COUNTER_CACHE_SIZE_IN_MB ${CASSANDRA_COUNTER_CACHE_SIZE_IN_MB} +echo CASSANDRA_DC ${CASSANDRA_DC} +echo CASSANDRA_DISK_OPTIMIZATION_STRATEGY ${CASSANDRA_DISK_OPTIMIZATION_STRATEGY} +echo CASSANDRA_ENDPOINT_SNITCH ${CASSANDRA_ENDPOINT_SNITCH} +echo CASSANDRA_GC_WARN_THRESHOLD_IN_MS ${CASSANDRA_GC_WARN_THRESHOLD_IN_MS} +echo CASSANDRA_INTERNODE_COMPRESSION ${CASSANDRA_INTERNODE_COMPRESSION} +echo CASSANDRA_KEY_CACHE_SIZE_IN_MB ${CASSANDRA_KEY_CACHE_SIZE_IN_MB} +echo CASSANDRA_LISTEN_ADDRESS ${CASSANDRA_LISTEN_ADDRESS} +echo CASSANDRA_LISTEN_INTERFACE ${CASSANDRA_LISTEN_INTERFACE} +echo CASSANDRA_MEMTABLE_ALLOCATION_TYPE ${CASSANDRA_MEMTABLE_ALLOCATION_TYPE} +echo CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD ${CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD} +echo CASSANDRA_MEMTABLE_FLUSH_WRITERS ${CASSANDRA_MEMTABLE_FLUSH_WRITERS} +echo CASSANDRA_MIGRATION_WAIT ${CASSANDRA_MIGRATION_WAIT} +echo CASSANDRA_NUM_TOKENS ${CASSANDRA_NUM_TOKENS} +echo CASSANDRA_RACK ${CASSANDRA_RACK} +echo CASSANDRA_RING_DELAY ${CASSANDRA_RING_DELAY} +echo CASSANDRA_RPC_ADDRESS ${CASSANDRA_RPC_ADDRESS} +echo CASSANDRA_RPC_INTERFACE ${CASSANDRA_RPC_INTERFACE} +echo CASSANDRA_SEEDS ${CASSANDRA_SEEDS} +echo CASSANDRA_SEED_PROVIDER ${CASSANDRA_SEED_PROVIDER} + + +# if DC and RACK are set, use GossipingPropertyFileSnitch +if [[ $CASSANDRA_DC && $CASSANDRA_RACK ]]; then + echo "dc=$CASSANDRA_DC" > $CASSANDRA_CONF_DIR/cassandra-rackdc.properties + echo "rack=$CASSANDRA_RACK" >> $CASSANDRA_CONF_DIR/cassandra-rackdc.properties + CASSANDRA_ENDPOINT_SNITCH="GossipingPropertyFileSnitch" +fi + +if [ -n "$CASSANDRA_MAX_HEAP" ]; then + sed -ri "s/^(#)?-Xmx[0-9]+.*/-Xmx$CASSANDRA_MAX_HEAP/" "$CASSANDRA_CONF_DIR/jvm.options" + sed -ri "s/^(#)?-Xms[0-9]+.*/-Xms$CASSANDRA_MAX_HEAP/" "$CASSANDRA_CONF_DIR/jvm.options" +fi + +if [ -n "$CASSANDRA_REPLACE_NODE" ]; then + echo "-Dcassandra.replace_address=$CASSANDRA_REPLACE_NODE/" >> "$CASSANDRA_CONF_DIR/jvm.options" +fi + +for rackdc in dc rack; do + var="CASSANDRA_${rackdc^^}" + val="${!var}" + if [ "$val" ]; then + sed -ri 's/^('"$rackdc"'=).*/\1 '"$val"'/' "$CASSANDRA_CONF_DIR/cassandra-rackdc.properties" + fi +done + +# TODO what else needs to be modified +for yaml in \ + broadcast_address \ + broadcast_rpc_address \ + cluster_name \ + disk_optimization_strategy \ + endpoint_snitch \ + listen_address \ + num_tokens \ + rpc_address \ + start_rpc \ + key_cache_size_in_mb \ + concurrent_reads \ + concurrent_writes \ + memtable_cleanup_threshold \ + memtable_allocation_type \ + memtable_flush_writers \ + concurrent_compactors \ + compaction_throughput_mb_per_sec \ + counter_cache_size_in_mb \ + internode_compression \ + endpoint_snitch \ + gc_warn_threshold_in_ms \ + listen_interface \ + rpc_interface \ + ; do + var="CASSANDRA_${yaml^^}" + val="${!var}" + if [ "$val" ]; then + sed -ri 's/^(# )?('"$yaml"':).*/\2 '"$val"'/' "$CASSANDRA_CFG" + fi +done + +echo "auto_bootstrap: ${CASSANDRA_AUTO_BOOTSTRAP}" >> $CASSANDRA_CFG + +# set the seed to itself. This is only for the first pod, otherwise +# it will be able to get seeds from the seed provider +if [[ $CASSANDRA_SEEDS == 'false' ]]; then + sed -ri 's/- seeds:.*/- seeds: "'"$POD_IP"'"/' $CASSANDRA_CFG +else # if we have seeds set them. Probably StatefulSet + sed -ri 's/- seeds:.*/- seeds: "'"$CASSANDRA_SEEDS"'"/' $CASSANDRA_CFG +fi + +sed -ri 's/- class_name: SEED_PROVIDER/- class_name: '"$CASSANDRA_SEED_PROVIDER"'/' $CASSANDRA_CFG + +# send gc to stdout +if [[ $CASSANDRA_GC_STDOUT == 'true' ]]; then + sed -ri 's/ -Xloggc:\/var\/log\/cassandra\/gc\.log//' $CASSANDRA_CONF_DIR/cassandra-env.sh +fi + +# enable RMI and JMX to work on one port +echo "JVM_OPTS=\"\$JVM_OPTS -Djava.rmi.server.hostname=$POD_IP\"" >> $CASSANDRA_CONF_DIR/cassandra-env.sh + +# getting WARNING messages with Migration Service +echo "-Dcassandra.migration_task_wait_in_seconds=${CASSANDRA_MIGRATION_WAIT}" >> $CASSANDRA_CONF_DIR/jvm.options +echo "-Dcassandra.ring_delay_ms=${CASSANDRA_RING_DELAY}" >> $CASSANDRA_CONF_DIR/jvm.options + +if [[ $CASSANDRA_OPEN_JMX == 'true' ]]; then + export LOCAL_JMX=no + sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.authenticate=true/ -Dcom\.sun\.management\.jmxremote\.authenticate=false/' $CASSANDRA_CONF_DIR/cassandra-env.sh + sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.password\.file=\/etc\/cassandra\/jmxremote\.password//' $CASSANDRA_CONF_DIR/cassandra-env.sh +fi + +chmod 700 "${CASSANDRA_DATA}" +chown -c -R cassandra "${CASSANDRA_DATA}" "${CASSANDRA_CONF_DIR}" + +export CLASSPATH=/kubernetes-cassandra.jar + +su cassandra -c "$CASSANDRA_HOME/bin/cassandra -f" diff --git a/storage/cassandra/java/.gitignore b/storage/cassandra/java/.gitignore new file mode 100644 index 000000000..eb5a316cb --- /dev/null +++ b/storage/cassandra/java/.gitignore @@ -0,0 +1 @@ +target diff --git a/storage/cassandra/java/README.md b/storage/cassandra/java/README.md new file mode 100644 index 000000000..45950f59d --- /dev/null +++ b/storage/cassandra/java/README.md @@ -0,0 +1,34 @@ +# Cassandra on Kubernetes Custom Seed Provider: releases.k8s.io/HEAD + +Within any deployment of Cassandra a Seed Provider is used to for node discovery and communication. When a Cassandra node first starts it must discover which nodes, or seeds, for the information about the Cassandra nodes in the ring / rack / datacenter. + +This Java project provides a custom Seed Provider which communicates with the Kubernetes API to discover the required information. This provider is bundled with the Docker provided in this example. + +# Configuring the Seed Provider + +The following environment variables may be used to override the default configurations: + +| ENV VAR | DEFAULT VALUE | NOTES | +| ------------- |:-------------: |:-------------:| +| KUBERNETES_PORT_443_TCP_ADDR | kubernetes.default.svc.cluster.local | The hostname of the API server | +| KUBERNETES_PORT_443_TCP_PORT | 443 | API port number | +| CASSANDRA_SERVICE | cassandra | Default service name for lookup | +| POD_NAMESPACE | default | Default pod service namespace | +| K8S_ACCOUNT_TOKEN | /var/run/secrets/kubernetes.io/serviceaccount/token | Default path to service token | + +# Using + + +If no endpoints are discovered from the API the seeds configured in the cassandra.yaml file are used. + +# Provider limitations + +This Cassandra Provider implements `SeedProvider`. and utilizes `SimpleSnitch`. This limits a Cassandra Ring to a single Cassandra Datacenter and ignores Rack setup. Datastax provides more documentation on the use of [_SNITCHES_](https://docs.datastax.com/en/cassandra/3.x/cassandra/architecture/archSnitchesAbout.html). Further development is planned to +expand this capability. + +This in affect makes every node a seed provider, which is not a recommended best practice. This increases maintenance and reduces gossip performance. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/cassandra/java/README.md?pixel)]() + diff --git a/storage/cassandra/java/pom.xml b/storage/cassandra/java/pom.xml new file mode 100644 index 000000000..2a2d21f23 --- /dev/null +++ b/storage/cassandra/java/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + io.k8s.cassandra + kubernetes-cassandra + 1.0.2 + + + + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + + + + 1.1.3 + 3.9 + + + + + junit + junit + 4.11 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + + + org.slf4j + slf4j-api + 1.7.5 + provided + + + ch.qos.logback + logback-classic + ${logback.version} + provided + + + + ch.qos.logback + logback-core + ${logback.version} + provided + + + + org.codehaus.jackson + jackson-core-asl + 1.6.3 + provided + + + + org.codehaus.jackson + jackson-mapper-asl + 1.6.3 + provided + + + + org.apache.cassandra + cassandra-all + ${cassandra.version} + provided + + + + diff --git a/storage/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java b/storage/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java new file mode 100644 index 000000000..830e9e17d --- /dev/null +++ b/storage/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.k8s.cassandra; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.ConfigurationLoader; +import org.apache.cassandra.config.YamlConfigurationLoader; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.locator.SeedProvider; +import org.apache.cassandra.locator.SimpleSeedProvider; +import org.apache.cassandra.utils.FBUtilities; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.*; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Self discovery {@link SeedProvider} that creates a list of Cassandra Seeds by + * communicating with the Kubernetes API. + *

Various System Variable can be used to configure this provider: + *

    + *
  • KUBERNETES_PORT_443_TCP_ADDR defaults to kubernetes.default.svc.cluster.local
    • + *
  • KUBERNETES_PORT_443_TCP_PORT defaults to 443
    • + *
  • CASSANDRA_SERVICE defaults to cassandra
    • + *
  • POD_NAMESPACE defaults to 'default'
    • + *
  • CASSANDRA_SERVICE_NUM_SEEDS defaults to 8 seeds
    • + *
  • K8S_ACCOUNT_TOKEN defaults to the path for the default token
    • + *
+ */ +public class KubernetesSeedProvider implements SeedProvider { + + private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProvider.class); + + /** + * default seeds to fall back on + */ + private List defaultSeeds; + + private TrustManager[] trustAll; + + private HostnameVerifier trustAllHosts; + + /** + * Create new Seeds + * @param params + */ + public KubernetesSeedProvider(Map params) { + + // Create default seeds + defaultSeeds = createDefaultSeeds(); + + // TODO: Load the CA cert when it is available on all platforms. + trustAll = new TrustManager[] { + new X509TrustManager() { + public void checkServerTrusted(X509Certificate[] certs, String authType) {} + public void checkClientTrusted(X509Certificate[] certs, String authType) {} + public X509Certificate[] getAcceptedIssuers() { return null; } + } + }; + + trustAllHosts = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + } + + /** + * Call kubernetes API to collect a list of seed providers + * @return list of seed providers + */ + public List getSeeds() { + + String host = getEnvOrDefault("KUBERNETES_PORT_443_TCP_ADDR", "kubernetes.default.svc.cluster.local"); + String port = getEnvOrDefault("KUBERNETES_PORT_443_TCP_PORT", "443"); + String serviceName = getEnvOrDefault("CASSANDRA_SERVICE", "cassandra"); + String podNamespace = getEnvOrDefault("POD_NAMESPACE", "default"); + String path = String.format("/api/v1/namespaces/%s/endpoints/", podNamespace); + String seedSizeVar = getEnvOrDefault("CASSANDRA_SERVICE_NUM_SEEDS", "8"); + Integer seedSize = Integer.valueOf(seedSizeVar); + String accountToken = getEnvOrDefault("K8S_ACCOUNT_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token"); + + List seeds = new ArrayList(); + try { + String token = getServiceAccountToken(accountToken); + + SSLContext ctx = SSLContext.getInstance("SSL"); + ctx.init(null, trustAll, new SecureRandom()); + + String PROTO = "https://"; + URL url = new URL(PROTO + host + ":" + port + path + serviceName); + logger.info("Getting endpoints from " + url); + HttpsURLConnection conn = (HttpsURLConnection)url.openConnection(); + + // TODO: Remove this once the CA cert is propagated everywhere, and replace + // with loading the CA cert. + conn.setHostnameVerifier(trustAllHosts); + + conn.setSSLSocketFactory(ctx.getSocketFactory()); + conn.addRequestProperty("Authorization", "Bearer " + token); + ObjectMapper mapper = new ObjectMapper(); + Endpoints endpoints = mapper.readValue(conn.getInputStream(), Endpoints.class); + + if (endpoints != null) { + // Here is a problem point, endpoints.subsets can be null in first node cases. + if (endpoints.subsets != null && !endpoints.subsets.isEmpty()){ + for (Subset subset : endpoints.subsets) { + if (subset.addresses != null && !subset.addresses.isEmpty()) { + for (Address address : subset.addresses) { + seeds.add(InetAddress.getByName(address.ip)); + + if(seeds.size() >= seedSize) { + logger.info("Available num endpoints: " + seeds.size()); + return Collections.unmodifiableList(seeds); + } + } + } + } + } + logger.info("Available num endpoints: " + seeds.size()); + } else { + logger.warn("Endpoints are not available using default seeds in cassandra.yaml"); + return Collections.unmodifiableList(defaultSeeds); + } + } catch (Exception ex) { + logger.warn("Request to kubernetes apiserver failed, using default seeds in cassandra.yaml", ex); + return Collections.unmodifiableList(defaultSeeds); + } + + if (seeds.size() == 0) { + // If we got nothing, we might be the first instance, in that case + // fall back on the seeds that were passed in cassandra.yaml. + logger.warn("Seeds are not available using default seeds in cassandra.yaml"); + return Collections.unmodifiableList(defaultSeeds); + } + + return Collections.unmodifiableList(seeds); + } + + /** + * Code taken from {@link SimpleSeedProvider}. This is used as a fall back + * incase we don't find seeds + * @return + */ + protected List createDefaultSeeds() + { + Config conf; + try { + conf = loadConfig(); + } + catch (Exception e) { + throw new AssertionError(e); + } + String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1); + List seeds = new ArrayList(); + for (String host : hosts) { + try { + seeds.add(InetAddress.getByName(host.trim())); + } + catch (UnknownHostException ex) { + // not fatal... DD will bark if there end up being zero seeds. + logger.warn("Seed provider couldn't lookup host {}", host); + } + } + + if(seeds.size() == 0) { + try { + seeds.add(InetAddress.getLocalHost()); + } catch (UnknownHostException e) { + logger.warn("Seed provider couldn't lookup localhost"); + } + } + return Collections.unmodifiableList(seeds); + } + + /** + * Code taken from {@link SimpleSeedProvider} + * @return + */ + protected static Config loadConfig() throws ConfigurationException + { + String loaderClass = System.getProperty("cassandra.config.loader"); + ConfigurationLoader loader = loaderClass == null + ? new YamlConfigurationLoader() + : FBUtilities.construct(loaderClass, "configuration loading"); + return loader.loadConfig(); + } + + private static String getEnvOrDefault(String var, String def) { + String val = System.getenv(var); + if (val == null) { + val = def; + } + return val; + } + + private static String getServiceAccountToken(String file) { + try { + return new String(Files.readAllBytes(Paths.get(file))); + } catch (IOException e) { + logger.warn("unable to load service account token" + file); + throw new RuntimeException("Unable to load services account token " + file); + } + } + + protected List getDefaultSeeds() { + return defaultSeeds; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class Address { + public String ip; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class Subset { + public List
addresses; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class Endpoints { + public List subsets; + } +} diff --git a/storage/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java b/storage/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java new file mode 100644 index 000000000..987622f66 --- /dev/null +++ b/storage/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.k8s.cassandra; + +import com.google.common.collect.ImmutableMap; +import org.apache.cassandra.locator.SeedProvider; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.hamcrest.Matchers.*; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.*; + +public class KubernetesSeedProviderTest { + + private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProviderTest.class); + + @Test + @Ignore("has to be run inside of a kube cluster") + public void getSeeds() throws Exception { + SeedProvider provider = new KubernetesSeedProvider(new HashMap()); + List seeds = provider.getSeeds(); + + assertThat(seeds, is(not(empty()))); + + } + + @Test + public void testDefaultSeeds() throws Exception { + + KubernetesSeedProvider provider = new KubernetesSeedProvider(new HashMap()); + List seeds = provider.getDefaultSeeds(); + List seedsTest = new ArrayList<>(); + seedsTest.add(InetAddress.getByName("8.4.4.4")); + seedsTest.add(InetAddress.getByName("8.8.8.8")); + assertThat(seeds, is(not(empty()))); + assertThat(seeds, is(seedsTest)); + logger.debug("seeds loaded {}", seeds); + + } + + +} \ No newline at end of file diff --git a/storage/cassandra/java/src/test/resources/cassandra.yaml b/storage/cassandra/java/src/test/resources/cassandra.yaml new file mode 100644 index 000000000..791d31036 --- /dev/null +++ b/storage/cassandra/java/src/test/resources/cassandra.yaml @@ -0,0 +1,57 @@ +# Copyright (C) 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# Warning! +# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file. +# +cluster_name: Test Cluster +# memtable_allocation_type: heap_buffers +memtable_allocation_type: offheap_objects +commitlog_sync: batch +commitlog_sync_batch_window_in_ms: 1.0 +commitlog_segment_size_in_mb: 5 +commitlog_directory: target/cassandra/commitlog +hints_directory: target/cassandra/hints +partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner +listen_address: 127.0.0.1 +storage_port: 7010 +rpc_port: 9170 +start_native_transport: true +native_transport_port: 9042 +column_index_size_in_kb: 4 +saved_caches_directory: target/cassandra/saved_caches +data_file_directories: + - target/cassandra/data +disk_access_mode: mmap +seed_provider: + - class_name: io.k8s.cassandra.KubernetesSeedProvider + parameters: + - seeds: "8.4.4.4,8.8.8.8" +endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch +dynamic_snitch: true +request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler +request_scheduler_id: keyspace +server_encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra +incremental_backups: true +concurrent_compactors: 4 +compaction_throughput_mb_per_sec: 0 +row_cache_class_name: org.apache.cassandra.cache.OHCProvider +row_cache_size_in_mb: 16 +enable_user_defined_functions: true +enable_scripted_user_defined_functions: true diff --git a/storage/cassandra/java/src/test/resources/logback-test.xml b/storage/cassandra/java/src/test/resources/logback-test.xml new file mode 100644 index 000000000..893b2d391 --- /dev/null +++ b/storage/cassandra/java/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %-5level %date{HH:mm:ss,SSS} %msg%n + + + DEBUG + + + + + + + + + + diff --git a/storage/hazelcast/README.md b/storage/hazelcast/README.md new file mode 100644 index 000000000..e62b7a765 --- /dev/null +++ b/storage/hazelcast/README.md @@ -0,0 +1,233 @@ +## Cloud Native Deployments of Hazelcast using Kubernetes + +The following document describes the development of a _cloud native_ [Hazelcast](http://hazelcast.org/) deployment on Kubernetes. When we say _cloud native_ we mean an application which understands that it is running within a cluster manager, and uses this cluster management infrastructure to help implement the application. In particular, in this instance, a custom Hazelcast ```bootstrapper``` is used to enable Hazelcast to dynamically discover Hazelcast nodes that have already joined the cluster. + +Any topology changes are communicated and handled by Hazelcast nodes themselves. + +This document also attempts to describe the core components of Kubernetes: _Pods_, _Services_, and _Deployments_. + +### Prerequisites + +This example assumes that you have a Kubernetes cluster installed and running, and that you have installed the `kubectl` command line tool somewhere in your path. Please see the [getting started](../../../docs/getting-started-guides/) for installation instructions for your platform. + +### A note for the impatient + +This is a somewhat long tutorial. If you want to jump straight to the "do it now" commands, please see the [tl; dr](#tl-dr) at the end. + +### Sources + +Source is freely available at: +* Hazelcast Discovery - https://github.com/pires/hazelcast-kubernetes-bootstrapper +* Dockerfile - https://github.com/pires/hazelcast-kubernetes +* Docker Trusted Build - https://quay.io/repository/pires/hazelcast-kubernetes + +### Simple Single Pod Hazelcast Node + +In Kubernetes, the atomic unit of an application is a [_Pod_](../../../docs/user-guide/pods.md). A Pod is one or more containers that _must_ be scheduled onto the same host. All containers in a pod share a network namespace, and may optionally share mounted volumes. + +In this case, we shall not run a single Hazelcast pod, because the discovery mechanism now relies on a service definition. + + +### Adding a Hazelcast Service + +In Kubernetes a _[Service](../../../docs/user-guide/services.md)_ describes a set of Pods that perform the same task. For example, the set of nodes in a Hazelcast cluster. An important use for a Service is to create a load balancer which distributes traffic across members of the set. But a _Service_ can also be used as a standing query which makes a dynamically changing set of Pods available via the Kubernetes API. This is actually how our discovery mechanism works, by relying on the service to discover other Hazelcast pods. + +Here is the service description: + + + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + name: hazelcast + name: hazelcast +spec: + ports: + - port: 5701 + selector: + name: hazelcast +``` + +[Download example](hazelcast-service.yaml?raw=true) + + +The important thing to note here is the `selector`. It is a query over labels, that identifies the set of _Pods_ contained by the _Service_. In this case the selector is `name: hazelcast`. If you look at the Replication Controller specification below, you'll see that the pod has the corresponding label, so it will be selected for membership in this Service. + +Create this service as follows: + +```sh +$ kubectl create -f examples/storage/hazelcast/hazelcast-service.yaml +``` + +### Adding replicated nodes + +The real power of Kubernetes and Hazelcast lies in easily building a replicated, resizable Hazelcast cluster. + +In Kubernetes a _[_Deployment_](../../../docs/user-guide/deployments.md)_ is responsible for replicating sets of identical pods. Like a _Service_ it has a selector query which identifies the members of its set. Unlike a _Service_ it also has a desired number of replicas, and it will create or delete _Pods_ to ensure that the number of _Pods_ matches up with its desired state. + +Deployments will "adopt" existing pods that match their selector query, so let's create a Deployment with a single replica to adopt our existing Hazelcast Pod. + + + +```yaml +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: hazelcast + labels: + name: hazelcast +spec: + template: + metadata: + labels: + name: hazelcast + spec: + containers: + - name: hazelcast + image: quay.io/pires/hazelcast-kubernetes:0.8.0 + imagePullPolicy: Always + env: + - name: "DNS_DOMAIN" + value: "cluster.local" + ports: + - name: hazelcast + containerPort: 5701 +``` + +[Download example](hazelcast-deployment.yaml?raw=true) + + +You may note that we tell Kubernetes that the container exposes the `hazelcast` port. + +The bulk of the replication controller config is actually identical to the Hazelcast pod declaration above, it simply gives the controller a recipe to use when creating new pods. The other parts are the `selector` which contains the controller's selector query, and the `replicas` parameter which specifies the desired number of replicas, in this case 1. + +Last but not least, we set `DNS_DOMAIN` environment variable according to your Kubernetes clusters DNS configuration. + +Create this controller: + +```sh +$ kubectl create -f examples/storage/hazelcast/hazelcast-deployment.yaml +``` + +After the controller provisions successfully the pod, you can query the service endpoints: +```sh +$ kubectl get endpoints hazelcast -o yaml +apiVersion: v1 +kind: Endpoints +metadata: + creationTimestamp: 2017-03-15T09:40:11Z + labels: + name: hazelcast + name: hazelcast + namespace: default + resourceVersion: "65060" + selfLink: /api/v1/namespaces/default/endpoints/hazelcast + uid: 62645b71-0963-11e7-b39c-080027985ce6 +subsets: +- addresses: + - ip: 172.17.0.2 + nodeName: minikube + targetRef: + kind: Pod + name: hazelcast-4195412960-mgqtk + namespace: default + resourceVersion: "65058" + uid: 7043708f-0963-11e7-b39c-080027985ce6 + ports: + - port: 5701 + protocol: TCP + +``` + +You can see that the _Service_ has found the pod created by the replication controller. + +Now it gets even more interesting. Let's scale our cluster to 2 pods: +```sh +$ kubectl scale deployment hazelcast --replicas 2 +``` + +Now if you list the pods in your cluster, you should see two hazelcast pods: + +```sh +$ kubectl get deployment,pods +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +deploy/hazelcast 2 2 2 2 2m + +NAME READY STATUS RESTARTS AGE +po/hazelcast-4195412960-0tl3w 1/1 Running 0 7s +po/hazelcast-4195412960-mgqtk 1/1 Running 0 2m +``` + +To prove that this all works, you can use the `log` command to examine the logs of one pod, for example: + +```sh +kubectl logs -f hazelcast-4195412960-0tl3w +2017-03-15 09:42:45.046 INFO 7 --- [ main] com.github.pires.hazelcast.Application : Starting Application on hazelcast-4195412960-0tl3w with PID 7 (/bootstrapper.jar started by root in /) +2017-03-15 09:42:45.060 INFO 7 --- [ main] com.github.pires.hazelcast.Application : No active profile set, falling back to default profiles: default +2017-03-15 09:42:45.128 INFO 7 --- [ main] s.c.a.AnnotationConfigApplicationContext : Refreshing org.springframework.context.annotation.AnnotationConfigApplicationContext@14514713: startup date [Wed Mar 15 09:42:45 GMT 2017]; root of context hierarchy +2017-03-15 09:42:45.989 INFO 7 --- [ main] o.s.j.e.a.AnnotationMBeanExporter : Registering beans for JMX exposure on startup +2017-03-15 09:42:46.001 INFO 7 --- [ main] c.g.p.h.HazelcastDiscoveryController : Asking k8s registry at https://kubernetes.default.svc.cluster.local.. +2017-03-15 09:42:46.376 INFO 7 --- [ main] c.g.p.h.HazelcastDiscoveryController : Found 2 pods running Hazelcast. +2017-03-15 09:42:46.458 INFO 7 --- [ main] c.h.instance.DefaultAddressPicker : [LOCAL] [someGroup] [3.8] Interfaces is disabled, trying to pick one address from TCP-IP config addresses: [172.17.0.6, 172.17.0.2] +2017-03-15 09:42:46.458 INFO 7 --- [ main] c.h.instance.DefaultAddressPicker : [LOCAL] [someGroup] [3.8] Prefer IPv4 stack is true. +2017-03-15 09:42:46.464 INFO 7 --- [ main] c.h.instance.DefaultAddressPicker : [LOCAL] [someGroup] [3.8] Picked [172.17.0.6]:5701, using socket ServerSocket[addr=/0:0:0:0:0:0:0:0,localport=5701], bind any local is true +2017-03-15 09:42:46.484 INFO 7 --- [ main] com.hazelcast.system : [172.17.0.6]:5701 [someGroup] [3.8] Hazelcast 3.8 (20170217 - d7998b4) starting at [172.17.0.6]:5701 +2017-03-15 09:42:46.484 INFO 7 --- [ main] com.hazelcast.system : [172.17.0.6]:5701 [someGroup] [3.8] Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved. +2017-03-15 09:42:46.485 INFO 7 --- [ main] com.hazelcast.system : [172.17.0.6]:5701 [someGroup] [3.8] Configured Hazelcast Serialization version : 1 +2017-03-15 09:42:46.679 INFO 7 --- [ main] c.h.s.i.o.impl.BackpressureRegulator : [172.17.0.6]:5701 [someGroup] [3.8] Backpressure is disabled +2017-03-15 09:42:47.069 INFO 7 --- [ main] com.hazelcast.instance.Node : [172.17.0.6]:5701 [someGroup] [3.8] Creating TcpIpJoiner +2017-03-15 09:42:47.182 INFO 7 --- [ main] c.h.s.i.o.impl.OperationExecutorImpl : [172.17.0.6]:5701 [someGroup] [3.8] Starting 2 partition threads +2017-03-15 09:42:47.189 INFO 7 --- [ main] c.h.s.i.o.impl.OperationExecutorImpl : [172.17.0.6]:5701 [someGroup] [3.8] Starting 3 generic threads (1 dedicated for priority tasks) +2017-03-15 09:42:47.197 INFO 7 --- [ main] com.hazelcast.core.LifecycleService : [172.17.0.6]:5701 [someGroup] [3.8] [172.17.0.6]:5701 is STARTING +2017-03-15 09:42:47.253 INFO 7 --- [cached.thread-3] c.hazelcast.nio.tcp.InitConnectionTask : [172.17.0.6]:5701 [someGroup] [3.8] Connecting to /172.17.0.2:5701, timeout: 0, bind-any: true +2017-03-15 09:42:47.262 INFO 7 --- [cached.thread-3] c.h.nio.tcp.TcpIpConnectionManager : [172.17.0.6]:5701 [someGroup] [3.8] Established socket connection between /172.17.0.6:58073 and /172.17.0.2:5701 +2017-03-15 09:42:54.260 INFO 7 --- [ration.thread-0] com.hazelcast.system : [172.17.0.6]:5701 [someGroup] [3.8] Cluster version set to 3.8 +2017-03-15 09:42:54.262 INFO 7 --- [ration.thread-0] c.h.internal.cluster.ClusterService : [172.17.0.6]:5701 [someGroup] [3.8] + +Members [2] { + Member [172.17.0.2]:5701 - 170f6924-7888-442a-9875-ad4d25659a8a + Member [172.17.0.6]:5701 - b1b82bfa-86c2-4931-af57-325c10c03b3b this +} + +2017-03-15 09:42:56.285 INFO 7 --- [ main] com.hazelcast.core.LifecycleService : [172.17.0.6]:5701 [someGroup] [3.8] [172.17.0.6]:5701 is STARTED +2017-03-15 09:42:56.287 INFO 7 --- [ main] com.github.pires.hazelcast.Application : Started Application in 11.831 seconds (JVM running for 12.219) +``` + +Now let's scale our cluster to 4 nodes: +```sh +$ kubectl scale deployment hazelcast --replicas 4 +``` + +Examine the status again by checking a node's logs and you should see the 4 members connected. Something like: +``` +(...) + +Members [4] { + Member [172.17.0.2]:5701 - 170f6924-7888-442a-9875-ad4d25659a8a + Member [172.17.0.6]:5701 - b1b82bfa-86c2-4931-af57-325c10c03b3b this + Member [172.17.0.9]:5701 - 0c7530d3-1b5a-4f40-bd59-7187e43c1110 + Member [172.17.0.10]:5701 - ad5c3000-7fd0-4ce7-8194-e9b1c2ed6dda +} +``` + +### tl; dr; + +For those of you who are impatient, here is the summary of the commands we ran in this tutorial. + +```sh +kubectl create -f service.yaml +kubectl create -f deployment.yaml +kubectl scale deployment hazelcast --replicas 2 +kubectl scale deployment hazelcast --replicas 4 +``` + +### Hazelcast Discovery Source + +See [here](https://github.com/pires/hazelcast-kubernetes-bootstrapper/blob/master/src/main/java/com/github/pires/hazelcast/HazelcastDiscoveryController.java) + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/hazelcast/README.md?pixel)]() + diff --git a/storage/hazelcast/hazelcast-deployment.yaml b/storage/hazelcast/hazelcast-deployment.yaml new file mode 100644 index 000000000..bbf61b092 --- /dev/null +++ b/storage/hazelcast/hazelcast-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: hazelcast + labels: + name: hazelcast +spec: + template: + metadata: + labels: + name: hazelcast + spec: + containers: + - name: hazelcast + image: quay.io/pires/hazelcast-kubernetes:3.8_1 + imagePullPolicy: Always + env: + - name: "DNS_DOMAIN" + value: "cluster.local" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: hazelcast + containerPort: 5701 diff --git a/storage/hazelcast/hazelcast-service.yaml b/storage/hazelcast/hazelcast-service.yaml new file mode 100644 index 000000000..0c9dc55da --- /dev/null +++ b/storage/hazelcast/hazelcast-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: hazelcast + name: hazelcast +spec: + ports: + - port: 5701 + selector: + name: hazelcast diff --git a/storage/minio/README.md b/storage/minio/README.md new file mode 100644 index 000000000..6a45c683f --- /dev/null +++ b/storage/minio/README.md @@ -0,0 +1,345 @@ +# Cloud Native Deployment of Minio using Kubernetes + +## Table of Contents + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Minio Standalone Server Deployment](#minio-standalone-server-deployment) + - [Standalone Quickstart](#standalone-quickstart) + - [Step 1: Create Persistent Volume Claim](#step-1-create-persistent-volume-claim) + - [Step 2: Create Deployment](#step-2-create-minio-deployment) + - [Step 3: Create LoadBalancer Service](#step-3-create-minio-service) + - [Step 4: Resource cleanup](#step-4-resource-cleanup) +- [Minio Distributed Server Deployment](#minio-distributed-server-deployment) + - [Distributed Quickstart](#distributed-quickstart) + - [Step 1: Create Minio Headless Service](#step-1-create-minio-headless-service) + - [Step 2: Create Minio Statefulset](#step-2-create-minio-statefulset) + - [Step 3: Create LoadBalancer Service](#step-3-create-minio-service) + - [Step 4: Resource cleanup](#step-4-resource-cleanup) + +## Introduction +Minio is an AWS S3 compatible, object storage server built for cloud applications and devops. Minio is _cloud native_, meaning Minio understands that it is running within a cluster manager, and uses the cluster management infrastructure for allocation of compute and storage resources. + +## Prerequisites + +This example assumes that you have a Kubernetes version >=1.4 cluster installed and running, and that you have installed the [`kubectl`](https://kubernetes.io/docs/tasks/kubectl/install/) command line tool in your path. Please see the +[getting started guides](https://kubernetes.io/docs/getting-started-guides/) for installation instructions for your platform. + +## Minio Standalone Server Deployment + +The following section describes the process to deploy standalone [Minio](https://minio.io/) server on Kubernetes. The deployment uses the [official Minio Docker image](https://hub.docker.com/r/minio/minio/~/dockerfile/) from Docker Hub. + +This section uses following core components of Kubernetes: + +- [_Pods_](https://kubernetes.io/docs/user-guide/pods/) +- [_Services_](https://kubernetes.io/docs/user-guide/services/) +- [_Deployments_](https://kubernetes.io/docs/user-guide/deployments/) +- [_Persistent Volume Claims_](https://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims) + +### Standalone Quickstart + +Run the below commands to get started quickly + +```sh +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-pvc.yaml?raw=true +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-deployment.yaml?raw=true +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-service.yaml?raw=true +``` + +### Step 1: Create Persistent Volume Claim + +Minio needs persistent storage to store objects. If there is no +persistent storage, the data stored in Minio instance will be stored in the container file system and will be wiped off as soon as the container restarts. + +Create a persistent volume claim (PVC) to request storage for the Minio instance. Kubernetes looks out for PVs matching the PVC request in the cluster and binds it to the PVC automatically. + +This is the PVC description. + +```sh +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + # This name uniquely identifies the PVC. Will be used in deployment below. + name: minio-pv-claim + annotations: + volume.alpha.kubernetes.io/storage-class: anything + labels: + app: minio-storage-claim +spec: + # Read more about access modes here: http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes + accessModes: + - ReadWriteOnce + resources: + # This is the request for storage. Should be available in the cluster. + requests: + storage: 10Gi +``` + +Create the PersistentVolumeClaim + +```sh +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-pvc.yaml?raw=true +persistentvolumeclaim "minio-pv-claim" created +``` + +### Step 2: Create Minio Deployment + +A deployment encapsulates replica sets and pods — so, if a pod goes down, replication controller makes sure another pod comes up automatically. This way you won’t need to bother about pod failures and will have a stable Minio service available. + +This is the deployment description. + +```sh +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + # This name uniquely identifies the Deployment + name: minio-deployment +spec: + strategy: + type: Recreate + template: + metadata: + labels: + # Label is used as selector in the service. + app: minio + spec: + # Refer to the PVC created earlier + volumes: + - name: storage + persistentVolumeClaim: + # Name of the PVC created earlier + claimName: minio-pv-claim + containers: + - name: minio + # Pulls the default Minio image from Docker Hub + image: minio/minio + command: + - minio + args: + - server + - /storage + env: + # Minio access key and secret key + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + ports: + - containerPort: 9000 + hostPort: 9000 + # Mount the volume into the pod + volumeMounts: + - name: storage # must match the volume name, above + mountPath: "/storage" +``` + +Create the Deployment + +```sh +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-deployment.yaml?raw=true +deployment "minio-deployment" created +``` + +### Step 3: Create Minio Service + +Now that you have a Minio deployment running, you may either want to access it internally (within the cluster) or expose it as a Service onto an external (outside of your cluster, maybe public internet) IP address, depending on your use case. You can achieve this using Services. There are 3 major service types — default type is ClusterIP, which exposes a service to connection from inside the cluster. NodePort and LoadBalancer are two types that expose services to external traffic. + +In this example, we expose the Minio Deployment by creating a LoadBalancer service. This is the service description. + +```sh +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio +``` +Create the Minio service + +```sh +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-standalone-service.yaml?raw=true +service "minio-service" created +``` + +The `LoadBalancer` service takes couple of minutes to launch. To check if the service was created successfully, run the command + +```sh +kubectl get svc minio-service +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +minio-service 10.55.248.23 104.199.249.165 9000:31852/TCP 1m +``` + +### Step 4: Resource cleanup + +Once you are done, cleanup the cluster using +```sh +kubectl delete deployment minio-deployment \ +&& kubectl delete pvc minio-pv-claim \ +&& kubectl delete svc minio-service +``` + +## Minio Distributed Server Deployment + +The following document describes the process to deploy [distributed Minio](https://docs.minio.io/docs/distributed-minio-quickstart-guide) server on Kubernetes. This example uses the [official Minio Docker image](https://hub.docker.com/r/minio/minio/~/dockerfile/) from Docker Hub. + +This example uses following core components of Kubernetes: + +- [_Pods_](https://kubernetes.io/docs/concepts/workloads/pods/pod/) +- [_Services_](https://kubernetes.io/docs/concepts/services-networking/service/) +- [_Statefulsets_](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/) + +### Distributed Quickstart + +Run the below commands to get started quickly + +```sh +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-headless-service.yaml?raw=true +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-statefulset.yaml?raw=true +kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-service.yaml?raw=true +``` + +### Step 1: Create Minio Headless Service + +Headless Service controls the domain within which StatefulSets are created. The domain managed by this Service takes the form: `$(service name).$(namespace).svc.cluster.local` (where “cluster.local” is the cluster domain), and the pods in this domain take the form: `$(pod-name-{i}).$(service name).$(namespace).svc.cluster.local`. This is required to get a DNS resolvable URL for each of the pods created within the Statefulset. + +This is the Headless service description. + +```sh +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio +spec: + clusterIP: None + ports: + - port: 9000 + name: minio + selector: + app: minio +``` + +Create the Headless Service + +```sh +$ kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-headless-service.yaml?raw=true +service "minio" created +``` + +### Step 2: Create Minio Statefulset + +A StatefulSet provides a deterministic name and a unique identity to each pod, making it easy to deploy stateful distributed applications. To launch distributed Minio you need to pass drive locations as parameters to the minio server command. Then, you’ll need to run the same command on all the participating pods. StatefulSets offer a perfect way to handle this requirement. + +This is the Statefulset description. + +```sh +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: minio +spec: + serviceName: minio + replicas: 4 + template: + metadata: + annotations: + pod.alpha.kubernetes.io/initialized: "true" + labels: + app: minio + spec: + containers: + - name: minio + env: + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + image: minio/minio + command: + - minio + args: + - server + - http://minio-0.minio.default.svc.cluster.local/data + - http://minio-1.minio.default.svc.cluster.local/data + - http://minio-2.minio.default.svc.cluster.local/data + - http://minio-3.minio.default.svc.cluster.local/data + ports: + - containerPort: 9000 + hostPort: 9000 + # These volume mounts are persistent. Each pod in the PetSet + # gets a volume mounted based on this field. + volumeMounts: + - name: data + mountPath: /data + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + volumeClaimTemplates: + - metadata: + name: data + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +``` + +Create the Statefulset + +```sh +$ kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-statefulset.yaml?raw=true +statefulset "minio" created +``` + +### Step 3: Create Minio Service + +Now that you have a Minio statefulset running, you may either want to access it internally (within the cluster) or expose it as a Service onto an external (outside of your cluster, maybe public internet) IP address, depending on your use case. You can achieve this using Services. There are 3 major service types — default type is ClusterIP, which exposes a service to connection from inside the cluster. NodePort and LoadBalancer are two types that expose services to external traffic. + +In this example, we expose the Minio Deployment by creating a LoadBalancer service. This is the service description. + +```sh +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio +``` +Create the Minio service + +```sh +$ kubectl create -f https://github.com/kubernetes/kubernetes/blob/master/examples/storage/minio/minio-distributed-service.yaml?raw=true +service "minio-service" created +``` + +The `LoadBalancer` service takes couple of minutes to launch. To check if the service was created successfully, run the command + +```sh +$ kubectl get svc minio-service +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +minio-service 10.55.248.23 104.199.249.165 9000:31852/TCP 1m +``` + +### Step 4: Resource cleanup + +You can cleanup the cluster using +```sh +kubectl delete statefulset minio \ +&& kubectl delete svc minio \ +&& kubectl delete svc minio-service +``` diff --git a/storage/minio/minio-distributed-headless-service.yaml b/storage/minio/minio-distributed-headless-service.yaml new file mode 100644 index 000000000..a822d76eb --- /dev/null +++ b/storage/minio/minio-distributed-headless-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio +spec: + clusterIP: None + ports: + - port: 9000 + name: minio + selector: + app: minio diff --git a/storage/minio/minio-distributed-service.yaml b/storage/minio/minio-distributed-service.yaml new file mode 100644 index 000000000..60514a863 --- /dev/null +++ b/storage/minio/minio-distributed-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio diff --git a/storage/minio/minio-distributed-statefulset.yaml b/storage/minio/minio-distributed-statefulset.yaml new file mode 100644 index 000000000..0d3f3adcf --- /dev/null +++ b/storage/minio/minio-distributed-statefulset.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: minio +spec: + serviceName: minio + replicas: 4 + template: + metadata: + annotations: + pod.alpha.kubernetes.io/initialized: "true" + labels: + app: minio + spec: + containers: + - name: minio + env: + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + image: minio/minio + command: + - minio + args: + - server + - http://minio-0.minio.default.svc.cluster.local/data + - http://minio-1.minio.default.svc.cluster.local/data + - http://minio-2.minio.default.svc.cluster.local/data + - http://minio-3.minio.default.svc.cluster.local/data + ports: + - containerPort: 9000 + hostPort: 9000 + # These volume mounts are persistent. Each pod in the PetSet + # gets a volume mounted based on this field. + volumeMounts: + - name: data + mountPath: /data + # These are converted to volume claims by the controller + # and mounted at the paths mentioned above. + volumeClaimTemplates: + - metadata: + name: data + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/storage/minio/minio-standalone-deployment.yaml b/storage/minio/minio-standalone-deployment.yaml new file mode 100644 index 000000000..1ec25a5eb --- /dev/null +++ b/storage/minio/minio-standalone-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + # This name uniquely identifies the Deployment + name: minio-deployment +spec: + strategy: + type: Recreate + template: + metadata: + labels: + # Label is used as selector in the service. + app: minio + spec: + # Refer to the PVC created earlier + volumes: + - name: storage + persistentVolumeClaim: + # Name of the PVC created earlier + claimName: minio-pv-claim + containers: + - name: minio + # Pulls the default Minio image from Docker Hub + image: minio/minio + command: + - minio + args: + - server + - /storage + env: + # Minio access key and secret key + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + ports: + - containerPort: 9000 + hostPort: 9000 + # Mount the volume into the pod + volumeMounts: + - name: storage # must match the volume name, above + mountPath: "/storage" diff --git a/storage/minio/minio-standalone-pvc.yaml b/storage/minio/minio-standalone-pvc.yaml new file mode 100644 index 000000000..edd05215a --- /dev/null +++ b/storage/minio/minio-standalone-pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + # This name uniquely identifies the PVC. Will be used in deployment below. + name: minio-pv-claim + annotations: + volume.alpha.kubernetes.io/storage-class: anything + labels: + app: minio-storage-claim +spec: + # Read more about access modes here: http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes + accessModes: + - ReadWriteOnce + resources: + # This is the request for storage. Should be available in the cluster. + requests: + storage: 10Gi diff --git a/storage/minio/minio-standalone-service.yaml b/storage/minio/minio-standalone-service.yaml new file mode 100644 index 000000000..60514a863 --- /dev/null +++ b/storage/minio/minio-standalone-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio diff --git a/storage/mysql-galera/README.md b/storage/mysql-galera/README.md new file mode 100644 index 000000000..2fe17bf89 --- /dev/null +++ b/storage/mysql-galera/README.md @@ -0,0 +1,137 @@ +## Galera Replication for MySQL on Kubernetes + +This document explains a simple demonstration example of running MySQL synchronous replication using Galera, specifically, Percona XtraDB cluster. The example is simplistic and used a fixed number (3) of nodes but the idea can be built upon and made more dynamic as Kubernetes matures. + +### Prerequisites + +This example assumes that you have a Kubernetes cluster installed and running, and that you have installed the ```kubectl``` command line tool somewhere in your path. Please see the [getting started](../../../docs/getting-started-guides/) for installation instructions for your platform. + +Also, this example requires the image found in the ```image``` directory. For your convenience, it is built and available on Docker's public image repository as ```capttofu/percona_xtradb_cluster_5_6```. It can also be built which would merely require that the image in the pod or replication controller files is updated. + +This example was tested on OS X with a Galera cluster running on VMWare using the fine repo developed by Paulo Pires [https://github.com/pires/kubernetes-vagrant-coreos-cluster] and client programs built for OS X. + +### Basic concept + +The basic idea is this: three replication controllers with a single pod, corresponding services, and a single overall service to connect to all three nodes. One of the important design goals of MySQL replication and/or clustering is that you don't want a single-point-of-failure, hence the need to distribute each node or slave across hosts or even geographical locations. Kubernetes is well-suited for facilitating this design pattern using the service and replication controller configuration files in this example. + +By defaults, there are only three pods (hence replication controllers) for this cluster. This number can be increased using the variable NUM_NODES, specified in the replication controller configuration file. It's important to know the number of nodes must always be odd. + +When the replication controller is created, it results in the corresponding container to start, run an entrypoint script that installs the MySQL system tables, set up users, and build up a list of servers that is used with the galera parameter ```wsrep_cluster_address```. This is a list of running nodes that galera uses for election of a node to obtain SST (Single State Transfer) from. + +Note: Kubernetes best-practices is to pre-create the services for each controller, and the configuration files which contain the service and replication controller for each node, when created, will result in both a service and replication contrller running for the given node. An important thing to know is that it's important that initially pxc-node1.yaml be processed first and no other pxc-nodeN services that don't have corresponding replication controllers should exist. The reason for this is that if there is a node in ```wsrep_clsuter_address``` without a backing galera node there will be nothing to obtain SST from which will cause the node to shut itself down and the container in question to exit (and another soon relaunched, repeatedly). + +First, create the overall cluster service that will be used to connect to the cluster: + +```kubectl create -f examples/storage/mysql-galera/pxc-cluster-service.yaml``` + +Create the service and replication controller for the first node: + +```kubectl create -f examples/storage/mysql-galera/pxc-node1.yaml``` + +### Create services and controllers for the remaining nodes + +Repeat the same previous steps for ```pxc-node2``` and ```pxc-node3``` + +When complete, you should be able connect with a MySQL client to the IP address + service ```pxc-cluster``` to find a working cluster + +### An example of creating a cluster + +Shown below are examples of Using ```kubectl``` from within the ```./examples/storage/mysql-galera``` directory, the status of the lauched replication controllers and services can be confirmed + +``` +$ kubectl create -f examples/storage/mysql-galera/pxc-cluster-service.yaml +services/pxc-cluster + +$ kubectl create -f examples/storage/mysql-galera/pxc-node1.yaml +services/pxc-node1 +replicationcontrollers/pxc-node1 + +$ kubectl create -f examples/storage/mysql-galera/pxc-node2.yaml +services/pxc-node2 +replicationcontrollers/pxc-node2 + +$ kubectl create -f examples/storage/mysql-galera/pxc-node3.yaml +services/pxc-node3 +replicationcontrollers/pxc-node3 + +``` + +### Confirm a running cluster + +Verify everything is running: + +``` +$ kubectl get rc,pods,services +CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS +pxc-node1 pxc-node1 capttofu/percona_xtradb_cluster_5_6:beta name=pxc-node1 1 +pxc-node2 pxc-node2 capttofu/percona_xtradb_cluster_5_6:beta name=pxc-node2 1 +pxc-node3 pxc-node3 capttofu/percona_xtradb_cluster_5_6:beta name=pxc-node3 1 +NAME READY STATUS RESTARTS AGE +pxc-node1-h6fqr 1/1 Running 0 41m +pxc-node2-sfqm6 1/1 Running 0 41m +pxc-node3-017b3 1/1 Running 0 40m +NAME LABELS SELECTOR IP(S) PORT(S) +pxc-cluster unit=pxc-cluster 10.100.179.58 3306/TCP +pxc-node1 name=pxc-node1 10.100.217.202 3306/TCP + 4444/TCP + 4567/TCP + 4568/TCP +pxc-node2 name=pxc-node2 10.100.47.212 3306/TCP + 4444/TCP + 4567/TCP + 4568/TCP +pxc-node3 name=pxc-node3 10.100.200.14 3306/TCP + 4444/TCP + 4567/TCP + 4568/TCP + +``` + +The cluster should be ready for use! + +### Connecting to the cluster + +Using the name of ```pxc-cluster``` service running interactively using ```kubernetes exec```, it is possible to connect to any of the pods using the mysql client on the pod's container to verify the cluster size, which should be ```3```. In this example below, pxc-node3 replication controller is chosen, and to find out the pod name, ```kubectl get pods``` and ```awk``` are employed: + +``` +$ kubectl get pods|grep pxc-node3|awk '{ print $1 }' +pxc-node3-0b5mc + +$ kubectl exec pxc-node3-0b5mc -i -t -- mysql -u root -p -h pxc-cluster + +Enter password: +Welcome to the MySQL monitor. Commands end with ; or \g. +Your MySQL connection id is 5 +Server version: 5.6.24-72.2-56-log Percona XtraDB Cluster (GPL), Release rel72.2, Revision 43abf03, WSREP version 25.11, wsrep_25.11 + +Copyright (c) 2009-2015 Percona LLC and/or its affiliates +Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + +Oracle is a registered trademark of Oracle Corporation and/or its +affiliates. Other names may be trademarks of their respective +owners. + +Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + +mysql> show status like 'wsrep_cluster_size'; ++--------------------+-------+ +| Variable_name | Value | ++--------------------+-------+ +| wsrep_cluster_size | 3 | ++--------------------+-------+ +1 row in set (0.06 sec) + +``` + +At this point, there is a working cluster that can begin being used via the pxc-cluster service IP address! + +### TODO + +This setup certainly can become more fluid and dynamic. One idea is to perhaps use an etcd container to store information about node state. Originally, there was a read-only kubernetes API available to each container but that has since been removed. Also, Kelsey Hightower is working on moving the functionality of confd to Kubernetes. This could replace the shell duct tape that builds the cluster configuration file for the image. + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/mysql-galera/README.md?pixel)]() + diff --git a/storage/mysql-galera/image/Dockerfile b/storage/mysql-galera/image/Dockerfile new file mode 100644 index 000000000..53a068c8c --- /dev/null +++ b/storage/mysql-galera/image/Dockerfile @@ -0,0 +1,56 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:trusty + +# add our user and group first to make sure their IDs get assigned +# consistently, regardless of whatever dependencies get added +RUN groupadd -r mysql && useradd -r -g mysql mysql + +ENV PERCONA_XTRADB_VERSION 5.6 +ENV MYSQL_VERSION 5.6 +ENV TERM linux + +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y perl --no-install-recommends && rm -rf /var/lib/apt/lists/* + +RUN apt-key adv --keyserver keys.gnupg.net --recv-keys 8507EFA5 + +RUN echo "deb http://repo.percona.com/apt trusty main" > /etc/apt/sources.list.d/percona.list +RUN echo "deb-src http://repo.percona.com/apt trusty main" >> /etc/apt/sources.list.d/percona.list + +# the "/var/lib/mysql" stuff here is because the mysql-server +# postinst doesn't have an explicit way to disable the +# mysql_install_db codepath besides having a database already +# "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter +RUN { \ + echo percona-server-server-5.6 percona-server-server/data-dir select ''; \ + echo percona-server-server-5.6 percona-server-server/root_password password ''; \ + } | debconf-set-selections \ + && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y percona-xtradb-cluster-client-"${MYSQL_VERSION}" \ + percona-xtradb-cluster-common-"${MYSQL_VERSION}" percona-xtradb-cluster-server-"${MYSQL_VERSION}" \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql && chown -R mysql:mysql /var/lib/mysql + +VOLUME /var/lib/mysql + +COPY my.cnf /etc/mysql/my.cnf +COPY cluster.cnf /etc/mysql/conf.d/cluster.cnf + +COPY docker-entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 3306 4444 4567 4568 +CMD ["mysqld"] diff --git a/storage/mysql-galera/image/cluster.cnf b/storage/mysql-galera/image/cluster.cnf new file mode 100644 index 000000000..87d70442a --- /dev/null +++ b/storage/mysql-galera/image/cluster.cnf @@ -0,0 +1,12 @@ +[mysqld] + +wsrep_provider=/usr/lib/libgalera_smm.so +wsrep_cluster_address=gcomm:// +binlog_format=ROW +default_storage_engine=InnoDB +innodb_autoinc_lock_mode=2 + +wsrep_sst_method=xtrabackup-v2 +wsrep_node_address=127.0.0.1 +wsrep_cluster_name=galera_kubernetes +wsrep_sst_auth=sstuser:changethis diff --git a/storage/mysql-galera/image/docker-entrypoint.sh b/storage/mysql-galera/image/docker-entrypoint.sh new file mode 100755 index 000000000..50185562f --- /dev/null +++ b/storage/mysql-galera/image/docker-entrypoint.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script does the following: +# +# 1. Sets up database privileges by building an SQL script +# 2. MySQL is initially started with this script a first time +# 3. Modify my.cnf and cluster.cnf to reflect available nodes to join +# + +# if NUM_NODES not passed, default to 3 +if [ -z "$NUM_NODES" ]; then + NUM_NODES=3 +fi + +if [ "${1:0:1}" = '-' ]; then + set -- mysqld "$@" +fi + +# if the command passed is 'mysqld' via CMD, then begin processing. +if [ "$1" = 'mysqld' ]; then + # read DATADIR from the MySQL config + DATADIR="$("$@" --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" + + # only check if system tables not created from mysql_install_db and permissions + # set with initial SQL script before proceeding to build SQL script + if [ ! -d "$DATADIR/mysql" ]; then + # fail if user didn't supply a root password + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" ]; then + echo >&2 'error: database is uninitialized and MYSQL_ROOT_PASSWORD not set' + echo >&2 ' Did you forget to add -e MYSQL_ROOT_PASSWORD=... ?' + exit 1 + fi + + # mysql_install_db installs system tables + echo 'Running mysql_install_db ...' + mysql_install_db --datadir="$DATADIR" + echo 'Finished mysql_install_db' + + # this script will be run once when MySQL first starts to set up + # prior to creating system tables and will ensure proper user permissions + tempSqlFile='/tmp/mysql-first-time.sql' + cat > "$tempSqlFile" <<-EOSQL +DELETE FROM mysql.user ; +CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; +GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; +EOSQL + + if [ "$MYSQL_DATABASE" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" + fi + + if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + + if [ "$MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi + fi + + # Add SST (Single State Transfer) user if Clustering is turned on + if [ -n "$GALERA_CLUSTER" ]; then + # this is the Single State Transfer user (SST, initial dump or xtrabackup user) + WSREP_SST_USER=${WSREP_SST_USER:-"sst"} + if [ -z "$WSREP_SST_PASSWORD" ]; then + echo >&2 'error: Galera cluster is enabled and WSREP_SST_PASSWORD is not set' + echo >&2 ' Did you forget to add -e WSREP_SST__PASSWORD=... ?' + exit 1 + fi + # add single state transfer (SST) user privileges + echo "CREATE USER '${WSREP_SST_USER}'@'localhost' IDENTIFIED BY '${WSREP_SST_PASSWORD}';" >> "$tempSqlFile" + echo "GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '${WSREP_SST_USER}'@'localhost';" >> "$tempSqlFile" + fi + + echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" + + # Add the SQL file to mysqld's command line args + set -- "$@" --init-file="$tempSqlFile" + fi + + chown -R mysql:mysql "$DATADIR" +fi + +# if cluster is turned on, then proceed to build cluster setting strings +# that will be interpolated into the config files +if [ -n "$GALERA_CLUSTER" ]; then + # this is the Single State Transfer user (SST, initial dump or xtrabackup user) + WSREP_SST_USER=${WSREP_SST_USER:-"sst"} + if [ -z "$WSREP_SST_PASSWORD" ]; then + echo >&2 'error: database is uninitialized and WSREP_SST_PASSWORD not set' + echo >&2 ' Did you forget to add -e WSREP_SST_PASSWORD=xxx ?' + exit 1 + fi + + # user/password for SST user + sed -i -e "s|^wsrep_sst_auth=sstuser:changethis|wsrep_sst_auth=${WSREP_SST_USER}:${WSREP_SST_PASSWORD}|" /etc/mysql/conf.d/cluster.cnf + + # set nodes own address + WSREP_NODE_ADDRESS=`ip addr show | grep -E '^[ ]*inet' | grep -m1 global | awk '{ print $2 }' | sed -e 's/\/.*//'` + if [ -n "$WSREP_NODE_ADDRESS" ]; then + sed -i -e "s|^wsrep_node_address=.*$|wsrep_node_address=${WSREP_NODE_ADDRESS}|" /etc/mysql/conf.d/cluster.cnf + fi + + # if the string is not defined or it only is 'gcomm://', this means bootstrap + if [ -z "$WSREP_CLUSTER_ADDRESS" -o "$WSREP_CLUSTER_ADDRESS" == "gcomm://" ]; then + # if empty, set to 'gcomm://' + # NOTE: this list does not imply membership. + # It only means "obtain SST and join from one of these..." + if [ -z "$WSREP_CLUSTER_ADDRESS" ]; then + WSREP_CLUSTER_ADDRESS="gcomm://" + fi + + # loop through number of nodes + for NUM in `seq 1 $NUM_NODES`; do + NODE_SERVICE_HOST="PXC_NODE${NUM}_SERVICE_HOST" + + # if set + if [ -n "${!NODE_SERVICE_HOST}" ]; then + # if not its own IP, then add it + if [ $(expr "$HOSTNAME" : "pxc-node${NUM}") -eq 0 ]; then + # if not the first bootstrap node add comma + if [ $WSREP_CLUSTER_ADDRESS != "gcomm://" ]; then + WSREP_CLUSTER_ADDRESS="${WSREP_CLUSTER_ADDRESS}," + fi + # append + # if user specifies USE_IP, use that + if [ -n "${USE_IP}" ]; then + WSREP_CLUSTER_ADDRESS="${WSREP_CLUSTER_ADDRESS}"${!NODE_SERVICE_HOST} + # otherwise use DNS + else + WSREP_CLUSTER_ADDRESS="${WSREP_CLUSTER_ADDRESS}pxc-node${NUM}" + fi + fi + fi + done + fi + + # WSREP_CLUSTER_ADDRESS is now complete and will be interpolated into the + # cluster address string (wsrep_cluster_address) in the cluster + # configuration file, cluster.cnf + if [ -n "$WSREP_CLUSTER_ADDRESS" -a "$WSREP_CLUSTER_ADDRESS" != "gcomm://" ]; then + sed -i -e "s|^wsrep_cluster_address=gcomm://|wsrep_cluster_address=${WSREP_CLUSTER_ADDRESS}|" /etc/mysql/conf.d/cluster.cnf + fi +fi + +# random server ID needed +sed -i -e "s/^server\-id=.*$/server-id=${RANDOM}/" /etc/mysql/my.cnf + +# finally, start mysql +exec "$@" diff --git a/storage/mysql-galera/image/my.cnf b/storage/mysql-galera/image/my.cnf new file mode 100644 index 000000000..078bd6fe6 --- /dev/null +++ b/storage/mysql-galera/image/my.cnf @@ -0,0 +1,55 @@ +[client] +port=3306 +socket=/var/run/mysqld/mysqld.sock + +[mysqld_safe] +socket=/var/run/mysqld/mysqld.sock +nice=0 + +[mysqld] +user=mysql +pid-file=/var/run/mysqld/mysqld.pid +socket=/var/run/mysqld/mysqld.sock +port=3306 +basedir=/usr +datadir=/var/lib/mysql +tmpdir=/tmp +lc-messages-dir=/usr/share/mysql +skip-external-locking + +key_buffer=16M +max_allowed_packet=16M +thread_stack=192K +thread_cache_size=8 + +myisam-recover=BACKUP +#max_connections=100 +query_cache_limit=1M +query_cache_size=16M +slow_query_log=1 +slow_query_log_file=/var/log/mysql/mysql-slow.log +long_query_time=2 +log-queries-not-using-indexes + +server-id=12345 +log_bin=/var/log/mysql/mysql-bin.log +expire_logs_days=4 +max_binlog_size=100M + +default_storage_engine=InnoDB +innodb_file_per_table +innodb_log_file_size=100M +innodb_log_buffer_size=10M +innodb_log_files_in_group=2 +innodb_buffer_pool_instances=4 +innodb_buffer_pool_size=100M + +[mysqldump] +quick +quote-names +max_allowed_packet=16M + +[isamchk] +key_buffer=16M + +!includedir /etc/mysql/conf.d/ diff --git a/storage/mysql-galera/pxc-cluster-service.yaml b/storage/mysql-galera/pxc-cluster-service.yaml new file mode 100644 index 000000000..f0bfd5e9a --- /dev/null +++ b/storage/mysql-galera/pxc-cluster-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: pxc-cluster + labels: + unit: pxc-cluster +spec: + ports: + - port: 3306 + name: mysql + selector: + unit: pxc-cluster \ No newline at end of file diff --git a/storage/mysql-galera/pxc-node1.yaml b/storage/mysql-galera/pxc-node1.yaml new file mode 100644 index 000000000..fa1163bdc --- /dev/null +++ b/storage/mysql-galera/pxc-node1.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: pxc-node1 + labels: + node: pxc-node1 +spec: + ports: + - port: 3306 + name: mysql + - port: 4444 + name: state-snapshot-transfer + - port: 4567 + name: replication-traffic + - port: 4568 + name: incremental-state-transfer + selector: + node: pxc-node1 +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: pxc-node1 +spec: + replicas: 1 + template: + metadata: + labels: + node: pxc-node1 + unit: pxc-cluster + spec: + containers: + - resources: + limits: + cpu: 0.3 + image: capttofu/percona_xtradb_cluster_5_6:beta + name: pxc-node1 + ports: + - containerPort: 3306 + - containerPort: 4444 + - containerPort: 4567 + - containerPort: 4568 + env: + - name: GALERA_CLUSTER + value: "true" + - name: WSREP_CLUSTER_ADDRESS + value: gcomm:// + - name: WSREP_SST_USER + value: sst + - name: WSREP_SST_PASSWORD + value: sst + - name: MYSQL_USER + value: mysql + - name: MYSQL_PASSWORD + value: mysql + - name: MYSQL_ROOT_PASSWORD + value: c-krit diff --git a/storage/mysql-galera/pxc-node2.yaml b/storage/mysql-galera/pxc-node2.yaml new file mode 100644 index 000000000..ead3675d7 --- /dev/null +++ b/storage/mysql-galera/pxc-node2.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Service +metadata: + name: pxc-node2 + labels: + node: pxc-node2 +spec: + ports: + - port: 3306 + name: mysql + - port: 4444 + name: state-snapshot-transfer + - port: 4567 + name: replication-traffic + - port: 4568 + name: incremental-state-transfer + selector: + node: pxc-node2 + +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: pxc-node2 +spec: + replicas: 1 + template: + metadata: + labels: + node: pxc-node2 + unit: pxc-cluster + spec: + containers: + - resources: + limits: + cpu: 0.3 + image: capttofu/percona_xtradb_cluster_5_6:beta + name: pxc-node2 + ports: + - containerPort: 3306 + - containerPort: 4444 + - containerPort: 4567 + - containerPort: 4568 + env: + - name: GALERA_CLUSTER + value: "true" + - name: WSREP_CLUSTER_ADDRESS + value: gcomm:// + - name: WSREP_SST_USER + value: sst + - name: WSREP_SST_PASSWORD + value: sst + - name: MYSQL_USER + value: mysql + - name: MYSQL_PASSWORD + value: mysql + - name: MYSQL_ROOT_PASSWORD + value: c-krit diff --git a/storage/mysql-galera/pxc-node3.yaml b/storage/mysql-galera/pxc-node3.yaml new file mode 100644 index 000000000..fbb368b27 --- /dev/null +++ b/storage/mysql-galera/pxc-node3.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Service +metadata: + name: pxc-node3 + labels: + node: pxc-node3 +spec: + ports: + - port: 3306 + name: mysql + - port: 4444 + name: state-snapshot-transfer + - port: 4567 + name: replication-traffic + - port: 4568 + name: incremental-state-transfer + selector: + node: pxc-node3 + +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: pxc-node3 +spec: + replicas: 1 + template: + metadata: + labels: + node: pxc-node3 + unit: pxc-cluster + spec: + containers: + - resources: + limits: + cpu: 0.3 + image: capttofu/percona_xtradb_cluster_5_6:beta + name: pxc-node3 + ports: + - containerPort: 3306 + - containerPort: 4444 + - containerPort: 4567 + - containerPort: 4568 + env: + - name: GALERA_CLUSTER + value: "true" + - name: WSREP_CLUSTER_ADDRESS + value: gcomm:// + - name: WSREP_SST_USER + value: sst + - name: WSREP_SST_PASSWORD + value: sst + - name: MYSQL_USER + value: mysql + - name: MYSQL_PASSWORD + value: mysql + - name: MYSQL_ROOT_PASSWORD + value: c-krit diff --git a/storage/redis/README.md b/storage/redis/README.md new file mode 100644 index 000000000..321d9c8aa --- /dev/null +++ b/storage/redis/README.md @@ -0,0 +1,133 @@ +## Reliable, Scalable Redis on Kubernetes + +The following document describes the deployment of a reliable, multi-node Redis on Kubernetes. It deploys a master with replicated slaves, as well as replicated redis sentinels which are use for health checking and failover. + +### Prerequisites + +This example assumes that you have a Kubernetes cluster installed and running, and that you have installed the ```kubectl``` command line tool somewhere in your path. Please see the [getting started](../../../docs/getting-started-guides/) for installation instructions for your platform. + +### A note for the impatient + +This is a somewhat long tutorial. If you want to jump straight to the "do it now" commands, please see the [tl; dr](#tl-dr) at the end. + +### Turning up an initial master/sentinel pod. + +A [_Pod_](../../../docs/user-guide/pods.md) is one or more containers that _must_ be scheduled onto the same host. All containers in a pod share a network namespace, and may optionally share mounted volumes. + +We will use the shared network namespace to bootstrap our Redis cluster. In particular, the very first sentinel needs to know how to find the master (subsequent sentinels just ask the first sentinel). Because all containers in a Pod share a network namespace, the sentinel can simply look at ```$(hostname -i):6379```. + +Here is the config for the initial master and sentinel pod: [redis-master.yaml](redis-master.yaml) + + +Create this master as follows: + +```sh +kubectl create -f examples/storage/redis/redis-master.yaml +``` + +### Turning up a sentinel service + +In Kubernetes a [_Service_](../../../docs/user-guide/services.md) describes a set of Pods that perform the same task. For example, the set of nodes in a Cassandra cluster, or even the single node we created above. An important use for a Service is to create a load balancer which distributes traffic across members of the set. But a _Service_ can also be used as a standing query which makes a dynamically changing set of Pods (or the single Pod we've already created) available via the Kubernetes API. + +In Redis, we will use a Kubernetes Service to provide a discoverable endpoints for the Redis sentinels in the cluster. From the sentinels Redis clients can find the master, and then the slaves and other relevant info for the cluster. This enables new members to join the cluster when failures occur. + +Here is the definition of the sentinel service: [redis-sentinel-service.yaml](redis-sentinel-service.yaml) + +Create this service: + +```sh +kubectl create -f examples/storage/redis/redis-sentinel-service.yaml +``` + +### Turning up replicated redis servers + +So far, what we have done is pretty manual, and not very fault-tolerant. If the ```redis-master``` pod that we previously created is destroyed for some reason (e.g. a machine dying) our Redis service goes away with it. + +In Kubernetes a [_Replication Controller_](../../../docs/user-guide/replication-controller.md) is responsible for replicating sets of identical pods. Like a _Service_ it has a selector query which identifies the members of it's set. Unlike a _Service_ it also has a desired number of replicas, and it will create or delete _Pods_ to ensure that the number of _Pods_ matches up with it's desired state. + +Replication Controllers will "adopt" existing pods that match their selector query, so let's create a Replication Controller with a single replica to adopt our existing Redis server. Here is the replication controller config: [redis-controller.yaml](redis-controller.yaml) + +The bulk of this controller config is actually identical to the redis-master pod definition above. It forms the template or "cookie cutter" that defines what it means to be a member of this set. + +Create this controller: + +```sh +kubectl create -f examples/storage/redis/redis-controller.yaml +``` + +We'll do the same thing for the sentinel. Here is the controller config: [redis-sentinel-controller.yaml](redis-sentinel-controller.yaml) + +We create it as follows: + +```sh +kubectl create -f examples/storage/redis/redis-sentinel-controller.yaml +``` + +### Scale our replicated pods + +Initially creating those pods didn't actually do anything, since we only asked for one sentinel and one redis server, and they already existed, nothing changed. Now we will add more replicas: + +```sh +kubectl scale rc redis --replicas=3 +``` + +```sh +kubectl scale rc redis-sentinel --replicas=3 +``` + +This will create two additional replicas of the redis server and two additional replicas of the redis sentinel. + +Unlike our original redis-master pod, these pods exist independently, and they use the ```redis-sentinel-service``` that we defined above to discover and join the cluster. + +### Delete our manual pod + +The final step in the cluster turn up is to delete the original redis-master pod that we created manually. While it was useful for bootstrapping discovery in the cluster, we really don't want the lifespan of our sentinel to be tied to the lifespan of one of our redis servers, and now that we have a successful, replicated redis sentinel service up and running, the binding is unnecessary. + +Delete the master as follows: + +```sh +kubectl delete pods redis-master +``` + +Now let's take a close look at what happens after this pod is deleted. There are three things that happen: + + 1. The redis replication controller notices that its desired state is 3 replicas, but there are currently only 2 replicas, and so it creates a new redis server to bring the replica count back up to 3 + 2. The redis-sentinel replication controller likewise notices the missing sentinel, and also creates a new sentinel. + 3. The redis sentinels themselves, realize that the master has disappeared from the cluster, and begin the election procedure for selecting a new master. They perform this election and selection, and chose one of the existing redis server replicas to be the new master. + +### Conclusion + +At this point we now have a reliable, scalable Redis installation. By scaling the replication controller for redis servers, we can increase or decrease the number of read-slaves in our cluster. Likewise, if failures occur, the redis-sentinels will perform master election and select a new master. + +**NOTE:** since redis 3.2 some security measures (bind to 127.0.0.1 and `--protected-mode`) are enabled by default. Please read about this in http://antirez.com/news/96 + + +### tl; dr + +For those of you who are impatient, here is the summary of commands we ran in this tutorial: + +``` +# Create a bootstrap master +kubectl create -f examples/storage/redis/redis-master.yaml + +# Create a service to track the sentinels +kubectl create -f examples/storage/redis/redis-sentinel-service.yaml + +# Create a replication controller for redis servers +kubectl create -f examples/storage/redis/redis-controller.yaml + +# Create a replication controller for redis sentinels +kubectl create -f examples/storage/redis/redis-sentinel-controller.yaml + +# Scale both replication controllers +kubectl scale rc redis --replicas=3 +kubectl scale rc redis-sentinel --replicas=3 + +# Delete the original master pod +kubectl delete pods redis-master +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/redis/README.md?pixel)]() + diff --git a/storage/redis/image/Dockerfile b/storage/redis/image/Dockerfile new file mode 100644 index 000000000..9f619a5ad --- /dev/null +++ b/storage/redis/image/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.4 + +RUN apk add --no-cache redis sed bash + +COPY redis-master.conf /redis-master/redis.conf +COPY redis-slave.conf /redis-slave/redis.conf +COPY run.sh /run.sh + +CMD [ "/run.sh" ] + +ENTRYPOINT [ "bash", "-c" ] diff --git a/storage/redis/image/redis-master.conf b/storage/redis/image/redis-master.conf new file mode 100644 index 000000000..29ae1bb41 --- /dev/null +++ b/storage/redis/image/redis-master.conf @@ -0,0 +1,828 @@ +# Redis configuration file example + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /var/run/redis.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# By default Redis listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 + +bind 0.0.0.0 + +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 60 + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving completely by commenting out all "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir /redis-master-data + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that trnasfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly yes + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/storage/redis/image/redis-slave.conf b/storage/redis/image/redis-slave.conf new file mode 100644 index 000000000..afd0a45fd --- /dev/null +++ b/storage/redis/image/redis-slave.conf @@ -0,0 +1,828 @@ +# Redis configuration file example + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize no + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /var/run/redis.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# By default Redis listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 + +bind 0.0.0.0 + +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 60 + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving completely by commenting out all "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir "./" + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +slaveof %master-ip% %master-port% + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that trnasfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly yes + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/storage/redis/image/run.sh b/storage/redis/image/run.sh new file mode 100755 index 000000000..2808c082b --- /dev/null +++ b/storage/redis/image/run.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function launchmaster() { + if [[ ! -e /redis-master-data ]]; then + echo "Redis master data doesn't exist, data won't be persistent!" + mkdir /redis-master-data + fi + redis-server /redis-master/redis.conf --protected-mode no +} + +function launchsentinel() { + while true; do + master=$(redis-cli -h ${REDIS_SENTINEL_SERVICE_HOST} -p ${REDIS_SENTINEL_SERVICE_PORT} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | cut -d' ' -f1) + if [[ -n ${master} ]]; then + master="${master//\"}" + else + master=$(hostname -i) + fi + + redis-cli -h ${master} INFO + if [[ "$?" == "0" ]]; then + break + fi + echo "Connecting to master failed. Waiting..." + sleep 10 + done + + sentinel_conf=sentinel.conf + + echo "sentinel monitor mymaster ${master} 6379 2" > ${sentinel_conf} + echo "sentinel down-after-milliseconds mymaster 60000" >> ${sentinel_conf} + echo "sentinel failover-timeout mymaster 180000" >> ${sentinel_conf} + echo "sentinel parallel-syncs mymaster 1" >> ${sentinel_conf} + echo "bind 0.0.0.0" + + redis-sentinel ${sentinel_conf} --protected-mode no +} + +function launchslave() { + while true; do + master=$(redis-cli -h ${REDIS_SENTINEL_SERVICE_HOST} -p ${REDIS_SENTINEL_SERVICE_PORT} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | cut -d' ' -f1) + if [[ -n ${master} ]]; then + master="${master//\"}" + else + echo "Failed to find master." + sleep 60 + exit 1 + fi + redis-cli -h ${master} INFO + if [[ "$?" == "0" ]]; then + break + fi + echo "Connecting to master failed. Waiting..." + sleep 10 + done + sed -i "s/%master-ip%/${master}/" /redis-slave/redis.conf + sed -i "s/%master-port%/6379/" /redis-slave/redis.conf + redis-server /redis-slave/redis.conf --protected-mode no +} + +if [[ "${MASTER}" == "true" ]]; then + launchmaster + exit 0 +fi + +if [[ "${SENTINEL}" == "true" ]]; then + launchsentinel + exit 0 +fi + +launchslave diff --git a/storage/redis/redis-controller.yaml b/storage/redis/redis-controller.yaml new file mode 100644 index 000000000..fcb5e67cd --- /dev/null +++ b/storage/redis/redis-controller.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: redis +spec: + replicas: 1 + selector: + name: redis + template: + metadata: + labels: + name: redis + spec: + containers: + - name: redis + image: gcr.io/google_containers/redis:v1 + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" + volumeMounts: + - mountPath: /redis-master-data + name: data + volumes: + - name: data + emptyDir: {} + diff --git a/storage/redis/redis-master.yaml b/storage/redis/redis-master.yaml new file mode 100644 index 000000000..57305a7a3 --- /dev/null +++ b/storage/redis/redis-master.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + name: redis + redis-sentinel: "true" + role: master + name: redis-master +spec: + containers: + - name: master + image: gcr.io/google_containers/redis:v1 + env: + - name: MASTER + value: "true" + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" + volumeMounts: + - mountPath: /redis-master-data + name: data + - name: sentinel + image: kubernetes/redis:v1 + env: + - name: SENTINEL + value: "true" + ports: + - containerPort: 26379 + volumes: + - name: data + emptyDir: {} diff --git a/storage/redis/redis-sentinel-controller.yaml b/storage/redis/redis-sentinel-controller.yaml new file mode 100644 index 000000000..da09e10cb --- /dev/null +++ b/storage/redis/redis-sentinel-controller.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: redis-sentinel +spec: + replicas: 1 + selector: + redis-sentinel: "true" + template: + metadata: + labels: + name: redis-sentinel + redis-sentinel: "true" + role: sentinel + spec: + containers: + - name: sentinel + image: gcr.io/google_containers/redis:v1 + env: + - name: SENTINEL + value: "true" + ports: + - containerPort: 26379 diff --git a/storage/redis/redis-sentinel-service.yaml b/storage/redis/redis-sentinel-service.yaml new file mode 100644 index 000000000..c670a8241 --- /dev/null +++ b/storage/redis/redis-sentinel-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: sentinel + role: service + name: redis-sentinel +spec: + ports: + - port: 26379 + targetPort: 26379 + selector: + redis-sentinel: "true" diff --git a/storage/rethinkdb/README.md b/storage/rethinkdb/README.md new file mode 100644 index 000000000..d065c4d00 --- /dev/null +++ b/storage/rethinkdb/README.md @@ -0,0 +1,130 @@ +RethinkDB Cluster on Kubernetes +============================== + +Setting up a [rethinkdb](http://rethinkdb.com/) cluster on [kubernetes](http://kubernetes.io) + +**Features** + + * Auto configuration cluster by querying info from k8s + * Simple + +Quick start +----------- + +**Step 1** + +Rethinkdb will discover its peer using endpoints provided by kubernetes service, +so first create a service so the following pod can query its endpoint + +```sh +$kubectl create -f examples/storage/rethinkdb/driver-service.yaml +``` + +check out: + +```sh +$kubectl get services +NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE +rethinkdb-driver 10.0.27.114 28015/TCP db=rethinkdb 10m +[...] +``` + +**Step 2** + +start the first server in the cluster + +```sh +$kubectl create -f examples/storage/rethinkdb/rc.yaml +``` + +Actually, you can start servers as many as you want at one time, just modify the `replicas` in `rc.ymal` + +check out again: + +```sh +$kubectl get pods +NAME READY REASON RESTARTS AGE +[...] +rethinkdb-rc-r4tb0 1/1 Running 0 1m +``` + +**Done!** + + +--- + +Scale +----- + +You can scale up your cluster using `kubectl scale`. The new pod will join to the existing cluster automatically, for example + + +```sh +$kubectl scale rc rethinkdb-rc --replicas=3 +scaled + +$kubectl get pods +NAME READY REASON RESTARTS AGE +[...] +rethinkdb-rc-f32c5 1/1 Running 0 1m +rethinkdb-rc-m4d50 1/1 Running 0 1m +rethinkdb-rc-r4tb0 1/1 Running 0 3m +``` + +Admin +----- + +You need a separate pod (labeled as role:admin) to access Web Admin UI + +```sh +kubectl create -f examples/storage/rethinkdb/admin-pod.yaml +kubectl create -f examples/storage/rethinkdb/admin-service.yaml +``` + +find the service + +```console +$kubectl get services +NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE +[...] +rethinkdb-admin 10.0.131.19 104.197.19.120 8080/TCP db=rethinkdb,role=admin 10m +rethinkdb-driver 10.0.27.114 28015/TCP db=rethinkdb 20m +``` + +We request an external load balancer in the [admin-service.yaml](admin-service.yaml) file: + +``` +type: LoadBalancer +``` + +The external load balancer allows us to access the service from outside the firewall via an external IP, 104.197.19.120 in this case. + +Note that you may need to create a firewall rule to allow the traffic, assuming you are using Google Compute Engine: + +```console +$ gcloud compute firewall-rules create rethinkdb --allow=tcp:8080 +``` + +Now you can open a web browser and access to *http://104.197.19.120:8080* to manage your cluster. + + + +**Why not just using pods in replicas?** + +This is because kube-proxy will act as a load balancer and send your traffic to different server, +since the ui is not stateless when playing with Web Admin UI will cause `Connection not open on server` error. + + +- - - + +**BTW** + + * `gen_pod.sh` is using to generate pod templates for my local cluster, +the generated pods which is using `nodeSelector` to force k8s to schedule containers to my designate nodes, for I need to access persistent data on my host dirs. Note that one needs to label the node before 'nodeSelector' can work, see this [tutorial](../../../docs/user-guide/node-selection/) + + * see [antmanler/rethinkdb-k8s](https://github.com/antmanler/rethinkdb-k8s) for detail + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/rethinkdb/README.md?pixel)]() + diff --git a/storage/rethinkdb/admin-pod.yaml b/storage/rethinkdb/admin-pod.yaml new file mode 100644 index 000000000..eac07f33a --- /dev/null +++ b/storage/rethinkdb/admin-pod.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + db: rethinkdb + role: admin + name: rethinkdb-admin +spec: + containers: + - image: gcr.io/google_containers/rethinkdb:1.16.0_1 + name: rethinkdb + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8080 + name: admin-port + - containerPort: 28015 + name: driver-port + - containerPort: 29015 + name: cluster-port + volumeMounts: + - mountPath: /data/rethinkdb_data + name: rethinkdb-storage + volumes: + - name: rethinkdb-storage + emptyDir: {} diff --git a/storage/rethinkdb/admin-service.yaml b/storage/rethinkdb/admin-service.yaml new file mode 100644 index 000000000..0ae37c026 --- /dev/null +++ b/storage/rethinkdb/admin-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + db: rethinkdb + name: rethinkdb-admin +spec: + ports: + - port: 8080 + targetPort: 8080 + type: LoadBalancer + selector: + db: rethinkdb + role: admin diff --git a/storage/rethinkdb/driver-service.yaml b/storage/rethinkdb/driver-service.yaml new file mode 100644 index 000000000..c2c559adf --- /dev/null +++ b/storage/rethinkdb/driver-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + db: rethinkdb + name: rethinkdb-driver +spec: + ports: + - port: 28015 + targetPort: 28015 + selector: + db: rethinkdb diff --git a/storage/rethinkdb/gen-pod.sh b/storage/rethinkdb/gen-pod.sh new file mode 100755 index 000000000..90a44f230 --- /dev/null +++ b/storage/rethinkdb/gen-pod.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +: ${VERSION:=1.16.0} + +readonly NAME=${1-} +if [[ -z "${NAME}" ]]; then + echo -e "\033[1;31mName must be specified\033[0m" + exit 1 +fi + +ADMIN="" +if [[ ${NAME} == "admin" ]]; then + ADMIN="role: admin" +fi + +NODE="" +# One needs to label a node with the same key/value pair, +# i.e., 'kubectl label nodes name=${2}' +if [[ ! -z "${2-}" ]]; then + NODE="nodeSelector: { name: ${2} }" +fi + +cat << EOF +apiVersion: v1 +kind: Pod +metadata: + labels: + ${ADMIN} + db: rethinkdb + name: rethinkdb-${NAME}-${VERSION} + namespace: rethinkdb +spec: + containers: + - image: antmanler/rethinkdb:${VERSION} + name: rethinkdb + ports: + - containerPort: 8080 + name: admin-port + protocol: TCP + - containerPort: 28015 + name: driver-port + protocol: TCP + - containerPort: 29015 + name: cluster-port + protocol: TCP + volumeMounts: + - mountPath: /data/rethinkdb_data + name: rethinkdb-storage + ${NODE} + restartPolicy: Always + volumes: + - hostPath: + path: /data/db/rethinkdb + name: rethinkdb-storage +EOF diff --git a/storage/rethinkdb/image/Dockerfile b/storage/rethinkdb/image/Dockerfile new file mode 100644 index 000000000..ff650cd24 --- /dev/null +++ b/storage/rethinkdb/image/Dockerfile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM rethinkdb:1.16.0 + + +RUN apt-get update && \ + apt-get install -yq curl && \ + rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* && \ + curl -L http://stedolan.github.io/jq/download/linux64/jq > /usr/bin/jq && \ + chmod u+x /usr/bin/jq + +COPY ./run.sh /usr/bin/run.sh +RUN chmod u+x /usr/bin/run.sh + +CMD "/usr/bin/run.sh" diff --git a/storage/rethinkdb/image/run.sh b/storage/rethinkdb/image/run.sh new file mode 100644 index 000000000..607eb59ed --- /dev/null +++ b/storage/rethinkdb/image/run.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +echo Checking for other nodes +IP="" +if [[ -n "${KUBERNETES_SERVICE_HOST}" ]]; then + + POD_NAMESPACE=${POD_NAMESPACE:-default} + MYHOST=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') + echo My host: ${MYHOST} + + URL="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/${POD_NAMESPACE}/endpoints/rethinkdb-driver" + echo "Endpont url: ${URL}" + echo "Looking for IPs..." + token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + # try to pick up first different ip from endpoints + IP=$(curl -s ${URL} --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer ${token}" \ + | jq -s -r --arg h "${MYHOST}" '.[0].subsets | .[].addresses | [ .[].ip ] | map(select(. != $h)) | .[0]') || exit 1 + [[ "${IP}" == null ]] && IP="" +fi + +if [[ -n "${IP}" ]]; then + ENDPOINT="${IP}:29015" + echo "Join to ${ENDPOINT}" + exec rethinkdb --bind all --join ${ENDPOINT} +else + echo "Start single instance" + exec rethinkdb --bind all +fi diff --git a/storage/rethinkdb/rc.yaml b/storage/rethinkdb/rc.yaml new file mode 100644 index 000000000..36b319191 --- /dev/null +++ b/storage/rethinkdb/rc.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + labels: + db: rethinkdb + name: rethinkdb-rc +spec: + replicas: 1 + selector: + db: rethinkdb + role: replicas + template: + metadata: + labels: + db: rethinkdb + role: replicas + spec: + containers: + - image: gcr.io/google_containers/rethinkdb:1.16.0_1 + name: rethinkdb + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8080 + name: admin-port + - containerPort: 28015 + name: driver-port + - containerPort: 29015 + name: cluster-port + volumeMounts: + - mountPath: /data/rethinkdb_data + name: rethinkdb-storage + volumes: + - name: rethinkdb-storage + emptyDir: {} diff --git a/storage/vitess/README.md b/storage/vitess/README.md new file mode 100644 index 000000000..0281f3df9 --- /dev/null +++ b/storage/vitess/README.md @@ -0,0 +1,113 @@ +## Vitess Example + +This example shows how to run a [Vitess](http://vitess.io) cluster in Kubernetes. +Vitess is a MySQL clustering system developed at YouTube that makes sharding +transparent to the application layer. It also makes scaling MySQL within +Kubernetes as simple as launching more pods. + +The example brings up a database with 2 shards, and then runs a pool of +[sharded guestbook](https://github.com/youtube/vitess/tree/master/examples/kubernetes/guestbook) +pods. The guestbook app was ported from the original +[guestbook](../../../examples/guestbook-go/) +example found elsewhere in this tree, modified to use Vitess as the backend. + +For a more detailed, step-by-step explanation of this example setup, see the +[Vitess on Kubernetes](http://vitess.io/getting-started/) guide. + +### Prerequisites + +You'll need to install [Go 1.4+](https://golang.org/doc/install) to build +`vtctlclient`, the command-line admin tool for Vitess. + +We also assume you have a running Kubernetes cluster with `kubectl` pointing to +it by default. See the [Getting Started guides](../../../docs/getting-started-guides/) +for how to get to that point. Note that your Kubernetes cluster needs to have +enough resources (CPU+RAM) to schedule all the pods. By default, this example +requires a cluster-wide total of at least 6 virtual CPUs and 10GiB RAM. You can +tune these requirements in the +[resource limits](../../../docs/user-guide/compute-resources.md) +section of each YAML file. + +Lastly, you need to open ports 30000-30001 (for the Vitess admin daemon) and 80 (for +the guestbook app) in your firewall. See the +[Services and Firewalls](../../../docs/user-guide/services-firewalls.md) +guide for examples of how to do that. + +### Configure site-local settings + +Run the `configure.sh` script to generate a `config.sh` file, which will be used +to customize your cluster settings. + +``` console +./configure.sh +``` + +Currently, we have out-of-the-box support for storing +[backups](http://vitess.io/user-guide/backup-and-restore.html) in +[Google Cloud Storage](https://cloud.google.com/storage/). +If you're using GCS, fill in the fields requested by the configure script. +Note that your Kubernetes cluster must be running on instances with the +`storage-rw` scope for this to work. With Container Engine, you can do this by +passing `--scopes storage-rw` to the `glcoud container clusters create` command. + +For other platforms, you'll need to choose the `file` backup storage plugin, +and mount a read-write network volume into the `vttablet` and `vtctld` pods. +For example, you can mount any storage service accessible through NFS into a +Kubernetes volume. Then provide the mount path to the configure script here. + +If you prefer to skip setting up a backup volume for the purpose of this example, +you can choose `file` mode and set the path to `/tmp`. + +### Start Vitess + +``` console +./vitess-up.sh +``` + +This will run through the steps to bring up Vitess. At the end, you should see +something like this: + +``` console +**************************** +* Complete! +* Use the following line to make an alias to kvtctl: +* alias kvtctl='$GOPATH/bin/vtctlclient -server 104.197.47.173:30001' +* See the vtctld UI at: http://104.197.47.173:30000 +**************************** +``` + +### Start the Guestbook app + +``` console +./guestbook-up.sh +``` + +The guestbook service is configured with `type: LoadBalancer` to tell Kubernetes +to expose it on an external IP. It may take a minute to set up, but you should +soon see the external IP show up under the internal one like this: + +``` console +$ kubectl get service guestbook +NAME LABELS SELECTOR IP(S) PORT(S) +guestbook name=guestbook 10.67.253.173 80/TCP + 104.197.151.132 +``` + +Visit the external IP in your browser to view the guestbook. Note that in this +modified guestbook, there are multiple pages to demonstrate range-based sharding +in Vitess. Each page number is assigned to one of the shards using a +[consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing) scheme. + +### Tear down + +``` console +./guestbook-down.sh +./vitess-down.sh +``` + +You may also want to remove any firewall rules you created. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storage/vitess/README.md?pixel)]() + diff --git a/storage/vitess/configure.sh b/storage/vitess/configure.sh new file mode 100755 index 000000000..7166c7fbb --- /dev/null +++ b/storage/vitess/configure.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script generates config.sh, which is a site-local config file that is not +# checked into source control. + +# Select and configure Backup Storage Implementation. +storage=gcs +read -p "Backup Storage (file, gcs) [gcs]: " +if [ -n "$REPLY" ]; then storage="$REPLY"; fi + +case "$storage" in +gcs) + # Google Cloud Storage + project=$(gcloud config list project | grep 'project\s*=' | sed -r 's/^.*=\s*(.*)$/\1/') + read -p "Google Developers Console Project [$project]: " + if [ -n "$REPLY" ]; then project="$REPLY"; fi + if [ -z "$project" ]; then + echo "ERROR: Project name must not be empty." + exit 1 + fi + + read -p "Google Cloud Storage bucket for Vitess backups: " bucket + if [ -z "$bucket" ]; then + echo "ERROR: Bucket name must not be empty." + exit 1 + fi + echo + echo "NOTE: If you haven't already created this bucket, you can do so by running:" + echo " gsutil mb gs://$bucket" + echo + + backup_flags=$(echo -backup_storage_implementation gcs \ + -gcs_backup_storage_project "'$project'" \ + -gcs_backup_storage_bucket "'$bucket'") + ;; +file) + # Mounted volume (e.g. NFS) + read -p "Root directory for backups (usually an NFS mount): " file_root + if [ -z "$file_root" ]; then + echo "ERROR: Root directory must not be empty." + exit 1 + fi + echo + echo "NOTE: You must add your NFS mount to the vtctld-controller-template" + echo " and vttablet-pod-template as described in the Kubernetes docs:" + echo " http://kubernetes.io/v1.0/docs/user-guide/volumes.html#nfs" + echo + + backup_flags=$(echo -backup_storage_implementation file \ + -file_backup_storage_root "'$file_root'") + ;; +*) + echo "ERROR: Unsupported backup storage implementation: $storage" + exit 1 +esac + +echo "Saving config.sh..." +echo "backup_flags=\"$backup_flags\"" > config.sh + diff --git a/storage/vitess/create_test_table.sql b/storage/vitess/create_test_table.sql new file mode 100644 index 000000000..0a6ef3609 --- /dev/null +++ b/storage/vitess/create_test_table.sql @@ -0,0 +1,8 @@ +CREATE TABLE messages ( + page BIGINT(20) UNSIGNED, + time_created_ns BIGINT(20) UNSIGNED, + keyspace_id BIGINT(20) UNSIGNED, + message VARCHAR(10000), + PRIMARY KEY (page, time_created_ns) +) ENGINE=InnoDB + diff --git a/storage/vitess/env.sh b/storage/vitess/env.sh new file mode 100644 index 000000000..49b06c800 --- /dev/null +++ b/storage/vitess/env.sh @@ -0,0 +1,63 @@ +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an include file used by the other scripts in this directory. + +# Most clusters will just be accessed with 'kubectl' on $PATH. +# However, some might require a different command. For example, GKE required +# KUBECTL='gcloud beta container kubectl' for a while. Now that most of our +# use cases just need KUBECTL=kubectl, we'll make that the default. +KUBECTL=${KUBECTL:-kubectl} + +# This should match the nodePort in vtctld-service.yaml +VTCTLD_PORT=${VTCTLD_PORT:-30001} + +# Customizable parameters +SHARDS=${SHARDS:-'-80,80-'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-2} +RDONLY_COUNT=${RDONLY_COUNT:-0} +MAX_TASK_WAIT_RETRIES=${MAX_TASK_WAIT_RETRIES:-300} +MAX_VTTABLET_TOPO_WAIT_RETRIES=${MAX_VTTABLET_TOPO_WAIT_RETRIES:-180} +VTTABLET_TEMPLATE=${VTTABLET_TEMPLATE:-'vttablet-pod-template.yaml'} +VTGATE_TEMPLATE=${VTGATE_TEMPLATE:-'vtgate-controller-template.yaml'} +VTGATE_COUNT=${VTGATE_COUNT:-1} +CELLS=${CELLS:-'test'} +ETCD_REPLICAS=3 + +VTGATE_REPLICAS=$VTGATE_COUNT + +# Get the ExternalIP of any node. +get_node_ip() { + $KUBECTL get -o template -t '{{range (index .items 0).status.addresses}}{{if eq .type "ExternalIP"}}{{.address}}{{end}}{{end}}' nodes +} + +# Try to find vtctld address if not provided. +get_vtctld_addr() { + if [ -z "$VTCTLD_ADDR" ]; then + node_ip=$(get_node_ip) + if [ -n "$node_ip" ]; then + VTCTLD_ADDR="$node_ip:$VTCTLD_PORT" + fi + fi + echo "$VTCTLD_ADDR" +} + +config_file=`dirname "${BASH_SOURCE}"`/config.sh +if [ ! -f $config_file ]; then + echo "Please run ./configure.sh first to generate config.sh file." + exit 1 +fi + +source $config_file + diff --git a/storage/vitess/etcd-controller-template.yaml b/storage/vitess/etcd-controller-template.yaml new file mode 100644 index 000000000..dcd7980bd --- /dev/null +++ b/storage/vitess/etcd-controller-template.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: etcd-{{cell}} +spec: + replicas: {{replicas}} + template: + metadata: + labels: + component: etcd + cell: {{cell}} + app: vitess + spec: + volumes: + - name: certs + hostPath: {path: /etc/ssl/certs} + containers: + - name: etcd + image: vitess/etcd:v2.0.13-lite + volumeMounts: + - name: certs + readOnly: true + mountPath: /etc/ssl/certs + resources: + limits: + memory: "128Mi" + cpu: "100m" + command: + - bash + - "-c" + - >- + ipaddr=$(hostname -i) + + global_etcd=$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + + cell="{{cell}}" && + local_etcd_host_var="ETCD_${cell^^}_SERVICE_HOST" && + local_etcd_port_var="ETCD_${cell^^}_SERVICE_PORT" && + local_etcd=${!local_etcd_host_var}:${!local_etcd_port_var} + + if [ "{{cell}}" != "global" ]; then + until etcdctl -C "http://$global_etcd" + set "/vt/cells/{{cell}}" "http://$local_etcd"; do + echo "[$(date)] waiting for global etcd to register cell '{{cell}}'"; + sleep 1; + done; + fi + + etcd -name $HOSTNAME -discovery {{discovery}} + -advertise-client-urls http://$ipaddr:4001 + -initial-advertise-peer-urls http://$ipaddr:7001 + -listen-client-urls http://$ipaddr:4001 + -listen-peer-urls http://$ipaddr:7001 + diff --git a/storage/vitess/etcd-down.sh b/storage/vitess/etcd-down.sh new file mode 100755 index 000000000..1f3ca258c --- /dev/null +++ b/storage/vitess/etcd-down.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that tears down the etcd servers started by +# etcd-up.sh. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +CELLS=${CELLS:-'test'} +cells=`echo $CELLS | tr ',' ' '` + +# Delete replication controllers +for cell in 'global' $cells; do + echo "Deleting etcd replicationcontroller for $cell cell..." + $KUBECTL delete replicationcontroller etcd-$cell + + echo "Deleting etcd service for $cell cell..." + $KUBECTL delete service etcd-$cell +done + diff --git a/storage/vitess/etcd-service-template.yaml b/storage/vitess/etcd-service-template.yaml new file mode 100644 index 000000000..817c3e132 --- /dev/null +++ b/storage/vitess/etcd-service-template.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: etcd-{{cell}} + labels: + component: etcd + cell: {{cell}} + app: vitess +spec: + ports: + - port: 4001 + selector: + component: etcd + cell: {{cell}} + app: vitess + diff --git a/storage/vitess/etcd-up.sh b/storage/vitess/etcd-up.sh new file mode 100755 index 000000000..b97e36900 --- /dev/null +++ b/storage/vitess/etcd-up.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates etcd clusters. +# Vitess requires a global cluster, as well as one for each cell. +# +# For automatic discovery, an etcd cluster can be bootstrapped from an +# existing cluster. In this example, we use an externally-run discovery +# service, but you can use your own. See the etcd docs for more: +# https://github.com/coreos/etcd/blob/v2.0.13/Documentation/clustering.md + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +replicas=${ETCD_REPLICAS:-3} + +CELLS=${CELLS:-'test'} +cells=`echo $CELLS | tr ',' ' '` + +for cell in 'global' $cells; do + # Generate a discovery token. + echo "Generating discovery token for $cell cell..." + discovery=$(curl -sL https://discovery.etcd.io/new?size=$replicas) + if [ -z "$discovery" ]; then + echo "Failed to get etcd discovery token for cell '$cell'." + exit 1 + fi + + # Create the client service, which will load-balance across all replicas. + echo "Creating etcd service for $cell cell..." + cat etcd-service-template.yaml | \ + sed -e "s/{{cell}}/$cell/g" | \ + $KUBECTL create -f - + + # Expand template variables + sed_script="" + for var in cell discovery replicas; do + sed_script+="s,{{$var}},${!var},g;" + done + + # Create the replication controller. + echo "Creating etcd replicationcontroller for $cell cell..." + cat etcd-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - +done + diff --git a/storage/vitess/guestbook-controller.yaml b/storage/vitess/guestbook-controller.yaml new file mode 100644 index 000000000..1c5ca5a18 --- /dev/null +++ b/storage/vitess/guestbook-controller.yaml @@ -0,0 +1,23 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: guestbook +spec: + replicas: 3 + template: + metadata: + labels: + component: guestbook + app: vitess + spec: + containers: + - name: guestbook + image: vitess/guestbook:v2.0.0-alpha5 + ports: + - name: http-server + containerPort: 8080 + resources: + limits: + memory: "128Mi" + cpu: "100m" + diff --git a/storage/vitess/guestbook-down.sh b/storage/vitess/guestbook-down.sh new file mode 100755 index 000000000..bac48ed32 --- /dev/null +++ b/storage/vitess/guestbook-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops guestbook. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Deleting guestbook replicationcontroller..." +$KUBECTL delete replicationcontroller guestbook + +echo "Deleting guestbook service..." +$KUBECTL delete service guestbook diff --git a/storage/vitess/guestbook-service.yaml b/storage/vitess/guestbook-service.yaml new file mode 100644 index 000000000..5435f7fa8 --- /dev/null +++ b/storage/vitess/guestbook-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: guestbook + labels: + component: guestbook + app: vitess +spec: + ports: + - port: 80 + targetPort: http-server + selector: + component: guestbook + app: vitess + type: LoadBalancer + diff --git a/storage/vitess/guestbook-up.sh b/storage/vitess/guestbook-up.sh new file mode 100755 index 000000000..b3afe9f05 --- /dev/null +++ b/storage/vitess/guestbook-up.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a guestbook replicationcontroller. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Creating guestbook service..." +$KUBECTL create -f guestbook-service.yaml + +echo "Creating guestbook replicationcontroller..." +$KUBECTL create -f guestbook-controller.yaml diff --git a/storage/vitess/vitess-down.sh b/storage/vitess/vitess-down.sh new file mode 100755 index 000000000..dc3884a35 --- /dev/null +++ b/storage/vitess/vitess-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +./vtgate-down.sh +SHARDS=$SHARDS CELLS=$CELLS TABLETS_PER_SHARD=$TABLETS_PER_SHARD ./vttablet-down.sh +./vtctld-down.sh +./etcd-down.sh diff --git a/storage/vitess/vitess-up.sh b/storage/vitess/vitess-up.sh new file mode 100755 index 000000000..4add4ee7e --- /dev/null +++ b/storage/vitess/vitess-up.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a fully functional vitess cluster. +# It performs the following steps: +# - Create etcd clusters +# - Create vtctld pod +# - Create vttablet pods +# - Perform vtctl initialization: +# SetKeyspaceShardingInfo, Rebuild Keyspace, Reparent Shard, Apply Schema +# - Create vtgate pods + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +cells=`echo $CELLS | tr ',' ' '` +num_cells=`echo $cells | wc -w` + +function update_spinner_value () { + spinner='-\|/' + cur_spinner=${spinner:$(($1%${#spinner})):1} +} + +function wait_for_running_tasks () { + # This function waits for pods to be in the "Running" state + # 1. task_name: Name that the desired task begins with + # 2. num_tasks: Number of tasks to wait for + # Returns: + # 0 if successful, -1 if timed out + task_name=$1 + num_tasks=$2 + counter=0 + + echo "Waiting for ${num_tasks}x $task_name to enter state Running" + + while [ $counter -lt $MAX_TASK_WAIT_RETRIES ]; do + # Get status column of pods with name starting with $task_name, + # count how many are in state Running + num_running=`$KUBECTL get pods | grep ^$task_name | grep Running | wc -l` + + echo -en "\r$task_name: $num_running out of $num_tasks in state Running..." + if [ $num_running -eq $num_tasks ] + then + echo Complete + return 0 + fi + update_spinner_value $counter + echo -n $cur_spinner + let counter=counter+1 + sleep 1 + done + echo Timed out + return -1 +} + +if [ -z "$GOPATH" ]; then + echo "ERROR: GOPATH undefined, can't obtain vtctlclient" + exit -1 +fi + +export KUBECTL='kubectl' + +echo "Downloading and installing vtctlclient..." +go get -u github.com/youtube/vitess/go/cmd/vtctlclient +num_shards=`echo $SHARDS | tr "," " " | wc -w` +total_tablet_count=$(($num_shards*$TABLETS_PER_SHARD*$num_cells)) +vtgate_count=$VTGATE_COUNT +if [ $vtgate_count -eq 0 ]; then + vtgate_count=$(($total_tablet_count/4>3?$total_tablet_count/4:3)) +fi + +echo "****************************" +echo "*Creating vitess cluster:" +echo "* Shards: $SHARDS" +echo "* Tablets per shard: $TABLETS_PER_SHARD" +echo "* Rdonly per shard: $RDONLY_COUNT" +echo "* VTGate count: $vtgate_count" +echo "* Cells: $cells" +echo "****************************" + +echo 'Running etcd-up.sh' && CELLS=$CELLS ./etcd-up.sh +wait_for_running_tasks etcd-global 3 +for cell in $cells; do + wait_for_running_tasks etcd-$cell 3 +done + +echo 'Running vtctld-up.sh' && ./vtctld-up.sh +echo 'Running vttablet-up.sh' && CELLS=$CELLS ./vttablet-up.sh +echo 'Running vtgate-up.sh' && ./vtgate-up.sh + +wait_for_running_tasks vtctld 1 +wait_for_running_tasks vttablet $total_tablet_count +wait_for_running_tasks vtgate $vtgate_count + +vtctld_port=30001 +vtctld_ip=`kubectl get -o yaml nodes | grep 'type: ExternalIP' -B 1 | head -1 | awk '{print $NF}'` +vtctl_server="$vtctld_ip:$vtctld_port" +kvtctl="$GOPATH/bin/vtctlclient -server $vtctl_server" + +echo Waiting for tablets to be visible in the topology +counter=0 +while [ $counter -lt $MAX_VTTABLET_TOPO_WAIT_RETRIES ]; do + num_tablets=0 + for cell in $cells; do + num_tablets=$(($num_tablets+`$kvtctl ListAllTablets $cell | wc -l`)) + done + echo -en "\r$num_tablets out of $total_tablet_count in topology..." + if [ $num_tablets -eq $total_tablet_count ] + then + echo Complete + break + fi + update_spinner_value $counter + echo -n $cur_spinner + let counter=counter+1 + sleep 1 + if [ $counter -eq $MAX_VTTABLET_TOPO_WAIT_RETRIES ] + then + echo Timed out + fi +done + +# split_shard_count = num_shards for sharded keyspace, 0 for unsharded +split_shard_count=$num_shards +if [ $split_shard_count -eq 1 ]; then + split_shard_count=0 +fi + +echo -n Setting Keyspace Sharding Info... +$kvtctl SetKeyspaceShardingInfo -force -split_shard_count $split_shard_count test_keyspace keyspace_id uint64 +echo Done +echo -n Rebuilding Keyspace Graph... +$kvtctl RebuildKeyspaceGraph test_keyspace +echo Done +echo -n Reparenting... +shard_num=1 +for shard in $(echo $SHARDS | tr "," " "); do + $kvtctl InitShardMaster -force test_keyspace/$shard `echo $cells | awk '{print $1}'`-0000000${shard_num}00 + let shard_num=shard_num+1 +done +echo Done +echo -n Applying Schema... +$kvtctl ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace +echo Done + +echo "****************************" +echo "* Complete!" +echo "* Use the following line to make an alias to kvtctl:" +echo "* alias kvtctl='\$GOPATH/bin/vtctlclient -server $vtctl_server'" +echo "* See the vtctld UI at: http://${vtctld_ip}:30000" +echo "****************************" + diff --git a/storage/vitess/vtctld-controller-template.yaml b/storage/vitess/vtctld-controller-template.yaml new file mode 100644 index 000000000..72fe245a2 --- /dev/null +++ b/storage/vitess/vtctld-controller-template.yaml @@ -0,0 +1,55 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: vtctld +spec: + replicas: 1 + template: + metadata: + labels: + component: vtctld + app: vitess + spec: + containers: + - name: vtctld + image: vitess/lite:v2.0.0-alpha5 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + - name: certs + readOnly: true + mountPath: /etc/ssl/certs + resources: + limits: + memory: "128Mi" + cpu: "100m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt && + su -p -c "/vt/bin/vtctld + -debug + -templates $VTTOP/go/cmd/vtctld/templates + -web_dir $VTTOP/web/vtctld + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port 15000 + -grpc_port 15001 + -service_map 'grpc-vtctl' + -topo_implementation etcd + -tablet_protocol grpc + -tablet_manager_protocol grpc + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + {{backup_flags}}" vitess + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} + - name: certs + hostPath: {path: /etc/ssl/certs} + diff --git a/storage/vitess/vtctld-down.sh b/storage/vitess/vtctld-down.sh new file mode 100755 index 000000000..d5a0dfb18 --- /dev/null +++ b/storage/vitess/vtctld-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtctld. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Deleting vtctld replicationcontroller..." +$KUBECTL delete replicationcontroller vtctld + +echo "Deleting vtctld service..." +$KUBECTL delete service vtctld diff --git a/storage/vitess/vtctld-service.yaml b/storage/vitess/vtctld-service.yaml new file mode 100644 index 000000000..70d619a87 --- /dev/null +++ b/storage/vitess/vtctld-service.yaml @@ -0,0 +1,22 @@ +kind: Service +apiVersion: v1 +metadata: + name: vtctld + labels: + component: vtctld + app: vitess +spec: + ports: + - port: 15000 + name: web + targetPort: 15000 + nodePort: 30000 + - port: 15001 + name: grpc + targetPort: 15001 + nodePort: 30001 + selector: + component: vtctld + app: vitess + type: NodePort + diff --git a/storage/vitess/vtctld-up.sh b/storage/vitess/vtctld-up.sh new file mode 100755 index 000000000..257b2d773 --- /dev/null +++ b/storage/vitess/vtctld-up.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts vtctld. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Creating vtctld service..." +$KUBECTL create -f vtctld-service.yaml + +echo "Creating vtctld replicationcontroller..." +# Expand template variables +sed_script="" +for var in backup_flags; do + sed_script+="s,{{$var}},${!var},g;" +done + +# Instantiate template and send to kubectl. +cat vtctld-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f - + +server=$(get_vtctld_addr) +echo +echo "vtctld address: http://$server" + diff --git a/storage/vitess/vtgate-controller-template.yaml b/storage/vitess/vtgate-controller-template.yaml new file mode 100644 index 000000000..03c9665b2 --- /dev/null +++ b/storage/vitess/vtgate-controller-template.yaml @@ -0,0 +1,45 @@ +kind: ReplicationController +apiVersion: v1 +metadata: + name: vtgate +spec: + replicas: {{replicas}} + template: + metadata: + labels: + component: vtgate + app: vitess + spec: + containers: + - name: vtgate + image: vitess/lite:v2.0.0-alpha5 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "512Mi" + cpu: "500m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt && + su -p -c "/vt/bin/vtgate + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port 15001 + -tablet_protocol grpc + -service_map 'bsonrpc-vt-vtgateservice' + -cell test" vitess + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} + diff --git a/storage/vitess/vtgate-down.sh b/storage/vitess/vtgate-down.sh new file mode 100755 index 000000000..cf72e840b --- /dev/null +++ b/storage/vitess/vtgate-down.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that stops vtgate. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +echo "Deleting vtgate replicationcontroller..." +$KUBECTL delete replicationcontroller vtgate + +echo "Deleting vtgate service..." +$KUBECTL delete service vtgate diff --git a/storage/vitess/vtgate-service.yaml b/storage/vitess/vtgate-service.yaml new file mode 100644 index 000000000..192968aa2 --- /dev/null +++ b/storage/vitess/vtgate-service.yaml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: vtgate + labels: + component: vtgate + app: vitess +spec: + ports: + - port: 15001 + selector: + component: vtgate + app: vitess + type: LoadBalancer + diff --git a/storage/vitess/vtgate-up.sh b/storage/vitess/vtgate-up.sh new file mode 100755 index 000000000..b7e327cc6 --- /dev/null +++ b/storage/vitess/vtgate-up.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that starts a vtgate replicationcontroller. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +VTGATE_REPLICAS=${VTGATE_REPLICAS:-3} +VTGATE_TEMPLATE=${VTGATE_TEMPLATE:-'vtgate-controller-template.yaml'} + +replicas=$VTGATE_REPLICAS + +echo "Creating vtgate service..." +$KUBECTL create -f vtgate-service.yaml + +sed_script="" +for var in replicas; do + sed_script+="s,{{$var}},${!var},g;" +done + +echo "Creating vtgate replicationcontroller..." +cat $VTGATE_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f - diff --git a/storage/vitess/vttablet-down.sh b/storage/vitess/vttablet-down.sh new file mode 100755 index 000000000..0683f1f7d --- /dev/null +++ b/storage/vitess/vttablet-down.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that tears down the vttablet pods started by +# vttablet-up.sh. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +server=$(get_vtctld_addr) + +# Delete the pods for all shards +CELLS=${CELLS:-'test'} +keyspace='test_keyspace' +SHARDS=${SHARDS:-'0'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5} +UID_BASE=${UID_BASE:-100} + +num_shards=`echo $SHARDS | tr "," " " | wc -w` +uid_base=$UID_BASE + +for shard in `seq 1 $num_shards`; do + cell_index=0 + for cell in `echo $CELLS | tr "," " "`; do + for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do + uid=$[$uid_base + $uid_index + $cell_index] + printf -v alias '%s-%010d' $cell $uid + + echo "Deleting pod for tablet $alias..." + $KUBECTL delete pod vttablet-$uid + done + let cell_index=cell_index+100000000 + done + let uid_base=uid_base+100 +done + diff --git a/storage/vitess/vttablet-pod-template.yaml b/storage/vitess/vttablet-pod-template.yaml new file mode 100644 index 000000000..d3d097e8f --- /dev/null +++ b/storage/vitess/vttablet-pod-template.yaml @@ -0,0 +1,128 @@ +kind: Pod +apiVersion: v1 +metadata: + name: vttablet-{{uid}} + labels: + component: vttablet + keyspace: "{{keyspace}}" + shard: "{{shard_label}}" + tablet: "{{alias}}" + app: vitess +spec: + containers: + - name: vttablet + image: vitess/lite:v2.0.0-alpha5 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + - name: certs + readOnly: true + mountPath: /etc/ssl/certs + resources: + limits: + memory: "1Gi" + cpu: "500m" + command: + - bash + - "-c" + - >- + set -e + + mysql_socket="$VTDATAROOT/{{tablet_subdir}}/mysql.sock" + + mkdir -p $VTDATAROOT/tmp + + chown -R vitess /vt + + while [ ! -e $mysql_socket ]; do + echo "[$(date)] waiting for $mysql_socket" ; + sleep 1 ; + done + + su -p -s /bin/bash -c "mysql -u vt_dba -S $mysql_socket + -e 'CREATE DATABASE IF NOT EXISTS vt_{{keyspace}}'" vitess + + su -p -s /bin/bash -c "/vt/bin/vttablet + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port {{port}} + -grpc_port {{grpc_port}} + -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' + -binlog_player_protocol grpc + -tablet-path {{alias}} + -tablet_hostname $(hostname -i) + -init_keyspace {{keyspace}} + -init_shard {{shard}} + -target_tablet_type {{tablet_type}} + -mysqlctl_socket $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app + -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba + -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl + -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered + -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 + -enable-rowcache + -rowcache-bin /usr/bin/memcached + -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock + -health_check_interval 5s + -restore_from_backup {{backup_flags}}" vitess + - name: mysql + image: vitess/lite:v2.0.0-alpha5 + volumeMounts: + - name: syslog + mountPath: /dev/log + - name: vtdataroot + mountPath: /vt/vtdataroot + resources: + limits: + memory: "1Gi" + cpu: "500m" + command: + - sh + - "-c" + - >- + mkdir -p $VTDATAROOT/tmp && + chown -R vitess /vt + + su -p -c "/vt/bin/mysqlctld + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -tablet_uid {{uid}} + -socket_file $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app + -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba + -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl + -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered + -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 + -bootstrap_archive mysql-db-dir_10.0.13-MariaDB.tbz" vitess + # The bootstrap archive above contains an empty mysql data dir + # with user permissions set up as required by Vitess. The archive is + # included in the Docker image. + env: + - name: EXTRA_MY_CNF + value: /vt/config/mycnf/master_mariadb.cnf + volumes: + - name: syslog + hostPath: {path: /dev/log} + - name: vtdataroot + emptyDir: {} + - name: certs + hostPath: {path: /etc/ssl/certs} + diff --git a/storage/vitess/vttablet-up.sh b/storage/vitess/vttablet-up.sh new file mode 100755 index 000000000..231b7589b --- /dev/null +++ b/storage/vitess/vttablet-up.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example script that creates a vttablet deployment. + +set -e + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +# Create the pods for shard-0 +CELLS=${CELLS:-'test'} +keyspace='test_keyspace' +SHARDS=${SHARDS:-'0'} +TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5} +port=15002 +grpc_port=16002 +UID_BASE=${UID_BASE:-100} +VTTABLET_TEMPLATE=${VTTABLET_TEMPLATE:-'vttablet-pod-template.yaml'} +RDONLY_COUNT=${RDONLY_COUNT:-2} + +uid_base=$UID_BASE +for shard in $(echo $SHARDS | tr "," " "); do + cell_index=0 + for cell in `echo $CELLS | tr ',' ' '`; do + echo "Creating $keyspace.shard-$shard pods in cell $CELL..." + for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do + uid=$[$uid_base + $uid_index + $cell_index] + printf -v alias '%s-%010d' $cell $uid + printf -v tablet_subdir 'vt_%010d' $uid + + echo "Creating pod for tablet $alias..." + + # Add xx to beginning or end if there is a dash. K8s does not allow for + # leading or trailing dashes for labels + shard_label=`echo $shard | sed s'/[-]$/-xx/' | sed s'/^-/xx-/'` + + tablet_type=replica + if [ $uid_index -gt $(($TABLETS_PER_SHARD-$RDONLY_COUNT-1)) ]; then + tablet_type=rdonly + fi + + # Expand template variables + sed_script="" + for var in alias cell uid keyspace shard shard_label port grpc_port tablet_subdir tablet_type backup_flags; do + sed_script+="s,{{$var}},${!var},g;" + done + + # Instantiate template and send to kubectl. + cat $VTTABLET_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f - + done + let cell_index=cell_index+100000000 + done + let uid_base=uid_base+100 +done diff --git a/storm/README.md b/storm/README.md new file mode 100644 index 000000000..b5b873f93 --- /dev/null +++ b/storm/README.md @@ -0,0 +1,172 @@ +# Storm example + +Following this example, you will create a functional [Apache +Storm](http://storm.apache.org/) cluster using Kubernetes and +[Docker](http://docker.io). + +You will setup an [Apache ZooKeeper](http://zookeeper.apache.org/) +service, a Storm master service (a.k.a. Nimbus server), and a set of +Storm workers (a.k.a. supervisors). + +For the impatient expert, jump straight to the [tl;dr](#tldr) +section. + +### Sources + +Source is freely available at: +* Docker image - https://github.com/mattf/docker-storm +* Docker Trusted Build - https://registry.hub.docker.com/search?q=mattf/storm + +## Step Zero: Prerequisites + +This example assumes you have a Kubernetes cluster installed and +running, and that you have installed the ```kubectl``` command line +tool somewhere in your path. Please see the [getting +started](../../docs/getting-started-guides/) for installation +instructions for your platform. + +## Step One: Start your ZooKeeper service + +ZooKeeper is a distributed coordination [service](../../docs/user-guide/services.md) that Storm uses as a +bootstrap and for state storage. + +Use the [`examples/storm/zookeeper.json`](zookeeper.json) file to create a [pod](../../docs/user-guide/pods.md) running +the ZooKeeper service. + +```sh +$ kubectl create -f examples/storm/zookeeper.json +``` + +Then, use the [`examples/storm/zookeeper-service.json`](zookeeper-service.json) file to create a +logical service endpoint that Storm can use to access the ZooKeeper +pod. + +```sh +$ kubectl create -f examples/storm/zookeeper-service.json +``` + +You should make sure the ZooKeeper pod is Running and accessible +before proceeding. + +### Check to see if ZooKeeper is running + +```sh +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +zookeeper 1/1 Running 0 43s +``` + +### Check to see if ZooKeeper is accessible + +```console +$ kubectl get services +NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE +zookeeper 10.254.139.141 2181/TCP name=zookeeper 10m +kubernetes 10.0.0.2 443/TCP 1d + +$ echo ruok | nc 10.254.139.141 2181; echo +imok +``` + +## Step Two: Start your Nimbus service + +The Nimbus service is the master (or head) service for a Storm +cluster. It depends on a functional ZooKeeper service. + +Use the [`examples/storm/storm-nimbus.json`](storm-nimbus.json) file to create a pod running +the Nimbus service. + +```sh +$ kubectl create -f examples/storm/storm-nimbus.json +``` + +Then, use the [`examples/storm/storm-nimbus-service.json`](storm-nimbus-service.json) file to +create a logical service endpoint that Storm workers can use to access +the Nimbus pod. + +```sh +$ kubectl create -f examples/storm/storm-nimbus-service.json +``` + +Ensure that the Nimbus service is running and functional. + +### Check to see if Nimbus is running and accessible + +```sh +$ kubectl get services +NAME LABELS SELECTOR IP(S) PORT(S) +kubernetes component=apiserver,provider=kubernetes 10.254.0.2 443 +zookeeper name=zookeeper name=zookeeper 10.254.139.141 2181 +nimbus name=nimbus name=nimbus 10.254.115.208 6627 + +$ sudo docker run -it -w /opt/apache-storm mattf/storm-base sh -c '/configure.sh 10.254.139.141 10.254.115.208; ./bin/storm list' +... +No topologies running. +``` + +## Step Three: Start your Storm workers + +The Storm workers (or supervisors) do the heavy lifting in a Storm +cluster. They run your stream processing topologies and are managed by +the Nimbus service. + +The Storm workers need both the ZooKeeper and Nimbus services to be +running. + +Use the [`examples/storm/storm-worker-controller.json`](storm-worker-controller.json) file to create a +[replication controller](../../docs/user-guide/replication-controller.md) that manages the worker pods. + +```sh +$ kubectl create -f examples/storm/storm-worker-controller.json +``` + +### Check to see if the workers are running + +One way to check on the workers is to get information from the +ZooKeeper service about how many clients it has. + +```sh +$ echo stat | nc 10.254.139.141 2181; echo +Zookeeper version: 3.4.6--1, built on 10/23/2014 14:18 GMT +Clients: + /192.168.48.0:44187[0](queued=0,recved=1,sent=0) + /192.168.45.0:39568[1](queued=0,recved=14072,sent=14072) + /192.168.86.1:57591[1](queued=0,recved=34,sent=34) + /192.168.8.0:50375[1](queued=0,recved=34,sent=34) + +Latency min/avg/max: 0/2/2570 +Received: 23199 +Sent: 23198 +Connections: 4 +Outstanding: 0 +Zxid: 0xa39 +Mode: standalone +Node count: 13 +``` + +There should be one client from the Nimbus service and one per +worker. Ideally, you should get ```stat``` output from ZooKeeper +before and after creating the replication controller. + +(Pull requests welcome for alternative ways to validate the workers) + +## tl;dr + +```kubectl create -f zookeeper.json``` + +```kubectl create -f zookeeper-service.json``` + +Make sure the ZooKeeper Pod is running (use: ```kubectl get pods```). + +```kubectl create -f storm-nimbus.json``` + +```kubectl create -f storm-nimbus-service.json``` + +Make sure the Nimbus Pod is running. + +```kubectl create -f storm-worker-controller.json``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/storm/README.md?pixel)]() + diff --git a/storm/storm-nimbus-service.json b/storm/storm-nimbus-service.json new file mode 100644 index 000000000..2a8f71349 --- /dev/null +++ b/storm/storm-nimbus-service.json @@ -0,0 +1,20 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "nimbus", + "labels": { + "name": "nimbus" + } + }, + "spec": { + "ports": [ + { + "port": 6627 + } + ], + "selector": { + "name": "nimbus" + } + } +} diff --git a/storm/storm-nimbus.json b/storm/storm-nimbus.json new file mode 100644 index 000000000..e9a2f927a --- /dev/null +++ b/storm/storm-nimbus.json @@ -0,0 +1,28 @@ +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "nimbus", + "labels": { + "name": "nimbus" + } + }, + "spec": { + "containers": [ + { + "name": "nimbus", + "image": "mattf/storm-nimbus", + "ports": [ + { + "containerPort": 6627 + } + ], + "resources": { + "limits": { + "cpu": "100m" + } + } + } + ] + } +} diff --git a/storm/storm-worker-controller.json b/storm/storm-worker-controller.json new file mode 100644 index 000000000..0806b2745 --- /dev/null +++ b/storm/storm-worker-controller.json @@ -0,0 +1,55 @@ +{ + "kind": "ReplicationController", + "apiVersion": "v1", + "metadata": { + "name": "storm-worker-controller", + "labels": { + "name": "storm-worker" + } + }, + "spec": { + "replicas": 2, + "selector": { + "name": "storm-worker" + }, + "template": { + "metadata": { + "labels": { + "name": "storm-worker", + "uses": "nimbus" + } + }, + "spec": { + "containers": [ + { + "name": "storm-worker", + "image": "mattf/storm-worker", + "ports": [ + { + "hostPort": 6700, + "containerPort": 6700 + }, + { + "hostPort": 6701, + "containerPort": 6701 + }, + { + "hostPort": 6702, + "containerPort": 6702 + }, + { + "hostPort": 6703, + "containerPort": 6703 + } + ], + "resources": { + "limits": { + "cpu": "200m" + } + } + } + ] + } + } + } +} diff --git a/storm/zookeeper-service.json b/storm/zookeeper-service.json new file mode 100644 index 000000000..bbfc515bd --- /dev/null +++ b/storm/zookeeper-service.json @@ -0,0 +1,20 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "zookeeper", + "labels": { + "name": "zookeeper" + } + }, + "spec": { + "ports": [ + { + "port": 2181 + } + ], + "selector": { + "name": "zookeeper" + } + } +} diff --git a/storm/zookeeper.json b/storm/zookeeper.json new file mode 100644 index 000000000..89b82b7bc --- /dev/null +++ b/storm/zookeeper.json @@ -0,0 +1,28 @@ +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "zookeeper", + "labels": { + "name": "zookeeper" + } + }, + "spec": { + "containers": [ + { + "name": "zookeeper", + "image": "mattf/zookeeper", + "ports": [ + { + "containerPort": 2181 + } + ], + "resources": { + "limits": { + "cpu": "100m" + } + } + } + ] + } +} diff --git a/sysdig-cloud/README.md b/sysdig-cloud/README.md new file mode 100644 index 000000000..14d431503 --- /dev/null +++ b/sysdig-cloud/README.md @@ -0,0 +1,27 @@ +[Sysdig Cloud](http://www.sysdig.com/) is a monitoring, alerting, and troubleshooting platform designed to natively support containerized and service-oriented applications. + +Sysdig Cloud comes with built-in, first class support for Kubernetes. In order to instrument your Kubernetes environment with Sysdig Cloud, you simply need to install the Sysdig Cloud agent container on each underlying host in your Kubernetes cluster. Sysdig Cloud will automatically begin monitoring all of your hosts, apps, pods, and services, and will also automatically connect to the Kubernetes API to pull relevant metadata about your environment. + +# Example Installation Files + +Provided here are two example sysdig.yaml files that can be used to automatically deploy the Sysdig Cloud agent container across a Kubernetes cluster. + +The recommended method is using daemon sets - minimum kubernetes version 1.1.1. + +If daemon sets are not available, then the replication controller method can be used (based on [this hack](https://stackoverflow.com/questions/33377054/how-to-require-one-pod-per-minion-kublet-when-configuring-a-replication-controll/33381862#33381862 )). + +# Latest Files + +See here for the latest maintained and updated versions of these example files: +https://github.com/draios/sysdig-cloud-scripts/tree/master/agent_deploy/kubernetes + +# Install instructions + +Please see the Sysdig Cloud support site for the latest documentation: +http://support.sysdigcloud.com/hc/en-us/sections/200959909 + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/sysdig-cloud/README.md?pixel)]() + diff --git a/sysdig-cloud/sysdig-daemonset.yaml b/sysdig-cloud/sysdig-daemonset.yaml new file mode 100644 index 000000000..e1fc1534a --- /dev/null +++ b/sysdig-cloud/sysdig-daemonset.yaml @@ -0,0 +1,72 @@ +#Use this sysdig.yaml when Daemon Sets are enabled on Kubernetes (minimum version 1.1.1). Otherwise use the RC method. + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: sysdig-agent + labels: + app: sysdig-agent +spec: + template: + metadata: + labels: + name: sysdig-agent + spec: + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + - name: dev-vol + hostPath: + path: /dev + - name: proc-vol + hostPath: + path: /proc + - name: boot-vol + hostPath: + path: /boot + - name: modules-vol + hostPath: + path: /lib/modules + - name: usr-vol + hostPath: + path: /usr + hostNetwork: true + hostPID: true + containers: + - name: sysdig-agent + image: sysdig/agent + securityContext: + privileged: true + env: + - name: ACCESS_KEY #REQUIRED - replace with your Sysdig Cloud access key + value: 8312341g-5678-abcd-4a2b2c-33bcsd655 +# - name: TAGS #OPTIONAL +# value: linux:ubuntu,dept:dev,local:nyc +# - name: COLLECTOR #OPTIONAL - on-prem install only +# value: 192.168.183.200 +# - name: SECURE #OPTIONAL - on-prem install only +# value: false +# - name: CHECK_CERTIFICATE #OPTIONAL - on-prem install only +# value: false +# - name: ADDITIONAL_CONF #OPTIONAL pass additional parameters to the agent such as authentication example provided here +# value: "k8s_uri: https://myacct:mypass@localhost:4430\nk8s_ca_certificate: k8s-ca.crt\nk8s_ssl_verify_certificate: true" + volumeMounts: + - mountPath: /host/var/run/docker.sock + name: docker-sock + readOnly: false + - mountPath: /host/dev + name: dev-vol + readOnly: false + - mountPath: /host/proc + name: proc-vol + readOnly: true + - mountPath: /host/boot + name: boot-vol + readOnly: true + - mountPath: /host/lib/modules + name: modules-vol + readOnly: true + - mountPath: /host/usr + name: usr-vol + readOnly: true diff --git a/sysdig-cloud/sysdig-rc.yaml b/sysdig-cloud/sysdig-rc.yaml new file mode 100644 index 000000000..d088cd535 --- /dev/null +++ b/sysdig-cloud/sysdig-rc.yaml @@ -0,0 +1,77 @@ +#Use this sysdig.yaml when Daemon Sets are NOT enabled on Kubernetes (minimum version 1.1.1). If Daemon Sets are available, use the other example sysdig.yaml - that is the recommended method. + +apiVersion: v1 +kind: ReplicationController +metadata: + name: sysdig-agent + labels: + app: sysdig-agent +spec: + replicas: 100 #REQUIRED - replace with the maximum number of slave nodes in the cluster + template: + spec: + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + - name: dev-vol + hostPath: + path: /dev + - name: proc-vol + hostPath: + path: /proc + - name: boot-vol + hostPath: + path: /boot + - name: modules-vol + hostPath: + path: /lib/modules + - name: usr-vol + hostPath: + path: /usr + hostNetwork: true + hostPID: true + containers: + - name: sysdig-agent + image: sysdig/agent + ports: + - containerPort: 6666 + hostPort: 6666 + securityContext: + privileged: true + env: + - name: ACCESS_KEY #REQUIRED - replace with your Sysdig Cloud access key + value: 8312341g-5678-abcd-4a2b2c-33bcsd655 +# - name: K8S_DELEGATED_NODE #OPTIONAL - only necessary when connecting remotely to API server +# value: +# - name: K8S_API_URI #OPTIONAL - only necessary when connecting remotely to API server +# value: "http[s]://[username:passwd@]host[:port]" +# - name: TAGS #OPTIONAL +# value: linux:ubuntu,dept:dev,local:nyc +# - name: COLLECTOR #OPTIONAL +# value: 192.168.183.200 +# - name: SECURE #OPTIONAL +# value: false +# - name: CHECK_CERTIFICATE #OPTIONAL +# value: false +# - name: ADDITIONAL_CONF #OPTIONAL +# value: "app_checks:\n - name: nginx\n check_module: nginx\n pattern:\n comm: nginx\n conf:\n nginx_status_url: "http://localhost:{port}/nginx_status\"" + volumeMounts: + - mountPath: /host/var/run/docker.sock + name: docker-sock + readOnly: false + - mountPath: /host/dev + name: dev-vol + readOnly: false + - mountPath: /host/proc + name: proc-vol + readOnly: true + - mountPath: /host/boot + name: boot-vol + readOnly: true + - mountPath: /host/lib/modules + name: modules-vol + readOnly: true + - mountPath: /host/usr + name: usr-vol + readOnly: true diff --git a/volumes/aws_ebs/README.md b/volumes/aws_ebs/README.md new file mode 100644 index 000000000..87ee0fc76 --- /dev/null +++ b/volumes/aws_ebs/README.md @@ -0,0 +1,37 @@ +This is a simple web server pod which serves HTML from an AWS EBS +volume. + +If you did not use kube-up script, make sure that your minions have the following IAM permissions ([Amazon IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role-console)): + +```shell + ec2:AttachVolume + ec2:DetachVolume + ec2:DescribeInstances + ec2:DescribeVolumes +``` + +Create a volume in the same region as your node. + +Add your volume information in the pod description file aws-ebs-web.yaml then create the pod: + +```shell + $ kubectl create -f examples/volumes/aws_ebs/aws-ebs-web.yaml +``` + +Add some data to the volume if is empty: + +```sh + $ echo "Hello World" >& /var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/{Region}/{Volume ID}/index.html +``` + +You should now be able to query your web server: + +```sh + $ curl + $ Hello World +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/aws_ebs/README.md?pixel)]() + diff --git a/volumes/aws_ebs/aws-ebs-web.yaml b/volumes/aws_ebs/aws-ebs-web.yaml new file mode 100644 index 000000000..56667f538 --- /dev/null +++ b/volumes/aws_ebs/aws-ebs-web.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: aws-web +spec: + containers: + - name: web + image: nginx + ports: + - name: web + containerPort: 80 + protocol: tcp + volumeMounts: + - name: html-volume + mountPath: "/usr/share/nginx/html" + volumes: + - name: html-volume + awsElasticBlockStore: + # Enter the volume ID below + volumeID: volume_ID + fsType: ext4 diff --git a/volumes/azure_disk/README.md b/volumes/azure_disk/README.md new file mode 100644 index 000000000..35edc7ca8 --- /dev/null +++ b/volumes/azure_disk/README.md @@ -0,0 +1,22 @@ +# How to Use it? + +On Azure VM, create a Pod using the volume spec based on [azure](azure.yaml). + +In the pod, you need to provide the following information: + +- *diskName*: (required) the name of the VHD blob object. +- *diskURI*: (required) the URI of the vhd blob object. +- *cachingMode*: (optional) disk caching mode. Must be one of None, ReadOnly, or ReadWrite. Default is None. +- *fsType*: (optional) the filesytem type to mount. Default is ext4. +- *readOnly*: (optional) whether the filesystem is used as readOnly. Default is false. + + +Launch the Pod: + +```console + # kubectl create -f examples/volumes/azure_disk/azure.yaml +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/azure_disk/README.md?pixel)]() + diff --git a/volumes/azure_disk/azure.yaml b/volumes/azure_disk/azure.yaml new file mode 100644 index 000000000..04df7fb9b --- /dev/null +++ b/volumes/azure_disk/azure.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: azure +spec: + containers: + - image: kubernetes/pause + name: azure + volumeMounts: + - name: azure + mountPath: /mnt/azure + volumes: + - name: azure + azureDisk: + diskName: test.vhd + diskURI: https://someaccount.blob.microsoft.net/vhds/test.vhd diff --git a/volumes/azure_file/README.md b/volumes/azure_file/README.md new file mode 100644 index 000000000..165a58166 --- /dev/null +++ b/volumes/azure_file/README.md @@ -0,0 +1,35 @@ +# How to Use it? + +Install *cifs-utils* on the Kubernetes host. For example, on Fedora based Linux + + # yum -y install cifs-utils + +Note, as explained in [Azure File Storage for Linux](https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/), the Linux hosts and the file share must be in the same Azure region. + +Obtain an Microsoft Azure storage account and create a [secret](secret/azure-secret.yaml) that contains the base64 encoded Azure Storage account name and key. In the secret file, base64-encode Azure Storage account name and pair it with name *azurestorageaccountname*, and base64-encode Azure Storage access key and pair it with name *azurestorageaccountkey*. + +Then create a Pod using the volume spec based on [azure](azure.yaml). + +In the pod, you need to provide the following information: + +- *secretName*: the name of the secret that contains both Azure storage account name and key. +- *shareName*: The share name to be used. +- *readOnly*: Whether the filesystem is used as readOnly. + +Create the secret: + +```console + # kubectl create -f examples/volumes/azure_file/secret/azure-secret.yaml +``` + +You should see the account name and key from `kubectl get secret` + +Then create the Pod: + +```console + # kubectl create -f examples/volumes/azure_file/azure.yaml +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/azure_file/README.md?pixel)]() + diff --git a/volumes/azure_file/azure.yaml b/volumes/azure_file/azure.yaml new file mode 100644 index 000000000..6567f3079 --- /dev/null +++ b/volumes/azure_file/azure.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: azure +spec: + containers: + - image: kubernetes/pause + name: azure + volumeMounts: + - name: azure + mountPath: /mnt/azure + volumes: + - name: azure + azureFile: + secretName: azure-secret + shareName: k8stest + readOnly: false diff --git a/volumes/azure_file/secret/azure-secret.yaml b/volumes/azure_file/secret/azure-secret.yaml new file mode 100644 index 000000000..bf448bd9c --- /dev/null +++ b/volumes/azure_file/secret/azure-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: azure-secret +type: Opaque +data: + azurestorageaccountname: azhzdGVzdA== + azurestorageaccountkey: eElGMXpKYm5ub2pGTE1Ta0JwNTBteDAyckhzTUsyc2pVN21GdDRMMTNob0I3ZHJBYUo4akQ2K0E0NDNqSm9nVjd5MkZVT2hRQ1dQbU02WWFOSHk3cWc9PQ== diff --git a/volumes/cephfs/README.md b/volumes/cephfs/README.md new file mode 100644 index 000000000..ce6263cda --- /dev/null +++ b/volumes/cephfs/README.md @@ -0,0 +1,38 @@ +# How to Use it? + +Install Ceph on the Kubernetes host. For example, on Fedora 21 + + # yum -y install ceph + +If you don't have a Ceph cluster, you can set up a [containerized Ceph cluster](https://github.com/ceph/ceph-docker/tree/master/examples/kubernetes) + +Then get the keyring from the Ceph cluster and copy it to */etc/ceph/keyring*. + +Once you have installed Ceph and a Kubernetes cluster, you can create a pod based on my examples [cephfs.yaml](cephfs.yaml) and [cephfs-with-secret.yaml](cephfs-with-secret.yaml). In the pod yaml, you need to provide the following information. + +- *monitors*: Array of Ceph monitors. +- *path*: Used as the mounted root, rather than the full Ceph tree. If not provided, default */* is used. +- *user*: The RADOS user name. If not provided, default *admin* is used. +- *secretFile*: The path to the keyring file. If not provided, default */etc/ceph/user.secret* is used. +- *secretRef*: Reference to Ceph authentication secrets. If provided, *secret* overrides *secretFile*. +- *readOnly*: Whether the filesystem is used as readOnly. + + +Here are the commands: + +```console + # kubectl create -f examples/volumes/cephfs/cephfs.yaml + + # create a secret if you want to use Ceph secret instead of secret file + # kubectl create -f examples/volumes/cephfs/secret/ceph-secret.yaml + + # kubectl create -f examples/volumes/cephfs/cephfs-with-secret.yaml + # kubectl get pods +``` + + If you ssh to that machine, you can run `docker ps` to see the actual pod and `docker inspect` to see the volumes used by the container. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/cephfs/README.md?pixel)]() + diff --git a/volumes/cephfs/cephfs-with-secret.yaml b/volumes/cephfs/cephfs-with-secret.yaml new file mode 100644 index 000000000..c3d7a02cb --- /dev/null +++ b/volumes/cephfs/cephfs-with-secret.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cephfs2 +spec: + containers: + - name: cephfs-rw + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/cephfs" + name: cephfs + volumes: + - name: cephfs + cephfs: + monitors: + - 10.16.154.78:6789 + - 10.16.154.82:6789 + - 10.16.154.83:6789 + user: admin + secretRef: + name: ceph-secret + readOnly: true diff --git a/volumes/cephfs/cephfs.yaml b/volumes/cephfs/cephfs.yaml new file mode 100644 index 000000000..e4eb395b6 --- /dev/null +++ b/volumes/cephfs/cephfs.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cephfs +spec: + containers: + - name: cephfs-rw + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/cephfs" + name: cephfs + volumes: + - name: cephfs + cephfs: + monitors: + - 10.16.154.78:6789 + - 10.16.154.82:6789 + - 10.16.154.83:6789 + # by default the path is /, but you can override and mount a specific path of the filesystem by using the path attribute + # path: /some/path/in/side/cephfs + user: admin + secretFile: "/etc/ceph/admin.secret" + readOnly: true diff --git a/volumes/cephfs/secret/ceph-secret.yaml b/volumes/cephfs/secret/ceph-secret.yaml new file mode 100644 index 000000000..e29a5535a --- /dev/null +++ b/volumes/cephfs/secret/ceph-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ceph-secret +data: + key: QVFCMTZWMVZvRjVtRXhBQTVrQ1FzN2JCajhWVUxSdzI2Qzg0SEE9PQ== diff --git a/volumes/fibre_channel/README.md b/volumes/fibre_channel/README.md new file mode 100644 index 000000000..0e1bcf9b3 --- /dev/null +++ b/volumes/fibre_channel/README.md @@ -0,0 +1,73 @@ +## Step 1. Setting up Fibre Channel Target + +On your FC SAN Zone manager, allocate and mask LUNs so Kubernetes hosts can access them. + +## Step 2. Creating the Pod with Fibre Channel persistent storage + +Once you have installed Fibre Channel initiator and new Kubernetes, you can create a pod based on my example [fc.yaml](fc.yaml). In the pod JSON, you need to provide *targetWWNs* (array of Fibre Channel target's World Wide Names), *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. + +Once your pod is created, run it on the Kubernetes master: + +```console +kubectl create -f ./your_new_pod.json +``` + +Here is my command and output: + +```console +# kubectl create -f examples/volumes/fibre_channel/fc.yaml +# kubectl get pods +NAME READY STATUS RESTARTS AGE +fcpd 2/2 Running 0 10m +``` + +On the Kubernetes host, I got these in mount output + +```console +#mount |grep /var/lib/kubelet/plugins/kubernetes.io +/dev/mapper/360a98000324669436c2b45666c567946 on /var/lib/kubelet/plugins/kubernetes.io/fc/500a0982991b8dc5-lun-2 type ext4 (ro,relatime,seclabel,stripe=16,data=ordered) +/dev/mapper/360a98000324669436c2b45666c567944 on /var/lib/kubelet/plugins/kubernetes.io/fc/500a0982991b8dc5-lun-1 type ext4 (rw,relatime,seclabel,stripe=16,data=ordered) +``` + +If you ssh to that machine, you can run `docker ps` to see the actual pod. + +```console +# docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +090ac457ddc2 kubernetes/pause "/pause" 12 minutes ago Up 12 minutes k8s_fcpd-rw.aae720ec_fcpd_default_4024318f-4121-11e5-a294-e839352ddd54_99eb5415 +5e2629cf3e7b kubernetes/pause "/pause" 12 minutes ago Up 12 minutes k8s_fcpd-ro.857720dc_fcpd_default_4024318f-4121-11e5-a294-e839352ddd54_c0175742 +2948683253f7 gcr.io/google_containers/pause:0.8.0 "/pause" 12 minutes ago Up 12 minutes k8s_POD.7be6d81d_fcpd_default_4024318f-4121-11e5-a294-e839352ddd54_8d9dd7bf +``` + +## Multipath + +To leverage multiple paths for block storage, it is important to perform the +multipath configuration on the host. +If your distribution does not provide `/etc/multipath.conf`, then you can +either use the following minimalistic one: + + defaults { + find_multipaths yes + user_friendly_names yes + } + +or create a new one by running: + + $ mpathconf --enable + +Finally you'll need to ensure to start or reload and enable multipath: + + $ systemctl enable multipathd.service + $ systemctl restart multipathd.service + +**Note:** Any change to `multipath.conf` or enabling multipath can lead to +inaccessible block devices, because they'll be claimed by multipath and +exposed as a device in /dev/mapper/*. + +Some additional informations about multipath can be found in the +[iSCSI documentation](../iscsi/README.md) + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/fibre_channel/README.md?pixel)]() + diff --git a/volumes/fibre_channel/fc.yaml b/volumes/fibre_channel/fc.yaml new file mode 100644 index 000000000..ac28bee4a --- /dev/null +++ b/volumes/fibre_channel/fc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: fc +spec: + containers: + - image: kubernetes/pause + name: fc + volumeMounts: + - name: fc-vol + mountPath: /mnt/fc + volumes: + - name: fc-vol + fc: + targetWWNs: ['500a0982991b8dc5', '500a0982891b8dc5'] + lun: 2 + fsType: ext4 + readOnly: true diff --git a/volumes/flexvolume/README.md b/volumes/flexvolume/README.md new file mode 100644 index 000000000..8f88b690c --- /dev/null +++ b/volumes/flexvolume/README.md @@ -0,0 +1 @@ +Please refer to https://github.com/kubernetes/community/tree/master/contributors/devel/flexvolume.md for documentation. \ No newline at end of file diff --git a/volumes/flexvolume/lvm b/volumes/flexvolume/lvm new file mode 100755 index 000000000..04a207789 --- /dev/null +++ b/volumes/flexvolume/lvm @@ -0,0 +1,197 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Notes: +# - Please install "jq" package before using this driver. +usage() { + err "Invalid usage. Usage: " + err "\t$0 init" + err "\t$0 attach " + err "\t$0 detach " + err "\t$0 waitforattach " + err "\t$0 mountdevice " + err "\t$0 unmountdevice " + err "\t$0 getvolumename " + err "\t$0 isattached " + exit 1 +} + +err() { + echo -ne $* 1>&2 +} + +log() { + echo -ne $* >&1 +} + +ismounted() { + MOUNT=`findmnt -n ${MNTPATH} 2>/dev/null | cut -d' ' -f1` + if [ "${MOUNT}" == "${MNTPATH}" ]; then + echo "1" + else + echo "0" + fi +} + +getdevice() { + VOLUMEID=$(echo ${JSON_PARAMS} | jq -r '.volumeID') + VG=$(echo ${JSON_PARAMS}|jq -r '.volumegroup') + + # LVM substitutes - with -- + VOLUMEID=`echo $VOLUMEID|sed s/-/--/g` + VG=`echo $VG|sed s/-/--/g` + + DMDEV="/dev/mapper/${VG}-${VOLUMEID}" + echo ${DMDEV} +} + +attach() { + JSON_PARAMS=$1 + SIZE=$(echo $1 | jq -r '.size') + + DMDEV=$(getdevice) + if [ ! -b "${DMDEV}" ]; then + err "{\"status\": \"Failure\", \"message\": \"Volume ${VOLUMEID} does not exist\"}" + exit 1 + fi + log "{\"status\": \"Success\", \"device\":\"${DMDEV}\"}" + exit 0 +} + +detach() { + log "{\"status\": \"Success\"}" + exit 0 +} + +waitforattach() { + shift + attach $* +} + +domountdevice() { + MNTPATH=$1 + DMDEV=$2 + FSTYPE=$(echo $3|jq -r '.["kubernetes.io/fsType"]') + + if [ ! -b "${DMDEV}" ]; then + err "{\"status\": \"Failure\", \"message\": \"${DMDEV} does not exist\"}" + exit 1 + fi + + if [ $(ismounted) -eq 1 ] ; then + log "{\"status\": \"Success\"}" + exit 0 + fi + + VOLFSTYPE=`blkid -o udev ${DMDEV} 2>/dev/null|grep "ID_FS_TYPE"|cut -d"=" -f2` + if [ "${VOLFSTYPE}" == "" ]; then + mkfs -t ${FSTYPE} ${DMDEV} >/dev/null 2>&1 + if [ $? -ne 0 ]; then + err "{ \"status\": \"Failure\", \"message\": \"Failed to create fs ${FSTYPE} on device ${DMDEV}\"}" + exit 1 + fi + fi + + mkdir -p ${MNTPATH} &> /dev/null + + mount ${DMDEV} ${MNTPATH} &> /dev/null + if [ $? -ne 0 ]; then + err "{ \"status\": \"Failure\", \"message\": \"Failed to mount device ${DMDEV} at ${MNTPATH}\"}" + exit 1 + fi + log "{\"status\": \"Success\"}" + exit 0 +} + +unmountdevice() { + MNTPATH=$1 + if [ ! -d ${MNTPATH} ]; then + log "{\"status\": \"Success\"}" + exit 0 + fi + + if [ $(ismounted) -eq 0 ] ; then + log "{\"status\": \"Success\"}" + exit 0 + fi + + umount ${MNTPATH} &> /dev/null + if [ $? -ne 0 ]; then + err "{ \"status\": \"Failed\", \"message\": \"Failed to unmount volume at ${MNTPATH}\"}" + exit 1 + fi + + log "{\"status\": \"Success\"}" + exit 0 +} + +getvolumename() { + JSON_PARAMS=$1 + DMDEV=$(getdevice) + + # get lvm device UUID + UUID=`lvs -o lv_uuid --noheadings ${DMDEV} 2>/dev/null|tr -d " "` + + log "{\"status\": \"Success\", \"volumeName\":\"${UUID}\"}" + exit 0 +} + +isattached() { + log "{\"status\": \"Success\", \"attached\":true}" + exit 0 +} + +op=$1 + +if [ "$op" = "init" ]; then + log "{\"status\": \"Success\"}" + exit 0 +fi + +if [ $# -lt 2 ]; then + usage +fi + +shift + +case "$op" in + attach) + attach $* + ;; + detach) + detach $* + ;; + waitforattach) + waitforattach $* + ;; + mountdevice) + domountdevice $* + ;; + unmountdevice) + unmountdevice $* + ;; + getvolumename) + getvolumename $* + ;; + isattached) + isattached $* + ;; + *) + err "{ \"status\": \"Not supported\" }" + exit 1 +esac + +exit 1 diff --git a/volumes/flexvolume/nfs b/volumes/flexvolume/nfs new file mode 100755 index 000000000..290af49d5 --- /dev/null +++ b/volumes/flexvolume/nfs @@ -0,0 +1,120 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Notes: +# - Please install "jq" package before using this driver. +usage() { + err "Invalid usage. Usage: " + err "\t$0 init" + err "\t$0 mount " + err "\t$0 unmount " + err "\t$0 getvolumename " + exit 1 +} + +err() { + echo -ne $* 1>&2 +} + +log() { + echo -ne $* >&1 +} + +ismounted() { + MOUNT=`findmnt -n ${MNTPATH} 2>/dev/null | cut -d' ' -f1` + if [ "${MOUNT}" == "${MNTPATH}" ]; then + echo "1" + else + echo "0" + fi +} + +domount() { + MNTPATH=$1 + + NFS_SERVER=$(echo $2 | jq -r '.server') + SHARE=$(echo $2 | jq -r '.share') + + if [ $(ismounted) -eq 1 ] ; then + log "{\"status\": \"Success\"}" + exit 0 + fi + + mkdir -p ${MNTPATH} &> /dev/null + + mount -t nfs ${NFS_SERVER}:/${SHARE} ${MNTPATH} &> /dev/null + if [ $? -ne 0 ]; then + err "{ \"status\": \"Failure\", \"message\": \"Failed to mount ${NFS_SERVER}:${SHARE} at ${MNTPATH}\"}" + exit 1 + fi + log "{\"status\": \"Success\"}" + exit 0 +} + +unmount() { + MNTPATH=$1 + if [ $(ismounted) -eq 0 ] ; then + log "{\"status\": \"Success\"}" + exit 0 + fi + + umount ${MNTPATH} &> /dev/null + if [ $? -ne 0 ]; then + err "{ \"status\": \"Failed\", \"message\": \"Failed to unmount volume at ${MNTPATH}\"}" + exit 1 + fi + + log "{\"status\": \"Success\"}" + exit 0 +} + +getvolumename() { + NFS_SERVER=$(echo $1 | jq -r '.server') + SHARE=$(echo $1 | jq -r '.share') + + log "{\"status\": \"Success\", \"volumeName\": \"${NFS_SERVER}/${SHARE}\"}" + exit 0 +} + +op=$1 + +if [ "$op" = "init" ]; then + log "{\"status\": \"Success\"}" + exit 0 +fi + +if [ $# -lt 2 ]; then + usage +fi + +shift + +case "$op" in + mount) + domount $* + ;; + unmount) + unmount $* + ;; + getvolumename) + getvolumename $* + ;; + *) + err "{ \"status\": \"Not supported\" }" + exit 1 +esac + +exit 1 diff --git a/volumes/flexvolume/nginx-nfs.yaml b/volumes/flexvolume/nginx-nfs.yaml new file mode 100644 index 000000000..177c1e8f4 --- /dev/null +++ b/volumes/flexvolume/nginx-nfs.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-nfs + namespace: default +spec: + containers: + - name: nginx-nfs + image: nginx + volumeMounts: + - name: test + mountPath: /data + ports: + - containerPort: 80 + volumes: + - name: test + flexVolume: + driver: "k8s/nfs" + fsType: "nfs" + options: + server: "172.16.0.25" + share: "dws_nas_scratch" diff --git a/volumes/flexvolume/nginx.yaml b/volumes/flexvolume/nginx.yaml new file mode 100644 index 000000000..c7d7859cc --- /dev/null +++ b/volumes/flexvolume/nginx.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx + namespace: default +spec: + containers: + - name: nginx + image: nginx + volumeMounts: + - name: test + mountPath: /data + ports: + - containerPort: 80 + volumes: + - name: test + flexVolume: + driver: "kubernetes.io/lvm" + fsType: "ext4" + options: + volumeID: "vol1" + size: "1000m" + volumegroup: "kube_vg" diff --git a/volumes/flocker/README.md b/volumes/flocker/README.md new file mode 100644 index 000000000..72ad47bc3 --- /dev/null +++ b/volumes/flocker/README.md @@ -0,0 +1,115 @@ +## Using Flocker volumes + +[Flocker](https://clusterhq.com/flocker) is an open-source clustered container data volume manager. It provides management +and orchestration of data volumes backed by a variety of storage backends. + +This example provides information about how to set-up a Flocker installation and configure it in Kubernetes, as well as how to use the plugin to use Flocker datasets as volumes in Kubernetes. + +### Prerequisites + +A Flocker cluster is required to use Flocker with Kubernetes. A Flocker cluster comprises: + +- *Flocker Control Service*: provides a REST over HTTP API to modify the desired configuration of the cluster; +- *Flocker Dataset Agent(s)*: a convergence agent that modifies the cluster state to match the desired configuration; +- *Flocker Container Agent(s)*: a convergence agent that modifies the cluster state to match the desired configuration (unused in this configuration but still required in the cluster). + +The Flocker cluster can be installed on the same nodes you are using for Kubernetes. For instance, you can install the Flocker Control Service on the same node as Kubernetes Master and Flocker Dataset/Container Agents on every Kubernetes Slave node. + +It is recommended to follow [Installing Flocker](https://docs.clusterhq.com/en/latest/install/index.html) and the instructions below to set-up the Flocker cluster to be used with Kubernetes. + +#### Flocker Control Service + +The Flocker Control Service should be installed manually on a host. In the future, this may be deployed in pod(s) and exposed as a Kubernetes service. + +#### Flocker Agent(s) + +The Flocker Agents should be manually installed on *all* Kubernetes nodes. These agents are responsible for (de)attachment and (un)mounting and are therefore services that should be run with appropriate privileges on these hosts. + +In order for the plugin to connect to Flocker (via REST API), several environment variables must be specified on *all* Kubernetes nodes. This may be specified in an init script for the node's Kubelet service, for example, you could store the below environment variables in a file called `/etc/flocker/env` and place `EnvironmentFile=/etc/flocker/env` into `/etc/systemd/system/kubelet.service` or wherever the `kubelet.service` file lives. + +The environment variables that need to be set are: + +- `FLOCKER_CONTROL_SERVICE_HOST` should refer to the hostname of the Control Service +- `FLOCKER_CONTROL_SERVICE_PORT` should refer to the port of the Control Service (the API service defaults to 4523 but this must still be specified) + +The following environment variables should refer to keys and certificates on the host that are specific to that host. + +- `FLOCKER_CONTROL_SERVICE_CA_FILE` should refer to the full path to the cluster certificate file +- `FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE` should refer to the full path to the [api key](https://docs.clusterhq.com/en/latest/config/generate-api-plugin.html) file for the API user +- `FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE` should refer to the full path to the [api certificate](https://docs.clusterhq.com/en/latest/config/generate-api-plugin.html) file for the API user + +More details regarding cluster authentication can be found at the documentation: [Flocker Cluster Security & Authentication](https://docs.clusterhq.com/en/latest/concepts/security.html) and [Configuring Cluster Authentication](https://docs.clusterhq.com/en/latest/config/configuring-authentication.html). + +### Create a pod with a Flocker volume + +**Note**: A new dataset must first be provisioned using the Flocker tools or Docker CLI *(To use the Docker CLI, you need the [Flocker plugin for Docker](https://clusterhq.com/docker-plugin/) installed along with Docker 1.9+)*. For example, using the [Volumes CLI](https://docs.clusterhq.com/en/latest/labs/volumes-cli.html), create a new dataset called 'my-flocker-vol' of size 10GB: + +```sh +flocker-volumes create -m name=my-flocker-vol -s 10G -n + +# -n or --node= Is the initial primary node for dataset (any unique +# prefix of node uuid, see flocker-volumes list-nodes) +``` + +The following *volume* spec from the [example pod](flocker-pod.yml) illustrates how to use this Flocker dataset as a volume. + +> Note, the [example pod](flocker-pod.yml) used here does not include a replication controller, therefore the POD will not be rescheduled upon failure. If your looking for an example that does include a replication controller and service spec you can use [this example pod including a replication controller](flocker-pod-with-rc.yml) + +```yaml + volumes: + - name: www-root + flocker: + datasetName: my-flocker-vol +``` + +- **datasetName** is the unique name for the Flocker dataset and should match the *name* in the metadata. + +Use `kubetctl` to create the pod. + +```sh +$ kubectl create -f examples/volumes/flocker/flocker-pod.yml +``` + +You should now verify that the pod is running and determine it's IP address: + +```sh +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +flocker 1/1 Running 0 3m +$ kubectl get pods flocker -t '{{.status.hostIP}}{{"\n"}}' +172.31.25.62 +``` + +An `ls` of the `/flocker` directory on the host (identified by the IP as above) will show the mount point for the volume. + +```sh +$ ls /flocker +0cf8789f-00da-4da0-976a-b6b1dc831159 +``` + +You can also see the mountpoint by inspecting the docker container on that host. + +```sh +$ docker inspect -f "{{.Mounts}}" | grep flocker +...{ /flocker/0cf8789f-00da-4da0-976a-b6b1dc831159 /usr/share/nginx/html true} +``` + +Add an index.html inside this directory and use `curl` to see this HTML file served up by nginx. + +```sh +$ echo "

Hello, World

" | tee /flocker/0cf8789f-00da-4da0-976a-b6b1dc831159/index.html +$ curl ip + +``` + +### More Info + +Read more about the [Flocker Cluster Architecture](https://docs.clusterhq.com/en/latest/concepts/architecture.html) and learn more about Flocker by visiting the [Flocker Documentation](https://docs.clusterhq.com/). + +#### Video Demo + +To see a demo example of using Kubernetes and Flocker, visit [Flocker's blog post on High Availability with Kubernetes and Flocker](https://clusterhq.com/2015/12/22/ha-demo-kubernetes-flocker/) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/flocker/README.md?pixel)]() + diff --git a/volumes/flocker/flocker-pod-with-rc.yml b/volumes/flocker/flocker-pod-with-rc.yml new file mode 100644 index 000000000..01a40f881 --- /dev/null +++ b/volumes/flocker/flocker-pod-with-rc.yml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: flocker-ghost + labels: + app: flocker-ghost +spec: + ports: + # the port that this service should serve on + - port: 80 + targetPort: 80 + selector: + app: flocker-ghost +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: flocker-ghost + # these labels can be applied automatically + # from the labels in the pod template if not set + labels: + purpose: demo +spec: + replicas: 1 + template: + metadata: + labels: + app: flocker-ghost + spec: + containers: + - name: flocker-ghost + image: ghost:0.7.1 + env: + - name: GET_HOSTS_FROM + value: dns + ports: + - containerPort: 2368 + hostPort: 80 + protocol: TCP + volumeMounts: + # name must match the volume name below + - name: ghost-data + mountPath: "/var/lib/ghost" + volumes: + - name: ghost-data + flocker: + datasetName: my-flocker-vol diff --git a/volumes/flocker/flocker-pod.yml b/volumes/flocker/flocker-pod.yml new file mode 100644 index 000000000..fb923cd49 --- /dev/null +++ b/volumes/flocker/flocker-pod.yml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: flocker-web +spec: + containers: + - name: web + image: nginx + ports: + - name: web + containerPort: 80 + volumeMounts: + # name must match the volume name below + - name: www-root + mountPath: "/usr/share/nginx/html" + volumes: + - name: www-root + flocker: + datasetName: my-flocker-vol diff --git a/volumes/glusterfs/README.md b/volumes/glusterfs/README.md new file mode 100644 index 000000000..d67bd8650 --- /dev/null +++ b/volumes/glusterfs/README.md @@ -0,0 +1,103 @@ +## GlusterFS + +[GlusterFS](http://www.gluster.org) is an open source scale-out filesystem. These examples provide information about how to allow containers use GlusterFS volumes. + +The example assumes that you have already set up a GlusterFS server cluster and have a working GlusterFS volume ready to use in the containers. + +### Prerequisites + +* Set up a GlusterFS server cluster +* Create a GlusterFS volume +* If you are not using hyperkube, you may need to install the GlusterFS client package on the Kubernetes nodes ([Guide](http://gluster.readthedocs.io/en/latest/Administrator%20Guide/)) + +### Create endpoints + +The first step is to create the GlusterFS endpoints definition in Kubernetes. Here is a snippet of [glusterfs-endpoints.json](glusterfs-endpoints.json): + +``` + "subsets": [ + { + "addresses": [{ "ip": "10.240.106.152" }], + "ports": [{ "port": 1 }] + }, + { + "addresses": [{ "ip": "10.240.79.157" }], + "ports": [{ "port": 1 }] + } + ] +``` + +The `subsets` field should be populated with the addresses of the nodes in the GlusterFS cluster. It is fine to provide any valid value (from 1 to 65535) in the `port` field. + +Create the endpoints: + +```sh +$ kubectl create -f examples/volumes/glusterfs/glusterfs-endpoints.json +``` + +You can verify that the endpoints are successfully created by running + +```sh +$ kubectl get endpoints +NAME ENDPOINTS +glusterfs-cluster 10.240.106.152:1,10.240.79.157:1 +``` + +We also need to create a service for these endpoints, so that they will persist. We will add this service without a selector to tell Kubernetes we want to add its endpoints manually. You can see [glusterfs-service.json](glusterfs-service.json) for details. + +Use this command to create the service: + +```sh +$ kubectl create -f examples/volumes/glusterfs/glusterfs-service.json +``` + + +### Create a Pod + +The following *volume* spec in [glusterfs-pod.json](glusterfs-pod.json) illustrates a sample configuration: + +```json +"volumes": [ + { + "name": "glusterfsvol", + "glusterfs": { + "endpoints": "glusterfs-cluster", + "path": "kube_vol", + "readOnly": true + } + } +] +``` + +The parameters are explained as the followings. + +- **endpoints** is the name of the Endpoints object that represents a Gluster cluster configuration. *kubelet* is optimized to avoid mount storm, it will randomly pick one from the endpoints to mount. If this host is unresponsive, the next Gluster host in the endpoints is automatically selected. +- **path** is the Glusterfs volume name. +- **readOnly** is the boolean that sets the mountpoint readOnly or readWrite. + +Create a pod that has a container using Glusterfs volume, + +```sh +$ kubectl create -f examples/volumes/glusterfs/glusterfs-pod.json +``` + +You can verify that the pod is running: + +```sh +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +glusterfs 1/1 Running 0 3m +``` + +You may execute the command `mount` inside the container to see if the GlusterFS volume is mounted correctly: + +```sh +$ kubectl exec glusterfs -- mount | grep gluster +10.240.106.152:kube_vol on /mnt/glusterfs type fuse.glusterfs (rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)``` + +You may also run `docker ps` on the host to see the actual container. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/glusterfs/README.md?pixel)]() + diff --git a/volumes/glusterfs/glusterfs-endpoints.json b/volumes/glusterfs/glusterfs-endpoints.json new file mode 100644 index 000000000..740ce4258 --- /dev/null +++ b/volumes/glusterfs/glusterfs-endpoints.json @@ -0,0 +1,33 @@ +{ + "kind": "Endpoints", + "apiVersion": "v1", + "metadata": { + "name": "glusterfs-cluster" + }, + "subsets": [ + { + "addresses": [ + { + "ip": "10.240.106.152" + } + ], + "ports": [ + { + "port": 1 + } + ] + }, + { + "addresses": [ + { + "ip": "10.240.79.157" + } + ], + "ports": [ + { + "port": 1 + } + ] + } + ] +} diff --git a/volumes/glusterfs/glusterfs-pod.json b/volumes/glusterfs/glusterfs-pod.json new file mode 100644 index 000000000..63c075be6 --- /dev/null +++ b/volumes/glusterfs/glusterfs-pod.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "glusterfs" + }, + "spec": { + "containers": [ + { + "name": "glusterfs", + "image": "nginx", + "volumeMounts": [ + { + "mountPath": "/mnt/glusterfs", + "name": "glusterfsvol" + } + ] + } + ], + "volumes": [ + { + "name": "glusterfsvol", + "glusterfs": { + "endpoints": "glusterfs-cluster", + "path": "kube_vol", + "readOnly": true + } + } + ] + } +} diff --git a/volumes/glusterfs/glusterfs-service.json b/volumes/glusterfs/glusterfs-service.json new file mode 100644 index 000000000..79139febd --- /dev/null +++ b/volumes/glusterfs/glusterfs-service.json @@ -0,0 +1,12 @@ +{ + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "glusterfs-cluster" + }, + "spec": { + "ports": [ + {"port": 1} + ] + } +} diff --git a/volumes/iscsi/README.md b/volumes/iscsi/README.md new file mode 100644 index 000000000..7f6090eff --- /dev/null +++ b/volumes/iscsi/README.md @@ -0,0 +1,136 @@ +## Introduction + +The Kubernetes iSCSI implementation can connect to iSCSI devices via open-iscsi and multipathd on Linux. +Currently supported features are + * Connecting to one portal + * Mounting a device directly or via multipathd + * Formatting and partitioning any new device connected + * CHAP authentication + +## Prerequisites + +This example expects there to be a working iSCSI target to connect to. +If there isn't one in place then it is possible to setup a software version on Linux by following these guides + + * [Setup a iSCSI target on Fedora](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi) + * [Install the iSCSI initiator on Fedora](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi&f=2) + * [Install multipathd for mpio support if required](http://www.linuxstories.eu/2014/07/how-to-setup-dm-multipath-on-rhel.html) + + +## Creating the pod with iSCSI persistent storage + +Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. If you have more than one target portals for a single IQN, you can mention other portal IPs in *portals* field. + +If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod YAML. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information. + +**Note:** If you have followed the instructions in the links above you +may have partitioned the device, the iSCSI volume plugin does not +currently support partitions so format the device as one partition or leave the device raw and Kubernetes will partition and format it one first mount. + +### CHAP Authentication + +To enable one-way or two-way CHAP authentication for discovery or session, following these steps. + + * Set `chapAuthDiscovery` to `true` for discovery authentication. + * Set `chapAuthSession` to `true` for session authentication. + * Create a CHAP secret and set `secretRef` to reference the CHAP secret. + + +Example can be found at [iscsi-chap.yaml](iscsi-chap.yaml) + +### CHAP Secret + +As illustrated in [chap-secret.yaml](chap-secret.yaml), the secret must have type `kubernetes.io/iscsi-chap` and consists of the following keys: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: chap-secret +type: "kubernetes.io/iscsi-chap" +data: + discovery.sendtargets.auth.username: + discovery.sendtargets.auth.password: + discovery.sendtargets.auth.username_in: + discovery.sendtargets.auth.password_in: + node.session.auth.username: + node.session.auth.password: + node.session.auth.username_in: + node.session.auth.password_in: +``` + +These keys map to those used by Open-iSCSI initiator. Detailed documents on these keys can be found at [Open-iSCSI](https://github.com/open-iscsi/open-iscsi/blob/master/etc/iscsid.conf) + +#### Create CHAP secret before creating iSCSI volumes and Pods + +```console +# kubectl create -f examples/volumes/iscsi/chap-iscsi.yaml +``` + + + +Once the pod config is created, run it on the Kubernetes master: + +```console +kubectl create -f ./your_new_pod.yaml +``` + +Here is the example pod created and expected output: + +```console +# kubectl create -f examples/volumes/iscsi/iscsi.yaml +# kubectl get pods +NAME READY STATUS RESTARTS AGE +iscsipd 2/2 RUNNING 0 2m +``` + +On the Kubernetes node, verify the mount output + +For a non mpio device the output should look like the following + +```console +# mount |grep kub +/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (ro,relatime,data=ordered) +/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/sdc on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +/dev/sdd on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/sdd on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +``` + +And for a node with mpio enabled the expected output would be similar to the following + +```console +# mount |grep kub +/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpatha on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +``` + + +If you ssh to that machine, you can run `docker ps` to see the actual pod. + +```console +# docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +3b8a772515d2 kubernetes/pause "/pause" 6 minutes ago Up 6 minutes k8s_iscsipd-rw.ed58ec4e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_d25592c5 +``` + +Run *docker inspect* and verify the container mounted the host directory into the their */mnt/iscsipd* directory. + +```console +# docker inspect --format '{{ range .Mounts }}{{ if eq .Destination "/mnt/iscsipd" }}{{ .Source }}{{ end }}{{ end }}' f855336407f4 +/var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro + +# docker inspect --format '{{ range .Mounts }}{{ if eq .Destination "/mnt/iscsipd" }}{{ .Source }}{{ end }}{{ end }}' 3b8a772515d2 +/var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw +``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/iscsi/README.md?pixel)]() + diff --git a/volumes/iscsi/chap-secret.yaml b/volumes/iscsi/chap-secret.yaml new file mode 100644 index 000000000..5bc9cc874 --- /dev/null +++ b/volumes/iscsi/chap-secret.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: chap-secret +type: "kubernetes.io/iscsi-chap" +data: + discovery.sendtargets.auth.username: dXNlcg== + discovery.sendtargets.auth.password: ZGVtbw== + discovery.sendtargets.auth.username_in: bXVzZXI= + discovery.sendtargets.auth.password_in: bXBhc3M= + node.session.auth.username: dXNlcm5hbWU= + node.session.auth.password: cGFzc3dvcmQ= + node.session.auth.username_in: bXVzZXIy + node.session.auth.password_in: bXBhc3My diff --git a/volumes/iscsi/iscsi-chap.yaml b/volumes/iscsi/iscsi-chap.yaml new file mode 100644 index 000000000..1ddc2f02c --- /dev/null +++ b/volumes/iscsi/iscsi-chap.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: iscsipd +spec: + containers: + - name: iscsipd-ro + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/iscsipd" + name: iscsivol + volumes: + - name: iscsivol + iscsi: + targetPortal: 127.0.0.1 + iqn: iqn.2015-02.example.com:test + lun: 0 + fsType: ext4 + readOnly: true + chapAuthDiscovery: true + chapAuthSession: true + secretRef: + name: chap-secret diff --git a/volumes/iscsi/iscsi.yaml b/volumes/iscsi/iscsi.yaml new file mode 100644 index 000000000..46736eda8 --- /dev/null +++ b/volumes/iscsi/iscsi.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: iscsipd +spec: + containers: + - name: iscsipd-rw + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/iscsipd" + name: iscsipd-rw + volumes: + - name: iscsipd-rw + iscsi: + targetPortal: 10.0.2.15:3260 + portals: ['10.0.2.16:3260', '10.0.2.17:3260'] + iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz + lun: 0 + fsType: ext4 + readOnly: true diff --git a/volumes/nfs/README.md b/volumes/nfs/README.md new file mode 100644 index 000000000..2c7c72e80 --- /dev/null +++ b/volumes/nfs/README.md @@ -0,0 +1,165 @@ +# Outline + +This example describes how to create Web frontend server, an auto-provisioned persistent volume on GCE, and an NFS-backed persistent claim. + +Demonstrated Kubernetes Concepts: + +* [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) to + define persistent disks (disk lifecycle not tied to the Pods). +* [Services](https://kubernetes.io/docs/concepts/services-networking/service/) to enable Pods to + locate one another. + +![alt text][nfs pv example] + +As illustrated above, two persistent volumes are used in this example: + +- Web frontend Pod uses a persistent volume based on NFS server, and +- NFS server uses an auto provisioned [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) from GCE PD or AWS EBS. + +Note, this example uses an NFS container that doesn't support NFSv4. + +[nfs pv example]: nfs-pv.png + + +## Quickstart + +```console +$ kubectl create -f examples/volumes/nfs/provisioner/nfs-server-gce-pv.yaml +$ kubectl create -f examples/volumes/nfs/nfs-server-rc.yaml +$ kubectl create -f examples/volumes/nfs/nfs-server-service.yaml +# get the cluster IP of the server using the following command +$ kubectl describe services nfs-server +# use the NFS server IP to update nfs-pv.yaml and execute the following +$ kubectl create -f examples/volumes/nfs/nfs-pv.yaml +$ kubectl create -f examples/volumes/nfs/nfs-pvc.yaml +# run a fake backend +$ kubectl create -f examples/volumes/nfs/nfs-busybox-rc.yaml +# get pod name from this command +$ kubectl get pod -l name=nfs-busybox +# use the pod name to check the test file +$ kubectl exec nfs-busybox-jdhf3 -- cat /mnt/index.html +``` + +## Example of NFS based persistent volume + +See [NFS Service and Replication Controller](nfs-web-rc.yaml) for a quick example of how to use an NFS +volume claim in a replication controller. It relies on the +[NFS persistent volume](nfs-pv.yaml) and +[NFS persistent volume claim](nfs-pvc.yaml) in this example as well. + +## Complete setup + +The example below shows how to export a NFS share from a single pod replication +controller and import it into two replication controllers. + +### NFS server part + +Define [the NFS Service and Replication Controller](nfs-server-rc.yaml) and +[NFS service](nfs-server-service.yaml): + +The NFS server exports an an auto-provisioned persistent volume backed by GCE PD: + +```console +$ kubectl create -f examples/volumes/nfs/provisioner/nfs-server-gce-pv.yaml +``` + +```console +$ kubectl create -f examples/volumes/nfs/nfs-server-rc.yaml +$ kubectl create -f examples/volumes/nfs/nfs-server-service.yaml +``` + +The directory contains dummy `index.html`. Wait until the pod is running +by checking `kubectl get pods -l role=nfs-server`. + +### Create the NFS based persistent volume claim + +The [NFS busybox controller](nfs-busybox-rc.yaml) uses a simple script to +generate data written to the NFS server we just started. First, you'll need to +find the cluster IP of the server: + +```console +$ kubectl describe services nfs-server +``` + +Replace the invalid IP in the [nfs PV](nfs-pv.yaml). (In the future, +we'll be able to tie these together using the service names, but for +now, you have to hardcode the IP.) + +Create the the [persistent volume](../../../docs/user-guide/persistent-volumes.md) +and the persistent volume claim for your NFS server. The persistent volume and +claim gives us an indirection that allow multiple pods to refer to the NFS +server using a symbolic name rather than the hardcoded server address. + +```console +$ kubectl create -f examples/volumes/nfs/nfs-pv.yaml +$ kubectl create -f examples/volumes/nfs/nfs-pvc.yaml +``` + +## Setup the fake backend + +The [NFS busybox controller](nfs-busybox-rc.yaml) updates `index.html` on the +NFS server every 10 seconds. Let's start that now: + +```console +$ kubectl create -f examples/volumes/nfs/nfs-busybox-rc.yaml +``` + +Conveniently, it's also a `busybox` pod, so we can get an early check +that our mounts are working now. Find a busybox pod and exec: + +```console +$ kubectl get pod -l name=nfs-busybox +NAME READY STATUS RESTARTS AGE +nfs-busybox-jdhf3 1/1 Running 0 25m +nfs-busybox-w3s4t 1/1 Running 0 25m +$ kubectl exec nfs-busybox-jdhf3 -- cat /mnt/index.html +Thu Oct 22 19:20:18 UTC 2015 +nfs-busybox-w3s4t +``` + +You should see output similar to the above if everything is working well. If +it's not, make sure you changed the invalid IP in the [NFS PV](nfs-pv.yaml) file +and make sure the `describe services` command above had endpoints listed +(indicating the service was associated with a running pod). + +### Setup the web server + +The [web server controller](nfs-web-rc.yaml) is an another simple replication +controller demonstrates reading from the NFS share exported above as a NFS +volume and runs a simple web server on it. + +Define the pod: + +```console +$ kubectl create -f examples/volumes/nfs/nfs-web-rc.yaml +``` + +This creates two pods, each of which serve the `index.html` from above. We can +then use a simple service to front it: + +```console +kubectl create -f examples/volumes/nfs/nfs-web-service.yaml +``` + +We can then use the busybox container we launched before to check that `nginx` +is serving the data appropriately: + +```console +$ kubectl get pod -l name=nfs-busybox +NAME READY STATUS RESTARTS AGE +nfs-busybox-jdhf3 1/1 Running 0 1h +nfs-busybox-w3s4t 1/1 Running 0 1h +$ kubectl get services nfs-web +NAME LABELS SELECTOR IP(S) PORT(S) +nfs-web role=web-frontend 10.0.68.37 80/TCP +$ kubectl exec nfs-busybox-jdhf3 -- wget -qO- http://10.0.68.37 +Thu Oct 22 19:28:55 UTC 2015 +nfs-busybox-w3s4t +``` + + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/nfs/README.md?pixel)]() + diff --git a/volumes/nfs/nfs-busybox-rc.yaml b/volumes/nfs/nfs-busybox-rc.yaml new file mode 100644 index 000000000..617d02755 --- /dev/null +++ b/volumes/nfs/nfs-busybox-rc.yaml @@ -0,0 +1,32 @@ +# This mounts the nfs volume claim into /mnt and continuously +# overwrites /mnt/index.html with the time and hostname of the pod. + +apiVersion: v1 +kind: ReplicationController +metadata: + name: nfs-busybox +spec: + replicas: 2 + selector: + name: nfs-busybox + template: + metadata: + labels: + name: nfs-busybox + spec: + containers: + - image: busybox + command: + - sh + - -c + - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done' + imagePullPolicy: IfNotPresent + name: busybox + volumeMounts: + # name must match the volume name below + - name: nfs + mountPath: "/mnt" + volumes: + - name: nfs + persistentVolumeClaim: + claimName: nfs diff --git a/volumes/nfs/nfs-data/Dockerfile b/volumes/nfs/nfs-data/Dockerfile new file mode 100644 index 000000000..96986bd80 --- /dev/null +++ b/volumes/nfs/nfs-data/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos +RUN yum -y install /usr/bin/ps nfs-utils && yum clean all +RUN mkdir -p /exports +ADD run_nfs.sh /usr/local/bin/ +ADD index.html /tmp/index.html +RUN chmod 644 /tmp/index.html + +# expose mountd 20048/tcp and nfsd 2049/tcp and rpcbind 111/tcp +EXPOSE 2049/tcp 20048/tcp 111/tcp 111/udp + +ENTRYPOINT ["/usr/local/bin/run_nfs.sh", "/exports"] diff --git a/volumes/nfs/nfs-data/README.md b/volumes/nfs/nfs-data/README.md new file mode 100644 index 000000000..df4b26d08 --- /dev/null +++ b/volumes/nfs/nfs-data/README.md @@ -0,0 +1,13 @@ +# NFS-exporter container with a file + +This container exports /exports with index.html in it via NFS. Based on +../exports. Since some Linux kernels have issues running NFSv4 daemons in containers, +only NFSv3 is opened in this container. + +Available as `gcr.io/google-samples/nfs-server` + + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/nfs/nfs-data/README.md?pixel)]() + diff --git a/volumes/nfs/nfs-data/index.html b/volumes/nfs/nfs-data/index.html new file mode 100644 index 000000000..cd0875583 --- /dev/null +++ b/volumes/nfs/nfs-data/index.html @@ -0,0 +1 @@ +Hello world! diff --git a/volumes/nfs/nfs-data/run_nfs.sh b/volumes/nfs/nfs-data/run_nfs.sh new file mode 100755 index 000000000..fa7b165c0 --- /dev/null +++ b/volumes/nfs/nfs-data/run_nfs.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function start() +{ + + # prepare /etc/exports + for i in "$@"; do + # fsid=0: needed for NFSv4 + echo "$i *(rw,fsid=0,insecure,no_root_squash)" >> /etc/exports + # move index.html to here + /bin/cp /tmp/index.html $i/ + chmod 644 $i/index.html + echo "Serving $i" + done + + # start rpcbind if it is not started yet + /usr/sbin/rpcinfo 127.0.0.1 > /dev/null; s=$? + if [ $s -ne 0 ]; then + echo "Starting rpcbind" + /usr/sbin/rpcbind -w + fi + + mount -t nfsd nfds /proc/fs/nfsd + + # -N 4.x: disable NFSv4 + # -V 3: enable NFSv3 + /usr/sbin/rpc.mountd -N 2 -V 3 -N 4 -N 4.1 + + /usr/sbin/exportfs -r + # -G 10 to reduce grace time to 10 seconds (the lowest allowed) + /usr/sbin/rpc.nfsd -G 10 -N 2 -V 3 -N 4 -N 4.1 2 + /usr/sbin/rpc.statd --no-notify + echo "NFS started" +} + +function stop() +{ + echo "Stopping NFS" + + /usr/sbin/rpc.nfsd 0 + /usr/sbin/exportfs -au + /usr/sbin/exportfs -f + + kill $( pidof rpc.mountd ) + umount /proc/fs/nfsd + echo > /etc/exports + exit 0 +} + + +trap stop TERM + +start "$@" + +# Ugly hack to do nothing and wait for SIGTERM +while true; do + sleep 5 +done diff --git a/volumes/nfs/nfs-pv.png b/volumes/nfs/nfs-pv.png new file mode 100644 index 000000000..1ac5fc0d1 Binary files /dev/null and b/volumes/nfs/nfs-pv.png differ diff --git a/volumes/nfs/nfs-pv.yaml b/volumes/nfs/nfs-pv.yaml new file mode 100644 index 000000000..258f4d4c9 --- /dev/null +++ b/volumes/nfs/nfs-pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs +spec: + capacity: + storage: 1Mi + accessModes: + - ReadWriteMany + nfs: + # FIXME: use the right IP + server: 10.244.1.4 + path: "/exports" diff --git a/volumes/nfs/nfs-pvc.yaml b/volumes/nfs/nfs-pvc.yaml new file mode 100644 index 000000000..9c1821f7c --- /dev/null +++ b/volumes/nfs/nfs-pvc.yaml @@ -0,0 +1,10 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: nfs +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi diff --git a/volumes/nfs/nfs-server-rc.yaml b/volumes/nfs/nfs-server-rc.yaml new file mode 100644 index 000000000..c83ed1db8 --- /dev/null +++ b/volumes/nfs/nfs-server-rc.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: nfs-server +spec: + replicas: 1 + selector: + role: nfs-server + template: + metadata: + labels: + role: nfs-server + spec: + containers: + - name: nfs-server + image: gcr.io/google-samples/nfs-server:1.1 + ports: + - name: nfs + containerPort: 2049 + - name: mountd + containerPort: 20048 + - name: rpcbind + containerPort: 111 + securityContext: + privileged: true + volumeMounts: + - mountPath: /exports + name: mypvc + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: nfs-pv-provisioning-demo diff --git a/volumes/nfs/nfs-server-service.yaml b/volumes/nfs/nfs-server-service.yaml new file mode 100644 index 000000000..9654d1583 --- /dev/null +++ b/volumes/nfs/nfs-server-service.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: nfs-server +spec: + ports: + - name: nfs + port: 2049 + - name: mountd + port: 20048 + - name: rpcbind + port: 111 + selector: + role: nfs-server diff --git a/volumes/nfs/nfs-web-rc.yaml b/volumes/nfs/nfs-web-rc.yaml new file mode 100644 index 000000000..6c96682cb --- /dev/null +++ b/volumes/nfs/nfs-web-rc.yaml @@ -0,0 +1,30 @@ +# This pod mounts the nfs volume claim into /usr/share/nginx/html and +# serves a simple web page. + +apiVersion: v1 +kind: ReplicationController +metadata: + name: nfs-web +spec: + replicas: 2 + selector: + role: web-frontend + template: + metadata: + labels: + role: web-frontend + spec: + containers: + - name: web + image: nginx + ports: + - name: web + containerPort: 80 + volumeMounts: + # name must match the volume name below + - name: nfs + mountPath: "/usr/share/nginx/html" + volumes: + - name: nfs + persistentVolumeClaim: + claimName: nfs diff --git a/volumes/nfs/nfs-web-service.yaml b/volumes/nfs/nfs-web-service.yaml new file mode 100644 index 000000000..b73cac2bc --- /dev/null +++ b/volumes/nfs/nfs-web-service.yaml @@ -0,0 +1,9 @@ +kind: Service +apiVersion: v1 +metadata: + name: nfs-web +spec: + ports: + - port: 80 + selector: + role: web-frontend diff --git a/volumes/nfs/provisioner/nfs-server-gce-pv.yaml b/volumes/nfs/provisioner/nfs-server-gce-pv.yaml new file mode 100644 index 000000000..92f9f5735 --- /dev/null +++ b/volumes/nfs/provisioner/nfs-server-gce-pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nfs-pv-provisioning-demo + labels: + demo: nfs-pv-provisioning + annotations: + volume.alpha.kubernetes.io/storage-class: any +spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 200Gi diff --git a/volumes/portworx/README.md b/volumes/portworx/README.md new file mode 100644 index 000000000..36e87b4ad --- /dev/null +++ b/volumes/portworx/README.md @@ -0,0 +1,370 @@ +# Portworx Volume + + - [Portworx](#portworx) + - [Prerequisites](#prerequisites) + - [Examples](#examples) + - [Using Pre-provisioned Portworx Volumes](#pre-provisioned) + - [Running Pod](#running-pod) + - [Persistent Volumes](#persistent-volumes) + - [Using Dynamic Provisioning](#dynamic-provisioning) + - [Storage Class](#storage-class) + +## Portworx + +[Portworx](http://www.portworx.com) can be used as a storage provider for your Kubernetes cluster. Portworx pools your servers capacity and turns your servers +or cloud instances into converged, highly available compute and storage nodes + +## Prerequisites + +- A Portworx instance running on all of your Kubernetes nodes. For + more information on how you can install Portworx can be found [here](http://docs.portworx.com) + +## Examples + +The following examples assumes that you already have a running Kubernetes cluster with Portworx installed on all nodes. + +### Using Pre-provisioned Portworx Volumes + + Create a Volume using Portworx CLI. + On one of the Kubernetes nodes with Portworx installed run the following command + + ```shell + /opt/pwx/bin/pxctl volume create --size --fs + ``` + +#### Running Pods + + Create Pod which uses Portworx Volumes + + Example spec: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: test-portworx-volume-pod + spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: test-container + volumeMounts: + - mountPath: /test-portworx-volume + name: test-volume + volumes: + - name: test-volume + # This Portworx volume must already exist. + portworxVolume: + volumeID: "" + fsType: "" + ``` + + [Download example](portworx-volume-pod.yaml?raw=true) + + Make sure to replace and in the above spec with + the ones that you used while creating the volume. + + Create the Pod. + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pod.yaml + ``` + + Verify that pod is running: + + ```bash + $ kubectl.sh get pods + NAME READY STATUS RESTARTS AGE + test-portworx-volume-pod 1/1 Running 0 16s + ``` + +#### Persistent Volumes + + 1. Create Persistent Volume. + + Example spec: + + ```yaml + apiVersion: v1 + kind: PersistentVolume + metadata: + name: + spec: + capacity: + storage: Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + portworxVolume: + volumeID: "" + fsType: "" + ``` + + Make sure to replace , and in the above spec with + the ones that you used while creating the volume. + + [Download example](portworx-volume-pv.yaml?raw=true) + + Creating the persistent volume: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pv.yaml + ``` + + Verifying persistent volume is created: + + ``` bash + $ kubectl describe pv pv0001 + Name: pv0001 + Labels: + StorageClass: + Status: Available + Claim: + Reclaim Policy: Retain + Access Modes: RWO + Capacity: 2Gi + Message: + Source: + Type: PortworxVolume (a Portworx Persistent Volume resource) + VolumeID: pv0001 + FSType: ext4 + No events. + ``` + + 2. Create Persistent Volume Claim. + + Example spec: + + ```yaml + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: pvc0001 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: Gi + ``` + + [Download example](portworx-volume-pvc.yaml?raw=true) + + Creating the persistent volume claim: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pvc.yaml + ``` + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pvc pvc0001 + Name: pvc0001 + Namespace: default + Status: Bound + Volume: pv0001 + Labels: + Capacity: 2Gi + Access Modes: RWO + No events. + ``` + + 3. Create Pod which uses Persistent Volume Claim. + + See example: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: pvpod + spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-portworx-volume + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvc0001 + ``` + + [Download example](portworx-volume-pvcpod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pvcpod.yaml + ``` + + Verifying pod is created: + + ``` bash + $ kubectl get pod pvpod + NAME READY STATUS RESTARTS AGE + pvpod 1/1 Running 0 48m + ``` + +### Using Dynamic Provisioning + +Using Dynamic Provisioning and Storage Classes you don't need to +create Portworx volumes out of band and they will be created automatically. + +#### Storage Class + + Using Storage Classes objects an admin can define the different classes of Portworx Volumes + that are offered in a cluster. Following are the different parameters that can be used to define a Portworx + Storage Class + + * `fs`: filesystem to be laid out: none|xfs|ext4 (default: `ext4`) + * `block_size`: block size in Kbytes (default: `32`) + * `repl`: replication factor [1..3] (default: `1`) + * `io_priority`: IO Priority: [high|medium|low] (default: `low`) + * `snap_interval`: snapshot interval in minutes, 0 disables snaps (default: `0`) + * `aggregation_level`: specifies the number of replication sets the volume can be aggregated from (default: `1`) + * `ephemeral`: ephemeral storage [true|false] (default `false`) + + + 1. Create Storage Class. + + See example: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: portworx-io-priority-high + provisioner: kubernetes.io/portworx-volume + parameters: + repl: "1" + snap_interval: "70" + io_priority: "high" + ``` + + [Download example](portworx-volume-sc-high.yaml?raw=true) + + Creating the storageclass: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-sc-high.yaml + ``` + + Verifying storage class is created: + + ``` bash + $ kubectl describe storageclass portworx-io-priority-high + Name: portworx-io-priority-high + IsDefaultClass: No + Annotations: + Provisioner: kubernetes.io/portworx-volume + Parameters: io_priority=high,repl=1,snapshot_interval=70 + No events. + ``` + + 2. Create Persistent Volume Claim. + + See example: + + ```yaml + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: pvcsc001 + annotations: + volume.beta.kubernetes.io/storage-class: portworx-io-priority-high + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + ``` + + [Download example](portworx-volume-pvcsc.yaml?raw=true) + + Creating the persistent volume claim: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pvcsc.yaml + ``` + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pvc pvcsc001 + Name: pvcsc001 + Namespace: default + StorageClass: portworx-io-priority-high + Status: Bound + Volume: pvc-e5578707-c626-11e6-baf6-08002729a32b + Labels: + Capacity: 2Gi + Access Modes: RWO + No Events + ``` + + Persistent Volume is automatically created and is bounded to this pvc. + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pv pvc-e5578707-c626-11e6-baf6-08002729a32b + Name: pvc-e5578707-c626-11e6-baf6-08002729a32b + Labels: + StorageClass: portworx-io-priority-high + Status: Bound + Claim: default/pvcsc001 + Reclaim Policy: Delete + Access Modes: RWO + Capacity: 2Gi + Message: + Source: + Type: PortworxVolume (a Portworx Persistent Volume resource) + VolumeID: 374093969022973811 + No events. + ``` + + 3. Create Pod which uses Persistent Volume Claim with storage class. + + See example: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: pvpod + spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-portworx-volume + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvcsc001 + ``` + + [Download example](portworx-volume-pvcscpod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/portworx/portworx-volume-pvcscpod.yaml + ``` + + Verifying pod is created: + + ``` bash + $ kubectl get pod pvpod + NAME READY STATUS RESTARTS AGE + pvpod 1/1 Running 0 48m + ``` + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/portworx/README.md?pixel)]() + diff --git a/volumes/portworx/portworx-volume-pod.yaml b/volumes/portworx/portworx-volume-pod.yaml new file mode 100644 index 000000000..c5f195911 --- /dev/null +++ b/volumes/portworx/portworx-volume-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-portworx-volume-pod +spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: test-container + volumeMounts: + - mountPath: /test-portworx-volume + name: test-volume + volumes: + - name: test-volume + # This Portworx volume must already exist. + portworxVolume: + volumeID: "vol1" diff --git a/volumes/portworx/portworx-volume-pv.yaml b/volumes/portworx/portworx-volume-pv.yaml new file mode 100644 index 000000000..af4e0114f --- /dev/null +++ b/volumes/portworx/portworx-volume-pv.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv0001 +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + portworxVolume: + volumeID: "pv0001" diff --git a/volumes/portworx/portworx-volume-pvc.yaml b/volumes/portworx/portworx-volume-pvc.yaml new file mode 100644 index 000000000..181a3848d --- /dev/null +++ b/volumes/portworx/portworx-volume-pvc.yaml @@ -0,0 +1,10 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc0001 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/volumes/portworx/portworx-volume-pvcpod.yaml b/volumes/portworx/portworx-volume-pvcpod.yaml new file mode 100644 index 000000000..fb92b320f --- /dev/null +++ b/volumes/portworx/portworx-volume-pvcpod.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pvpod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-portworx-volume + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvc0001 diff --git a/volumes/portworx/portworx-volume-pvcsc.yaml b/volumes/portworx/portworx-volume-pvcsc.yaml new file mode 100644 index 000000000..b07ddb302 --- /dev/null +++ b/volumes/portworx/portworx-volume-pvcsc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvcsc001 + annotations: + volume.beta.kubernetes.io/storage-class: portworx-io-priority-high +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/volumes/portworx/portworx-volume-pvcscpod.yaml b/volumes/portworx/portworx-volume-pvcscpod.yaml new file mode 100644 index 000000000..464bf5d8f --- /dev/null +++ b/volumes/portworx/portworx-volume-pvcscpod.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pvpod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-portworx-volume + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvcsc001 diff --git a/volumes/portworx/portworx-volume-sc-high.yaml b/volumes/portworx/portworx-volume-sc-high.yaml new file mode 100644 index 000000000..b9a0a51dc --- /dev/null +++ b/volumes/portworx/portworx-volume-sc-high.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: portworx-io-priority-high +provisioner: kubernetes.io/portworx-volume +parameters: + repl: "1" + snap_interval: "70" + io_priority: "high" diff --git a/volumes/quobyte/Readme.md b/volumes/quobyte/Readme.md new file mode 100644 index 000000000..b2689f11e --- /dev/null +++ b/volumes/quobyte/Readme.md @@ -0,0 +1,98 @@ + + +- [Quobyte Volume](#quobyte-volume) + - [Quobyte](#quobyte) + - [Prerequisites](#prerequisites) + - [Fixed user Mounts](#fixed-user-mounts) + - [Creating a pod](#creating-a-pod) + + + +# Quobyte Volume + +## Quobyte + +[Quobyte](http://www.quobyte.com) is software that turns commodity servers into a reliable and highly automated multi-data center file system. + +The example assumes that you already have a running Kubernetes cluster and you already have setup Quobyte-Client (1.3+) on each Kubernetes node. + +### Prerequisites + +- Running Quobyte storage cluster +- Quobyte client (1.3+) installed on the Kubernetes nodes more information how you can install Quobyte on your Kubernetes nodes, can be found in the [documentation](https://support.quobyte.com) of Quobyte. +- To get access to Quobyte and the documentation please [contact us](http://www.quobyte.com/get-quobyte) +- Already created Quobyte Volume +- Added the line `allow-usermapping-in-volumename` in `/etc/quobyte/client.cfg` to allow the fixed user mounts + +### Fixed user Mounts + +Quobyte supports since 1.3 fixed user mounts. The fixed-user mounts simply allow to mount all Quobyte Volumes inside one directory and use them as different users. All access to the Quobyte Volume will be rewritten to the specified user and group – both are optional, independent of the user inside the container. You can read more about it [here](https://blog.inovex.de/docker-plugins) under the section "Quobyte Mount and Docker — what’s special" + +## Creating a pod + +See example: + + + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: quobyte +spec: + containers: + - name: quobyte + image: kubernetes/pause + volumeMounts: + - mountPath: /mnt + name: quobytevolume + volumes: + - name: quobytevolume + quobyte: + registry: registry:7861 + volume: testVolume + readOnly: false + user: root + group: root +``` + +[Download example](quobyte-pod.yaml?raw=true) + + +Parameters: +* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as : pair or if you want to specify multiple registries you just have to put a comma between them e.q. :,:,:. The host can be an IP address or if you have a working DNS you can also provide the DNS names. +* **volume** volume represents a Quobyte volume which must be created before usage. +* **readOnly** is the boolean that sets the mountpoint readOnly or readWrite. +* **user** maps all access to this user. Default is `root`. +* **group** maps all access to this group. Default is `nfsnobody`. + +Creating the pod: + +```bash +$ kubectl create -f examples/volumes/quobyte/quobyte-pod.yaml +``` + +Verify that the pod is running: + +```bash +$ kubectl get pods quobyte +NAME READY STATUS RESTARTS AGE +quobyte 1/1 Running 0 48m + +$ kubectl get pods quobyte --template '{{.status.hostIP}}{{"\n"}}' +10.245.1.3 +``` + +SSH onto the Machine and validate that quobyte is mounted: + +```bash +$ mount | grep quobyte +quobyte@10.239.10.21:7861/ on /var/lib/kubelet/plugins/kubernetes.io~quobyte type fuse (rw,nosuid,nodev,noatime,user_id=0,group_id=0,default_permissions,allow_other) + +$ docker inspect --format '{{ range .Mounts }}{{ if eq .Destination "/mnt"}}{{ .Source }}{{ end }}{{ end }}' 55ab97593cd3 +/var/lib/kubelet/plugins/kubernetes.io~quobyte/root#root@testVolume +``` + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/quobyte/Readme.md?pixel)]() + diff --git a/volumes/quobyte/quobyte-pod.yaml b/volumes/quobyte/quobyte-pod.yaml new file mode 100644 index 000000000..f731a53ea --- /dev/null +++ b/volumes/quobyte/quobyte-pod.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: quobyte +spec: + containers: + - name: quobyte + image: kubernetes/pause + volumeMounts: + - mountPath: /mnt + name: quobytevolume + volumes: + - name: quobytevolume + quobyte: + registry: registry:7861 + volume: testVolume + readOnly: false + user: root + group: root diff --git a/volumes/rbd/README.md b/volumes/rbd/README.md new file mode 100644 index 000000000..bf4f9852e --- /dev/null +++ b/volumes/rbd/README.md @@ -0,0 +1,59 @@ +# How to Use it? + +Install Ceph on the Kubernetes host. For example, on Fedora 21 + + # yum -y install ceph-common + +If you don't have a Ceph cluster, you can set up a [containerized Ceph cluster](https://github.com/ceph/ceph-docker) + +Then get the keyring from the Ceph cluster and copy it to */etc/ceph/keyring*. + +Once you have installed Ceph and new Kubernetes, you can create a pod based on my examples [rbd.json](rbd.json) [rbd-with-secret.json](rbd-with-secret.json). In the pod JSON, you need to provide the following information. + +- *monitors*: Ceph monitors. +- *pool*: The name of the RADOS pool, if not provided, default *rbd* pool is used. +- *image*: The image name that rbd has created. +- *user*: The RADOS user name. If not provided, default *admin* is used. +- *keyring*: The path to the keyring file. If not provided, default */etc/ceph/keyring* is used. +- *secretName*: The name of the authentication secrets. If provided, *secretName* overrides *keyring*. Note, see below about how to create a secret. +- *fsType*: The filesystem type (ext4, xfs, etc) that formatted on the device. +- *readOnly*: Whether the filesystem is used as readOnly. + +# Use Ceph Authentication Secret + +If Ceph authentication secret is provided, the secret should be first be *base64 encoded*, then encoded string is placed in a secret yaml. For example, getting Ceph user `kube`'s base64 encoded secret can use the following command: + +```console + # grep key /etc/ceph/ceph.client.kube.keyring |awk '{printf "%s", $NF}'|base64 +QVFBTWdYaFZ3QkNlRGhBQTlubFBhRnlmVVNhdEdENGRyRldEdlE9PQ== +``` + +An example yaml is provided [here](secret/ceph-secret.yaml). Then post the secret through ```kubectl``` in the following command. + +```console + # kubectl create -f examples/volumes/rbd/secret/ceph-secret.yaml +``` + +# Get started + +Here are my commands: + +```console + # kubectl create -f examples/volumes/rbd/rbd.json + # kubectl get pods +``` + +On the Kubernetes host, I got these in mount output + +```console + #mount |grep kub + /dev/rbd0 on /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/kube-image-foo type ext4 (ro,relatime,stripe=4096,data=ordered) + /dev/rbd0 on /var/lib/kubelet/pods/ec2166b4-de07-11e4-aaf5-d4bed9b39058/volumes/kubernetes.io~rbd/rbdpd type ext4 (ro,relatime,stripe=4096,data=ordered) +``` + + If you ssh to that machine, you can run `docker ps` to see the actual pod and `docker inspect` to see the volumes used by the container. + + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/rbd/README.md?pixel)]() + diff --git a/volumes/rbd/rbd-with-secret.json b/volumes/rbd/rbd-with-secret.json new file mode 100644 index 000000000..30375583d --- /dev/null +++ b/volumes/rbd/rbd-with-secret.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "rbd2" + }, + "spec": { + "containers": [ + { + "name": "rbd-rw", + "image": "kubernetes/pause", + "volumeMounts": [ + { + "mountPath": "/mnt/rbd", + "name": "rbdpd" + } + ] + } + ], + "volumes": [ + { + "name": "rbdpd", + "rbd": { + "monitors": [ + "10.16.154.78:6789", + "10.16.154.82:6789", + "10.16.154.83:6789" + ], + "pool": "kube", + "image": "foo", + "user": "admin", + "secretRef": { + "name": "ceph-secret" + }, + "fsType": "ext4", + "readOnly": true + } + } + ] + } +} diff --git a/volumes/rbd/rbd.json b/volumes/rbd/rbd.json new file mode 100644 index 000000000..68033bffd --- /dev/null +++ b/volumes/rbd/rbd.json @@ -0,0 +1,39 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "rbd" + }, + "spec": { + "containers": [ + { + "name": "rbd-rw", + "image": "kubernetes/pause", + "volumeMounts": [ + { + "mountPath": "/mnt/rbd", + "name": "rbdpd" + } + ] + } + ], + "volumes": [ + { + "name": "rbdpd", + "rbd": { + "monitors": [ + "10.16.154.78:6789", + "10.16.154.82:6789", + "10.16.154.83:6789" + ], + "pool": "kube", + "image": "foo", + "user": "admin", + "keyring": "/etc/ceph/keyring", + "fsType": "ext4", + "readOnly": true + } + } + ] + } +} diff --git a/volumes/rbd/secret/ceph-secret.yaml b/volumes/rbd/secret/ceph-secret.yaml new file mode 100644 index 000000000..f717f9005 --- /dev/null +++ b/volumes/rbd/secret/ceph-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ceph-secret +type: "kubernetes.io/rbd" +data: + key: QVFCMTZWMVZvRjVtRXhBQTVrQ1FzN2JCajhWVUxSdzI2Qzg0SEE9PQ== diff --git a/volumes/scaleio/README.md b/volumes/scaleio/README.md new file mode 100644 index 000000000..b5fc8f42e --- /dev/null +++ b/volumes/scaleio/README.md @@ -0,0 +1,302 @@ + + + + +WARNING +WARNING +WARNING +WARNING +WARNING + +

PLEASE NOTE: This document applies to the HEAD of the source tree

+ +If you are using a released version of Kubernetes, you should +refer to the docs that go with that version. + +Documentation for other releases can be found at +[releases.k8s.io](http://releases.k8s.io). + +-- + + + + + +# Dell EMC ScaleIO Volume Plugin for Kubernetes + +This document shows how to configure Kubernetes resources to consume storage from volumes hosted on ScaleIO cluster. + +## Pre-Requisites + +* Kubernetes ver 1.6 or later +* ScaleIO ver 2.0 or later +* A ScaleIO cluster with an API gateway +* ScaleIO SDC binary installed/configured on each Kubernetes node that will consume storage + +## ScaleIO Setup + +This document assumes you are familiar with ScaleIO and have a cluster ready to go. If you are *not familiar* with ScaleIO, please review *Learn how to setup a 3-node* [ScaleIO cluster on Vagrant](https://github.com/codedellemc/labs/tree/master/setup-scaleio-vagrant) and see *General instructions on* [setting up ScaleIO](https://www.emc.com/products-solutions/trial-software-download/scaleio.htm) + +For this demonstration, ensure the following: + + - The ScaleIO `SDC` component is installed and properly configured on all Kubernetes nodes where deployed pods will consume ScaleIO-backed volumes. + - You have a configured ScaleIO gateway that is accessible from the Kubernetes nodes. + +## Deploy Kubernetes Secret for ScaleIO + +The ScaleIO plugin uses a Kubernetes Secret object to store the `username` and `password` credentials. +Kuberenetes requires the secret values to be base64-encoded to simply obfuscate (not encrypt) the clear text as shown below. + +``` +$> echo -n "siouser" | base64 +c2lvdXNlcg== +$> echo -n "sc@l3I0" | base64 +c2NAbDNJMA== +``` +The previous will generate `base64-encoded` values for the username and password. +Remember to generate the credentials for your own environment and copy them in a secret file similar to the following. + +File: [secret.yaml](secret.yaml) + +``` +apiVersion: v1 +kind: Secret +metadata: + name: sio-secret +type: kubernetes.io/scaleio +data: + username: c2lvdXNlcg== + password: c2NAbDNJMA== +``` + +Notice the name of the secret specified above as `sio-secret`. It will be referred in other YAML files. Next, deploy the secret. + +``` +$ kubectl create -f ./examples/volumes/scaleio/secret.yaml +``` + +## Deploying Pods with Persistent Volumes + +The example presented in this section shows how the ScaleIO volume plugin can automatically attach, format, and mount an existing ScaleIO volume for pod. +The Kubernetes ScaleIO volume spec supports the following attributes: + +| Attribute | Description | +|-----------|-------------| +| gateway | address to a ScaleIO API gateway (required)| +| system | the name of the ScaleIO system (required)| +| protectionDomain| the name of the ScaleIO protection domain (default `default`)| +| storagePool| the name of the volume storage pool (default `default`)| +| storageMode| the storage provision mode: `ThinProvisionned` (default) or `ThickProvisionned`| +| volumeName| the name of an existing volume in ScaleIO (required)| +| secretRef:name| reference to a configuered Secret object (required, see Secret earlier)| +| readOnly| specifies the access mode to the mounted volume (default `false`)| +| fsType| the file system to use for the volume (default `ext4`)| + +### Create Volume + +Static persistent volumes require that the volume, to be consumed by the pod, be already created in ScaleIO. You can use your ScaleIO tooling to create a new volume or use the name of a volume that already exists in ScaleIO. For this demo, we assume there's a volume named `vol-0`. If you want to use an existing volume, ensure its name is reflected properly in the `volumeName` attribute below. + +### Deploy Pod YAML + +Create a pod YAML file that declares the volume (above) to be used. + +File: [pod.yaml](pod.yaml) + +``` +apiVersion: v1 +kind: Pod +metadata: + name: pod-0 +spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: pod-0 + volumeMounts: + - mountPath: /test-pd + name: vol-0 + volumes: + - name: vol-0 + scaleIO: + gateway: https://localhost:443/api + system: scaleio + volumeName: vol-0 + secretRef: + name: sio-secret + fsType: xfs +``` +Notice the followings in the previous YAML: + +- Update the `gatewway` to point to your ScaleIO gateway endpoint. +- The `volumeName` attribute refers to the name of an existing volume in ScaleIO. +- The `secretRef:name` attribute references the name of the secret object deployed earlier. + +Next, deploy the pod. + +``` +$> kubectl create -f examples/volumes/scaleio/pod.yaml +``` +You can verify the pod: +``` +$> kubectl get pod +NAME READY STATUS RESTARTS AGE +pod-0 1/1 Running 0 33s +``` +Or for more detail, use +``` +kubectl describe pod pod-0 +``` +You can see the attached/mapped volume on the node: +``` +$> lsblk +NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +... +scinia 252:0 0 8G 0 disk /var/lib/kubelet/pods/135986c7-dcb7-11e6-9fbf-080027c990a7/volumes/kubernetes.io~scaleio/vol-0 +``` + +## StorageClass and Dynamic Provisioning + +In the example in this section, we will see how the ScaleIO volume plugin can automatically provision described in a `StorageClass`. +The ScaleIO volume plugin is a dynamic provisioner identified as `kubernetes.io/scaleio` and supports the following parameters: + +| Parameter | Description | +|-----------|-------------| +| gateway | address to a ScaleIO API gateway (required)| +| system | the name of the ScaleIO system (required)| +| protectionDomain| the name of the ScaleIO protection domain (default `default`)| +| storagePool| the name of the volume storage pool (default `default`)| +| storageMode| the storage provision mode: `ThinProvisionned` (default) or `ThickProvisionned`| +| secretRef| reference to the name of a configuered Secret object (required)| +| readOnly| specifies the access mode to the mounted volume (default `false`)| +| fsType| the file system to use for the volume (default `ext4`)| + + +### ScaleIO StorageClass + +Define a new `StorageClass` as shown in the following YAML. + +File [sc.yaml](sc.yaml) + +``` +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: sio-small +provisioner: kubernetes.io/scaleio +parameters: + gateway: https://localhost:443/api + system: scaleio + protectionDomain: default + secretRef: sio-secret + fsType: xfs +``` +Note the followings: + +- The `name` attribute is set to sio-small . It will be referenced later. +- The `secretRef` attribute matches the name of the Secret object created earlier. + +Next, deploy the storage class file. + +``` +$> kubectl create -f examples/volumes/scaleio/sc.yaml + +$> kubectl get sc +NAME TYPE +sio-small kubernetes.io/scaleio +``` + +### PVC for the StorageClass + +The next step is to define/deploy a `PeristentVolumeClaim` that will use the StorageClass. + +File [sc-pvc.yaml](sc-pvc.yaml) + +``` +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc-sio-small + annotations: + volume.beta.kubernetes.io/storage-class: sio-small +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +``` + +Note the `annotations:` entry which specifies annotation `volume.beta.kubernetes.io/storage-class: sio-small` which references the name of the storage class defined earlier. + +Next, we deploy PVC file for the storage class. This step will cause the Kubernetes ScaleIO plugin to create the volume in the storage system. +``` +$> kubectl create -f examples/volumes/scaleio/sc-pvc.yaml +``` +You verify that a new volume created in the ScaleIO dashboard. You can also verify the newly created volume as follows. +``` + kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESSMODES AGE +pvc-sio-small Bound pvc-5fc78518-dcae-11e6-a263-080027c990a7 10Gi RWO 1h +``` + +###Pod for PVC and SC +At this point, the volume is created (by the claim) in the storage system. To use it, we must define a pod that references the volume as done in this YAML. + +File [pod-sc-pvc.yaml](pod-sc-pvc.yaml) + +``` +kind: Pod +apiVersion: v1 +metadata: + name: pod-sio-small +spec: + containers: + - name: pod-sio-small-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - mountPath: /test + name: test-data + volumes: + - name: test-data + persistentVolumeClaim: + claimName: pvc-sio-small +``` + +Notice that the `claimName:` attribute refers to the name of the PVC defined and deployed earlier. Next, let us deploy the file. + +``` +$> kubectl create -f examples/volumes/scaleio/pod-sc-pvc.yaml +``` +We can now verify that the new pod is deployed OK. +``` +kubectl get pod +NAME READY STATUS RESTARTS AGE +pod-0 1/1 Running 0 23m +pod-sio-small 1/1 Running 0 5s +``` +You can use the ScaleIO dashboard to verify that the new volume has one attachment. You can verify the volume information for the pod: +``` +$> kubectl describe pod pod-sio-small +... +Volumes: + test-data: + Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace) + ClaimName: pvc-sio-small + ReadOnly: false +... +``` +Lastly, you can see the volume's attachment on the Kubernetes node: +``` +$> lsblk +... +scinia 252:0 0 8G 0 disk /var/lib/kubelet/pods/135986c7-dcb7-11e6-9fbf-080027c990a7/volumes/kubernetes.io~scaleio/vol-0 +scinib 252:16 0 16G 0 disk /var/lib/kubelet/pods/62db442e-dcba-11e6-9fbf-080027c990a7/volumes/kubernetes.io~scaleio/sio-5fc9154ddcae11e68db708002 + +``` + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/scaleio/README.md?pixel)]() + diff --git a/volumes/scaleio/pod-sc-pvc.yaml b/volumes/scaleio/pod-sc-pvc.yaml new file mode 100644 index 000000000..ceed7b567 --- /dev/null +++ b/volumes/scaleio/pod-sc-pvc.yaml @@ -0,0 +1,15 @@ +kind: Pod +apiVersion: v1 +metadata: + name: pod-sio-small +spec: + containers: + - name: pod-sio-small-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - mountPath: /test + name: test-data + volumes: + - name: test-data + persistentVolumeClaim: + claimName: pvc-sio-small diff --git a/volumes/scaleio/pod.yaml b/volumes/scaleio/pod.yaml new file mode 100644 index 000000000..4b53b2b53 --- /dev/null +++ b/volumes/scaleio/pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-0 +spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: pod-0 + volumeMounts: + - mountPath: /test-pd + name: vol-0 + volumes: + - name: vol-0 + scaleIO: + gateway: https://localhost:443/api + system: scaleio + volumeName: vol-0 + secretRef: + name: sio-secret + fsType: xfs diff --git a/volumes/scaleio/sc-pvc.yaml b/volumes/scaleio/sc-pvc.yaml new file mode 100644 index 000000000..3937a1e82 --- /dev/null +++ b/volumes/scaleio/sc-pvc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc-sio-small + annotations: + volume.beta.kubernetes.io/storage-class: sio-small +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/volumes/scaleio/sc.yaml b/volumes/scaleio/sc.yaml new file mode 100644 index 000000000..85de382bf --- /dev/null +++ b/volumes/scaleio/sc.yaml @@ -0,0 +1,11 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: sio-small +provisioner: kubernetes.io/scaleio +parameters: + gateway: https://localhost:443/api + system: scaleio + protectionDomain: default + secretRef: sio-secret + fsType: xfs diff --git a/volumes/scaleio/secret.yaml b/volumes/scaleio/secret.yaml new file mode 100644 index 000000000..b2fad68af --- /dev/null +++ b/volumes/scaleio/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: sio-secret +type: kubernetes.io/scaleio +data: + username: YWRtaW4= + password: c0NhbGVpbzEyMw== diff --git a/volumes/vsphere/README.md b/volumes/vsphere/README.md new file mode 100644 index 000000000..d50fbbe37 --- /dev/null +++ b/volumes/vsphere/README.md @@ -0,0 +1,621 @@ +# vSphere Volume + + - [Prerequisites](#prerequisites) + - [Examples](#examples) + - [Volumes](#volumes) + - [Persistent Volumes](#persistent-volumes) + - [Storage Class](#storage-class) + - [Virtual SAN policy support inside Kubernetes](#virtual-san-policy-support-inside-kubernetes) + - [Stateful Set](#stateful-set) + +## Prerequisites + +- Kubernetes with vSphere Cloud Provider configured. + For cloudprovider configuration please refer [vSphere getting started guide](http://kubernetes.io/docs/getting-started-guides/vsphere/). + +## Examples + +### Volumes + + 1. Create VMDK. + + First ssh into ESX and then use following command to create vmdk, + + ```shell + vmkfstools -c 2G /vmfs/volumes/datastore1/volumes/myDisk.vmdk + ``` + + 2. Create Pod which uses 'myDisk.vmdk'. + + See example + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: test-vmdk + spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: test-container + volumeMounts: + - mountPath: /test-vmdk + name: test-volume + volumes: + - name: test-volume + # This VMDK volume must already exist. + vsphereVolume: + volumePath: "[datastore1] volumes/myDisk" + fsType: ext4 + ``` + + [Download example](vsphere-volume-pod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pod.yaml + ``` + + Verify that pod is running: + + ```bash + $ kubectl get pods test-vmdk + NAME READY STATUS RESTARTS AGE + test-vmdk 1/1 Running 0 48m + ``` + +### Persistent Volumes + + 1. Create VMDK. + + First ssh into ESX and then use following command to create vmdk, + + ```shell + vmkfstools -c 2G /vmfs/volumes/datastore1/volumes/myDisk.vmdk + ``` + + 2. Create Persistent Volume. + + See example: + + ```yaml + apiVersion: v1 + kind: PersistentVolume + metadata: + name: pv0001 + spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + vsphereVolume: + volumePath: "[datastore1] volumes/myDisk" + fsType: ext4 + ``` + + [Download example](vsphere-volume-pv.yaml?raw=true) + + Creating the persistent volume: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pv.yaml + ``` + + Verifying persistent volume is created: + + ``` bash + $ kubectl describe pv pv0001 + Name: pv0001 + Labels: + Status: Available + Claim: + Reclaim Policy: Retain + Access Modes: RWO + Capacity: 2Gi + Message: + Source: + Type: vSphereVolume (a Persistent Disk resource in vSphere) + VolumePath: [datastore1] volumes/myDisk + FSType: ext4 + No events. + ``` + + 3. Create Persistent Volume Claim. + + See example: + + ```yaml + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: pvc0001 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + ``` + + [Download example](vsphere-volume-pvc.yaml?raw=true) + + Creating the persistent volume claim: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvc.yaml + ``` + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pvc pvc0001 + Name: pvc0001 + Namespace: default + Status: Bound + Volume: pv0001 + Labels: + Capacity: 2Gi + Access Modes: RWO + No events. + ``` + + 3. Create Pod which uses Persistent Volume Claim. + + See example: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: pvpod + spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-vmdk + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvc0001 + ``` + + [Download example](vsphere-volume-pvcpod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvcpod.yaml + ``` + + Verifying pod is created: + + ``` bash + $ kubectl get pod pvpod + NAME READY STATUS RESTARTS AGE + pvpod 1/1 Running 0 48m + ``` + +### Storage Class + + __Note: Here you don't need to create vmdk it is created for you.__ + 1. Create Storage Class. + + Example 1: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: fast + provisioner: kubernetes.io/vsphere-volume + parameters: + diskformat: zeroedthick + fstype: ext3 + ``` + + [Download example](vsphere-volume-sc-fast.yaml?raw=true) + + You can also specify the datastore in the Storageclass as shown in example 2. The volume will be created on the datastore specified in the storage class. + This field is optional. If not specified as shown in example 1, the volume will be created on the datastore specified in the vsphere config file used to initialize the vSphere Cloud Provider. + + Example 2: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: fast + provisioner: kubernetes.io/vsphere-volume + parameters: + diskformat: zeroedthick + datastore: VSANDatastore + ``` + + [Download example](vsphere-volume-sc-with-datastore.yaml?raw=true) + Creating the storageclass: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-sc-fast.yaml + ``` + + Verifying storage class is created: + + ``` bash + $ kubectl describe storageclass fast + Name: fast + IsDefaultClass: No + Annotations: + Provisioner: kubernetes.io/vsphere-volume + Parameters: diskformat=zeroedthick,fstype=ext3 + No events. + ``` + + 2. Create Persistent Volume Claim. + + See example: + + ```yaml + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: pvcsc001 + annotations: + volume.beta.kubernetes.io/storage-class: fast + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + ``` + + [Download example](vsphere-volume-pvcsc.yaml?raw=true) + + Creating the persistent volume claim: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvcsc.yaml + ``` + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pvc pvcsc001 + Name: pvcsc001 + Namespace: default + StorageClass: fast + Status: Bound + Volume: pvc-83295256-f8e0-11e6-8263-005056b2349c + Labels: + Capacity: 2Gi + Access Modes: RWO + Events: + FirstSeen LastSeen Count From SubObjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 1m 1m 1 persistentvolume-controller Normal ProvisioningSucceeded Successfully provisioned volume pvc-83295256-f8e0-11e6-8263-005056b2349c using kubernetes.io/vsphere-volume + + ``` + + Persistent Volume is automatically created and is bounded to this pvc. + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pv pvc-83295256-f8e0-11e6-8263-005056b2349c + Name: pvc-83295256-f8e0-11e6-8263-005056b2349c + Labels: + StorageClass: fast + Status: Bound + Claim: default/pvcsc001 + Reclaim Policy: Delete + Access Modes: RWO + Capacity: 2Gi + Message: + Source: + Type: vSphereVolume (a Persistent Disk resource in vSphere) + VolumePath: [datastore1] kubevols/kubernetes-dynamic-pvc-83295256-f8e0-11e6-8263-005056b2349c.vmdk + FSType: ext3 + No events. + ``` + + __Note: VMDK is created inside ```kubevols``` folder in datastore which is mentioned in 'vsphere' cloudprovider configuration. + The cloudprovider config is created during setup of Kubernetes cluster on vSphere.__ + + 3. Create Pod which uses Persistent Volume Claim with storage class. + + See example: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: pvpod + spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-vmdk + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvcsc001 + ``` + + [Download example](vsphere-volume-pvcscpod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvcscpod.yaml + ``` + + Verifying pod is created: + + ``` bash + $ kubectl get pod pvpod + NAME READY STATUS RESTARTS AGE + pvpod 1/1 Running 0 48m + ``` + +### Virtual SAN policy support inside Kubernetes + + Vsphere Infrastructure(VI) Admins will have the ability to specify custom Virtual SAN Storage Capabilities during dynamic volume provisioning. You can now define storage requirements, such as performance and availability, in the form of storage capabilities during dynamic volume provisioning. The storage capability requirements are converted into a Virtual SAN policy which are then pushed down to the Virtual SAN layer when a persistent volume (virtual disk) is being created. The virtual disk is distributed across the Virtual SAN datastore to meet the requirements. + + The official [VSAN policy documentation](https://pubs.vmware.com/vsphere-65/index.jsp?topic=%2Fcom.vmware.vsphere.virtualsan.doc%2FGUID-08911FD3-2462-4C1C-AE81-0D4DBC8F7990.html) describes in detail about each of the individual storage capabilities that are supported by VSAN. The user can specify these storage capabilities as part of storage class defintion based on his application needs. + + The policy settings can be one or more of the following: + + * *hostFailuresToTolerate*: represents NumberOfFailuresToTolerate + * *diskStripes*: represents NumberofDiskStripesPerObject + * *objectSpaceReservation*: represents ObjectSpaceReservation + * *cacheReservation*: represents FlashReadCacheReservation + * *iopsLimit*: represents IOPSLimitForObject + * *forceProvisioning*: represents if volume must be Force Provisioned + + __Note: Here you don't need to create persistent volume it is created for you.__ + 1. Create Storage Class. + + Example 1: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: fast + provisioner: kubernetes.io/vsphere-volume + parameters: + diskformat: zeroedthick + hostFailuresToTolerate: "2" + cachereservation: "20" + ``` + [Download example](vsphere-volume-sc-vsancapabilities.yaml?raw=true) + + Here a persistent volume will be created with the Virtual SAN capabilities - hostFailuresToTolerate to 2 and cachereservation is 20% read cache reserved for storage object. Also the persistent volume will be *zeroedthick* disk. + The official [VSAN policy documentation](https://pubs.vmware.com/vsphere-65/index.jsp?topic=%2Fcom.vmware.vsphere.virtualsan.doc%2FGUID-08911FD3-2462-4C1C-AE81-0D4DBC8F7990.html) describes in detail about each of the individual storage capabilities that are supported by VSAN and can be configured on the virtual disk. + + You can also specify the datastore in the Storageclass as shown in example 2. The volume will be created on the datastore specified in the storage class. + This field is optional. If not specified as shown in example 1, the volume will be created on the datastore specified in the vsphere config file used to initialize the vSphere Cloud Provider. + + Example 2: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: fast + provisioner: kubernetes.io/vsphere-volume + parameters: + diskformat: zeroedthick + datastore: VSANDatastore + hostFailuresToTolerate: "2" + cachereservation: "20" + ``` + + [Download example](vsphere-volume-sc-vsancapabilities-with-datastore.yaml?raw=true) + + __Note: If you do not apply a storage policy during dynamic provisioning on a VSAN datastore, it will use a default Virtual SAN policy.__ + + Creating the storageclass: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-sc-vsancapabilities.yaml + ``` + + Verifying storage class is created: + + ``` bash + $ kubectl describe storageclass fast + Name: fast + Annotations: + Provisioner: kubernetes.io/vsphere-volume + Parameters: diskformat=zeroedthick, hostFailuresToTolerate="2", cachereservation="20" + No events. + ``` + + 2. Create Persistent Volume Claim. + + See example: + + ```yaml + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: pvcsc-vsan + annotations: + volume.beta.kubernetes.io/storage-class: fast + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + ``` + + [Download example](vsphere-volume-pvcsc.yaml?raw=true) + + Creating the persistent volume claim: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvcsc.yaml + ``` + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pvc pvcsc-vsan + Name: pvcsc-vsan + Namespace: default + Status: Bound + Volume: pvc-80f7b5c1-94b6-11e6-a24f-005056a79d2d + Labels: + Capacity: 2Gi + Access Modes: RWO + No events. + ``` + + Persistent Volume is automatically created and is bounded to this pvc. + + Verifying persistent volume claim is created: + + ``` bash + $ kubectl describe pv pvc-80f7b5c1-94b6-11e6-a24f-005056a79d2d + Name: pvc-80f7b5c1-94b6-11e6-a24f-005056a79d2d + Labels: + Status: Bound + Claim: default/pvcsc-vsan + Reclaim Policy: Delete + Access Modes: RWO + Capacity: 2Gi + Message: + Source: + Type: vSphereVolume (a Persistent Disk resource in vSphere) + VolumePath: [VSANDatastore] kubevols/kubernetes-dynamic-pvc-80f7b5c1-94b6-11e6-a24f-005056a79d2d.vmdk + FSType: ext4 + No events. + ``` + + __Note: VMDK is created inside ```kubevols``` folder in datastore which is mentioned in 'vsphere' cloudprovider configuration. + The cloudprovider config is created during setup of Kubernetes cluster on vSphere.__ + + 3. Create Pod which uses Persistent Volume Claim with storage class. + + See example: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: pvpod + spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvcsc-vsan + ``` + + [Download example](vsphere-volume-pvcscpod.yaml?raw=true) + + Creating the pod: + + ``` bash + $ kubectl create -f examples/volumes/vsphere/vsphere-volume-pvcscpod.yaml + ``` + + Verifying pod is created: + + ``` bash + $ kubectl get pod pvpod + NAME READY STATUS RESTARTS AGE + pvpod 1/1 Running 0 48m + ``` + +### Stateful Set + +vSphere volumes can be consumed by Stateful Sets. + + 1. Create a storage class that will be used by the ```volumeClaimTemplates``` of a Stateful Set. + + See example: + + ```yaml + kind: StorageClass + apiVersion: storage.k8s.io/v1beta1 + metadata: + name: thin-disk + provisioner: kubernetes.io/vsphere-volume + parameters: + diskformat: thin + ``` + + [Download example](simple-storageclass.yaml) + + 2. Create a Stateful set that consumes storage from the Storage Class created. + + See example: + ```yaml + --- + apiVersion: v1 + kind: Service + metadata: + name: nginx + labels: + app: nginx + spec: + ports: + - port: 80 + name: web + clusterIP: None + selector: + app: nginx + --- + apiVersion: apps/v1beta1 + kind: StatefulSet + metadata: + name: web + spec: + serviceName: "nginx" + replicas: 14 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.beta.kubernetes.io/storage-class: thin-disk + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + ``` + This will create Persistent Volume Claims for each replica and provision a volume for each claim if an existing volume could be bound to the claim. + + [Download example](simple-statefulset.yaml) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/volumes/vsphere/README.md?pixel)]() + diff --git a/volumes/vsphere/deployment.yaml b/volumes/vsphere/deployment.yaml new file mode 100644 index 000000000..1ea779b8a --- /dev/null +++ b/volumes/vsphere/deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: deployment +spec: + replicas: 1 + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis + volumeMounts: + - name: vmfs-vmdk-storage + mountPath: /data/ + volumes: + - name: vmfs-vmdk-storage + vsphereVolume: + volumePath: "[Datastore] volumes/testdir" + fsType: ext4 \ No newline at end of file diff --git a/volumes/vsphere/simple-statefulset.yaml b/volumes/vsphere/simple-statefulset.yaml new file mode 100644 index 000000000..465c7c5ed --- /dev/null +++ b/volumes/vsphere/simple-statefulset.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + clusterIP: None + selector: + app: nginx +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 14 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + annotations: + volume.beta.kubernetes.io/storage-class: thin-disk + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/volumes/vsphere/simple-storageclass.yaml b/volumes/vsphere/simple-storageclass.yaml new file mode 100644 index 000000000..bfa4dae4c --- /dev/null +++ b/volumes/vsphere/simple-storageclass.yaml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: thin-disk +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: thin diff --git a/volumes/vsphere/vsphere-volume-pod.yaml b/volumes/vsphere/vsphere-volume-pod.yaml new file mode 100644 index 000000000..8660d62e4 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pod.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-vmdk +spec: + containers: + - image: gcr.io/google_containers/test-webserver + name: test-container + volumeMounts: + - mountPath: /test-vmdk + name: test-volume + volumes: + - name: test-volume + # This VMDK volume must already exist. + vsphereVolume: + volumePath: "[DatastoreName] volumes/myDisk" + fsType: ext4 \ No newline at end of file diff --git a/volumes/vsphere/vsphere-volume-pv.yaml b/volumes/vsphere/vsphere-volume-pv.yaml new file mode 100644 index 000000000..5bc278288 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv0001 +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + vsphereVolume: + volumePath: "[DatastoreName] volumes/myDisk" + fsType: ext4 \ No newline at end of file diff --git a/volumes/vsphere/vsphere-volume-pvc.yaml b/volumes/vsphere/vsphere-volume-pvc.yaml new file mode 100644 index 000000000..181a3848d --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pvc.yaml @@ -0,0 +1,10 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvc0001 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/volumes/vsphere/vsphere-volume-pvcpod.yaml b/volumes/vsphere/vsphere-volume-pvcpod.yaml new file mode 100644 index 000000000..291664ada --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pvcpod.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pvpod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-vmdk + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvc0001 diff --git a/volumes/vsphere/vsphere-volume-pvcsc.yaml b/volumes/vsphere/vsphere-volume-pvcsc.yaml new file mode 100644 index 000000000..f73ed91b5 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pvcsc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pvcsc001 + annotations: + volume.beta.kubernetes.io/storage-class: fast +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/volumes/vsphere/vsphere-volume-pvcscpod.yaml b/volumes/vsphere/vsphere-volume-pvcscpod.yaml new file mode 100644 index 000000000..036aeb280 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-pvcscpod.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pvpod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/test-webserver + volumeMounts: + - name: test-volume + mountPath: /test-vmdk + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: pvcsc001 diff --git a/volumes/vsphere/vsphere-volume-sc-fast.yaml b/volumes/vsphere/vsphere-volume-sc-fast.yaml new file mode 100644 index 000000000..b2b436f8e --- /dev/null +++ b/volumes/vsphere/vsphere-volume-sc-fast.yaml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: zeroedthick + fstype: ext3 \ No newline at end of file diff --git a/volumes/vsphere/vsphere-volume-sc-vsancapabilities-with-datastore.yaml b/volumes/vsphere/vsphere-volume-sc-vsancapabilities-with-datastore.yaml new file mode 100644 index 000000000..2fd931666 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-sc-vsancapabilities-with-datastore.yaml @@ -0,0 +1,10 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: zeroedthick + datastore: vsanDatastore + hostFailuresToTolerate: "2" + cachereservation: "20" diff --git a/volumes/vsphere/vsphere-volume-sc-vsancapabilities.yaml b/volumes/vsphere/vsphere-volume-sc-vsancapabilities.yaml new file mode 100644 index 000000000..ad2ff9d15 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-sc-vsancapabilities.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: zeroedthick + hostFailuresToTolerate: "2" + cachereservation: "20" diff --git a/volumes/vsphere/vsphere-volume-sc-with-datastore.yaml b/volumes/vsphere/vsphere-volume-sc-with-datastore.yaml new file mode 100644 index 000000000..5e468ef77 --- /dev/null +++ b/volumes/vsphere/vsphere-volume-sc-with-datastore.yaml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: fast +provisioner: kubernetes.io/vsphere-volume +parameters: + diskformat: zeroedthick + datastore: vsanDatastore