Merge pull request #136 from Altinity/0.4.2

Altinity · Jul 15, 2019 · 4b9101a · 4b9101a
2 parents 3d3526e + fd2b523
commit 4b9101a
Show file tree

Hide file tree

Showing 22 changed files with 391 additions and 300 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # === Builder ===
 
-FROM golang:1.11.5 AS builder
+FROM golang:1.12 AS builder
 
 RUN apt-get update && apt-get install -y -q apt-utils && apt-get install -y -q gettext-base
 WORKDIR $GOPATH/src/github.com/altinity/clickhouse-operator

diff --git a/docs/custom_resource_explained.md b/docs/custom_resource_explained.md
@@ -385,6 +385,16 @@ Another example with selectively described replicas. Note - `replicasCount` spec
         # 10. {replicaD} - short hashed replica name (BEWARE, this is an experimental feature)
         # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature)
         generateName: "service-{chi}"
+        # type ObjectMeta struct from k8s.io/meta/v1
+        metadata:
+          labels:
+            custom.label: "custom.value"
+          annotations:
+            cloud.google.com/load-balancer-type: "Internal"
+            service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
+            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"
         # type ServiceSpec struct from k8s.io/core/v1
         spec:
           ports:

diff --git a/docs/examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml b/docs/examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml
@@ -33,6 +33,15 @@ spec:
     serviceTemplates:
       - name: ch-service
         generateName: chendpoint
+        metadata:
+          labels:
+            custom.label: "custom.value"
+          annotations:
+            cloud.google.com/load-balancer-type: "Internal"
+            service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
+            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"
         spec:
           ports:
             - name: http

diff --git a/docs/examples/99-clickhouseinstallation-max.yaml b/docs/examples/99-clickhouseinstallation-max.yaml
@@ -171,6 +171,16 @@ s    zookeeper:
         # 10. {replicaD} - short hashed replica name (BEWARE, this is an experimental feature)
         # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature)
         generateName: "service-{chi}"
+        # type ObjectMeta struct from k8s.io/meta/v1
+        metadata:
+          labels:
+            custom.label: "custom.value"
+          annotations:
+            cloud.google.com/load-balancer-type: "Internal"
+            service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
+            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"
         # type ServiceSpec struct from k8s.io/core/v1
         spec:
           ports:

diff --git a/docs/examples/label_nodes.sh b/docs/examples/label_nodes.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+kubectl get nodes | \
+grep -v master | \
+awk '{print $1}' | \
+tail -n +2 | \
+while read -r line; do
+	NODE=$line
+	#kubectl label nodes <node-name> <label-key>=<label-value>
+	#kubectl label nodes --overwrite=true $NODE clickhouse=allow
+	kubectl label nodes $NODE clickhouse=allow
+done
+
diff --git a/docs/replication_setup.md b/docs/replication_setup.md
@@ -6,166 +6,89 @@
 1. ClickHouse operator [installed](operator_installation_details.md)
 1. Zookeeper installed as described in [Zookeeper Setup](zookeeper_setup.md)
 
-## Installation
-Let's create ClickHouse installation with replication cluster. There are two example manifests available - depending on what Zookeeper installation was made:
- 1. One-node Zookeeper installation - based in [./examples/04-zookeeper-replication-01-minimal.yaml](./examples/04-zookeeper-replication-01-minimal.yaml)
- 1. Three-nodes Zookeeper installation - based in [./examples/04-zookeeper-replication-02-medium.yaml](./examples/04-zookeeper-replication-02-medium.yaml) 
-We'd like to create all resources inside separated namespace, which is convenient to manage.
-
-Create namespace where all replication cluster would live
-```bash
-kubectl create namespace replcluster
-```
-
-We'll go here with medium 3-nodes Zookeeper installation.
-Create ClickHouse installation object inside namespace:
-```bash
-kubectl -n replcluster apply -f ./examples/04-zookeeper-replication-02-medium.yaml 
-```
 
 ## Manifest
 
+The example below creates a cluster with 2 shards and 2 replicas and persistent storage.
+
 ```yaml
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+
+metadata:
+  name: test
+
+spec:
+  defaults:
+    templates: 
+      volumeClaimTemplate: default
+      podTemplate: clickhouse:19.6
+
   configuration:
-    zookeeper:
+    zookeeper: # Add Zookeeper
       nodes:
-        - host: zookeeper-0.zookeepers.zoo3ns
-          port: 2181
-        - host: zookeeper-1.zookeepers.zoo3ns
-          port: 2181
-        - host: zookeeper-2.zookeepers.zoo3ns
-          port: 2181
-```
-
-Zookeeper nodes are DNS names [assigned to Zookeeper](zookeeper_setup.md#dns-names) cluster as described in details in [Zookeeper Setup](zookeeper_setup.md) docs
-
-Verify cluster - show available ClickHouse cluster from within ClickHouse node instance
-```bash
-kubectl -n replcluster exec -it chi-e557e4-54a2-0-0-0 -- clickhouse-client -q "select cluster, shard_num, replica_num, host_name, host_address from system.clusters where cluster='replcluster'"
+      - host: zookeeper.zoo1ns
+        port: 2181
+    clusters:
+      - name: replicated
+        layout:
+          shardsCount: 2
+          replicasCount: 2
+
+  templates:
+    volumeClaimTemplates:
+      - name: default
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 500Mi
+    podTemplates:
+      - name: clickhouse:19.6
+        spec:
+          containers:
+            - name: clickhouse-pod
+              image: yandex/clickhouse-server:19.6.2.11
 ```
 
-where `chi-e557e4-54a2-0-0-0` is pod's name obtained by `kubectl -n replcluster get pod`
-
-Now let's create replicated table.
 
 ## Replicated table setup
 
 ### Macros
 Operator provides set of [macros](https://clickhouse.yandex/docs/en/operations/server_settings/settings/#macros), which are:
- 1. `{installation}` - which refers to `.metadata.name` of ClickHouse Installation custom resource
- 1. Multiple macros to specify name (i.e. - `.spec.configuration.clusters.name`) of each cluster, named as <"`.spec.configuration.clusters.name` value">  
- 1. Multiple macros to specify name of each shard of of each cluster, named as <"`.spec.configuration.clusters.name` value"-shard>
- 1. `{replica}` - which specifies replica name of each shard. Replica should be unique within each shard, so we can use StatefulSet name for this purpose
-
-For example, our manifest [./examples/04-zookeeper-replication-02-medium.yaml](./examples/04-zookeeper-replication-02-medium.yaml) we'll have the following macros specified:
-- for the first pod (`chi-e557e4-54a2-0-0-0`) we have shard with index `0`
-```xml
-    <macros>
-        <installation>repl-02</installation>
-        <replcluster>replcluster</replcluster>
-        <replcluster-shard>0</replcluster-shard>
-        <replica>chi-e557e4-54a2-0-0</replica>
-    </macros>
-```
-- for the second pod (`chi-e557e4-54a2-0-1-0`) we have shard with index `0` (it is replica of the first pod)
-```xml
-    <macros>
-        <installation>repl-02</installation>
-        <replcluster>replcluster</replcluster>
-        <replcluster-shard>0</replcluster-shard>
-        <replica>chi-e557e4-54a2-0-1</replica>
-    </macros>
-```
-- for the third pod (`chi-e557e4-54a2-1-0-0`) we have shard with index `1`
-```xml
-    <macros>
-        <installation>repl-02</installation>
-        <replcluster>replcluster</replcluster>
-        <replcluster-shard>1</replcluster-shard>
-        <replica>chi-e557e4-54a2-1-0</replica>
-    </macros>
-```
-etc for the rest of the pods.
+ 1. `{installation}` -- ClickHouse Installation name
+ 1. `{cluster}` -- primary cluster name
+ 1. `{replica}` -- replica name in the cluster, maps to pod service name
+ 1. `{shard}` -- shard id
+
+ClickHouse also supports internal macros {database} and {table} that maps to current database.table.
 
 ### Create replicated table
 
 Now we can create [replicated table](https://clickhouse.yandex/docs/en/operations/table_engines/replication/), using specified macros
-`CREATE TABLE` query example for replicated table looks like the following (do not copy-paste this, real copy-paste-ready query goes later):
-```sql
-CREATE TABLE articles_repl.events_repl_local (
-    event_date  Date,
-    event_type  Int32,
-    article_id  Int32,
-    title       String
-) engine=ReplicatedMergeTree('/clickhouse/{installation}/{replcluster}/tables/{replcluster-shard}/articles_repl/events_repl_local', '{replica}', event_date, (event_type, article_id), 8192);
-```
-`CREATE TABLE` query example for distributed table looks like the following (do not copy-paste this, real copy-paste-ready query goes later):
-```sql
-CREATE TABLE IF NOT EXISTS  
-   articles_repl.events_replicated AS articles_repl.events_repl_local
-ENGINE = Distributed( replcluster, articles_repl, events_repl_local, rand());
-```
 
-So, on each of all pod we have to run these queries (copy+paste from here on):
 ```sql
-CREATE DATABASE articles_repl;
-CREATE TABLE articles_repl.events_repl_local (
+CREATE TABLE events_local on cluster '{cluster}' (
     event_date  Date,
     event_type  Int32,
     article_id  Int32,
     title       String
-) engine=ReplicatedMergeTree('/clickhouse/{installation}/{replcluster}/tables/{replcluster-shard}/articles_repl/events_repl_local', '{replica}', event_date, (event_type, article_id), 8192);
-CREATE TABLE IF NOT EXISTS  
-   articles_repl.events_replicated AS articles_repl.events_repl_local
-ENGINE = Distributed( replcluster, articles_repl, events_repl_local, rand());
-```
-
-Run queries:
-```bash
-kubectl -n replcluster exec -it chi-e557e4-54a2-0-0-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
-kubectl -n replcluster exec -it chi-e557e4-54a2-0-1-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
-kubectl -n replcluster exec -it chi-e557e4-54a2-1-0-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
-kubectl -n replcluster exec -it chi-e557e4-54a2-1-1-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
-kubectl -n replcluster exec -it chi-e557e4-54a2-2-0-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
-kubectl -n replcluster exec -it chi-e557e4-54a2-2-1-0 -- clickhouse-client
-... paste CREATE DATABASE + 2 CREATE TABLE SQLs ...
+) engine=ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}', event_date, (event_type, article_id), 8192);
 ```
 
-and insert some test data on any of replicas
 ```sql
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 1, 1, concat('events_dist_1_', toString(now())));
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 1, 2, concat('events_dist_2_', toString(now())));
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 1, 3, concat('events_dist_3_', toString(now())));
-
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 2, 1, concat('events_dist_1_', toString(now())));
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 2, 2, concat('events_dist_2_', toString(now())));
-INSERT INTO articles_repl.events_replicated VALUES ('2010-01-02', 2, 3, concat('events_dist_3_', toString(now())));
+CREATE TABLE events on cluster '{cluster}' AS events_local
+ENGINE = Distributed('{cluster}', default, events_local, rand());
 ```
 
-and watch how these data are distributed over the cluster
+We can generate some data:
 ```sql
-SELECT count(*) FROM articles_repl.events_replicated;
-SELECT count(*) FROM articles_repl.events_repl_local;
+insert into events select today(), rand()%3, number, 'my title' from numbers(100);
 ```
 
-## Delete cluster
-
-As soon as cluster is not required any more we can delete it from k8s 
-
-- delete ClickHouse Installation namespace with cluster inside
-```bash
-kubectl delete namespaces replcluster 
-```
-- delete Zookeeper namespace with Zookeeper inside 
-```bash
-kubectl delete namespaces zoo3ns 
-```
-- uninstall operator
-```bash
-kubectl delete -f clickhouse-operator-install.yaml
+And check how these data are distributed over the cluster
+```sql
+SELECT count() FROM events;
+SELECT count() FROM events_local;
 ```
diff --git a/docs/schema_migration.md b/docs/schema_migration.md
@@ -1,19 +1,23 @@
-# Schema migration
+# Schema maintenance
 
-`clickhouse-operator` can resize cluster. After cluster is scaled up and new ClickHouse instances are ready, `clickhouse-operator` migrates schema.
-Schema is unified among all replicas within each cluster. So, suppose we have the following configuration:
- * **replica1** has table `table1`
- * **replica2** has table `table2`
-
-Please, pay attention, that each replica has its own table. 
+`clickhouse-operator` automates schema management when cluster is scaled up or down
 
-Having specified setup, we decide to add one more replica.
+# Schema auto-creation
 
-`clickhouse-operator` adds one more ClickHouse instance, and, after scale process completed, all three nodes would have both tables created:
- * **replica1** has tables: `table1`, `table2`
- * **replica2** has tables: `table1`, `table2`
- * **replica3** has tables: `table1`, `table2`
+After cluster is scaled up and new ClickHouse instances are ready, `clickhouse-operator` automatically creates schema.
 
-Please note - `clickhouse-operator` does nothing with data, table definition only is migrated. 
-However, in case newly added replica is properly configured, ClickHouse itself would replicate data from master to newly added replica.
-
+If shard is added:
+  * Other shards in the cluster are analyzed for distributed and corresponding local tables
+  * Databases for local and distributed tables are created
+  * Local tables are created
+  * Distributed tables are created
+
+If replica is added:
+  * Other replicas **at the same shard** are analyzed for replicated tables
+  * Databases for replicated tables are created
+  * Replicated tables are created
+  * Then the same logic as to adding shard applies
+
+# Schema auto-deletion
+
+If cluster is scaled down and some shards or replicas are deleted, `clickhouse-operator` drops replicated table to make sure nothing is left in ZooKeeper.
diff --git a/docs/zookeeper_setup.md b/docs/zookeeper_setup.md
@@ -1,7 +1,6 @@
-# Task: Setup Zookeeper
+# Setting up Zookeeper
 
-We are going to setup Zookeeper in k8s environment.
-This document assumes k8s cluster already setup and `kubectl` has access to it.
+This document describes how to setup ZooKeeper in k8s environment.
 
 Zookeeper installation is available in two options:
 1. [Quick start](#quick-start) - just run it quickly and ask no questions

diff --git a/manifests/dev/clickhouse-operator-template-01-section-crd.yaml b/manifests/dev/clickhouse-operator-template-01-section-crd.yaml
@@ -256,6 +256,9 @@ spec:
                         type: string
                       generateName:
                         type: string
+                      metadata:
+                        # TODO specify ObjectMeta
+                        type: object
                       spec:
                         # TODO specify ServiceSpec
                         type: object
diff --git a/manifests/operator/clickhouse-operator-install.yaml b/manifests/operator/clickhouse-operator-install.yaml
@@ -256,6 +256,9 @@ spec:
                         type: string
                       generateName:
                         type: string
+                      metadata:
+                        # TODO specify ObjectMeta
+                        type: object
                       spec:
                         # TODO specify ServiceSpec
                         type: object

diff --git a/manifests/zookeeper/advanced/05-stateful-set-persistent-volume.yaml b/manifests/zookeeper/advanced/05-stateful-set-persistent-volume.yaml
@@ -51,6 +51,9 @@ spec:
               name: server
             - containerPort: 3888
               name: leader-election
+# See those links for proper startup settings:
+# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
+# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
           command:
             - sh
             - -c
@@ -63,13 +66,13 @@ spec:
           --election_port=3888 \
           --server_port=2888 \
           --tick_time=2000 \
-          --init_limit=10 \
-          --sync_limit=5 \
-          --heap=512M \
-          --max_client_cnxns=60 \
-          --snap_retain_count=3 \
-          --purge_interval=12 \
-          --max_session_timeout=40000 \
+          --init_limit=30000 \
+          --sync_limit=10 \
+          --heap=1G \
+          --max_client_cnxns=2000 \
+          --snap_retain_count=10 \
+          --purge_interval=1 \
+          --max_session_timeout=60000000 \
           --min_session_timeout=4000 \
           --log_level=INFO"
           readinessProbe: