Merge pull request #683 from Altinity/0.14.0

Merge 0.14.0 to master

Vagrantfile

@@ -6,11 +6,26 @@ def total_cpus
   Etc.nprocessors
 end
 
+def get_provider
+    provider='virtualbox'
+    for arg in ARGV
+        if ['hyperv','docker'].include? arg
+            provider=arg
+        end
+    end
+    return provider
+end
+
 
 Vagrant.configure(2) do |config|
   config.vm.box = "generic/ubuntu2004"
   config.vm.box_check_update = false
-  config.vm.synced_folder ".", "/vagrant"
+
+  if get_provider == "hyperv"
+    config.vm.synced_folder ".", "/vagrant", type: "smb", smb_username: ENV['USERNAME'], smb_password: ENV['PASSWORD'], mount_options: ["domain="+ENV['USERDOMAIN'], "user="+ENV['USERNAME'], "vers=3.0"," mfsymlinks"]
+  else
+    config.vm.synced_folder ".", "/vagrant"
+  end
 
   if Vagrant.has_plugin?("vagrant-vbguest")
     config.vbguest.auto_update = false
@@ -20,6 +35,7 @@ Vagrant.configure(2) do |config|
     config.timezone.value = "UTC"
   end
 
+
   config.vm.provider "virtualbox" do |vb|
     vb.gui = false
     vb.cpus = total_cpus
@@ -32,6 +48,19 @@ Vagrant.configure(2) do |config|
     vb.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
   end
 
+  config.vm.provider "hyperv" do |hv|
+    # hv.gui = false
+    # hv.default_nic_type = "virtio"
+    hv.cpus = total_cpus
+    hv.maxmemory = "6144"
+    hv.memory = "2048"
+    hv.enable_virtualization_extensions = true
+    hv.linked_clone = true
+    hv.vm_integration_services = {
+        time_synchronization: true,
+    }
+  end
+
   config.vm.define :clickhouse_operator do |clickhouse_operator|
     clickhouse_operator.vm.network "private_network", ip: "172.16.2.99", nic_type: "virtio"
     # port forwarding works only when pair with kubectl port-forward
@@ -52,24 +81,40 @@ Vagrant.configure(2) do |config|
   config.vm.provision "shell", inline: <<-SHELL
     set -xeuo pipefail
     export DEBIAN_FRONTEND=noninteractive
+    # make linux fast again
+    if [[ "0" == $(grep "mitigations" /etc/default/grub | wc -l) ]]; then
+        echo 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off"' >> /etc/default/grub
+        echo 'GRUB_CMDLINE_LINUX_DEFAULT="quiet splash noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off"' >> /etc/default/grub
+        grub-mkconfig
+    fi
+    systemctl enable systemd-timesyncd
+    systemctl start systemd-timesyncd
 
     apt-get update
     apt-get install --no-install-recommends -y apt-transport-https ca-certificates software-properties-common curl
-    apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git ntp
+    apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git
 
     # yq
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys CC86BB64
     add-apt-repository ppa:rmescandon/yq
     apt-get install --no-install-recommends -y yq
 
     # clickhouse
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E0C56BD4
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
     add-apt-repository "deb http://repo.clickhouse.tech/deb/stable/ main/"
     apt-get install --no-install-recommends -y clickhouse-client
 
+    # golang
+    export GOLANG_VERSION=1.16
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6BC817356A3D45E
+    add-apt-repository ppa:longsleep/golang-backports
+    apt-get install --no-install-recommends -y golang-${GOLANG_VERSION}-go
+    ln -nvsf /usr/lib/go-${GOLANG_VERSION}/bin/go /bin/go
+    ln -nvsf /usr/lib/go-${GOLANG_VERSION}/bin/gofmt /bin/gofmt
+
     # docker
-    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8D81803C0EBFCD88
-    add-apt-repository "deb https://download.docker.com/linux/ubuntu focal test"
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) test"
     apt-get install --no-install-recommends -y docker-ce pigz
 
     # docker compose
@@ -88,9 +133,13 @@ Vagrant.configure(2) do |config|
     sha256sum -c /usr/local/bin/k9s.sha256
     tar --verbose -zxvf /usr/local/bin/k9s_${K9S_VERSION}_Linux_x86_64.tar.gz -C /usr/local/bin k9s
 
+    # audit2rbac
+    AUDIT2RBAC_VERSION=0.8.0
+    curl -sL https://github.com/liggitt/audit2rbac/releases/download/v${AUDIT2RBAC_VERSION}/audit2rbac-linux-amd64.tar.gz | tar -zxvf - -C /usr/local/bin
 
     # minikube
-    MINIKUBE_VERSION=1.17.1
+    MINIKUBE_VERSION=1.18.1
+    # MINIKUBE_VERSION=1.19.0
     wget -c --progress=bar:force:noscroll -O /usr/local/bin/minikube https://github.com/kubernetes/minikube/releases/download/v${MINIKUBE_VERSION}/minikube-linux-amd64
     chmod +x /usr/local/bin/minikube
     # required for k8s 1.18+
@@ -100,10 +149,10 @@ Vagrant.configure(2) do |config|
 #    export VALIDATE_YAML=false # only for 1.14
 #    K8S_VERSION=${K8S_VERSION:-1.15.12}
 #    K8S_VERSION=${K8S_VERSION:-1.16.15}
-#    K8S_VERSION=${K8S_VERSION:-1.17.14}
-#    K8S_VERSION=${K8S_VERSION:-1.18.12}
-#    K8S_VERSION=${K8S_VERSION:-1.19.7}
-    K8S_VERSION=${K8S_VERSION:-1.20.2}
+#    K8S_VERSION=${K8S_VERSION:-1.17.17}
+#    K8S_VERSION=${K8S_VERSION:-1.18.18}
+#    K8S_VERSION=${K8S_VERSION:-1.19.10}
+    K8S_VERSION=${K8S_VERSION:-1.20.6}
     export VALIDATE_YAML=true
 
     killall kubectl || true
@@ -114,6 +163,16 @@ Vagrant.configure(2) do |config|
     mkdir -p /home/vagrant/.minikube
     ln -svf /home/vagrant/.minikube /root/.minikube
 
+    mkdir -p /root/.minikube/files/etc/ssl/certs
+
+cat <<EOF >/root/.minikube/files/etc/ssl/certs/audit-policy.yaml
+    # Log all requests at the Metadata level.
+    apiVersion: audit.k8s.io/v1
+    kind: Policy
+    rules:
+    - level: Metadata
+EOF
+
     mkdir -p /home/vagrant/.kube
     ln -svf /home/vagrant/.kube /root/.kube
 
@@ -123,14 +182,16 @@ Vagrant.configure(2) do |config|
     sudo -H -u vagrant minikube config set memory 5G
     sudo -H -u vagrant minikube config set driver docker
     sudo -H -u vagrant minikube config set kubernetes-version ${K8S_VERSION}
-    sudo -H -u vagrant minikube start
+    sudo -H -u vagrant minikube start --extra-config=apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml --extra-config=apiserver.audit-log-path=-
     sudo -H -u vagrant minikube addons enable ingress
     sudo -H -u vagrant minikube addons enable ingress-dns
     sudo -H -u vagrant minikube addons enable metrics-server
 
+#     minikube delete
+#     rm -rf /tmp/juju*
 #     minikube config set vm-driver none
 #     minikube config set kubernetes-version ${K8S_VERSION}
-#     minikube start --vm=true
+#     minikube start
 #     minikube addons enable ingress
 #     minikube addons enable ingress-dns
 #     minikube addons enable metrics-server
@@ -147,13 +208,14 @@ Vagrant.configure(2) do |config|
     source $HOME/.bashrc
     export KREW_ROOT=/home/vagrant/.krew
     kubectl krew install tap
-    kubectl krew install debug
     kubectl krew install sniff
     kubectl krew install flame
+    kubectl krew install minio
+    # look to https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/#ephemeral-container
+    # kubectl krew install debug
 
     cd /vagrant/
-
-    git_branch=$(git rev-parse --abbrev-ref HEAD)
+    git_branch=$(git rev-parse --abbrev-ref HEAD /vagrant/)
     export OPERATOR_RELEASE=$(cat release)
     export BRANCH=${BRANCH:-$git_branch}
     export OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE:-kube-system}
@@ -204,9 +266,16 @@ Vagrant.configure(2) do |config|
 
     pip3 install -r /vagrant/tests/requirements.txt
 
+    python3 /vagrant/tests/test_metrics_alerts.py
     python3 /vagrant/tests/test.py --only=operator/*
     python3 /vagrant/tests/test_examples.py
     python3 /vagrant/tests/test_metrics_exporter.py
-    python3 /vagrant/tests/test_metrics_alerts.py
+
+    # audit2rbac
+    kubectl logs kube-apiserver-minikube -n kube-system | grep audit.k8s.io/v1 > /tmp/audit2rbac.log
+    audit2rbac -f /tmp/audit2rbac.log --serviceaccount kube-system:clickhouse-operator > /tmp/audit2rbac.yaml
+    # cp -fv /tmp/audit2rbac.yaml /vagrant/deploy/dev/clickhouse-operator-install-yaml-template-02-section-rbac-restricted.yaml
+
   SHELL
+
 end

cmd/metrics_exporter/app/metrics_exporter.go

@@ -90,10 +90,10 @@ func Run() {
 	log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt)
 
 	// Initialize k8s API clients
-	_, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
+	kubeClient, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
 
 	// Create operator instance
-	chop := chop.GetCHOp(chopClient, chopConfigFile)
+	chop := chop.GetCHOp(kubeClient, chopClient, chopConfigFile)
 	chop.SetupLog()
 	log.Info(chop.Config().String(true))
 

cmd/operator/app/clickhouse_operator.go

@@ -98,7 +98,7 @@ func Run() {
 	kubeClient, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
 
 	// Create operator instance
-	chop := chop.GetCHOp(chopClient, chopConfigFile)
+	chop := chop.GetCHOp(kubeClient, chopClient, chopConfigFile)
 	chop.SetupLog()
 	log.V(1).A().Info("Log options parsed")
 	log.Info(chop.Config().String(true))

config/config-dev.yaml

@@ -94,6 +94,13 @@ chConfigNetworksHostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi
 # located in `chUsersConfigsPath` folder
 chUsername: clickhouse_operator
 chPassword: clickhouse_operator_password
+
+# Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances
+# Can be used instead of explicitly specified username and password
+chCredentialsSecretNamespace: "dev"
+chCredentialsSecretName: "chop-clickhouse-credentials"
+
+# Port where to connect to ClickHouse instances to
 chPort: 8123
 
 ################################################
@@ -104,7 +111,7 @@ chPort: 8123
 
 logtostderr: "true"
 alsologtostderr: "false"
-v: "2"
+v: "3"
 stderrthreshold: ""
 vmodule: ""
 log_backtrace_at: ""
@@ -119,3 +126,29 @@ log_backtrace_at: ""
 reconcileThreadsNumber: 1
 reconcileWaitExclude: false
 reconcileWaitInclude: false
+
+################################################
+##
+## Labels management parameters
+##
+################################################
+
+# When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`,
+# exclude labels from the following list:
+#excludeFromPropagationLabels:
+#  - "labelA"
+#  - "labelB"
+
+# Whether to append *Scope* labels to StatefulSet and Pod.
+# Full list of available *scope* labels check in labeler.go
+#  LabelShardScopeIndex
+#  LabelReplicaScopeIndex
+#  LabelCHIScopeIndex
+#  LabelCHIScopeCycleSize
+#  LabelCHIScopeCycleIndex
+#  LabelCHIScopeCycleOffset
+#  LabelClusterScopeIndex
+#  LabelClusterScopeCycleSize
+#  LabelClusterScopeCycleIndex
+#  LabelClusterScopeCycleOffset
+appendScopeLabels: "yes"

config/config.yaml

@@ -86,14 +86,22 @@ chConfigNetworksHostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi
 ##
 ################################################
 
-# ClickHouse credentials (username, password and port) to be used by operator to connect to ClickHouse instances for:
+# ClickHouse credentials (username, password and port) to be used by operator to connect to ClickHouse instances
+# for:
 # 1. Metrics requests
 # 2. Schema maintenance
 # 3. DROP DNS CACHE
 # User with such credentials can be specified in additional ClickHouse .xml config files,
 # located in `chUsersConfigsPath` folder
 chUsername: clickhouse_operator
 chPassword: clickhouse_operator_password
+
+# Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances
+# Can be used instead of explicitly specified username and password
+chCredentialsSecretNamespace: ""
+chCredentialsSecretName: ""
+
+# Port where to connect to ClickHouse instances to
 chPort: 8123
 
 ################################################
@@ -119,3 +127,29 @@ log_backtrace_at: ""
 reconcileThreadsNumber: 10
 reconcileWaitExclude: false
 reconcileWaitInclude: false
+
+################################################
+##
+## Labels management parameters
+##
+################################################
+
+# When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`,
+# exclude labels from the following list:
+#excludeFromPropagationLabels:
+#  - "labelA"
+#  - "labelB"
+
+# Whether to append *Scope* labels to StatefulSet and Pod.
+# Full list of available *scope* labels check in labeler.go
+#  LabelShardScopeIndex
+#  LabelReplicaScopeIndex
+#  LabelCHIScopeIndex
+#  LabelCHIScopeCycleSize
+#  LabelCHIScopeCycleIndex
+#  LabelCHIScopeCycleOffset
+#  LabelClusterScopeIndex
+#  LabelClusterScopeCycleSize
+#  LabelClusterScopeCycleIndex
+#  LabelClusterScopeCycleOffset
+appendScopeLabels: "no"

config/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: dev
+  name: chop-clickhouse-credentials
+type: Opaque
+stringData:
+  username: chuser
+  password: chpassword