From 93372b52d4b35254f1bd855d3c1c76e2f371141b Mon Sep 17 00:00:00 2001 From: Yanjun Zhou Date: Fri, 8 Apr 2022 16:15:34 -0700 Subject: [PATCH] Support ClickHouse deployment with Persistent Volume Signed-off-by: Yanjun Zhou --- build/yamls/flow-visibility.yml | 45 +++---- .../yamls/flow-visibility/base/clickhouse.yml | 8 +- .../provisioning/datasources/create_table.sh | 6 +- .../patches/pv/createLocalPv.yml | 30 +++++ .../patches/pv/createNfsPv.yml | 23 ++++ .../flow-visibility/patches/pv/mountPv.yml | 14 +++ .../flow-visibility/patches/ram/mountRam.yml | 12 ++ docs/network-flow-visibility.md | 116 +++++++++++++++++- hack/generate-manifest-flow-visibility.sh | 93 +++++++++++++- .../clickhouse-monitor/main.go | 100 +++++++++++++-- 10 files changed, 402 insertions(+), 45 deletions(-) create mode 100644 build/yamls/flow-visibility/patches/pv/createLocalPv.yml create mode 100644 build/yamls/flow-visibility/patches/pv/createNfsPv.yml create mode 100644 build/yamls/flow-visibility/patches/pv/mountPv.yml create mode 100644 build/yamls/flow-visibility/patches/ram/mountRam.yml diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 7217afb84f7..3f8ba45984d 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -86,26 +86,27 @@ data: UInt64,\n reverseThroughputFromDestinationNode UInt64,\n trusted UInt8 DEFAULT 0\n ) engine=MergeTree\n ORDER BY (timeInserted, flowEndSeconds)\n \ TTL timeInserted + INTERVAL 1 HOUR\n SETTINGS merge_with_ttl_timeout = - 3600;\n\n CREATE MATERIALIZED VIEW flows_pod_view\n ENGINE = SummingMergeTree\n - \ ORDER BY (\n timeInserted,\n flowEndSeconds,\n flowEndSecondsFromSourceNode,\n - \ flowEndSecondsFromDestinationNode,\n sourcePodName,\n destinationPodName,\n - \ destinationIP,\n destinationServicePortName,\n flowType,\n - \ sourcePodNamespace,\n destinationPodNamespace)\n TTL timeInserted - + INTERVAL 1 HOUR\n SETTINGS merge_with_ttl_timeout = 3600\n POPULATE\n - \ AS SELECT\n timeInserted,\n flowEndSeconds,\n flowEndSecondsFromSourceNode,\n - \ flowEndSecondsFromDestinationNode,\n sourcePodName,\n destinationPodName,\n - \ destinationIP,\n destinationServicePortName,\n flowType,\n - \ sourcePodNamespace,\n destinationPodNamespace,\n sum(octetDeltaCount) - AS octetDeltaCount,\n sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount,\n - \ sum(throughput) AS throughput,\n sum(reverseThroughput) AS reverseThroughput,\n - \ sum(throughputFromSourceNode) AS throughputFromSourceNode,\n sum(throughputFromDestinationNode) - AS throughputFromDestinationNode\n FROM flows\n GROUP BY\n timeInserted,\n + 3600;\n\n CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view\n ENGINE + = SummingMergeTree\n ORDER BY (\n timeInserted,\n flowEndSeconds,\n + \ flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n + \ sourcePodName,\n destinationPodName,\n destinationIP,\n + \ destinationServicePortName,\n flowType,\n sourcePodNamespace,\n + \ destinationPodNamespace)\n TTL timeInserted + INTERVAL 1 HOUR\n SETTINGS + merge_with_ttl_timeout = 3600\n POPULATE\n AS SELECT\n timeInserted,\n \ flowEndSeconds,\n flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n \ sourcePodName,\n destinationPodName,\n destinationIP,\n \ destinationServicePortName,\n flowType,\n sourcePodNamespace,\n - \ destinationPodNamespace;\n\n CREATE MATERIALIZED VIEW flows_node_view\n - \ ENGINE = SummingMergeTree\n ORDER BY (\n timeInserted,\n flowEndSeconds,\n + \ destinationPodNamespace,\n sum(octetDeltaCount) AS octetDeltaCount,\n + \ sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount,\n sum(throughput) + AS throughput,\n sum(reverseThroughput) AS reverseThroughput,\n sum(throughputFromSourceNode) + AS throughputFromSourceNode,\n sum(throughputFromDestinationNode) AS throughputFromDestinationNode\n + \ FROM flows\n GROUP BY\n timeInserted,\n flowEndSeconds,\n \ flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n + \ sourcePodName,\n destinationPodName,\n destinationIP,\n + \ destinationServicePortName,\n flowType,\n sourcePodNamespace,\n + \ destinationPodNamespace;\n\n CREATE MATERIALIZED VIEW IF NOT EXISTS + flows_node_view\n ENGINE = SummingMergeTree\n ORDER BY (\n timeInserted,\n + \ flowEndSeconds,\n flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n \ sourceNodeName,\n destinationNodeName,\n sourcePodNamespace,\n \ destinationPodNamespace)\n TTL timeInserted + INTERVAL 1 HOUR\n SETTINGS merge_with_ttl_timeout = 3600\n POPULATE\n AS SELECT\n timeInserted,\n @@ -120,9 +121,9 @@ data: AS reverseThroughputFromDestinationNode\n FROM flows\n GROUP BY\n timeInserted,\n \ flowEndSeconds,\n flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n \ sourceNodeName,\n destinationNodeName,\n sourcePodNamespace,\n - \ destinationPodNamespace;\n\n CREATE MATERIALIZED VIEW flows_policy_view\n - \ ENGINE = SummingMergeTree\n ORDER BY (\n timeInserted,\n flowEndSeconds,\n - \ flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n + \ destinationPodNamespace;\n\n CREATE MATERIALIZED VIEW IF NOT EXISTS + flows_policy_view\n ENGINE = SummingMergeTree\n ORDER BY (\n timeInserted,\n + \ flowEndSeconds,\n flowEndSecondsFromSourceNode,\n flowEndSecondsFromDestinationNode,\n \ egressNetworkPolicyName,\n egressNetworkPolicyRuleAction,\n ingressNetworkPolicyName,\n \ ingressNetworkPolicyRuleAction,\n sourcePodNamespace,\n destinationPodNamespace)\n \ TTL timeInserted + INTERVAL 1 HOUR\n SETTINGS merge_with_ttl_timeout = @@ -145,7 +146,7 @@ data: \ ORDER BY (timeCreated);\n \nEOSQL\n" kind: ConfigMap metadata: - name: clickhouse-mounted-configmap-dkbmg82ctg + name: clickhouse-mounted-configmap-58fkkt9b56 namespace: flow-visibility --- apiVersion: v1 @@ -4934,12 +4935,14 @@ spec: value: default.flows - name: MV_NAMES value: default.flows_pod_view default.flows_node_view default.flows_policy_view + - name: STORAGE_SIZE + value: 8Gi image: projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor:latest imagePullPolicy: IfNotPresent name: clickhouse-monitor volumes: - configMap: - name: clickhouse-mounted-configmap-dkbmg82ctg + name: clickhouse-mounted-configmap-58fkkt9b56 name: clickhouse-configmap-volume - emptyDir: medium: Memory diff --git a/build/yamls/flow-visibility/base/clickhouse.yml b/build/yamls/flow-visibility/base/clickhouse.yml index 18fe72ee747..bf251312edc 100644 --- a/build/yamls/flow-visibility/base/clickhouse.yml +++ b/build/yamls/flow-visibility/base/clickhouse.yml @@ -45,8 +45,6 @@ spec: volumeMounts: - name: clickhouse-configmap-volume mountPath: /docker-entrypoint-initdb.d - - name: clickhouse-storage-volume - mountPath: /var/lib/clickhouse - name: clickhouse-monitor image: flow-visibility-clickhouse-monitor env: @@ -66,11 +64,9 @@ spec: value: "default.flows" - name: MV_NAMES value: "default.flows_pod_view default.flows_node_view default.flows_policy_view" + - name: STORAGE_SIZE + value: "8Gi" volumes: - name: clickhouse-configmap-volume configMap: name: $(CLICKHOUSE_CONFIG_MAP_NAME) - - name: clickhouse-storage-volume - emptyDir: - medium: Memory - sizeLimit: 8Gi diff --git a/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh b/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh index 9f135579460..423fa45d00d 100644 --- a/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh +++ b/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh @@ -72,7 +72,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL TTL timeInserted + INTERVAL 1 HOUR SETTINGS merge_with_ttl_timeout = 3600; - CREATE MATERIALIZED VIEW flows_pod_view + CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view ENGINE = SummingMergeTree ORDER BY ( timeInserted, @@ -121,7 +121,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL sourcePodNamespace, destinationPodNamespace; - CREATE MATERIALIZED VIEW flows_node_view + CREATE MATERIALIZED VIEW IF NOT EXISTS flows_node_view ENGINE = SummingMergeTree ORDER BY ( timeInserted, @@ -163,7 +163,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL sourcePodNamespace, destinationPodNamespace; - CREATE MATERIALIZED VIEW flows_policy_view + CREATE MATERIALIZED VIEW IF NOT EXISTS flows_policy_view ENGINE = SummingMergeTree ORDER BY ( timeInserted, diff --git a/build/yamls/flow-visibility/patches/pv/createLocalPv.yml b/build/yamls/flow-visibility/patches/pv/createLocalPv.yml new file mode 100644 index 00000000000..a33411b67b9 --- /dev/null +++ b/build/yamls/flow-visibility/patches/pv/createLocalPv.yml @@ -0,0 +1,30 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: clickhouse-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +allowVolumeExpansion: True +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clickhouse-pv +spec: + storageClassName: clickhouse-storage + capacity: + storage: 8Gi + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + local: + path: LOCAL_PATH + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: clickhouse/instance + operator: In + values: + - data diff --git a/build/yamls/flow-visibility/patches/pv/createNfsPv.yml b/build/yamls/flow-visibility/patches/pv/createNfsPv.yml new file mode 100644 index 00000000000..17bdf41b782 --- /dev/null +++ b/build/yamls/flow-visibility/patches/pv/createNfsPv.yml @@ -0,0 +1,23 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: clickhouse-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +allowVolumeExpansion: True +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clickhouse-pv +spec: + storageClassName: clickhouse-storage + capacity: + storage: 8Gi + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + nfs: + path: NFS_SERVER_PATH + server: NFS_SERVER_ADDRESS diff --git a/build/yamls/flow-visibility/patches/pv/mountPv.yml b/build/yamls/flow-visibility/patches/pv/mountPv.yml new file mode 100644 index 00000000000..b836378851b --- /dev/null +++ b/build/yamls/flow-visibility/patches/pv/mountPv.yml @@ -0,0 +1,14 @@ +- op: add + path: /spec/defaults/templates/dataVolumeClaimTemplate + value: clickhouse-storage-template +- op: add + path: /spec/templates/volumeClaimTemplates + value: + - name: clickhouse-storage-template + spec: + storageClassName: STORAGECLASS_NAME + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi diff --git a/build/yamls/flow-visibility/patches/ram/mountRam.yml b/build/yamls/flow-visibility/patches/ram/mountRam.yml new file mode 100644 index 00000000000..a500acfcd9f --- /dev/null +++ b/build/yamls/flow-visibility/patches/ram/mountRam.yml @@ -0,0 +1,12 @@ +- op: add + path: /spec/templates/podTemplates/0/spec/volumes/- + value: + name: clickhouse-storage-volume + emptyDir: + medium: Memory + sizeLimit: 8Gi +- op: add + path: /spec/templates/podTemplates/0/spec/containers/0/volumeMounts/- + value: + name: clickhouse-storage-volume + mountPath: /var/lib/clickhouse diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md index 3d202f5f47a..320993b59e8 100644 --- a/docs/network-flow-visibility.md +++ b/docs/network-flow-visibility.md @@ -33,7 +33,10 @@ - [About Grafana and ClickHouse](#about-grafana-and-clickhouse) - [Deployment Steps](#deployment-steps-1) - [Credentials Configuration](#credentials-configuration) - - [ClickHouse Configuration](#clickhouse-configuration) + - [ClickHouse Configuration](#clickhouse-configuration) + - [Service Customization](#service-customization) + - [Performance Configuration](#performance-configuration) + - [Persistent Volumes](#persistent-volumes) - [Pre-built Dashboards](#pre-built-dashboards) - [Flow Records Dashboard](#flow-records-dashboard) - [Pod-to-Pod Flows Dashboard](#pod-to-pod-flows-dashboard) @@ -750,7 +753,9 @@ type: Opaque We recommend changing all the credentials above if you are going to run the Flow Collector in production. -##### ClickHouse Configuration +#### ClickHouse Configuration + +##### Service Customization The ClickHouse database can be accessed through the Service `clickhouse-clickhouse`. The Pod exposes HTTP port at 8123 and TCP port at 9000 by default. The ports are @@ -800,6 +805,8 @@ metadata: namespace: flow-visibility ``` +##### Performance Configuration + The ClickHouse throughput depends on two factors - the storage size of the ClickHouse and the time interval between the batch commits to the ClickHouse. Larger storage size and longer commit interval provide higher throughput. @@ -819,11 +826,114 @@ storage size, please modify the `sizeLimit` in the following section. name: clickhouse-storage-volume ``` +To deploy ClickHouse with Persistent Volumes and limited storage size, please refer +to [Persistent Volumes](#persistent-volumes). + The time interval between the batch commits to the ClickHouse is specified in the [Flow Aggregator Configuration](#configuration-1) as `commitInterval`. The ClickHouse throughput grows sightly when the commit interval grows from 1s to 8s. A commit interval larger than 8s provides little improvement on the throughput. +##### Persistent Volumes + +By default, ClickHouse is deployed in memory. From Antrea v1.7, the ClickHouse +supports deployment with Persistent Volumes. + +[`PersistentVolume`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +(PV) is a piece of storage in the K8s cluster, which requires to be manually +provisioned by an administrator or dynamically provisioned using Storage Classes. +A `PersistentVolumeClaim`(PVC) is a request for storage which consumes PV. As +the ClickHouse is deployed as a StatefulSet, the volume can be claimed using +`volumeClaimTemplate`. + +To deploy the ClickHouse with Persistent Volumes, please follow the steps below: + +1. Provision the `PersistentVolume`. K8s supports a great number of +[`PersistentVolume` types](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes). +You can provision your own `PersistentVolume` per your requirements. Here are +two simple samples for your reference. + + - Local PV allows you to store the ClickHouse data at a pre-defined path on + a specific node. Refer to [createLocalPv.yml][local_pv_yaml] to create the + PV. Please replace `LOCAL_PATH` with the path to store the ClickHouse data + and label the node used to store the ClickHouse data with + `clickhouse/instance=data`. + + - NFS PV allows you to store the ClickHouse data on an existing NFS server. + Refer to [createNfsPv.yml][nfs_pv_yaml] to create the PV. Please replace + `NFS_SERVER_ADDRESS` with the host name of the NFS server and `NFS_SERVER_PATH` + with the exported path on the NFS server. + + In both examples, you can set `.spec.capacity.storage` in `PersistentVolume` + to your storage size. This value is for informative purpose as K8s does not + enforce the capacity of PVs. If you want to limit the storage usage, you need + to ask for your storage system to enforce that. For example, you can create + a Local PV on a partition with the limited size. We recommend using a dedicate + saving space for the ClickHouse if you are going to run the Flow Collector in + production. + +1. Request the PV for the ClickHouse. Please add a `volumeClaimTemplate` section +under `.spec.templates` to the resource `ClickHouseInstallation` in +`flow-visibility.yml` as the sample below. `storageClassName` should be filled +as your own `StorageClass` name, and `.resources.requests.storage` should be set +to your storage size. + + ```yaml + volumeClaimTemplates: + - name: clickhouse-storage-template + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: clickhouse-storage + ``` + + Then add this template as `dataVolumeClaimTemplate` to the section below. + + ```yaml + defaults: + templates: + dataVolumeClaimTemplate: clickhouse-storage-template + podTemplate: pod-template + serviceTemplate: service-template + ``` + +1. Remove the in-memory deployement related contents, which is defined as +`volumeMounts` + `volumes` in the resource `ClickHouseInstallation` in +`flow-visibility.yml`. + + The `volumeMounts` to be removed is shown as the following section. + + ```yaml + - mountPath: /var/lib/clickhouse + name: clickhouse-storage-volume + ``` + + The `volumes` to be removed is shown as the following section. + + ```yaml + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: clickhouse-storage-volume + ``` + +If you prefer to generate the manfitest automatically with default settings, +please clone the repository and run one of the following commands: + +```yaml +# To generate a manifest with Local PV for the ClickHouse +./hack/generate-manifest-flow-visibility.sh --volume pv --local > flow-visibility.yml + +# To generate a manifest with NFS PV for the ClickHouse +./hack/generate-manifest-flow-visibility.sh --volume pv --nfs :/ > flow-visibility.yml + +# To generate a manifest with a customized StorageClass for the ClickHouse +./hack/generate-manifest-flow-visibility.sh --volume pv --storageclass > flow-visibility.yml +``` + #### Pre-built Dashboards The following dashboards are pre-built and are recommended for Antrea flow @@ -1108,3 +1218,5 @@ With filters applied: Visualization Network Policy Dashboard"> [flow_visibility_kustomization_yaml]: ../build/yamls/flow-visibility/base/kustomization.yml +[local_pv_yaml]: ../build/yamls/flow-visibility/patches/pv/createLocalPv.yml +[nfs_pv_yaml]: ../build/yamls/flow-visibility/patches/pv/createNfsPv.yml diff --git a/hack/generate-manifest-flow-visibility.sh b/hack/generate-manifest-flow-visibility.sh index bcb11af965e..496d2d5d79b 100755 --- a/hack/generate-manifest-flow-visibility.sh +++ b/hack/generate-manifest-flow-visibility.sh @@ -23,8 +23,16 @@ function echoerr { _usage="Usage: $0 [--mode (dev|release)] [--keep] [--help|-h] Generate a YAML manifest for the Clickhouse-Grafana Flow-visibility Solution, using Kustomize, and print it to stdout. - --mode (dev|release) Choose the configuration variant that you need (default is 'dev') - --keep Debug flag which will preserve the generated kustomization.yml + --mode (dev|release) Choose the configuration variant that you need (default is 'dev'). + --keep Debug flag which will preserve the generated kustomization.yml. + --volume (ram|pv) Choose the volume provider that you need (default is 'ram'). + --storageclass -sc Provide the StorageClass used to dynamically provision the + Persistent Volume for ClickHouse storage. + --local Create the Persistent Volume for ClickHouse with a provided + local path. + --nfs Create the Persistent Volume for ClickHouse with a provided + NFS server hostname or IP address and the path exported in the + form of hostname:path. This tool uses kustomize (https://github.com/kubernetes-sigs/kustomize) to generate manifests for Clickhouse-Grafana Flow-visibility Solution. You can set the KUSTOMIZE environment variable to the @@ -41,11 +49,14 @@ function print_help { MODE="dev" KEEP=false +VOLUME="ram" +STORAGECLASS="" +LOCALPATH="" +NFSPATH="" while [[ $# -gt 0 ]] do key="$1" - case $key in --mode) MODE="$2" @@ -55,6 +66,22 @@ case $key in KEEP=true shift ;; + --volume) + VOLUME="$2" + shift 2 + ;; + -sc|--storageclass) + STORAGECLASS="$2" + shift 2 + ;; + --local) + LOCALPATH="$2" + shift 2 + ;; + --nfs) + NFSPATH="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -84,6 +111,33 @@ if [ "$MODE" == "release" ] && [ -z "$IMG_TAG" ]; then exit 1 fi +if [ "$VOLUME" != "ram" ] && [ "$VOLUME" != "pv" ]; then + echoerr "--volume must be one of 'ram' or 'pv'" + print_help + exit 1 +fi + +if [ "$VOLUME" == "pv" ] && [ "$LOCALPATH" == "" ] && [ "$NFSPATH" == "" ] && [ "$STORAGECLASS" == "" ]; then + echoerr "When deploy with 'pv', one of '--local', '--nfs', '--storageclass' should be set" + print_help + exit 1 +fi + +if [ "$LOCALPATH" != "" ] && [ "$NFSPATH" != "" ]; then + echoerr "Cannot set '--local' and '--nfs' at the same time" + print_help + exit 1 +fi + +if [ "$NFSPATH" != "" ]; then + pathPair=(${NFSPATH//:/ }) + if [ ${#pathPair[@]} != 2 ]; then + echoerr "--nfs must be in the form of hostname:path" + print_help + exit 1 + fi +fi + THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $THIS_DIR/verify-kustomize.sh @@ -118,6 +172,39 @@ fi if [ "$MODE" == "release" ]; then $KUSTOMIZE edit set image flow-visibility-clickhouse-monitor=$IMG_NAME:$IMG_TAG fi +BASE=../$MODE +cd .. + +if [ "$VOLUME" == "ram" ]; then + mkdir ram && cd ram + cp $KUSTOMIZATION_DIR/patches/ram/*.yml . + touch kustomization.yml + $KUSTOMIZE edit add base $BASE + $KUSTOMIZE edit add patch --path mountRam.yml --group clickhouse.altinity.com --version v1 --kind ClickHouseInstallation --name clickhouse +fi + +if [ "$VOLUME" == "pv" ]; then + mkdir pv && cd pv + cp $KUSTOMIZATION_DIR/patches/pv/*.yml . + touch kustomization.yml + $KUSTOMIZE edit add base $BASE + + if [[ $STORAGECLASS != "" ]]; then + sed -i.bak -E "s/STORAGECLASS_NAME/$STORAGECLASS/" mountPv.yml + else + sed -i.bak -E "s/STORAGECLASS_NAME/clickhouse-storage/" mountPv.yml + fi + if [[ $LOCALPATH != "" ]]; then + sed -i.bak -E "s~LOCAL_PATH~$LOCALPATH~" createLocalPv.yml + $KUSTOMIZE edit add base createLocalPv.yml + fi + if [[ $NFSPATH != "" ]]; then + sed -i.bak -E "s~NFS_SERVER_ADDRESS~${pathPair[0]}~" createNfsPv.yml + sed -i.bak -E "s~NFS_SERVER_PATH~${pathPair[1]}~" createNfsPv.yml + $KUSTOMIZE edit add base createNfsPv.yml + fi + $KUSTOMIZE edit add patch --path mountPv.yml --group clickhouse.altinity.com --version v1 --kind ClickHouseInstallation --name clickhouse +fi $KUSTOMIZE build diff --git a/plugins/flow-visibility/clickhouse-monitor/main.go b/plugins/flow-visibility/clickhouse-monitor/main.go index cc1f6c61e46..2abb9dcada3 100644 --- a/plugins/flow-visibility/clickhouse-monitor/main.go +++ b/plugins/flow-visibility/clickhouse-monitor/main.go @@ -17,7 +17,10 @@ package main import ( "database/sql" "fmt" + "math" "os" + "regexp" + "strconv" "strings" "time" @@ -48,6 +51,8 @@ const ( ) var ( + // Storage size allocated for the ClickHouse in number of byte + allocatedSpace uint64 // The name of the table to store the flow records tableName = os.Getenv("TABLE_NAME") // The names of the materialized views @@ -58,8 +63,16 @@ var ( func main() { // Check environment variables - if len(tableName) == 0 || len(mvNames) == 0 { - klog.ErrorS(nil, "Unable to load environment variables, TABLE_NAME and MV_NAMES must be defined") + allocatedSpaceStr := os.Getenv("STORAGE_SIZE") + + if len(tableName) == 0 || len(mvNames) == 0 || len(allocatedSpaceStr) == 0 { + klog.ErrorS(nil, "Unable to load environment variables, TABLE_NAME, MV_NAMES and STORAGE_SIZE must be defined") + return + } + var err error + allocatedSpace, err = parseSize(allocatedSpaceStr) + if err != nil { + klog.ErrorS(err, "Error when parsing STORAGE_SIZE") return } @@ -68,6 +81,7 @@ func main() { klog.ErrorS(err, "Error when connecting to ClickHouse") os.Exit(1) } + checkStorageCondition(connect) wait.Forever(func() { // The monitor stops working for several rounds after a deletion // as the release of memory space by the ClickHouse MergeTree engine requires time @@ -118,28 +132,69 @@ func connectLoop() (*sql.DB, error) { return connect, nil } -// Checks the memory usage in the ClickHouse, and deletes records when it exceeds the threshold. -func monitorMemory(connect *sql.DB) { +// Check if ClickHouse shares storage space with other software +func checkStorageCondition(connect *sql.DB) { var ( freeSpace uint64 + usedSpace uint64 totalSpace uint64 ) - // Get memory usage from ClickHouse system table + getDiskUsage(connect, &freeSpace, &totalSpace) + getClickHouseUsage(connect, &usedSpace) + availablePercentage := float64(freeSpace+usedSpace) / float64(totalSpace) + klog.InfoS("Low available percentage implies ClickHouse does not save data on a dedicated disk", "availablePercentage", availablePercentage) +} + +func getDiskUsage(connect *sql.DB, freeSpace *uint64, totalSpace *uint64) { + // Get free space from ClickHouse system table + if err := wait.PollImmediate(queryRetryInterval, queryTimeout, func() (bool, error) { + if err := connect.QueryRow("SELECT free_space, total_space FROM system.disks").Scan(freeSpace, totalSpace); err != nil { + klog.ErrorS(err, "Failed to get the disk usage") + return false, nil + } else { + return true, nil + } + }); err != nil { + klog.ErrorS(err, "Failed to get the disk usage", "timeout", queryTimeout) + return + } +} + +func getClickHouseUsage(connect *sql.DB, usedSpace *uint64) { + // Get space usage from ClickHouse system table if err := wait.PollImmediate(queryRetryInterval, queryTimeout, func() (bool, error) { - if err := connect.QueryRow("SELECT free_space, total_space FROM system.disks").Scan(&freeSpace, &totalSpace); err != nil { - klog.ErrorS(err, "Failed to get memory usage for ClickHouse") + if err := connect.QueryRow("SELECT SUM(bytes) FROM system.parts").Scan(usedSpace); err != nil { + klog.ErrorS(err, "Failed to get the used space size by the ClickHouse") return false, nil } else { return true, nil } }); err != nil { - klog.ErrorS(err, "Failed to get memory usage for ClickHouse", "timeout", queryTimeout) + klog.ErrorS(err, "Failed to get the used space size by the ClickHouse", "timeout", queryTimeout) return } +} + +// Checks the memory usage in the ClickHouse, and deletes records when it exceeds the threshold. +func monitorMemory(connect *sql.DB) { + var ( + freeSpace uint64 + usedSpace uint64 + totalSpace uint64 + ) + getDiskUsage(connect, &freeSpace, &totalSpace) + getClickHouseUsage(connect, &usedSpace) + + // Total space for ClickHouse is the smaller of user allocated space size and the actual space size on disk + if (freeSpace + usedSpace) < allocatedSpace { + totalSpace = freeSpace + usedSpace + } else { + totalSpace = allocatedSpace + } // Calculate the memory usage - usagePercentage := float64(totalSpace-freeSpace) / float64(totalSpace) - klog.InfoS("Memory usage", "total", totalSpace, "used", totalSpace-freeSpace, "percentage", usagePercentage) + usagePercentage := float64(usedSpace) / float64(totalSpace) + klog.InfoS("Memory usage", "total", totalSpace, "used", usedSpace, "percentage", usagePercentage) // Delete records when memory usage is larger than threshold if usagePercentage > threshold { timeBoundary, err := getTimeBoundary(connect) @@ -200,3 +255,28 @@ func getDeleteRowNum(connect *sql.DB) (uint64, error) { deleteRowNum = uint64(float64(count) * deletePercentage) return deleteRowNum, nil } + +// Parse human readable storage size to number in bytes +func parseSize(sizeString string) (uint64, error) { + sizeMap := map[string]float64{"k": 1, "m": 2, "g": 3, "t": 4, "p": 5} + sizeRegex := regexp.MustCompile(`^(\d+)([kKmMgGtTpP])?([iI])?[bB]?$`) + matches := sizeRegex.FindStringSubmatch(sizeString) + if len(matches) != 4 { + return 0, fmt.Errorf("invalid storage size: %s", sizeString) + } + size, err := strconv.ParseUint(matches[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("error when parsing storage size number: %v", err) + } + unit := strings.ToLower(matches[2]) + if exponent, ok := sizeMap[unit]; ok { + if matches[3] == "i" { + size = size * uint64(math.Pow(1024, exponent)) + } else { + size = size * uint64(math.Pow(1000, exponent)) + } + } else { + return 0, fmt.Errorf("error when parsing storage size unit: %s", unit) + } + return size, nil +}