From 90cde81f249ed4d9bdea2bb03d6341b892dc03f1 Mon Sep 17 00:00:00 2001 From: Adam Boguszewski Date: Mon, 28 Nov 2022 12:25:16 +0100 Subject: [PATCH 1/2] feat: drop v1 upgrade script --- CHANGELOG.md | 2 + ci/tests.sh | 4 - deploy/helm/sumologic/upgrade-1.0.0.sh | 518 ------------------ docs/v1-migration-doc.md | 293 ---------- tests/helm/upgrade_script/run.sh | 55 -- .../upgrade_script/static/basic.input.yaml | 3 - tests/helm/upgrade_script/static/basic.log | 5 - .../upgrade_script/static/basic.output.yaml | 3 - .../static/chart_envs.input.yaml | 33 -- .../helm/upgrade_script/static/chart_envs.log | 7 - .../static/chart_envs.output.yaml | 30 - .../static/falco_disabled.input.yaml | 2 - .../upgrade_script/static/falco_disabled.log | 5 - .../static/falco_disabled.output.yaml | 2 - .../static/falco_enabled.input.yaml | 2 - .../upgrade_script/static/falco_enabled.log | 5 - .../static/falco_enabled.output.yaml | 2 - .../static/filter_stream.input.yaml | 3 - .../upgrade_script/static/filter_stream.log | 8 - .../static/filter_stream.output.yaml | 9 - .../static/filter_stream_and_time.input.yaml | 3 - .../static/filter_stream_and_time.log | 8 - .../static/filter_stream_and_time.output.yaml | 9 - .../static/filter_time.input.yaml | 3 - .../upgrade_script/static/filter_time.log | 8 - .../static/filter_time.output.yaml | 9 - .../static/fluent_bit_casts.input.yaml | 9 - .../static/fluent_bit_casts.log | 7 - .../static/fluent_bit_casts.output.yaml | 9 - .../static/fluent_bit_sed_db.input.yaml | 71 --- .../static/fluent_bit_sed_db.log | 7 - .../static/fluent_bit_sed_db.output.yaml | 71 --- .../upgrade_script/static/hooks.input.yaml | 27 - tests/helm/upgrade_script/static/hooks.log | 11 - .../upgrade_script/static/hooks.output.yaml | 27 - .../static/key_mapping_empty.input.yaml | 14 - .../static/key_mapping_empty.log | 17 - .../static/key_mapping_empty.output.yaml | 1 - .../key_mapping_one_to_multiple.input.yaml | 28 - .../static/key_mapping_one_to_multiple.log | 84 --- .../key_mapping_one_to_multiple.output.yaml | 75 --- .../static/key_mapping_one_to_one.input.yaml | 33 -- .../static/key_mapping_one_to_one.log | 25 - .../static/key_mapping_one_to_one.output.yaml | 36 -- .../static/prometheus_remote_write.input.yaml | 76 --- .../static/prometheus_remote_write.log | 5 - .../prometheus_remote_write.output.yaml | 86 --- ...rometheus_remote_write_modified.input.yaml | 77 --- .../prometheus_remote_write_modified.log | 8 - ...ometheus_remote_write_modified.output.yaml | 86 --- .../prometheus_service_monitors.input.yaml | 59 -- .../static/prometheus_service_monitors.log | 6 - .../prometheus_service_monitors.output.yaml | 70 --- .../static/version_invalid.input.yaml | 2 - .../upgrade_script/static/version_invalid.log | 7 - .../static/version_invalid.output.yaml | 2 - .../static/version_valid.input.yaml | 2 - .../upgrade_script/static/version_valid.log | 6 - .../static/version_valid.output.yaml | 2 - 59 files changed, 2 insertions(+), 2075 deletions(-) delete mode 100755 deploy/helm/sumologic/upgrade-1.0.0.sh delete mode 100644 docs/v1-migration-doc.md delete mode 100755 tests/helm/upgrade_script/run.sh delete mode 100644 tests/helm/upgrade_script/static/basic.input.yaml delete mode 100644 tests/helm/upgrade_script/static/basic.log delete mode 100644 tests/helm/upgrade_script/static/basic.output.yaml delete mode 100644 tests/helm/upgrade_script/static/chart_envs.input.yaml delete mode 100644 tests/helm/upgrade_script/static/chart_envs.log delete mode 100644 tests/helm/upgrade_script/static/chart_envs.output.yaml delete mode 100644 tests/helm/upgrade_script/static/falco_disabled.input.yaml delete mode 100644 tests/helm/upgrade_script/static/falco_disabled.log delete mode 100644 tests/helm/upgrade_script/static/falco_disabled.output.yaml delete mode 100644 tests/helm/upgrade_script/static/falco_enabled.input.yaml delete mode 100644 tests/helm/upgrade_script/static/falco_enabled.log delete mode 100644 tests/helm/upgrade_script/static/falco_enabled.output.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_stream.input.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_stream.log delete mode 100644 tests/helm/upgrade_script/static/filter_stream.output.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_stream_and_time.input.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_stream_and_time.log delete mode 100644 tests/helm/upgrade_script/static/filter_stream_and_time.output.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_time.input.yaml delete mode 100644 tests/helm/upgrade_script/static/filter_time.log delete mode 100644 tests/helm/upgrade_script/static/filter_time.output.yaml delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_casts.input.yaml delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_casts.log delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_casts.output.yaml delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_sed_db.input.yaml delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_sed_db.log delete mode 100644 tests/helm/upgrade_script/static/fluent_bit_sed_db.output.yaml delete mode 100644 tests/helm/upgrade_script/static/hooks.input.yaml delete mode 100644 tests/helm/upgrade_script/static/hooks.log delete mode 100644 tests/helm/upgrade_script/static/hooks.output.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_empty.input.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_empty.log delete mode 100644 tests/helm/upgrade_script/static/key_mapping_empty.output.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_multiple.input.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_multiple.log delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_multiple.output.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_one.input.yaml delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_one.log delete mode 100644 tests/helm/upgrade_script/static/key_mapping_one_to_one.output.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write.input.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write.log delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write.output.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write_modified.input.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write_modified.log delete mode 100644 tests/helm/upgrade_script/static/prometheus_remote_write_modified.output.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_service_monitors.input.yaml delete mode 100644 tests/helm/upgrade_script/static/prometheus_service_monitors.log delete mode 100644 tests/helm/upgrade_script/static/prometheus_service_monitors.output.yaml delete mode 100644 tests/helm/upgrade_script/static/version_invalid.input.yaml delete mode 100644 tests/helm/upgrade_script/static/version_invalid.log delete mode 100644 tests/helm/upgrade_script/static/version_invalid.output.yaml delete mode 100644 tests/helm/upgrade_script/static/version_valid.input.yaml delete mode 100644 tests/helm/upgrade_script/static/version_valid.log delete mode 100644 tests/helm/upgrade_script/static/version_valid.output.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a4fc877fdb..5d7c095f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - feat: add parameter to configure additional Prometheus remote writes [#2611] - docs: rename user-provided config from values.yaml to user-values.yaml [#2619] - feat: update opentelemetry-operator chart and fix progagators list in instrumentation resource [#2628] +- feat: drop migration script for v1 [#2654] ### Fixed @@ -85,6 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2626]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2626 [#2622]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2622 [#2643]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2643 +[#2654]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2654 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.17.0...main [telegraf_operator_comapare_1.3.5_and_1.3.10]: https://github.com/influxdata/helm-charts/compare/telegraf-operator-1.3.5...telegraf-operator-1.3.10 [cert-manager-1.4]: https://github.com/cert-manager/cert-manager/releases/tag/v1.4.0 diff --git a/ci/tests.sh b/ci/tests.sh index d9a4e4239e..e326093a02 100755 --- a/ci/tests.sh +++ b/ci/tests.sh @@ -14,10 +14,6 @@ pushd "${ROOT_DIR}" || exit 1 echo "Test helm templates generation" "${TESTS_DIR}/run.sh" || (echo "Failed testing templates" && exit 1) -# Test upgrade script -echo "Test upgrade script..." -"${TESTS_DIR}/upgrade_script/run.sh" || (echo "Failed testing upgrade script" && exit 1) - # Test upgrade v2 script echo "Test upgrade v2 script..." "${TESTS_DIR}/upgrade_v2_script/run.sh" || (echo "Failed testing upgrade v2 script" && exit 1) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh deleted file mode 100755 index 2c49b854b2..0000000000 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ /dev/null @@ -1,518 +0,0 @@ -#!/bin/bash - -set -euo pipefail -IFS=$'\n\t' - -readonly OLD_VALUES_YAML="${1:---help}" -readonly HELM_RELEASE_NAME="${2:-collection}" -readonly NAMESPACE="${3:-sumologic}" -readonly PREVIOUS_VERSION=0.17 - -readonly TEMP_FILE=upgrade-1.0.0-temp-file - -readonly MIN_BASH_VERSION=4.0 -readonly MIN_YQ_VERSION=3.4.1 - -readonly KEY_MAPPINGS=" -eventsDeployment.nodeSelector:fluentd.events.statefulset.nodeSelector -eventsDeployment.resources.limits.cpu:fluentd.events.statefulset.resources.limits.cpu -eventsDeployment.resources.limits.memory:fluentd.events.statefulset.resources.limits.memory -eventsDeployment.resources.requests.cpu:fluentd.events.statefulset.resources.requests.cpu -eventsDeployment.resources.requests.memory:fluentd.events.statefulset.resources.requests.memory -eventsDeployment.tolerations:fluentd.events.statefulset.tolerations -sumologic.addTimestamp:fluentd.logs.output.addTimestamp -sumologic.chunkLimitSize:fluentd.buffer.chunkLimitSize -sumologic.eventCollectionEnabled:fluentd.events.enabled -sumologic.events.sourceCategory:fluentd.events.sourceCategory -sumologic.fluentd.buffer:fluentd.buffer.type -sumologic.fluentdLogLevel:fluentd.logLevel -sumologic.flushInterval:fluentd.buffer.flushInterval -sumologic.k8sMetadataFilter.cacheRefresh:fluentd.metadata.cacheRefresh -sumologic.k8sMetadataFilter.cacheSize:fluentd.metadata.cacheSize -sumologic.k8sMetadataFilter.cacheTtl:fluentd.metadata.cacheTtl -sumologic.k8sMetadataFilter.verifySsl:fluentd.logs.containers.k8sMetadataFilter.verifySsl -sumologic.k8sMetadataFilter.watch:fluentd.logs.containers.k8sMetadataFilter.watch -sumologic.logFormat:fluentd.logs.output.logFormat -sumologic.numThreads:fluentd.buffer.numThreads -sumologic.queueChunkLimitSize:fluentd.buffer.queueChunkLimitSize -sumologic.timestampKey:fluentd.logs.output.timestampKey -sumologic.totalLimitSize:fluentd.buffer.totalLimitSize -sumologic.verifySsl:fluentd.verifySsl -sumologic.watchResourceEventsOverrides:fluentd.events.watchResourceEventsOverrides" - -readonly KEY_MAPPINGS_MULTIPLE=" -deployment.affinity:fluentd.logs.statefulset.affinity:fluentd.metrics.statefulset.affinity -deployment.nodeSelector:fluentd.logs.statefulset.nodeSelector:fluentd.metrics.statefulset.nodeSelector -deployment.podAntiAffinity:fluentd.logs.statefulset.podAntiAffinity:fluentd.metrics.statefulset.podAntiAffinity -deployment.replicaCount:fluentd.logs.statefulset.replicaCount:fluentd.metrics.statefulset.replicaCount -deployment.resources.limits.cpu:fluentd.logs.statefulset.resources.limits.cpu:fluentd.metrics.statefulset.resources.limits.cpu -deployment.resources.limits.memory:fluentd.logs.statefulset.resources.limits.memory:fluentd.metrics.statefulset.resources.limits.memory -deployment.resources.requests.cpu:fluentd.logs.statefulset.resources.requests.cpu:fluentd.metrics.statefulset.resources.requests.cpu -deployment.resources.requests.memory:fluentd.logs.statefulset.resources.requests.memory:fluentd.metrics.statefulset.resources.requests.memory -deployment.tolerations:fluentd.logs.statefulset.tolerations:fluentd.metrics.statefulset.tolerations -sumologic.fluentd.autoscaling.enabled:fluentd.logs.autoscaling.enabled:fluentd.metrics.autoscaling.enabled -sumologic.fluentd.autoscaling.maxReplicas:fluentd.logs.autoscaling.maxReplicas:fluentd.metrics.autoscaling.maxReplicas -sumologic.fluentd.autoscaling.minReplicas:fluentd.logs.autoscaling.minReplicas:fluentd.metrics.autoscaling.minReplicas -sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage:fluentd.logs.autoscaling.targetCPUUtilizationPercentage:fluentd.metrics.autoscaling.targetCPUUtilizationPercentage -sumologic.excludeContainerRegex:fluentd.logs.containers.excludeContainerRegex:fluentd.logs.default.excludeContainerRegex:fluentd.logs.systemd.excludeContainerRegex:fluentd.logs.kubelet.excludeContainerRegex -sumologic.excludeHostRegex:fluentd.logs.containers.excludeHostRegex:fluentd.logs.default.excludeHostRegex:fluentd.logs.systemd.excludeHostRegex:fluentd.logs.kubelet.excludeHostRegex -sumologic.excludeNamespaceRegex:fluentd.logs.containers.excludeNamespaceRegex:fluentd.logs.default.excludeNamespaceRegex:fluentd.logs.systemd.excludeNamespaceRegex:fluentd.logs.kubelet.excludeNamespaceRegex -sumologic.excludePodRegex:fluentd.logs.containers.excludePodRegex:fluentd.logs.default.excludePodRegex:fluentd.logs.systemd.excludePodRegex:fluentd.logs.kubelet.excludePodRegex -sumologic.sourceCategory:fluentd.logs.containers.sourceCategory:fluentd.logs.default.sourceCategory:fluentd.logs.systemd.sourceCategory:fluentd.logs.kubelet.sourceCategory -sumologic.sourceCategoryPrefix:fluentd.logs.containers.sourceCategoryPrefix:fluentd.logs.default.sourceCategoryPrefix:fluentd.logs.systemd.sourceCategoryPrefix:fluentd.logs.kubelet.sourceCategoryPrefix -sumologic.sourceCategoryReplaceDash:fluentd.logs.containers.sourceCategoryReplaceDash:fluentd.logs.default.sourceCategoryReplaceDash:fluentd.logs.systemd.sourceCategoryReplaceDash:fluentd.logs.kubelet.sourceCategoryReplaceDash -sumologic.sourceName:fluentd.logs.containers.sourceName:fluentd.logs.default.sourceName:fluentd.logs.systemd.sourceName:fluentd.logs.kubelet.sourceName -" - -readonly KEY_MAPPINGS_EMPTY=" -deployment -eventsDeployment -fluentd.autoscaling -fluentd.rawConfig -fluentd.statefulset -sumologic.addStream -sumologic.addTime -sumologic.events -sumologic.fluentd -sumologic.k8sMetadataFilter -sumologic.kubernetesMeta -sumologic.kubernetesMetaReduce" - -readonly KEY_CASTS_STRING=" -fluent-bit.backend.forward.tls -fluent-bit.backend.forward.tls_verify -sumologic.setup.clusterRole.annotations.\"helm.sh/hook-weight\" -sumologic.setup.clusterRoleBinding.annotations.\"helm.sh/hook-weight\" -sumologic.setup.configMap.annotations.\"helm.sh/hook-weight\" -sumologic.setup.job.annotations.\"helm.sh/hook-weight\" -sumologic.setup.serviceAccount.annotations.\"helm.sh/hook-weight\" -" - -# Prometheus changes -# Diff of prometheus regexes: -# git diff v1.0.0 v0.17.1 deploy/helm/sumologic/values.yaml | \ -# grep -E '(regex|url)\:' | \ -# grep -E "^(\-|\+)\s+ (regex|url)\:" | \ -# sed 's|- url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888||' | \ -# sed 's/$/\\n/' -# -# minus means 1.0.0 regex -# plus means 0.17.2 regex - -readonly EXPECTED_PROMETHEUS_METRICS_CHANGES="/prometheus.metrics.state -- regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) -+ regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) -/prometheus.metrics.controller-manager -/prometheus.metrics.scheduler -/prometheus.metrics.apiserver -- regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:duration_seconds|latencies)_(?:count|sum)|apiserver_request_latencies_summary(?:|_count|_sum)|etcd_request_cache_(?:add|get)_(?:duration_seconds|latencies_summary)_(?:count|sum)|etcd_helper_cache_(?:hit|miss)_(?:count|total)) -+ regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_latenc(?:ies|y_seconds).*|etcd_request_cache_get_latenc(?:ies_summary|y_seconds).*|etcd_request_cache_add_latenc(?:ies_summary|y_seconds).*|etcd_helper_cache_hit_(?:count|total)|etcd_helper_cache_miss_(?:count|total)) -/prometheus.metrics.kubelet -- regex: kubelet;(?:kubelet_docker_operations_errors(?:|_total)|kubelet_(?:docker|runtime)_operations_duration_seconds_(?:count|sum)|kubelet_running_(?:container|pod)_count|kubelet_(:?docker|runtime)_operations_latency_microseconds(?:|_count|_sum)) -+ regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*) -/prometheus.metrics.container -- regex: kubelet;.+;(?:container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_fs_usage_bytes|container_fs_limit_bytes) -+ regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|) -/prometheus.metrics.container -- regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total) -+ regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total) -/prometheus.metrics.node -- regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total) -+ regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files) -/prometheus.metrics.operator.rule -- regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' -+ regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' -/prometheus.metrics" - -# https://slides.com/perk/how-to-train-your-bash#/41 -readonly LOG_FILE="/tmp/$(basename "$0").log" -info() { echo -e "[INFO] $*" | tee -a "${LOG_FILE}" >&2 ; } -warning() { echo -e "[WARNING] $*" | tee -a "${LOG_FILE}" >&2 ; } -error() { echo -e "[ERROR] $*" | tee -a "${LOG_FILE}" >&2 ; } -fatal() { echo -e "[FATAL] $*" | tee -a "${LOG_FILE}" >&2 ; exit 1 ; } - -function print_help_and_exit() { - readonly MAN="Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -As part of this major release, the format of the values.yaml file has changed. - -This script will automatically take the configurations of your existing values.yaml -and return one that is compatible with v1.0.0. - -Requirements: - yq (>= ${MIN_YQ_VERSION}) https://github.com/mikefarah/yq/releases/tag/3.2.1 - grep - sed - bash (>= ${MIN_BASH_VERSION}) - -Usage: - # for default helm release name 'collection' and namespace 'sumologic' - ./upgrade-1.0.0.sh /path/to/values.yaml - - # for non-default helm release name and k8s namespace - ./upgrade-1.0.0.sh /path/to/values.yaml helm_release_name k8s_namespace - -Returns: - new_values.yaml - -For more details, please refer to Migration steps and Changelog here: -https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v1.0/deploy/docs/v1_migration_doc.md" - - echo "${MAN}" - exit 0 -} - -function check_if_print_help_and_exit() { - if [[ "$1" == "--help" ]]; then - print_help_and_exit - fi -} - -function check_required_command() { - local command_to_check="$1" - command -v "${command_to_check}" >/dev/null 2>&1 || { error "Required command is missing: ${command_to_check}"; fatal "Please consult --help and install missing commands before continue. Aborting."; } -} - -function compare_versions() { - local no_lower_than="${1}" - local app_version="${2}" - - if [[ "$(printf '%s\n' "${app_version}" "${no_lower_than}" | sort -V | head -n 1)" == "${no_lower_than}" ]]; then - echo "pass" - else - echo "fail" - fi -} - -function check_app_version() { - local app_name="${1}" - local no_lower_than="${2}" - local app_version="${3}" - - if [[ -z ${app_version} ]] || [[ $(compare_versions "${no_lower_than}" "${app_version}") == "fail" ]]; then - error "${app_name} version: '${app_version}' is invalid - it should be no lower than ${no_lower_than}" - fatal "Please update your ${app_name} and retry." - fi -} - -function check_yq_version() { - local yq_version - yq_version=$(yq --version | grep -oE '[^[:space:]]+$') - - check_app_version "grep" "${MIN_YQ_VERSION}" "${yq_version}" -} - -function check_bash_version() { - check_app_version "bash" "${MIN_BASH_VERSION}" "${BASH_VERSION}" -} - -function create_temp_file() { - echo -n > "${TEMP_FILE}" -} - -function migrate_customer_keys() { - # Convert variables to arrays - set +e - IFS=$'\n' read -r -d ' ' -a MAPPINGS <<< "${KEY_MAPPINGS}" - readonly MAPPINGS - IFS=$'\n' read -r -d ' ' -a MAPPINGS_MULTIPLE <<< "${KEY_MAPPINGS_MULTIPLE}" - readonly MAPPINGS_MULTIPLE - IFS=$'\n' read -r -d ' ' -a MAPPINGS_EMPTY <<< "${KEY_MAPPINGS_EMPTY}" - readonly MAPPINGS_EMPTY - IFS=$'\n' read -r -d ' ' -a CASTS_STRING <<< "${KEY_CASTS_STRING}" - readonly CASTS_STRING - set -e - - readonly CUSTOMER_KEYS=$(yq --printMode p r "${OLD_VALUES_YAML}" -- '**') - - for key in ${CUSTOMER_KEYS}; do - if [[ ${MAPPINGS[*]} =~ ${key}: ]]; then - # whatever you want to do when arr contains value - for i in "${MAPPINGS[@]}"; do - IFS=':' read -r -a maps <<< "${i}" - if [[ ${maps[0]} == "${key}" ]]; then - info "Mapping ${key} into ${maps[1]}" - yq w -i "${TEMP_FILE}" -- "${maps[1]}" "$(yq r "${OLD_VALUES_YAML}" -- "${maps[0]}")" - yq d -i "${TEMP_FILE}" -- "${maps[0]}" - fi - done - elif [[ ${MAPPINGS_MULTIPLE[*]} =~ ${key}: ]]; then - # whatever you want to do when arr contains value - info "Mapping ${key} into:" - for i in "${MAPPINGS_MULTIPLE[@]}"; do - IFS=':' read -r -a maps <<< "${i}" - if [[ ${maps[0]} == "${key}" ]]; then - for element in "${maps[@]:1}"; do - info "- ${element}" - yq w -i "${TEMP_FILE}" -- "${element}" "$(yq r "${OLD_VALUES_YAML}" -- "${maps[0]}")" - yq d -i "${TEMP_FILE}" -- "${maps[0]}" - done - fi - done - else - yq w -i "${TEMP_FILE}" -- "${key}" "$(yq r "${OLD_VALUES_YAML}" -- "${key}")" - fi - - if [[ "${MAPPINGS_EMPTY[*]}" =~ ${key} ]]; then - info "Removing ${key}" - yq d -i "${TEMP_FILE}" -- "${key}" - fi - - if [[ "${CASTS_STRING[*]}" =~ ${key} ]]; then - info "Casting ${key} to str" - # As yq doesn't cast `on` and `off` from bool to cast, we use sed based casts - yq w -i "${TEMP_FILE}" -- "${key}" "$(yq r "${OLD_VALUES_YAML}" "${key}")__YQ_REPLACEMENT_CAST" - sed -i.bak 's/\(^.*: \)\(.*\)__YQ_REPLACEMENT_CAST/\1"\2"/g' "${TEMP_FILE}" - fi - done - echo -} - -function migrate_add_stream_and_add_time() { - # Preserve the functionality of addStream=false or addTime=false - if [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addStream)" == "false" ] && [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addTime)" == "false" ]; then - REMOVE="stream,time" - elif [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addStream)" == "false" ]; then - REMOVE="stream" - elif [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addTime)" == "false" ]; then - REMOVE="time" - else - REMOVE= - fi - -# Add filter on beginning of current filters -FILTER=" - @type record_modifier - remove_keys ${REMOVE} -" - - # Apply changes if required - if [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addStream)" == "false" ] || [ "$(yq r "${OLD_VALUES_YAML}" -- sumologic.addTime)" == "false" ]; then - info "Creating fluentd.logs.containers.extraFilterPluginConf to preserve addStream/addTime functionality" - yq w -i "${TEMP_FILE}" -- fluentd.logs.containers.extraFilterPluginConf "${FILTER}" - fi -} - -function migrate_pre_upgrade_hook() { - # Keep pre-upgrade hook - if [[ -n "$(yq r "${TEMP_FILE}" -- sumologic.setup)" ]]; then - info "Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade'" - yq w -i "${TEMP_FILE}" -- 'sumologic.setup.*.annotations[helm.sh/hook]' 'pre-install,pre-upgrade' - fi -} - -function check_falco_state() { - # Print information about falco state - if [[ "$(yq r "${TEMP_FILE}" -- falco.enabled)" == 'true' ]]; then - info 'falco will be enabled. Change "falco.enabled" to "false" if you want to disable it (default for 1.0)' - else - info 'falco will be disabled. Change "falco.enabled" to "true" if you want to enable it' - fi -} - -function get_regex() { - # Get regex from old yaml file and strip `'` and `"` from beginning/end of it - local write_index="${1}" - local relabel_index="${2}" - yq r "${OLD_VALUES_YAML}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${write_index}].writeRelabelConfigs[${relabel_index}].regex" | sed -e "s/^'//" -e 's/^"//' -e "s/'$//" -e 's/"$//' -} - -function get_release_regex() { - local metric_name="${1}" - local str_grep="${2}" - local filter="${3}" - - echo "${EXPECTED_PROMETHEUS_METRICS_CHANGES}" | grep -A 3 "${metric_name}$" | "${filter}" -n 3 | grep -E "${str_grep}" | grep -oE ': .*' | sed 's/: //' | sed -e "s/^'//" -e 's/^"//' -e "s/'$//" -e 's/"$//' -} - -function migrate_prometheus_metrics() { - - - metrics_length="$(yq r -l "${OLD_VALUES_YAML}" -- 'prometheus-operator.prometheus.prometheusSpec.remoteWrite')" - metrics_length="$(( metrics_length - 1))" - - for i in $(seq 0 "${metrics_length}"); do - metric_name="$(yq r "${OLD_VALUES_YAML}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].url" | grep -oE '/prometheus\.metrics.*' || true)" - metric_regex_length="$(yq r -l "${OLD_VALUES_YAML}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs")" - metric_regex_length="$(( metric_regex_length - 1))" - - for j in $(seq 0 "${metric_regex_length}"); do - metric_regex_action=$(yq r "${OLD_VALUES_YAML}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}].action") - if [[ "${metric_regex_action}" = "keep" ]]; then - break - fi - done - regexes_len="$( (echo "${EXPECTED_PROMETHEUS_METRICS_CHANGES}" | grep -A 2 "${metric_name}$" | grep regex || true ) | wc -l)" - if [[ "${regexes_len}" -eq "2" ]]; then - - regex_1_0="$(get_release_regex "${metric_name}" '^\s*-' 'head')" - regex_0_17="$(get_release_regex "${metric_name}" '^\s*\+' 'head')" - regex="$(get_regex "${i}" "${j}")" - if [[ "${regex_0_17}" = "${regex}" ]]; then - yq w -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}].regex" "${regex_1_0}" - else - warning "Changes of regex for 'prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}]' (${metric_name}) detected, please migrate it manually" - fi - fi - - if [[ "${metric_name}" = "/prometheus.metrics.container" ]]; then - regex_1_0="$(get_release_regex "${metric_name}" '^\s*-' 'head')" - regex_0_17="$(get_release_regex "${metric_name}" '^\s*\+' 'head')" - regex="$(get_regex "${i}" "${j}")" - if [[ "${regex_0_17}" = "${regex}" ]]; then - yq w -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}].regex" "${regex_1_0}" - else - regex_1_0="$(get_release_regex "${metric_name}" '^\s*-' 'tail')" - regex_0_17="$(get_release_regex "${metric_name}" '^\s*\+' 'tail')" - if [[ "${regex_0_17}" = "${regex}" ]]; then - yq w -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}].regex" "${regex_1_0}" - else - warning "Changes of regex for 'prometheus-operator.prometheus.prometheusSpec.remoteWrite[${i}].writeRelabelConfigs[${j}]' (${metric_name}) detected, please migrate it manually" - fi - fi - fi - done -} - -function fix_fluentbit_env() { - # Fix fluent-bit env - if [[ -n "$(yq r "${TEMP_FILE}" -- fluent-bit.env)" ]]; then - info "Patching fluent-bit CHART environmental variable" - yq w -i "${TEMP_FILE}" -- "fluent-bit.env(name==CHART).valueFrom.configMapKeyRef.key" "fluentdLogs" - fi -} - -function fix_prometheus_service_monitors() { - # Fix prometheus service monitors - if [[ -n "$(yq r "${TEMP_FILE}" -- prometheus-operator.prometheus.additionalServiceMonitors)" ]]; then - info "Patching prometheus-operator.prometheus.additionalServiceMonitors" - yq d -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.additionalServiceMonitors(name==${HELM_RELEASE_NAME}-${NAMESPACE})" - yq d -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.additionalServiceMonitors(name==${HELM_RELEASE_NAME}-${NAMESPACE}-otelcol)" - yq d -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.additionalServiceMonitors(name==${HELM_RELEASE_NAME}-${NAMESPACE}-events)" - echo "--- -prometheus-operator: - prometheus: - additionalServiceMonitors: - - name: ${HELM_RELEASE_NAME}-${NAMESPACE}-otelcol - additionalLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-otelcol - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - ${NAMESPACE} - selector: - matchLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-otelcol - - name: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-logs - additionalLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-logs - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - ${NAMESPACE} - selector: - matchLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-logs - - name: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-metrics - additionalLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-metrics - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - ${NAMESPACE} - selector: - matchLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-metrics - - name: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-events - additionalLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-events - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - ${NAMESPACE} - selector: - matchLabels: - app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-events" | yq m -a=append -i "${TEMP_FILE}" - - fi - - if [[ -n "$(yq r "${TEMP_FILE}" -- prometheus-operator.prometheus.prometheusSpec.containers)" ]]; then - info "Patching prometheus CHART environmental variable" - yq w -i "${TEMP_FILE}" -- "prometheus-operator.prometheus.prometheusSpec.containers(name==prometheus-config-reloader).env(name==CHART).valueFrom.configMapKeyRef.key" "fluentdMetrics" - fi -} - -function check_user_image() { - # Check user's image and echo warning if the image has been changed - readonly USER_VERSION="$(yq r "${OLD_VALUES_YAML}" -- image.tag)" - if [[ -n "${USER_VERSION}" ]]; then - if [[ "${USER_VERSION}" =~ ^"${PREVIOUS_VERSION}"\.[[:digit:]]+$ ]]; then - yq w -i "${TEMP_FILE}" -- image.tag 1.0.0 - info "Changing image.tag from '${USER_VERSION}' to '1.0.0'" - else - warning "You are using unsupported version: ${USER_VERSION}" - warning "Please upgrade to '${PREVIOUS_VERSION}.x' or ensure that new_values.yaml is valid" - fi - fi -} - -function migrate_fluentbit_db_path() { - grep 'tail-db/tail-containers-state.db' "${TEMP_FILE}" 1>/dev/null 2>&1 || return 0 - # New fluent-bit db path - info 'Replacing tail-db/tail-containers-state.db to tail-db/tail-containers-state-sumo.db' - warning 'Please ensure that new fluent-bit configuration is correct' - - sed -i.bak 's?tail-db/tail-containers-state.db?tail-db/tail-containers-state-sumo.db?g' "${TEMP_FILE}" -} - -function fix_yq() { - # account for yq bug that stringifies empty maps - sed -i.bak "s/'{}'/{}/g" "${TEMP_FILE}" -} - -function rename_temp_file() { - mv "${TEMP_FILE}" new_values.yaml -} - -function cleanup_bak_file() { - rm "${TEMP_FILE}.bak" -} - -function echo_footer() { - DONE="\nThank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart.\nA new yaml file has been generated for you. Please check the current directory for new_values.yaml." - echo -e "${DONE}" -} - -check_if_print_help_and_exit "${OLD_VALUES_YAML}" -check_bash_version -check_required_command yq -check_yq_version -check_required_command grep -check_required_command sed - -create_temp_file - -migrate_customer_keys - -migrate_add_stream_and_add_time -migrate_pre_upgrade_hook - -check_falco_state - -migrate_prometheus_metrics - -fix_fluentbit_env -fix_prometheus_service_monitors - -check_user_image -migrate_fluentbit_db_path - -fix_yq -rename_temp_file -cleanup_bak_file - -echo_footer - -exit 0 diff --git a/docs/v1-migration-doc.md b/docs/v1-migration-doc.md deleted file mode 100644 index 1cdd5bfe13..0000000000 --- a/docs/v1-migration-doc.md +++ /dev/null @@ -1,293 +0,0 @@ -# Kubernetes Collection 1.0.0 - Breaking Changes - -- [Helm Users](#helm-users) - - [Changes](#changes) - - [How to upgrade](#how-to-upgrade) - - [1. Upgrade to helm chart version `v0.17.4`](#1-upgrade-to-helm-chart-version-v0174) - - [2: Run upgrade script](#2-run-upgrade-script) - - [Troubleshooting Upgrade](#troubleshooting-upgrade) - - [Rollback](#rollback) -- [Non-Helm Users](#non-helm-users) - - [Breaking Changes](#breaking-changes) - - [How to upgrade for Non-helm Users](#how-to-upgrade-for-non-helm-users) - - [1. Tear down existing Fluentd, Prometheus, Fluent Bit and Falco resources](#1-tear-down-existing-fluentd-prometheus-fluent-bit-and-falco-resources) - - [2. Deploy Fluentd, Fluent Bit and Prometheus again with the version 1.0.0 yaml](#2-deploy-fluentd-fluent-bit-and-prometheus-again-with-the-version-100-yaml) -- [Kubernetes App dashboard update](#kubernetes-app-dashboard-update) - -Based on the feedback from our users, we will be introducing several changes -to the Sumo Logic Kubernetes Collection solution. -Here we detail the changes for both Helm and Non-Helm users, as well as -the exact steps for migration. - -## Helm Users - -### Changes - -- Falco installation disabled by Default. - If you want to enable Falco, modify the `enabled` flag for Falco in `user-values.yaml` as shown below: - - ```yaml - falco: - ## Set the enabled flag to false to disable falco. - enabled: true - ``` - -- Bumped the helm Falco chart version to v`1.1.6` which included a fix to disable - the bitcoin/crypto miner rule by default - -- Changes in Configuration Parameters - - The `user-values.yaml` file has had several configs moved and renamed to improve usability. - Namely, we introduced a new fluentd section into which we moved all of the - Fluentd specific configs, while configs for our dependency charts - (`prometheus-operator`, `fluent-bit`, `metrics-server`, `falco`) have not changed. - - | Old Config | New Config | - |:----------------------------------------:|:----------------------------------------------------------:| - | sumologic.eventCollectionEnabled | fluentd.events.enabled | - | sumologic.events.sourceCategory | fluentd.events.sourceCategory | - | sumologic.logFormat | fluentd.logs.output.logFormat | - | sumologic.flushInterval | fluentd.buffer.flushInterval | - | sumologic.numThreads | fluentd.buffer.numThreads | - | sumologic.chunkLimitSize | fluentd.buffer.chunkLimitSize | - | sumologic.queueChunkLimitSize | fluentd.buffer.queueChunkLimitSize | - | sumologic.totalLimitSize | fluentd.buffer.totalLimitSize | - | sumologic.sourceName | fluentd.logs.containers.sourceName | - | sumologic.sourceCategory | fluentd.logs.containers.sourceCategory | - | sumologic.sourceCategoryPrefix | fluentd.logs.containers.sourceCategoryPrefix | - | sumologic.sourceCategoryReplaceDash | fluentd.logs.containers.sourceCategoryReplaceDash | - | sumologic.addTimestamp | fluentd.logs.output.addTimestamp | - | sumologic.timestampKey | fluentd.logs.output.timestampKey | - | sumologic.verifySsl | fluentd.verifySsl | - | sumologic.excludeContainerRegex | fluentd.logs.containers.excludeContainerRegex | - | sumologic.excludeHostRegex | fluentd.logs.containers.excludeHostRegex | - | sumologic.excludeNamespaceRegex | fluentd.logs.containers.excludeNamespaceRegex | - | sumologic.excludePodRegex | fluentd.logs.containers.excludePodRegex | - | sumologic.fluentdLogLevel | fluentd.logLevel | - | sumologic.watchResourceEventsOverrides | fluentd.events.watchResourceEventsOverrides | - | sumologic.fluentd.buffer | fluentd.buffer.type | - | sumologic.fluentd.autoscaling.* | fluentd.logs.autoscaling.* , fluentd.metrics.autoscaling.* | - | sumologic.k8sMetadataFilter.watch | fluentd.logs.containers.k8sMetadataFilter.watch | - | sumologic.k8sMetadataFilter.verifySsl | fluentd.logs.containers.k8sMetadataFilter.verifySsl | - | sumologic.k8sMetadataFilter.cacheSize | fluentd.metadata.cacheSize | - | sumologic.k8sMetadataFilter.cacheTtl | fluentd.metadata.cacheTtl | - | sumologic.k8sMetadataFilter.cacheRefresh | fluentd.metadata.cacheRefresh | - | deployment.* | fluentd.logs.statefulset.* , fluentd.metrics.statefulset.* | - | eventsDeployment.* | fluentd.eventsStatefulset.* | - -- `sumologic.kubernetesMeta` and `sumologic.kubernetesMetaReduce` have been removed. - The default log format (`fluentd.logs.output.logFormat`) is `fields`, - which removes the relevant metadata from the JSON body of the logs, - making these configs no longer necessary. -- `sumologic.addStream` and `sumologic.addTime` (default values were `true`) have been removed; - the default behavior will remain the same. - To preserve the behavior of `addStream = false` or `addTime = false`, - you can add the following config to the `user-values.yaml` file: - - ```yaml - fluentd: - logs: - containers: - extraFilterPluginConf: - - @type record_modifier - remove_keys stream, time - - ``` - -Until now, Helm users have not been able to modify their Fluentd configuration -outside of the specific parameters that we exposed in the `user-values.yaml` file. -Now, we expose the ability to modify the Fluentd configuration as needed. - -Some use-cases include : - -- custom log pipelines, -- adding Fluentd filter plugins (ex: fluentd throttle plugin), or -- adding Fluentd output plugins (ex: forward to both Sumo and S3) - -You can look for example configurations [here](v1-conf-examples.md) - -The Fluentd `deployments` have been changed to `statefulsets` to support the use of persistent volumes. -This will allow better buffering behavior. -They also now include `“fluentd”` in their names. -This is not a breaking change for Helm users. - -The unified Fluentd `statefulsets` have been split into set of two different Fluentd's, -one for `logs` and the other one for `metrics`. - -### How to upgrade - -**Note: The below steps are using Helm 2. Helm 3 is not supported.** - -#### 1. Upgrade to helm chart version `v0.17.4` - -Run the below command to fetch the latest helm chart: - -```bash -helm repo update -``` - -For the users who are not already on `v0.17.4` of the helm chart, please upgrade -to that version first by running the below command. - -```bash -helm upgrade collection sumologic/sumologic --reuse-values --version=0.17.4 -``` - -#### 2: Run upgrade script - -For Helm users, the only breaking changes are the renamed config parameters. -For users who use a `user-values.yaml` file, we provide a script that users can run -to convert their existing `user-values.yaml` file into one that is compatible with the major release. - -- Get the existing values for the helm chart and store it as `current_user-values.yaml` - with the below command: - - ```bash - helm get values > current_user-values.yaml - ``` - -- Run `curl` the upgrade script as follows: - - ```bash - curl -LJO https://raw.githubusercontent.com/SumoLogic/sumologic-kubernetes-collection/release-v1.0/deploy/helm/sumologic/upgrade-1.0.0.sh - ``` - -- Run the upgrade script on the above file with the below command. - - ```bash - ./upgrade-1.0.0.sh current_user-values.yaml - ``` - -- At this point, users can then run: - - ```bash - helm upgrade collection sumologic/sumologic --version=1.0.0 -f new_user-values.yaml - ``` - -### Troubleshooting Upgrade - -If you receive the below error - -``` -Error: UPGRADE FAILED: error validating "": error validating data: [ValidationError(StatefulSet.spec.template.spec.containers[0].resources.limits.cpu fluentd): - invalid type for io.k8s.apimachinery.pkg.api.resource.Quantity: got "map", expected "string", ValidationError(StatefulSet.spec.template.spec.containers[0].resources.limits.memory fluentd): - invalid type for io.k8s.apimachinery.pkg.api.resource.Quantity: got "map", expected "string", ValidationError(StatefulSet.spec.template.spec.containers[0].resources.requests.cpu fluentd): - invalid type for io.k8s.apimachinery.pkg.api.resource.Quantity: got "map", expected "string", ValidationError(StatefulSet.spec.template.spec.containers[0].resources.requests.memory fluentd): - invalid type for io.k8s.apimachinery.pkg.api.resource.Quantity: got "map", expected "string"] -``` - -it likely means your OS is picking up an older version -of `bash` even though you may have upgraded. -Makes sure you are running a version of `bash` >= 4.4 by running `bash --version`. -If the version of bash is correct, you can rerun the upgrade script by running -`bash upgrade-1.0.0.sh current_user-values.yaml` and then rerun -`helm upgrade collection sumologic/sumologic --version=1.0.0 -f new_user-values.yaml` to resolve. - -### Rollback - -If something goes wrong, or you want to go back to the previous version, -you can [rollback changes using helm](https://helm.sh/docs/helm/helm_rollback/): - -``` -helm history collection -helm rollback collection -``` - -## Non-Helm Users - -### Breaking Changes - -- The use of environment variables to set configs has been removed to avoid the - extra layer of indirection and confusion. - Instead, configs will be set directly within the Fluentd pipeline. -- `kubernetesMeta` and `kubernetesMetaReduce` have been removed from - `logs.kubernetes.sumologic.filter.conf` of the Fluentd pipeline for the same reason - as above (Helm users) -- Similarly `addStream` and `addTime` (default values were `true`) have been removed - from `logs.kubernetes.sumologic.filter.conf` of the Fluentd pipeline; - the default behavior will remain the same. - To preserve the behavior of `addStream = false` or `addTime = false`, you can add: - - ```yaml - - @type record_modifier - remove_keys stream,time - - ``` - - above the output plugin section [here](https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v1.0/deploy/kubernetes/fluentd-sumologic.yaml.tmpl#L251) - -- The Fluentd deployments have been changed to **statefulsets** to support the use - of **persistent volumes**. This will allow better buffering behavior. - They also now include `“fluentd”` in their names. - This is a breaking change for non-Helm users as the deployments will not - be cleaned up upon upgrade, leading to duplicate events (logs and metrics will not - experience data duplication). -- The unified Fluentd `statefulsets` have been split into set of two different Fluentd's, - one for `logs` and the other one for `metrics`. -- We now support the collection of renamed metrics (for Kubernetes version `1.17+`). - -### How to upgrade for Non-helm Users - -#### 1. Tear down existing Fluentd, Prometheus, Fluent Bit and Falco resources - -You will need the YAML files you created when you first installed collection. -Run the following commands to remove Falco, Fluent-bit, Prometheus Operator and FluentD. -You do not need to delete the Namespace and Secret you originally created as they will still be used. - -```sh -kubectl delete -f falco.yaml -kubectl delete -f fluent-bit.yaml -kubectl delete -f prometheus.yaml -kubectl delete -f fluentd-sumologic.yaml -``` - -#### 2. Deploy Fluentd, Fluent Bit and Prometheus again with the version 1.0.0 yaml - -Follow the below steps to deploy new resources. - -##### 2.1 Deploy Fluentd - -- Non-Helm users who have made changes to configs in the - [environment variable sections] of the `fluentd-sumologic.yaml` file will need to - move those config changes directly into the Fluentd pipeline. - -- Run the below command to get the `fluentd-sumologic.yaml` manifest for version `v1.0.0` - and then make the changes identified in the above step. - - ```bash - curl https://raw.githubusercontent.com/SumoLogic/sumologic-kubernetes-collection/release-v1.0/deploy/kubernetes/fluentd-sumologic.yaml.tmpl | \ - sed 's/\$NAMESPACE'"//g" | \ - sed 's/cluster kubernetes/cluster /g' >> fluentd-sumologic.yaml - ``` - -- Non-Helm users running a Kubernetes version of 1.13 or older will need to - **remove the following filter plugin section from their Fluentd pipeline. - This is required to prevent data duplication.** - - ```yaml - # NOTE: Remove this filter if you are running Kubernetes 1.13 or below. - @type grep - - key @metric - pattern /^apiserver_request_count|^apiserver_request_latencies_summary|^kubelet_runtime_operations_latency_microseconds|^kubelet_docker_operations_latency_microseconds|^kubelet_docker_operations_errors$/ - - - ``` - -[environment variable sections]: https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v0.17/deploy/kubernetes/fluentd-sumologic.yaml.tmpl#L627-L678 - -##### 2.2 Deploy Prometheus - -- Follow steps mentioned [here](https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v1.0/deploy/docs/Non_Helm_Installation.md#deploy-prometheus) to deploy Prometheus. - -##### 2.3: Deploy Fluent Bit - -- Follow steps mentioned [here](https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v1.0/deploy/docs/Non_Helm_Installation.md#deploy-fluentbit) to deploy Fluent Bit. - -## Kubernetes App dashboard update - -After successful migration please make sure to [reinstall your Kubernetes App] -to the latest version. - -[reinstall your Kubernetes App]: https://help.sumologic.com/docs/integrations/containers-orchestration/kubernetes#installing-the-kubernetes-app diff --git a/tests/helm/upgrade_script/run.sh b/tests/helm/upgrade_script/run.sh deleted file mode 100755 index 75b22b14b2..0000000000 --- a/tests/helm/upgrade_script/run.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Test generation: -# export test_name=example_test; \ -# bash deploy/helm/sumologic/upgrade-1.0.0.sh \ -# tests/upgrade_script/static/${test_name}.input.yaml \ -# 1> tests/upgrade_script/static/${test_name}.log 2>&1 \ -# && cp new_values.yaml tests/upgrade_script/static/${test_name}.output.yaml - -SCRIPT_PATH="$( dirname "$(realpath "${0}")" )" - -# shellcheck disable=SC1090 -# shellcheck source=tests/helm/functions.sh -source "${SCRIPT_PATH}/../functions.sh" -readonly TEST_TMP_OUT="${SCRIPT_PATH}/tmp/out.log" - -set_variables "${SCRIPT_PATH}" - -prepare_tests - -TEST_SUCCESS=true -for input_file in ${TEST_INPUT_FILES}; do - test_name="${input_file//.input.yaml/}" - output_file="${test_name}.output.yaml" - log_file="${test_name}.log" - - test_start "${test_name}" - bash "${TEST_SCRIPT_PATH}/../../../deploy/helm/sumologic/upgrade-1.0.0.sh" "${TEST_STATICS_PATH}/${input_file}" 1>"${TEST_TMP_OUT}" 2>&1 - mv new_values.yaml "${TEST_OUT}" - - test_output=$(diff "${TEST_STATICS_PATH}/${output_file}" "${TEST_OUT}") - test_log=$(diff "${TEST_STATICS_PATH}/${log_file}" "${TEST_TMP_OUT}") - rm "${TEST_TMP_OUT}" "${TEST_OUT}" - - if [[ -n "${test_output}" || -n "${test_log}" ]]; then - if [[ -n "${test_output}" ]]; then - echo -e "\tOutput diff (${TEST_STATICS_PATH}/${output_file}):\n${test_output}" - fi - if [[ -n "${test_log}" ]]; then - echo -e "\tLog diff (${TEST_STATICS_PATH}/${log_file}):\n${test_log}" - fi - test_failed "${test_name}" - TEST_SUCCESS=false - else - test_passed "${test_name}" - fi -done - -cleanup_tests - -if [[ "${TEST_SUCCESS}" = "true" ]]; then - exit 0 -else - exit 1 -fi diff --git a/tests/helm/upgrade_script/static/basic.input.yaml b/tests/helm/upgrade_script/static/basic.input.yaml deleted file mode 100644 index 5be192280b..0000000000 --- a/tests/helm/upgrade_script/static/basic.input.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sumologic: - accessId: my-access-id - accessKey: my-access-key diff --git a/tests/helm/upgrade_script/static/basic.log b/tests/helm/upgrade_script/static/basic.log deleted file mode 100644 index 5d0fff4f6d..0000000000 --- a/tests/helm/upgrade_script/static/basic.log +++ /dev/null @@ -1,5 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/basic.output.yaml b/tests/helm/upgrade_script/static/basic.output.yaml deleted file mode 100644 index 5be192280b..0000000000 --- a/tests/helm/upgrade_script/static/basic.output.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sumologic: - accessId: my-access-id - accessKey: my-access-key diff --git a/tests/helm/upgrade_script/static/chart_envs.input.yaml b/tests/helm/upgrade_script/static/chart_envs.input.yaml deleted file mode 100644 index 21dc07fcae..0000000000 --- a/tests/helm/upgrade_script/static/chart_envs.input.yaml +++ /dev/null @@ -1,33 +0,0 @@ -fluent-bit: - env: - - name: CHART - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: release - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -prometheus-operator: - prometheus: - prometheusSpec: - ## Define resources requests and limits for single Pods. - resources: {} - # requests: - # memory: 400Mi - thanos: - version: v0.10.0 - containers: - - name: "prometheus-config-reloader" - env: - - name: CHART - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: release - - name: NAMESPACE - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: fluentdNamespace diff --git a/tests/helm/upgrade_script/static/chart_envs.log b/tests/helm/upgrade_script/static/chart_envs.log deleted file mode 100644 index 977cd92a08..0000000000 --- a/tests/helm/upgrade_script/static/chart_envs.log +++ /dev/null @@ -1,7 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[INFO] Patching fluent-bit CHART environmental variable -[INFO] Patching prometheus CHART environmental variable - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/chart_envs.output.yaml b/tests/helm/upgrade_script/static/chart_envs.output.yaml deleted file mode 100644 index e829fa18ab..0000000000 --- a/tests/helm/upgrade_script/static/chart_envs.output.yaml +++ /dev/null @@ -1,30 +0,0 @@ -fluent-bit: - env: - - name: CHART - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: fluentdLogs - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -prometheus-operator: - prometheus: - prometheusSpec: - resources: {} - thanos: - version: v0.10.0 - containers: - - name: prometheus-config-reloader - env: - - name: CHART - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: fluentdMetrics - - name: NAMESPACE - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: fluentdNamespace diff --git a/tests/helm/upgrade_script/static/falco_disabled.input.yaml b/tests/helm/upgrade_script/static/falco_disabled.input.yaml deleted file mode 100644 index 2a3f74ee4e..0000000000 --- a/tests/helm/upgrade_script/static/falco_disabled.input.yaml +++ /dev/null @@ -1,2 +0,0 @@ -falco: - enabled: false diff --git a/tests/helm/upgrade_script/static/falco_disabled.log b/tests/helm/upgrade_script/static/falco_disabled.log deleted file mode 100644 index 5d0fff4f6d..0000000000 --- a/tests/helm/upgrade_script/static/falco_disabled.log +++ /dev/null @@ -1,5 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/falco_disabled.output.yaml b/tests/helm/upgrade_script/static/falco_disabled.output.yaml deleted file mode 100644 index 2a3f74ee4e..0000000000 --- a/tests/helm/upgrade_script/static/falco_disabled.output.yaml +++ /dev/null @@ -1,2 +0,0 @@ -falco: - enabled: false diff --git a/tests/helm/upgrade_script/static/falco_enabled.input.yaml b/tests/helm/upgrade_script/static/falco_enabled.input.yaml deleted file mode 100644 index 2cbb945296..0000000000 --- a/tests/helm/upgrade_script/static/falco_enabled.input.yaml +++ /dev/null @@ -1,2 +0,0 @@ -falco: - enabled: true diff --git a/tests/helm/upgrade_script/static/falco_enabled.log b/tests/helm/upgrade_script/static/falco_enabled.log deleted file mode 100644 index 708ef49365..0000000000 --- a/tests/helm/upgrade_script/static/falco_enabled.log +++ /dev/null @@ -1,5 +0,0 @@ - -[INFO] falco will be enabled. Change "falco.enabled" to "false" if you want to disable it (default for 1.0) - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/falco_enabled.output.yaml b/tests/helm/upgrade_script/static/falco_enabled.output.yaml deleted file mode 100644 index 2cbb945296..0000000000 --- a/tests/helm/upgrade_script/static/falco_enabled.output.yaml +++ /dev/null @@ -1,2 +0,0 @@ -falco: - enabled: true diff --git a/tests/helm/upgrade_script/static/filter_stream.input.yaml b/tests/helm/upgrade_script/static/filter_stream.input.yaml deleted file mode 100644 index a2bda80a7f..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream.input.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sumologic: - addStream: false - addTime: true diff --git a/tests/helm/upgrade_script/static/filter_stream.log b/tests/helm/upgrade_script/static/filter_stream.log deleted file mode 100644 index fba2ddfd6c..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream.log +++ /dev/null @@ -1,8 +0,0 @@ -[INFO] Removing sumologic.addStream -[INFO] Removing sumologic.addTime - -[INFO] Creating fluentd.logs.containers.extraFilterPluginConf to preserve addStream/addTime functionality -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/filter_stream.output.yaml b/tests/helm/upgrade_script/static/filter_stream.output.yaml deleted file mode 100644 index 93fa5f5f5b..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream.output.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sumologic: {} -fluentd: - logs: - containers: - extraFilterPluginConf: |- - - @type record_modifier - remove_keys stream - diff --git a/tests/helm/upgrade_script/static/filter_stream_and_time.input.yaml b/tests/helm/upgrade_script/static/filter_stream_and_time.input.yaml deleted file mode 100644 index ffaeecc086..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream_and_time.input.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sumologic: - addStream: false - addTime: false diff --git a/tests/helm/upgrade_script/static/filter_stream_and_time.log b/tests/helm/upgrade_script/static/filter_stream_and_time.log deleted file mode 100644 index fba2ddfd6c..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream_and_time.log +++ /dev/null @@ -1,8 +0,0 @@ -[INFO] Removing sumologic.addStream -[INFO] Removing sumologic.addTime - -[INFO] Creating fluentd.logs.containers.extraFilterPluginConf to preserve addStream/addTime functionality -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/filter_stream_and_time.output.yaml b/tests/helm/upgrade_script/static/filter_stream_and_time.output.yaml deleted file mode 100644 index 6fdd0e81ba..0000000000 --- a/tests/helm/upgrade_script/static/filter_stream_and_time.output.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sumologic: {} -fluentd: - logs: - containers: - extraFilterPluginConf: |- - - @type record_modifier - remove_keys stream,time - diff --git a/tests/helm/upgrade_script/static/filter_time.input.yaml b/tests/helm/upgrade_script/static/filter_time.input.yaml deleted file mode 100644 index fe360f7b17..0000000000 --- a/tests/helm/upgrade_script/static/filter_time.input.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sumologic: - addStream: true - addTime: false diff --git a/tests/helm/upgrade_script/static/filter_time.log b/tests/helm/upgrade_script/static/filter_time.log deleted file mode 100644 index fba2ddfd6c..0000000000 --- a/tests/helm/upgrade_script/static/filter_time.log +++ /dev/null @@ -1,8 +0,0 @@ -[INFO] Removing sumologic.addStream -[INFO] Removing sumologic.addTime - -[INFO] Creating fluentd.logs.containers.extraFilterPluginConf to preserve addStream/addTime functionality -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/filter_time.output.yaml b/tests/helm/upgrade_script/static/filter_time.output.yaml deleted file mode 100644 index fc042b34e7..0000000000 --- a/tests/helm/upgrade_script/static/filter_time.output.yaml +++ /dev/null @@ -1,9 +0,0 @@ -sumologic: {} -fluentd: - logs: - containers: - extraFilterPluginConf: |- - - @type record_modifier - remove_keys time - diff --git a/tests/helm/upgrade_script/static/fluent_bit_casts.input.yaml b/tests/helm/upgrade_script/static/fluent_bit_casts.input.yaml deleted file mode 100644 index 5fcbc8c5ca..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_casts.input.yaml +++ /dev/null @@ -1,9 +0,0 @@ -fluent-bit: - backend: - type: forward - forward: - host: ${CHART}.${NAMESPACE}.svc.cluster.local. - port: 24321 - tls: "off" - tls_verify: "on" - tls_debug: 1 diff --git a/tests/helm/upgrade_script/static/fluent_bit_casts.log b/tests/helm/upgrade_script/static/fluent_bit_casts.log deleted file mode 100644 index d4d0acca43..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_casts.log +++ /dev/null @@ -1,7 +0,0 @@ -[INFO] Casting fluent-bit.backend.forward.tls to str -[INFO] Casting fluent-bit.backend.forward.tls_verify to str - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/fluent_bit_casts.output.yaml b/tests/helm/upgrade_script/static/fluent_bit_casts.output.yaml deleted file mode 100644 index 402eb3ec00..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_casts.output.yaml +++ /dev/null @@ -1,9 +0,0 @@ -fluent-bit: - backend: - type: forward - forward: - host: ${CHART}.${NAMESPACE}.svc.cluster.local. - port: 24321 - tls: "off" - tls_verify: "on" - tls_debug: 1 diff --git a/tests/helm/upgrade_script/static/fluent_bit_sed_db.input.yaml b/tests/helm/upgrade_script/static/fluent_bit_sed_db.input.yaml deleted file mode 100644 index 33d171640c..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_sed_db.input.yaml +++ /dev/null @@ -1,71 +0,0 @@ -fluent-bit: - rawConfig: |- - @INCLUDE fluent-bit-service.conf - - [INPUT] - Name tail - Path /var/log/containers/*.log - Multiline On - Parser_Firstline multi_line - Tag containers.* - Refresh_Interval 1 - Rotate_Wait 60 - Mem_Buf_Limit 5MB - Skip_Long_Lines On - DB /tail-db/tail-containers-state.db - DB.Sync Normal - [INPUT] - Name systemd - Tag host.* - Systemd_Filter _SYSTEMD_UNIT=addon-config.service - Systemd_Filter _SYSTEMD_UNIT=addon-run.service - Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service - Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service - Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=containerd.service - Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service - Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service - Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service - Systemd_Filter _SYSTEMD_UNIT=dbus.service - Systemd_Filter _SYSTEMD_UNIT=docker.service - Systemd_Filter _SYSTEMD_UNIT=efs.service - Systemd_Filter _SYSTEMD_UNIT=etcd-member.service - Systemd_Filter _SYSTEMD_UNIT=etcd.service - Systemd_Filter _SYSTEMD_UNIT=etcd2.service - Systemd_Filter _SYSTEMD_UNIT=etcd3.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service - Systemd_Filter _SYSTEMD_UNIT=flanneld.service - Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service - Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service - Systemd_Filter _SYSTEMD_UNIT=kubelet.service - Systemd_Filter _SYSTEMD_UNIT=ldconfig.service - Systemd_Filter _SYSTEMD_UNIT=locksmithd.service - Systemd_Filter _SYSTEMD_UNIT=logrotate.service - Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service - Systemd_Filter _SYSTEMD_UNIT=mdmon.service - Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-server.service - Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service - Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service - Systemd_Filter _SYSTEMD_UNIT=ntp.service - Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service - Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service - Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service - Systemd_Filter _SYSTEMD_UNIT=rpcbind.service - Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service - Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service - Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service - Max_Entries 1000 - Read_From_Tail true - - @INCLUDE fluent-bit-output.conf diff --git a/tests/helm/upgrade_script/static/fluent_bit_sed_db.log b/tests/helm/upgrade_script/static/fluent_bit_sed_db.log deleted file mode 100644 index e00e390af0..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_sed_db.log +++ /dev/null @@ -1,7 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[INFO] Replacing tail-db/tail-containers-state.db to tail-db/tail-containers-state-sumo.db -[WARNING] Please ensure that new fluent-bit configuration is correct - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/fluent_bit_sed_db.output.yaml b/tests/helm/upgrade_script/static/fluent_bit_sed_db.output.yaml deleted file mode 100644 index 4ab48de355..0000000000 --- a/tests/helm/upgrade_script/static/fluent_bit_sed_db.output.yaml +++ /dev/null @@ -1,71 +0,0 @@ -fluent-bit: - rawConfig: |- - @INCLUDE fluent-bit-service.conf - - [INPUT] - Name tail - Path /var/log/containers/*.log - Multiline On - Parser_Firstline multi_line - Tag containers.* - Refresh_Interval 1 - Rotate_Wait 60 - Mem_Buf_Limit 5MB - Skip_Long_Lines On - DB /tail-db/tail-containers-state-sumo.db - DB.Sync Normal - [INPUT] - Name systemd - Tag host.* - Systemd_Filter _SYSTEMD_UNIT=addon-config.service - Systemd_Filter _SYSTEMD_UNIT=addon-run.service - Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service - Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service - Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=containerd.service - Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service - Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service - Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service - Systemd_Filter _SYSTEMD_UNIT=dbus.service - Systemd_Filter _SYSTEMD_UNIT=docker.service - Systemd_Filter _SYSTEMD_UNIT=efs.service - Systemd_Filter _SYSTEMD_UNIT=etcd-member.service - Systemd_Filter _SYSTEMD_UNIT=etcd.service - Systemd_Filter _SYSTEMD_UNIT=etcd2.service - Systemd_Filter _SYSTEMD_UNIT=etcd3.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service - Systemd_Filter _SYSTEMD_UNIT=flanneld.service - Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service - Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service - Systemd_Filter _SYSTEMD_UNIT=kubelet.service - Systemd_Filter _SYSTEMD_UNIT=ldconfig.service - Systemd_Filter _SYSTEMD_UNIT=locksmithd.service - Systemd_Filter _SYSTEMD_UNIT=logrotate.service - Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service - Systemd_Filter _SYSTEMD_UNIT=mdmon.service - Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-server.service - Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service - Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service - Systemd_Filter _SYSTEMD_UNIT=ntp.service - Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service - Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service - Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service - Systemd_Filter _SYSTEMD_UNIT=rpcbind.service - Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service - Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service - Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service - Max_Entries 1000 - Read_From_Tail true - - @INCLUDE fluent-bit-output.conf diff --git a/tests/helm/upgrade_script/static/hooks.input.yaml b/tests/helm/upgrade_script/static/hooks.input.yaml deleted file mode 100644 index 7950e7ba49..0000000000 --- a/tests/helm/upgrade_script/static/hooks.input.yaml +++ /dev/null @@ -1,27 +0,0 @@ -sumologic: - setup: - clusterRole: - annotations: - helm.sh/hook: pre-install - helm.sh/hook-weight: "1" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - clusterRoleBinding: - annotations: - helm.sh/hook: pre-install - helm.sh/hook-weight: "2" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - configMap: - annotations: - helm.sh/hook: pre-install - helm.sh/hook-weight: "2" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - job: - annotations: - helm.sh/hook: pre-install - helm.sh/hook-weight: "3" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - serviceAccount: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/tests/helm/upgrade_script/static/hooks.log b/tests/helm/upgrade_script/static/hooks.log deleted file mode 100644 index a02393138c..0000000000 --- a/tests/helm/upgrade_script/static/hooks.log +++ /dev/null @@ -1,11 +0,0 @@ -[INFO] Casting sumologic.setup.clusterRole.annotations."helm.sh/hook-weight" to str -[INFO] Casting sumologic.setup.clusterRoleBinding.annotations."helm.sh/hook-weight" to str -[INFO] Casting sumologic.setup.configMap.annotations."helm.sh/hook-weight" to str -[INFO] Casting sumologic.setup.job.annotations."helm.sh/hook-weight" to str -[INFO] Casting sumologic.setup.serviceAccount.annotations."helm.sh/hook-weight" to str - -[INFO] Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade' -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/hooks.output.yaml b/tests/helm/upgrade_script/static/hooks.output.yaml deleted file mode 100644 index 326f9d8c5e..0000000000 --- a/tests/helm/upgrade_script/static/hooks.output.yaml +++ /dev/null @@ -1,27 +0,0 @@ -sumologic: - setup: - clusterRole: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "1" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - clusterRoleBinding: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "2" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - configMap: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "2" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - job: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "3" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - serviceAccount: - annotations: - helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/tests/helm/upgrade_script/static/key_mapping_empty.input.yaml b/tests/helm/upgrade_script/static/key_mapping_empty.input.yaml deleted file mode 100644 index 68aad8b770..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_empty.input.yaml +++ /dev/null @@ -1,14 +0,0 @@ -deployment: some-string -eventsDeployment: some-string -fluentd: - autoscaling: some-string - rawConfig: some-string - statefulset: some-string -sumologic: - addStream: some-string - addTime: some-string - events: some-string - fluentd: some-string - k8sMetadataFilter: some-string - kubernetesMeta: some-string - kubernetesMetaReduce: some-string diff --git a/tests/helm/upgrade_script/static/key_mapping_empty.log b/tests/helm/upgrade_script/static/key_mapping_empty.log deleted file mode 100644 index 75513fc29e..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_empty.log +++ /dev/null @@ -1,17 +0,0 @@ -[INFO] Removing deployment -[INFO] Removing eventsDeployment -[INFO] Removing fluentd.autoscaling -[INFO] Removing fluentd.rawConfig -[INFO] Removing fluentd.statefulset -[INFO] Removing sumologic.addStream -[INFO] Removing sumologic.addTime -[INFO] Removing sumologic.events -[INFO] Removing sumologic.fluentd -[INFO] Removing sumologic.k8sMetadataFilter -[INFO] Removing sumologic.kubernetesMeta -[INFO] Removing sumologic.kubernetesMetaReduce - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/key_mapping_empty.output.yaml b/tests/helm/upgrade_script/static/key_mapping_empty.output.yaml deleted file mode 100644 index 2c9fff3dcc..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_empty.output.yaml +++ /dev/null @@ -1 +0,0 @@ -{fluentd: {}, sumologic: {}} diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.input.yaml b/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.input.yaml deleted file mode 100644 index 4dbf8cc40e..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.input.yaml +++ /dev/null @@ -1,28 +0,0 @@ -deployment: - affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity - nodeSelector: fluentd.logs.statefulset.nodeSelector, fluentd.metrics.statefulset.nodeSelector - podAntiAffinity: fluentd.logs.statefulset.podAntiAffinity, fluentd.metrics.statefulset.podAntiAffinity - replicaCount: fluentd.logs.statefulset.replicaCount, fluentd.metrics.statefulset.replicaCount - resources: - limits: - cpu: fluentd.logs.statefulset.resources.limits.cpu, fluentd.metrics.statefulset.resources.limits.cpu - memory: fluentd.logs.statefulset.resources.limits.memory, fluentd.metrics.statefulset.resources.limits.memory - requests: - cpu: fluentd.logs.statefulset.resources.requests.cpu, fluentd.metrics.statefulset.resources.requests.cpu - memory: fluentd.logs.statefulset.resources.requests.memory, fluentd.metrics.statefulset.resources.requests.memory - tolerations: fluentd.logs.statefulset.tolerations, fluentd.metrics.statefulset.tolerations -sumologic: - fluentd: - autoscaling: - enabled: fluentd.logs.autoscaling.enabled, fluentd.metrics.autoscaling.enabled - maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas - minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas - targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.log b/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.log deleted file mode 100644 index eaf221e9f5..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.log +++ /dev/null @@ -1,84 +0,0 @@ -[INFO] Mapping deployment.affinity into: -[INFO] - fluentd.logs.statefulset.affinity -[INFO] - fluentd.metrics.statefulset.affinity -[INFO] Mapping deployment.nodeSelector into: -[INFO] - fluentd.logs.statefulset.nodeSelector -[INFO] - fluentd.metrics.statefulset.nodeSelector -[INFO] Mapping deployment.podAntiAffinity into: -[INFO] - fluentd.logs.statefulset.podAntiAffinity -[INFO] - fluentd.metrics.statefulset.podAntiAffinity -[INFO] Mapping deployment.replicaCount into: -[INFO] - fluentd.logs.statefulset.replicaCount -[INFO] - fluentd.metrics.statefulset.replicaCount -[INFO] Mapping deployment.resources.limits.cpu into: -[INFO] - fluentd.logs.statefulset.resources.limits.cpu -[INFO] - fluentd.metrics.statefulset.resources.limits.cpu -[INFO] Mapping deployment.resources.limits.memory into: -[INFO] - fluentd.logs.statefulset.resources.limits.memory -[INFO] - fluentd.metrics.statefulset.resources.limits.memory -[INFO] Mapping deployment.resources.requests.cpu into: -[INFO] - fluentd.logs.statefulset.resources.requests.cpu -[INFO] - fluentd.metrics.statefulset.resources.requests.cpu -[INFO] Mapping deployment.resources.requests.memory into: -[INFO] - fluentd.logs.statefulset.resources.requests.memory -[INFO] - fluentd.metrics.statefulset.resources.requests.memory -[INFO] Mapping deployment.tolerations into: -[INFO] - fluentd.logs.statefulset.tolerations -[INFO] - fluentd.metrics.statefulset.tolerations -[INFO] Mapping sumologic.fluentd.autoscaling.enabled into: -[INFO] - fluentd.logs.autoscaling.enabled -[INFO] - fluentd.metrics.autoscaling.enabled -[INFO] Mapping sumologic.fluentd.autoscaling.maxReplicas into: -[INFO] - fluentd.logs.autoscaling.maxReplicas -[INFO] - fluentd.metrics.autoscaling.maxReplicas -[INFO] Mapping sumologic.fluentd.autoscaling.minReplicas into: -[INFO] - fluentd.logs.autoscaling.minReplicas -[INFO] - fluentd.metrics.autoscaling.minReplicas -[INFO] Mapping sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage into: -[INFO] - fluentd.logs.autoscaling.targetCPUUtilizationPercentage -[INFO] - fluentd.metrics.autoscaling.targetCPUUtilizationPercentage -[INFO] Mapping sumologic.excludeContainerRegex into: -[INFO] - fluentd.logs.containers.excludeContainerRegex -[INFO] - fluentd.logs.default.excludeContainerRegex -[INFO] - fluentd.logs.systemd.excludeContainerRegex -[INFO] - fluentd.logs.kubelet.excludeContainerRegex -[INFO] Mapping sumologic.excludeHostRegex into: -[INFO] - fluentd.logs.containers.excludeHostRegex -[INFO] - fluentd.logs.default.excludeHostRegex -[INFO] - fluentd.logs.systemd.excludeHostRegex -[INFO] - fluentd.logs.kubelet.excludeHostRegex -[INFO] Mapping sumologic.excludeNamespaceRegex into: -[INFO] - fluentd.logs.containers.excludeNamespaceRegex -[INFO] - fluentd.logs.default.excludeNamespaceRegex -[INFO] - fluentd.logs.systemd.excludeNamespaceRegex -[INFO] - fluentd.logs.kubelet.excludeNamespaceRegex -[INFO] Mapping sumologic.excludePodRegex into: -[INFO] - fluentd.logs.containers.excludePodRegex -[INFO] - fluentd.logs.default.excludePodRegex -[INFO] - fluentd.logs.systemd.excludePodRegex -[INFO] - fluentd.logs.kubelet.excludePodRegex -[INFO] Mapping sumologic.sourceCategory into: -[INFO] - fluentd.logs.containers.sourceCategory -[INFO] - fluentd.logs.default.sourceCategory -[INFO] - fluentd.logs.systemd.sourceCategory -[INFO] - fluentd.logs.kubelet.sourceCategory -[INFO] Mapping sumologic.sourceCategoryPrefix into: -[INFO] - fluentd.logs.containers.sourceCategoryPrefix -[INFO] - fluentd.logs.default.sourceCategoryPrefix -[INFO] - fluentd.logs.systemd.sourceCategoryPrefix -[INFO] - fluentd.logs.kubelet.sourceCategoryPrefix -[INFO] Mapping sumologic.sourceCategoryReplaceDash into: -[INFO] - fluentd.logs.containers.sourceCategoryReplaceDash -[INFO] - fluentd.logs.default.sourceCategoryReplaceDash -[INFO] - fluentd.logs.systemd.sourceCategoryReplaceDash -[INFO] - fluentd.logs.kubelet.sourceCategoryReplaceDash -[INFO] Mapping sumologic.sourceName into: -[INFO] - fluentd.logs.containers.sourceName -[INFO] - fluentd.logs.default.sourceName -[INFO] - fluentd.logs.systemd.sourceName -[INFO] - fluentd.logs.kubelet.sourceName - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.output.yaml b/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.output.yaml deleted file mode 100644 index 42ddcaed47..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_multiple.output.yaml +++ /dev/null @@ -1,75 +0,0 @@ -fluentd: - logs: - statefulset: - affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity - nodeSelector: fluentd.logs.statefulset.nodeSelector, fluentd.metrics.statefulset.nodeSelector - podAntiAffinity: fluentd.logs.statefulset.podAntiAffinity, fluentd.metrics.statefulset.podAntiAffinity - replicaCount: fluentd.logs.statefulset.replicaCount, fluentd.metrics.statefulset.replicaCount - resources: - limits: - cpu: fluentd.logs.statefulset.resources.limits.cpu, fluentd.metrics.statefulset.resources.limits.cpu - memory: fluentd.logs.statefulset.resources.limits.memory, fluentd.metrics.statefulset.resources.limits.memory - requests: - cpu: fluentd.logs.statefulset.resources.requests.cpu, fluentd.metrics.statefulset.resources.requests.cpu - memory: fluentd.logs.statefulset.resources.requests.memory, fluentd.metrics.statefulset.resources.requests.memory - tolerations: fluentd.logs.statefulset.tolerations, fluentd.metrics.statefulset.tolerations - autoscaling: - enabled: fluentd.logs.autoscaling.enabled, fluentd.metrics.autoscaling.enabled - maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas - minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas - targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage - containers: - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName - default: - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName - systemd: - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName - kubelet: - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName - metrics: - statefulset: - affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity - nodeSelector: fluentd.logs.statefulset.nodeSelector, fluentd.metrics.statefulset.nodeSelector - podAntiAffinity: fluentd.logs.statefulset.podAntiAffinity, fluentd.metrics.statefulset.podAntiAffinity - replicaCount: fluentd.logs.statefulset.replicaCount, fluentd.metrics.statefulset.replicaCount - resources: - limits: - cpu: fluentd.logs.statefulset.resources.limits.cpu, fluentd.metrics.statefulset.resources.limits.cpu - memory: fluentd.logs.statefulset.resources.limits.memory, fluentd.metrics.statefulset.resources.limits.memory - requests: - cpu: fluentd.logs.statefulset.resources.requests.cpu, fluentd.metrics.statefulset.resources.requests.cpu - memory: fluentd.logs.statefulset.resources.requests.memory, fluentd.metrics.statefulset.resources.requests.memory - tolerations: fluentd.logs.statefulset.tolerations, fluentd.metrics.statefulset.tolerations - autoscaling: - enabled: fluentd.logs.autoscaling.enabled, fluentd.metrics.autoscaling.enabled - maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas - minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas - targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_one.input.yaml b/tests/helm/upgrade_script/static/key_mapping_one_to_one.input.yaml deleted file mode 100644 index f1b9c48db1..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_one.input.yaml +++ /dev/null @@ -1,33 +0,0 @@ -eventsDeployment: - nodeSelector: fluentd.events.statefulset.nodeSelector - resources: - limits: - cpu: fluentd.events.statefulset.resources.limits.cpu - memory: fluentd.events.statefulset.resources.limits.memory - requests: - cpu: fluentd.events.statefulset.resources.requests.cpu - memory: fluentd.events.statefulset.resources.requests.memory - tolerations: fluentd.events.statefulset.tolerations -sumologic: - addTimestamp: fluentd.logs.output.addTimestamp - chunkLimitSize: fluentd.buffer.chunkLimitSize - eventCollectionEnabled: fluentd.events.enabled - events: - sourceCategory: fluentd.events.sourceCategory - fluentd: - buffer: fluentd.buffer.type - fluentdLogLevel: fluentd.logLevel - flushInterval: fluentd.buffer.flushInterval - k8sMetadataFilter.: - cacheRefresh: fluentd.metadata.cacheRefresh - cacheSize: fluentd.metadata.cacheSize - cacheTtl: fluentd.metadata.cacheTtl - verifySsl: fluentd.logs.containers.k8sMetadataFilter.verifySsl - watch: fluentd.logs.containers.k8sMetadataFilter.watch - logFormat: fluentd.logs.output.logFormat - numThreads: fluentd.buffer.numThreads - queueChunkLimitSize: fluentd.buffer.queueChunkLimitSize - timestampKey: fluentd.logs.output.timestampKey - totalLimitSize: fluentd.buffer.totalLimitSize - verifySsl: fluentd.verifySsl - watchResourceEventsOverrides: fluentd.events.watchResourceEventsOverrides diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_one.log b/tests/helm/upgrade_script/static/key_mapping_one_to_one.log deleted file mode 100644 index 7c83f0ac0a..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_one.log +++ /dev/null @@ -1,25 +0,0 @@ -[INFO] Mapping eventsDeployment.nodeSelector into fluentd.events.statefulset.nodeSelector -[INFO] Mapping eventsDeployment.resources.limits.cpu into fluentd.events.statefulset.resources.limits.cpu -[INFO] Mapping eventsDeployment.resources.limits.memory into fluentd.events.statefulset.resources.limits.memory -[INFO] Mapping eventsDeployment.resources.requests.cpu into fluentd.events.statefulset.resources.requests.cpu -[INFO] Mapping eventsDeployment.resources.requests.memory into fluentd.events.statefulset.resources.requests.memory -[INFO] Mapping eventsDeployment.tolerations into fluentd.events.statefulset.tolerations -[INFO] Mapping sumologic.addTimestamp into fluentd.logs.output.addTimestamp -[INFO] Mapping sumologic.chunkLimitSize into fluentd.buffer.chunkLimitSize -[INFO] Mapping sumologic.eventCollectionEnabled into fluentd.events.enabled -[INFO] Mapping sumologic.events.sourceCategory into fluentd.events.sourceCategory -[INFO] Mapping sumologic.fluentd.buffer into fluentd.buffer.type -[INFO] Mapping sumologic.fluentdLogLevel into fluentd.logLevel -[INFO] Mapping sumologic.flushInterval into fluentd.buffer.flushInterval -[INFO] Mapping sumologic.logFormat into fluentd.logs.output.logFormat -[INFO] Mapping sumologic.numThreads into fluentd.buffer.numThreads -[INFO] Mapping sumologic.queueChunkLimitSize into fluentd.buffer.queueChunkLimitSize -[INFO] Mapping sumologic.timestampKey into fluentd.logs.output.timestampKey -[INFO] Mapping sumologic.totalLimitSize into fluentd.buffer.totalLimitSize -[INFO] Mapping sumologic.verifySsl into fluentd.verifySsl -[INFO] Mapping sumologic.watchResourceEventsOverrides into fluentd.events.watchResourceEventsOverrides - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/key_mapping_one_to_one.output.yaml b/tests/helm/upgrade_script/static/key_mapping_one_to_one.output.yaml deleted file mode 100644 index fbe39ff055..0000000000 --- a/tests/helm/upgrade_script/static/key_mapping_one_to_one.output.yaml +++ /dev/null @@ -1,36 +0,0 @@ -fluentd: - events: - statefulset: - nodeSelector: fluentd.events.statefulset.nodeSelector - resources: - limits: - cpu: fluentd.events.statefulset.resources.limits.cpu - memory: fluentd.events.statefulset.resources.limits.memory - requests: - cpu: fluentd.events.statefulset.resources.requests.cpu - memory: fluentd.events.statefulset.resources.requests.memory - tolerations: fluentd.events.statefulset.tolerations - enabled: fluentd.events.enabled - sourceCategory: fluentd.events.sourceCategory - watchResourceEventsOverrides: fluentd.events.watchResourceEventsOverrides - logs: - output: - addTimestamp: fluentd.logs.output.addTimestamp - logFormat: fluentd.logs.output.logFormat - timestampKey: fluentd.logs.output.timestampKey - buffer: - chunkLimitSize: fluentd.buffer.chunkLimitSize - type: fluentd.buffer.type - flushInterval: fluentd.buffer.flushInterval - numThreads: fluentd.buffer.numThreads - queueChunkLimitSize: fluentd.buffer.queueChunkLimitSize - totalLimitSize: fluentd.buffer.totalLimitSize - logLevel: fluentd.logLevel - verifySsl: fluentd.verifySsl -sumologic: - k8sMetadataFilter.: - cacheRefresh: fluentd.metadata.cacheRefresh - cacheSize: fluentd.metadata.cacheSize - cacheTtl: fluentd.metadata.cacheTtl - verifySsl: fluentd.logs.containers.k8sMetadataFilter.verifySsl - watch: fluentd.logs.containers.k8sMetadataFilter.watch diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write.input.yaml b/tests/helm/upgrade_script/static/prometheus_remote_write.input.yaml deleted file mode 100644 index 83ee733d62..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write.input.yaml +++ /dev/null @@ -1,76 +0,0 @@ -prometheus-operator: - prometheus: - prometheusSpec: - remoteWrite: - # kube state metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state - writeRelabelConfigs: - - action: keep - regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) - sourceLabels: [job, __name__] - # controller manager metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager - writeRelabelConfigs: - - action: keep - regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* - sourceLabels: [job, __name__] - # scheduler metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler - writeRelabelConfigs: - - action: keep - regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* - sourceLabels: [job, __name__] - # api server metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver - writeRelabelConfigs: - - action: keep - regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_latenc(?:ies|y_seconds).*|etcd_request_cache_get_latenc(?:ies_summary|y_seconds).*|etcd_request_cache_add_latenc(?:ies_summary|y_seconds).*|etcd_helper_cache_hit_(?:count|total)|etcd_helper_cache_miss_(?:count|total)) - sourceLabels: [job, __name__] - # kubelet metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*) - sourceLabels: [job, __name__] - # cadvisor container metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: labelmap - regex: container_name - replacement: container - - action: drop - regex: POD - sourceLabels: [container] - - action: keep - regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|) - sourceLabels: [job,container,__name__] - # cadvisor aggregate container metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total) - sourceLabels: [job,__name__] - # node exporter metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node - writeRelabelConfigs: - - action: keep - regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files) - sourceLabels: [job, __name__] - # prometheus operator rules - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: [__name__] - # My custome remote write - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.custom - writeRelabelConfigs: - - action: keep - regex: (?:custom_metric.*) - sourceLabels: [__name__] - # health - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics - writeRelabelConfigs: - - action: keep - regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*|otelcol.*) - sourceLabels: [__name__] diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write.log b/tests/helm/upgrade_script/static/prometheus_remote_write.log deleted file mode 100644 index 5d0fff4f6d..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write.log +++ /dev/null @@ -1,5 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write.output.yaml b/tests/helm/upgrade_script/static/prometheus_remote_write.output.yaml deleted file mode 100644 index 55c4d7b065..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write.output.yaml +++ /dev/null @@ -1,86 +0,0 @@ -prometheus-operator: - prometheus: - prometheusSpec: - remoteWrite: - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state - writeRelabelConfigs: - - action: keep - regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager - writeRelabelConfigs: - - action: keep - regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler - writeRelabelConfigs: - - action: keep - regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver - writeRelabelConfigs: - - action: keep - regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:duration_seconds|latencies)_(?:count|sum)|apiserver_request_latencies_summary(?:|_count|_sum)|etcd_request_cache_(?:add|get)_(?:duration_seconds|latencies_summary)_(?:count|sum)|etcd_helper_cache_(?:hit|miss)_(?:count|total)) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:kubelet_docker_operations_errors(?:|_total)|kubelet_(?:docker|runtime)_operations_duration_seconds_(?:count|sum)|kubelet_running_(?:container|pod)_count|kubelet_(:?docker|runtime)_operations_latency_microseconds(?:|_count|_sum)) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: labelmap - regex: container_name - replacement: container - - action: drop - regex: POD - sourceLabels: - - container - - action: keep - regex: kubelet;.+;(?:container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_fs_usage_bytes|container_fs_limit_bytes) - sourceLabels: - - job - - container - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node - writeRelabelConfigs: - - action: keep - regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.custom - writeRelabelConfigs: - - action: keep - regex: (?:custom_metric.*) - sourceLabels: - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics - writeRelabelConfigs: - - action: keep - regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*|otelcol.*) - sourceLabels: - - __name__ diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.input.yaml b/tests/helm/upgrade_script/static/prometheus_remote_write_modified.input.yaml deleted file mode 100644 index 651e621b2d..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.input.yaml +++ /dev/null @@ -1,77 +0,0 @@ -prometheus-operator: - prometheus: - prometheusSpec: - remoteWrite: - # kube state metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state - writeRelabelConfigs: - - action: keep - regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase|custom) - sourceLabels: [job, __name__] - # controller manager metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager - writeRelabelConfigs: - - action: keep - regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* - sourceLabels: [job, __name__] - # scheduler metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler - writeRelabelConfigs: - - action: keep - regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* - sourceLabels: [job, __name__] - # api server metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver - writeRelabelConfigs: - - action: keep - regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_latenc(?:ies|y_seconds).*|etcd_request_cache_get_latenc(?:ies_summary|y_seconds).*|etcd_request_cache_add_latenc(?:ies_summary|y_seconds).*|etcd_helper_cache_hit_(?:count|total)|etcd_helper_cache_miss_(?:count|total)) - sourceLabels: [job, __name__] - # kubelet metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*|custom) - sourceLabels: [job, __name__] - # cadvisor container metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: labelmap - regex: container_name - replacement: container - - action: drop - regex: POD - sourceLabels: [container] - - action: keep - regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|custom) - sourceLabels: [job,container,__name__] - # cadvisor aggregate container metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total) - sourceLabels: [job,__name__] - # node exporter metrics - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node - writeRelabelConfigs: - - action: keep - regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total|node_memory_MemAvailable_bytes|node_memory_MemTotal_bytes|node_memory_Buffers_bytes|node_memory_SwapCached_bytes|node_memory_Cached_bytes|node_memory_MemFree_bytes|node_memory_SwapFree_bytes|node_ipvs_incoming_bytes_total|node_ipvs_outgoing_bytes_total|node_ipvs_incoming_packets_total|node_ipvs_outgoing_packets_total|node_disk_reads_completed_total|node_disk_writes_completed_total|node_disk_read_bytes_total|node_disk_written_bytes_total|node_filesystem_avail_bytes|node_filesystem_free_bytes|node_filesystem_size_bytes|node_filesystem_files) - sourceLabels: [job, __name__] - # prometheus operator rules - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_cpu:rate:sum|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|instance:node_network_transmit_bytes:rate:sum|instance:node_cpu:ratio|cluster:node_cpu:sum_rate5m|cluster:node_cpu:ratio|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|:node_memory_MemFreeCachedBuffers_bytes:sum|:node_memory_MemTotal_bytes:sum|node:node_memory_bytes_available:sum|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: [__name__] - # My custome remote write - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.custom - writeRelabelConfigs: - - action: keep - regex: (?:custom_metric.*) - sourceLabels: [__name__] - # health - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics - writeRelabelConfigs: - - action: keep - regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*|otelcol.*) - sourceLabels: [__name__] - diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.log b/tests/helm/upgrade_script/static/prometheus_remote_write_modified.log deleted file mode 100644 index 31f6a27b74..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.log +++ /dev/null @@ -1,8 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[WARNING] Changes of regex for 'prometheus-operator.prometheus.prometheusSpec.remoteWrite[0].writeRelabelConfigs[0]' (/prometheus.metrics.state) detected, please migrate it manually -[WARNING] Changes of regex for 'prometheus-operator.prometheus.prometheusSpec.remoteWrite[4].writeRelabelConfigs[0]' (/prometheus.metrics.kubelet) detected, please migrate it manually -[WARNING] Changes of regex for 'prometheus-operator.prometheus.prometheusSpec.remoteWrite[5].writeRelabelConfigs[2]' (/prometheus.metrics.container) detected, please migrate it manually - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.output.yaml b/tests/helm/upgrade_script/static/prometheus_remote_write_modified.output.yaml deleted file mode 100644 index c08cb32134..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_remote_write_modified.output.yaml +++ /dev/null @@ -1,86 +0,0 @@ -prometheus-operator: - prometheus: - prometheusSpec: - remoteWrite: - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state - writeRelabelConfigs: - - action: keep - regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_daemonset_metadata_generation|kube_deployment_metadata_generation|kube_deployment_spec_paused|kube_deployment_spec_replicas|kube_deployment_spec_strategy_rollingupdate_max_unavailable|kube_deployment_status_replicas_available|kube_deployment_status_observed_generation|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_spec_unschedulable|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase|custom) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager - writeRelabelConfigs: - - action: keep - regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler - writeRelabelConfigs: - - action: keep - regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_latency_microseconds.* - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver - writeRelabelConfigs: - - action: keep - regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:duration_seconds|latencies)_(?:count|sum)|apiserver_request_latencies_summary(?:|_count|_sum)|etcd_request_cache_(?:add|get)_(?:duration_seconds|latencies_summary)_(?:count|sum)|etcd_helper_cache_(?:hit|miss)_(?:count|total)) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:kubelet_docker_operations_errors|kubelet_docker_operations_latency_microseconds|kubelet_running_container_count|kubelet_running_pod_count|kubelet_runtime_operations_latency_microseconds.*|custom) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: labelmap - regex: container_name - replacement: container - - action: drop - regex: POD - sourceLabels: - - container - - action: keep - regex: kubelet;.+;(?:container_cpu_load_average_10s|container_cpu_system_seconds_total|container_cpu_usage_seconds_total|container_cpu_cfs_throttled_seconds_total|container_memory_usage_bytes|container_memory_swap|container_memory_working_set_bytes|container_spec_memory_limit_bytes|container_spec_memory_swap_limit_bytes|container_spec_memory_reservation_limit_bytes|container_spec_cpu_quota|container_spec_cpu_period|container_fs_usage_bytes|container_fs_limit_bytes|container_fs_reads_bytes_total|container_fs_writes_bytes_total|custom) - sourceLabels: - - job - - container - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container - writeRelabelConfigs: - - action: keep - regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node - writeRelabelConfigs: - - action: keep - regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total) - sourceLabels: - - job - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.custom - writeRelabelConfigs: - - action: keep - regex: (?:custom_metric.*) - sourceLabels: - - __name__ - - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics - writeRelabelConfigs: - - action: keep - regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*|otelcol.*) - sourceLabels: - - __name__ diff --git a/tests/helm/upgrade_script/static/prometheus_service_monitors.input.yaml b/tests/helm/upgrade_script/static/prometheus_service_monitors.input.yaml deleted file mode 100644 index fdc7a3ca77..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_service_monitors.input.yaml +++ /dev/null @@ -1,59 +0,0 @@ -prometheus-operator: - prometheus: - additionalServiceMonitors: - - name: collection-sumologic - additionalLabels: - app: collection-sumologic - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic - - name: my-service-monitor - additionalLabels: - app: some-app - endpoints: - - port: 8080 - namespaceSelector: - matchNames: - - test-app - selector: - matchLabels: - test: service-monitors - - name: collection-sumologic-events - additionalLabels: - app: collection-sumologic-events - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-events - - name: collection-fluent-bit - additionalLabels: - app: collection-fluent-bit - endpoints: - - port: metrics - path: /api/v1/metrics/prometheus - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: fluent-bit - - name: collection-sumologic-otelcol - additionalLabels: - app: collection-sumologic-otelcol - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-otelcol diff --git a/tests/helm/upgrade_script/static/prometheus_service_monitors.log b/tests/helm/upgrade_script/static/prometheus_service_monitors.log deleted file mode 100644 index 8acc57af32..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_service_monitors.log +++ /dev/null @@ -1,6 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[INFO] Patching prometheus-operator.prometheus.additionalServiceMonitors - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/prometheus_service_monitors.output.yaml b/tests/helm/upgrade_script/static/prometheus_service_monitors.output.yaml deleted file mode 100644 index 1eae716a52..0000000000 --- a/tests/helm/upgrade_script/static/prometheus_service_monitors.output.yaml +++ /dev/null @@ -1,70 +0,0 @@ -prometheus-operator: - prometheus: - additionalServiceMonitors: - - name: my-service-monitor - additionalLabels: - app: some-app - endpoints: - - port: 8080 - namespaceSelector: - matchNames: - - test-app - selector: - matchLabels: - test: service-monitors - - name: collection-fluent-bit - additionalLabels: - app: collection-fluent-bit - endpoints: - - port: metrics - path: /api/v1/metrics/prometheus - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: fluent-bit - - name: collection-sumologic-otelcol - additionalLabels: - app: collection-sumologic-otelcol - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-otelcol - - name: collection-sumologic-fluentd-logs - additionalLabels: - app: collection-sumologic-fluentd-logs - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-fluentd-logs - - name: collection-sumologic-fluentd-metrics - additionalLabels: - app: collection-sumologic-fluentd-metrics - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-fluentd-metrics - - name: collection-sumologic-fluentd-events - additionalLabels: - app: collection-sumologic-fluentd-events - endpoints: - - port: metrics - namespaceSelector: - matchNames: - - sumologic - selector: - matchLabels: - app: collection-sumologic-fluentd-events diff --git a/tests/helm/upgrade_script/static/version_invalid.input.yaml b/tests/helm/upgrade_script/static/version_invalid.input.yaml deleted file mode 100644 index 7877b27d44..0000000000 --- a/tests/helm/upgrade_script/static/version_invalid.input.yaml +++ /dev/null @@ -1,2 +0,0 @@ -image: - tag: 0.17.3-customer-build diff --git a/tests/helm/upgrade_script/static/version_invalid.log b/tests/helm/upgrade_script/static/version_invalid.log deleted file mode 100644 index 7090ebe35d..0000000000 --- a/tests/helm/upgrade_script/static/version_invalid.log +++ /dev/null @@ -1,7 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[WARNING] You are using unsupported version: 0.17.3-customer-build -[WARNING] Please upgrade to '0.17.x' or ensure that new_values.yaml is valid - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/version_invalid.output.yaml b/tests/helm/upgrade_script/static/version_invalid.output.yaml deleted file mode 100644 index 7877b27d44..0000000000 --- a/tests/helm/upgrade_script/static/version_invalid.output.yaml +++ /dev/null @@ -1,2 +0,0 @@ -image: - tag: 0.17.3-customer-build diff --git a/tests/helm/upgrade_script/static/version_valid.input.yaml b/tests/helm/upgrade_script/static/version_valid.input.yaml deleted file mode 100644 index 7e19da47d5..0000000000 --- a/tests/helm/upgrade_script/static/version_valid.input.yaml +++ /dev/null @@ -1,2 +0,0 @@ -image: - tag: 0.17.45 diff --git a/tests/helm/upgrade_script/static/version_valid.log b/tests/helm/upgrade_script/static/version_valid.log deleted file mode 100644 index f27804d2d3..0000000000 --- a/tests/helm/upgrade_script/static/version_valid.log +++ /dev/null @@ -1,6 +0,0 @@ - -[INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it -[INFO] Changing image.tag from '0.17.45' to '1.0.0' - -Thank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart. -A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/helm/upgrade_script/static/version_valid.output.yaml b/tests/helm/upgrade_script/static/version_valid.output.yaml deleted file mode 100644 index 48152b6294..0000000000 --- a/tests/helm/upgrade_script/static/version_valid.output.yaml +++ /dev/null @@ -1,2 +0,0 @@ -image: - tag: 1.0.0 From 784228fedd086421b017b4f0ea0edbc5ab21c83d Mon Sep 17 00:00:00 2001 From: Adam Boguszewski Date: Mon, 28 Nov 2022 12:21:22 +0100 Subject: [PATCH 2/2] feat: drop v2 upgrade script --- CHANGELOG.md | 1 + ci/tests.sh | 4 - deploy/helm/sumologic/upgrade-2.0.0.sh | 1141 ----------------- docs/v2-migration-doc.md | 444 ------- tests/helm/upgrade_v2_script/run.sh | 55 - .../static/fluent_bit.input.yaml | 160 --- .../upgrade_v2_script/static/fluent_bit.log | 14 - .../static/fluent_bit.output.yaml | 149 --- .../static/fluent_bit_image.input.yaml | 6 - .../static/fluent_bit_image.log | 11 - .../static/fluent_bit_image.output.yaml | 5 - .../static/fluent_bit_no_image.input.yaml | 39 - .../static/fluent_bit_no_image.log | 5 - .../static/fluent_bit_no_image.output.yaml | 40 - .../fluentd_persistence_false.input.yaml | 3 - .../static/fluentd_persistence_false.log | 10 - .../fluentd_persistence_false.output.yaml | 3 - .../fluentd_persistence_true.input.yaml | 3 - .../static/fluentd_persistence_true.log | 5 - .../fluentd_persistence_true.output.yaml | 3 - .../static/fluentd_whole_config.input.yaml | 530 -------- .../static/fluentd_whole_config.log | 8 - .../static/fluentd_whole_config.output.yaml | 239 ---- .../static/image_rename.input.yaml | 27 - .../upgrade_v2_script/static/image_rename.log | 18 - .../static/image_rename.output.yaml | 21 - .../static/metrics_server.input.yaml | 9 - .../static/metrics_server.log | 6 - .../static/metrics_server.output.yaml | 5 - .../static/otelcol_image_name.input.yaml | 21 - .../static/otelcol_image_name.log | 6 - .../static/otelcol_image_name.output.yaml | 21 - .../prometheus_operator_valid.input.yaml | 56 - .../static/prometheus_operator_valid.log | 11 - .../prometheus_operator_valid.output.yaml | 261 ---- ...prometheus_remote_write_regexes.input.yaml | 102 -- .../prometheus_remote_write_regexes.log | 8 - ...rometheus_remote_write_regexes.output.yaml | 146 --- ...te_write_regexes_changed_values.input.yaml | 14 - ...us_remote_write_regexes_changed_values.log | 14 - ...e_write_regexes_changed_values.output.yaml | 16 - ...prometheus_remote_write_timeout.input.yaml | 117 -- .../prometheus_remote_write_timeout.log | 11 - ...rometheus_remote_write_timeout.output.yaml | 155 --- ...emote_write_timeout_unspecified.input.yaml | 118 -- ...theus_remote_write_timeout_unspecified.log | 11 - ...mote_write_timeout_unspecified.output.yaml | 155 --- .../prometheus_service_monitors.input.yaml | 81 -- .../static/prometheus_service_monitors.log | 10 - .../prometheus_service_monitors.output.yaml | 292 ----- .../static/prometheus_whole_config.input.yaml | 703 ---------- .../static/prometheus_whole_config.log | 13 - .../prometheus_whole_config.output.yaml | 568 -------- .../sumologic_collector_fields.input.yaml | 36 - .../static/sumologic_collector_fields.log | 6 - .../sumologic_collector_fields.output.yaml | 27 - .../static/sumologic_sources.input.yaml | 35 - .../static/sumologic_sources.log | 7 - .../static/sumologic_sources.output.yaml | 36 - 59 files changed, 1 insertion(+), 6020 deletions(-) delete mode 100755 deploy/helm/sumologic/upgrade-2.0.0.sh delete mode 100644 docs/v2-migration-doc.md delete mode 100755 tests/helm/upgrade_v2_script/run.sh delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_image.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_image.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_image.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_no_image.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_no_image.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluent_bit_no_image.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_false.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_false.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_false.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_true.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_true.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_persistence_true.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_whole_config.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_whole_config.log delete mode 100644 tests/helm/upgrade_v2_script/static/fluentd_whole_config.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/image_rename.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/image_rename.log delete mode 100644 tests/helm/upgrade_v2_script/static/image_rename.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/metrics_server.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/metrics_server.log delete mode 100644 tests/helm/upgrade_v2_script/static/metrics_server.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/otelcol_image_name.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/otelcol_image_name.log delete mode 100644 tests/helm/upgrade_v2_script/static/otelcol_image_name.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_operator_valid.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_operator_valid.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_operator_valid.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes_changed_values.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes_changed_values.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_regexes_changed_values.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout_unspecified.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout_unspecified.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_remote_write_timeout_unspecified.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_service_monitors.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_service_monitors.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_service_monitors.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_whole_config.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_whole_config.log delete mode 100644 tests/helm/upgrade_v2_script/static/prometheus_whole_config.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_collector_fields.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_collector_fields.log delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_collector_fields.output.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_sources.input.yaml delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_sources.log delete mode 100644 tests/helm/upgrade_v2_script/static/sumologic_sources.output.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d7c095f62..e56d493d70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - docs: rename user-provided config from values.yaml to user-values.yaml [#2619] - feat: update opentelemetry-operator chart and fix progagators list in instrumentation resource [#2628] - feat: drop migration script for v1 [#2654] +- feat: drop migration script for v2 [#2654] ### Fixed diff --git a/ci/tests.sh b/ci/tests.sh index e326093a02..6720df94d7 100755 --- a/ci/tests.sh +++ b/ci/tests.sh @@ -14,10 +14,6 @@ pushd "${ROOT_DIR}" || exit 1 echo "Test helm templates generation" "${TESTS_DIR}/run.sh" || (echo "Failed testing templates" && exit 1) -# Test upgrade v2 script -echo "Test upgrade v2 script..." -"${TESTS_DIR}/upgrade_v2_script/run.sh" || (echo "Failed testing upgrade v2 script" && exit 1) - popd || exit 1 echo "DONE" diff --git a/deploy/helm/sumologic/upgrade-2.0.0.sh b/deploy/helm/sumologic/upgrade-2.0.0.sh deleted file mode 100755 index 5c4ba21698..0000000000 --- a/deploy/helm/sumologic/upgrade-2.0.0.sh +++ /dev/null @@ -1,1141 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -IFS=$'\n\t' - -readonly OLD_VALUES_YAML="${1:---help}" -readonly PREVIOUS_VERSION=1.3 - -readonly TEMP_FILE=upgrade-2.0.0-temp-file - -readonly MIN_BASH_VERSION=4.0 -readonly MIN_YQ_VERSION=3.4.0 -readonly MAX_YQ_VERSION=4.0.0 - -readonly KEY_MAPPINGS=" -prometheus-operator.prometheusOperator.tlsProxy.enabled:kube-prometheus-stack.prometheusOperator.tls.enabled -otelcol.deployment.image.name:otelcol.deployment.image.repository -fluent-bit.image.fluent_bit.repository:fluent-bit.image.repository -fluent-bit.image.fluent_bit.tag:fluent-bit.image.tag -" - -readonly KEY_VALUE_MAPPINGS=" -" - -readonly KEY_MAPPINGS_MULTIPLE=" -image.repository:fluentd.image.repository:sumologic.setup.job.image.repository -image.tag:fluentd.image.tag:sumologic.setup.job.image.tag -image.pullPolicy:fluentd.image.pullPolicy:sumologic.setup.job.image.pullPolicy -" - -readonly KEYS_TO_DELETE=" -prometheus-operator -fluent-bit.metrics -fluent-bit.trackOffsets -fluent-bit.service.flush -fluent-bit.backend -fluent-bit.input -fluent-bit.parsers -fluent-bit.rawConfig -" - -# https://slides.com/perk/how-to-train-your-bash#/41 -readonly LOG_FILE="/tmp/$(basename "$0").log" -info() { echo -e "[INFO] $*" | tee -a "${LOG_FILE}" >&2 ; } -warning() { echo -e "[WARNING] $*" | tee -a "${LOG_FILE}" >&2 ; } -error() { echo -e "[ERROR] $*" | tee -a "${LOG_FILE}" >&2 ; } -fatal() { echo -e "[FATAL] $*" | tee -a "${LOG_FILE}" >&2 ; exit 1 ; } - -function print_help_and_exit() { - local MAN - set +e - read -r -d '' MAN < x >= ${MIN_YQ_VERSION}) https://github.com/mikefarah/yq/releases/tag/${MIN_YQ_VERSION} - grep - sed - bash (>= ${MIN_BASH_VERSION}) - -Usage: - # for default helm release name 'collection' and namespace 'sumologic' - ./upgrade-2.0.0.sh /path/to/values.yaml - - # for non-default helm release name and k8s namespace - ./upgrade-2.0.0.sh /path/to/values.yaml helm_release_name k8s_namespace - -Returns: - new_values.yaml - -For more details, please refer to Migration steps and Changelog here: -https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v2.0/deploy/docs/v2_migration_doc.md -EOF - set -e - - echo "${MAN}" - exit 0 -} - -function check_if_print_help_and_exit() { - if [[ "$1" == "--help" ]]; then - print_help_and_exit - fi -} - -function check_required_command() { - local command_to_check="$1" - command -v "${command_to_check}" >/dev/null 2>&1 || { error "Required command is missing: ${command_to_check}"; fatal "Please consult --help and install missing commands before continue. Aborting."; } -} - -function compare_versions() { - local no_lower_than="${1}" - local app_version="${2}" - - if [[ "$(printf '%s\n' "${app_version}" "${no_lower_than}" | sort -V | head -n 1)" == "${no_lower_than}" ]]; then - echo "pass" - else - echo "fail" - fi -} - -function check_app_version() { - local app_name="${1}" - local no_lower_than="${2}" - local app_version="${3}" - - if [[ -z ${app_version} ]] || [[ $(compare_versions "${no_lower_than}" "${app_version}") == "fail" ]]; then - error "${app_name} version: '${app_version}' is invalid - it should be no lower than ${no_lower_than}" - fatal "Please update your ${app_name} and retry." - fi -} - -function check_app_version_with_max() { - local app_name="${1}" - local no_lower_than="${2}" - local lower_than="${3}" - local app_version="${4}" - - if [[ -z ${app_version} ]] || [[ $(compare_versions "${no_lower_than}" "${app_version}") == "fail" ]]; then - error "${app_name} version: '${app_version}' is invalid - it should be no lower than ${no_lower_than}" - fatal "Please update your ${app_name} and retry." - fi - - if [[ "${app_version}" == "${lower_than}" ]] || [[ $(compare_versions "${app_version}" "${lower_than}") == "fail" ]]; then - error "${app_name} version: '${app_version}' is invalid - it should be lower than ${lower_than}" - fatal "Please downgrade '${app_name}' and retry." - fi -} - -function check_yq_version() { - local yq_version - readonly yq_version=$(yq --version | grep -oE '[^[:space:]]+$') - - check_app_version_with_max "yq" "${MIN_YQ_VERSION}" "${MAX_YQ_VERSION}" "${yq_version}" -} - -function check_bash_version() { - check_app_version "bash" "${MIN_BASH_VERSION}" "${BASH_VERSION}" -} - -function create_temp_file() { - echo -n > "${TEMP_FILE}" -} - -function migrate_prometheus_operator_to_kube_prometheus_stack() { - # Nothing to migrate, return - if [[ -z $(yq r "${TEMP_FILE}" prometheus-operator) ]] ; then - return - fi - - if [[ -n "$(yq r "${TEMP_FILE}" 'prometheus-operator.prometheus.prometheusSpec.containers.(name==prometheus-config-reloader)')" ]]; then - info "Migrating prometheus-config-reloader container to config-reloader in prometheusSpec" - yq m -i --arrays append \ - "${TEMP_FILE}" \ - <( - yq p <( - yq w <( - yq r "${TEMP_FILE}" -- 'prometheus-operator.prometheus.prometheusSpec.containers.(name==prometheus-config-reloader)' \ - ) name config-reloader \ - ) 'prometheus-operator.prometheus.prometheusSpec.containers[+]' - ) - yq d -i "${TEMP_FILE}" "prometheus-operator.prometheus.prometheusSpec.containers.(name==prometheus-config-reloader)" - fi - - info "Migrating from prometheus-operator to kube-prometheus-stack" - yq m -i \ - "${TEMP_FILE}" \ - <( - yq p \ - <(yq r "${TEMP_FILE}" "prometheus-operator") \ - "kube-prometheus-stack" \ - ) - yq d -i "${TEMP_FILE}" "prometheus-operator" -} - -function migrate_prometheus_retention_period() { - if [[ -z "$(yq r "${TEMP_FILE}" -- 'prometheus-operator')" ]]; then - return - fi - local RETENTION - readonly RETENTION="$(yq r "${TEMP_FILE}" -- 'prometheus-operator.prometheus.prometheusSpec.retention')" - - if [[ -z "${RETENTION}" ]]; then - return - fi - - if [[ "${RETENTION}" == "7d" ]]; then - info "Changing prometheus retention period from 7d to 1d" - yq w -i "${TEMP_FILE}" 'prometheus-operator.prometheus.prometheusSpec.retention' 1d - elif [[ "${RETENTION}" != "1d" ]]; then - warning "Prometheus retention set to ${RETENTION} (different than 7d - default for 1.3). Bailing migration to 1d (default for 2.0)" - fi -} - -function migrate_prometheus_recording_rules() { - if [[ -z "$(yq r "${TEMP_FILE}" -- 'prometheus-operator')" ]]; then - return - fi - - local RECORDING_RULES_OVERRIDE - readonly RECORDING_RULES_OVERRIDE=$(yq r "${TEMP_FILE}" -- 'prometheus-operator.kubeTargetVersionOverride') - - if [[ "${RECORDING_RULES_OVERRIDE}" == "1.13.0-0" ]]; then - add_prometheus_pre_1_14_recording_rules "${TEMP_FILE}" - info "Removing prometheus kubeTargetVersionOverride='1.13.0-0'" - yq d -i "${TEMP_FILE}" "prometheus-operator.kubeTargetVersionOverride" - elif [[ -n "${RECORDING_RULES_OVERRIDE}" ]]; then - warning "prometheus-operator.kubeTargetVersionOverride should be unset or set to '1.13.0-0' but it's set to: ${RECORDING_RULES_OVERRIDE}" - warning "Please unset it or set it to '1.13.0-0' and rerun this script" - fi -} - -function kube_prometheus_stack_update_remote_write_regexes() { - local URL_METRICS_OPERATOR_RULE - # shellcheck disable=SC2016 - readonly URL_METRICS_OPERATOR_RULE='http://$(FLUENTD_METRICS_SVC).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule' - - local PROMETHEUS_METRICS_OPERATOR_RULE_REGEX - readonly PROMETHEUS_METRICS_OPERATOR_RULE_REGEX="cluster_quantile:apiserver_request_latencies:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile|cluster_quantile:scheduler_binding_latency:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:" - - local TEMP_REWRITE_PROMETHEUS_METRICS_OPERATOR_RULE - readonly TEMP_REWRITE_PROMETHEUS_METRICS_OPERATOR_RULE="$( - yq r "${TEMP_FILE}" \ - "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.\"url==${URL_METRICS_OPERATOR_RULE}\"" - )" - - local CURRENT_METRICS_OPERATOR_RULE_REGEX - readonly CURRENT_METRICS_OPERATOR_RULE_REGEX="$( - yq r "${TEMP_FILE}" \ - "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.\"url==${URL_METRICS_OPERATOR_RULE}\".writeRelabelConfigs[0].regex" - )" - if [[ -n "${CURRENT_METRICS_OPERATOR_RULE_REGEX}" ]]; then - if [[ -n $(diff <(echo "${PROMETHEUS_METRICS_OPERATOR_RULE_REGEX}") <(echo "${CURRENT_METRICS_OPERATOR_RULE_REGEX}")) ]] ; then - info "Updating prometheus regex in rewrite rule for url: ${URL_METRICS_OPERATOR_RULE} but it has a different value than expected" - info "Actual: '${CURRENT_METRICS_OPERATOR_RULE_REGEX}'" - info "Expected: '${PROMETHEUS_METRICS_OPERATOR_RULE_REGEX}'" - fi - fi - - if [[ -n "${TEMP_REWRITE_PROMETHEUS_METRICS_OPERATOR_RULE}" ]]; then - info "Updating prometheus regex in rewrite rule for url: ${URL_METRICS_OPERATOR_RULE}..." - # shellcheck disable=SC2016 - yq delete -i "${TEMP_FILE}" 'kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite."url==http://$(FLUENTD_METRICS_SVC).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule"' - - local SCRIPT - SCRIPT="$(cat <<- EOF - - command: update - path: 'kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.[+]' - value: - url: ${URL_METRICS_OPERATOR_RULE} - remoteTimeout: 5s - writeRelabelConfigs: - - action: keep - regex: 'cluster_quantile:apiserver_request_duration_seconds:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile|cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' - sourceLabels: [__name__] - EOF -)" - - yq w -i "${TEMP_FILE}" --script <(echo "${SCRIPT}") - fi - - ############################################################################## - - local URL_METRICS_CONTROL_PLANE_COREDNS -# shellcheck disable=SC2016 - readonly URL_METRICS_CONTROL_PLANE_COREDNS='http://$(FLUENTD_METRICS_SVC).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.control-plane.coredns' - - local PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS_REGEX - readonly PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS_REGEX="coredns;(?:coredns_cache_(size|(hits|misses)_total)|coredns_dns_request_duration_seconds_(count|sum)|coredns_(dns_request|dns_response_rcode|forward_request)_count_total|process_(cpu_seconds_total|open_fds|resident_memory_bytes))" - - local TEMP_REWRITE_PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS - readonly TEMP_REWRITE_PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS="$( - yq r "${TEMP_FILE}" \ - "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.\"url==${URL_METRICS_CONTROL_PLANE_COREDNS}\"" - )" - - local CURRENT_METRICS_CONTROL_PLANE_COREDNS_REGEX - readonly CURRENT_METRICS_CONTROL_PLANE_COREDNS_REGEX="$( - yq r "${TEMP_FILE}" \ - "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.\"url==${URL_METRICS_CONTROL_PLANE_COREDNS}\".writeRelabelConfigs[0].regex" - )" - if [[ -n "${CURRENT_METRICS_CONTROL_PLANE_COREDNS_REGEX}" ]] ; then - if [[ -n $(diff <(echo "${PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS_REGEX}") <(echo "${CURRENT_METRICS_CONTROL_PLANE_COREDNS_REGEX}")) ]] ; then - info "Updating prometheus regex in rewrite rule for url: ${URL_METRICS_CONTROL_PLANE_COREDNS} but it has a different value than expected" - info "Actual: '${CURRENT_METRICS_CONTROL_PLANE_COREDNS_REGEX}'" - info "Expected: '${PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS_REGEX}'" - fi - fi - - if [[ -n "${TEMP_REWRITE_PROMETHEUS_METRICS_CONTROL_PLANE_COREDNS}" ]]; then - info "Updating prometheus regex in rewrite rule for url: ${URL_METRICS_CONTROL_PLANE_COREDNS}..." - yq delete -i "${TEMP_FILE}" "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.\"url==${URL_METRICS_CONTROL_PLANE_COREDNS}\"" - - local SCRIPT - SCRIPT="$(cat <<- EOF - - command: update - path: 'kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.[+]' - value: - url: ${URL_METRICS_CONTROL_PLANE_COREDNS} - remoteTimeout: 5s - writeRelabelConfigs: - - action: keep - regex: 'coredns;(?:coredns_cache_(size|entries|(hits|misses)_total)|coredns_dns_request_duration_seconds_(count|sum)|coredns_(dns_request|dns_response_rcode|forward_request)_count_total|coredns_(forward_requests|dns_requests|dns_responses)_total|process_(cpu_seconds_total|open_fds|resident_memory_bytes))' - sourceLabels: [job, __name__] - EOF -)" - - yq w -i "${TEMP_FILE}" --script <(echo "${SCRIPT}") - fi -} - -function kube_prometheus_stack_migrate_remote_write_urls() { - info "Migrating prometheus remote write urls" - - # shellcheck disable=SC2016 - sed -i'.bak' \ - 's#http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus#http://$(FLUENTD_METRICS_SVC).$(NAMESPACE).svc.cluster.local:9888/prometheus#g' \ - "${TEMP_FILE}" && \ - rm "${TEMP_FILE}".bak -} - -function kube_prometheus_stack_migrate_chart_env_variable() { - if [[ -z "$(yq r "${TEMP_FILE}" -- 'kube-prometheus-stack.prometheus.prometheusSpec.containers.(name==config-reloader).env.(name==CHART)')" ]]; then - return - fi - - yq w -i "${TEMP_FILE}" \ - 'kube-prometheus-stack.prometheus.prometheusSpec.containers.(name==config-reloader).env.(name==CHART).name' \ - FLUENTD_METRICS_SVC -} - -function add_prometheus_pre_1_14_recording_rules() { - local temp_file="${1}" - local PROMETHEUS_RULES - # Using tags below for heredoc - PROMETHEUS_RULES=$(cat <<- "EOF" - groups: - - name: node-pre-1.14.rules - rules: - - expr: 1 - avg(rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m])) - record: :node_cpu_utilisation:avg1m - - expr: |- - 1 - avg by (node) ( - rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info:) - record: node:node_cpu_utilisation:avg1m - - expr: |- - 1 - - sum( - node_memory_MemFree_bytes{job="node-exporter"} + - node_memory_Cached_bytes{job="node-exporter"} + - node_memory_Buffers_bytes{job="node-exporter"} - ) - / - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - record: ':node_memory_utilisation:' - - expr: |- - sum by (node) ( - ( - node_memory_MemFree_bytes{job="node-exporter"} + - node_memory_Cached_bytes{job="node-exporter"} + - node_memory_Buffers_bytes{job="node-exporter"} - ) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_available:sum - - expr: |- - (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) - / - node:node_memory_bytes_total:sum - record: node:node_memory_utilisation:ratio - - expr: |- - 1 - - sum by (node) ( - ( - node_memory_MemFree_bytes{job="node-exporter"} + - node_memory_Cached_bytes{job="node-exporter"} + - node_memory_Buffers_bytes{job="node-exporter"} - ) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: 'node:node_memory_utilisation:' - - expr: 1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum) - record: 'node:node_memory_utilisation_2:' - - expr: |- - max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_usage:' - - expr: |- - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_total:sum - - expr: |- - sum(irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) + - sum(irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) - record: :node_net_utilisation:sum_irate - - expr: |- - sum by (node) ( - (irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m]) + - irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_utilisation:sum_irate - - expr: |- - sum(irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m])) + - sum(irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) - record: :node_net_saturation:sum_irate - - expr: |- - sum by (node) ( - (irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m]) + - irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_saturation:sum_irate - - expr: |- - max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_usage:' - - expr: |- - sum(node_load1{job="node-exporter"}) - / - sum(node:node_num_cpu:sum) - record: ':node_cpu_saturation_load1:' - - expr: |- - sum by (node) ( - node_load1{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - node:node_num_cpu:sum - record: 'node:node_cpu_saturation_load1:' - - expr: avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) - record: :node_disk_saturation:avg_irate - - expr: |- - avg by (node) ( - irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_saturation:avg_irate - - expr: avg(irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) - record: :node_disk_utilisation:avg_irate - - expr: |- - avg by (node) ( - irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_utilisation:avg_irate - - expr: |- - 1e3 * sum( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - ) - record: :node_memory_swap_io_bytes:sum_rate - - expr: |- - 1e3 * sum by (node) ( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_swap_io_bytes:sum_rate - - expr: |- - node:node_cpu_utilisation:avg1m - * - node:node_num_cpu:sum - / - scalar(sum(node:node_num_cpu:sum)) - record: node:cluster_cpu_utilisation:ratio - - expr: |- - (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) - / - scalar(sum(node:node_memory_bytes_total:sum)) - record: node:cluster_memory_utilisation:ratio - - expr: |- - sum by (node) ( - node_load1{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - node:node_num_cpu:sum - record: 'node:node_cpu_saturation_load1:' - - expr: |- - max by (instance, namespace, pod, device) ( - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - / - node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - ) - record: 'node:node_filesystem_avail:' - - expr: |- - max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_usage:' - - expr: |- - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - ( - max(node_filesystem_files{job="node-exporter", mountpoint="/"}) - by (instance) - ), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_total:' - - expr: |- - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - ( - max(node_filesystem_files_free{job="node-exporter", mountpoint="/"}) - by (instance) - ), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_free:' - EOF -) - - info "Adding 'additionalPrometheusRulesMap.pre-1.14-node-rules' to prometheus configuration" - yq w -i "${temp_file}" 'prometheus-operator.additionalPrometheusRulesMap."pre-1.14-node-rules"' \ - --from <(echo "${PROMETHEUS_RULES}") -} - -function add_new_scrape_labels_to_prometheus_service_monitors(){ - if [[ -z "$(yq r "${TEMP_FILE}" -- 'prometheus-operator.prometheus.additionalServiceMonitors')" ]]; then - return - fi - - info "Adding 'sumologic.com/scrape: \"true\"' scrape labels to prometheus service monitors" - yq w --style double -i "${TEMP_FILE}" \ - 'prometheus-operator.prometheus.additionalServiceMonitors.[*].selector.matchLabels."sumologic.com/scrape"' true -} - -function kube_prometheus_stack_set_remote_write_timeout_to_5s() { - local prometheus_spec - readonly prometheus_spec="$(yq r "${TEMP_FILE}" -- "kube-prometheus-stack.prometheus.prometheusSpec")" - - if [[ -z "$(yq r - 'remoteWrite' <<< "${prometheus_spec}")" ]]; then - # No remoteWrite to migrate - return - fi - - if [[ -n "$(yq r - 'remoteWrite.[*].remoteTimeout' <<< "${prometheus_spec}")" ]]; then - echo - info "kube-prometheus-stack.prometheus.prometheusSpec.remoteWrite.[*].remoteTimeout is set" - info "Please note that we've set it by default to 5s in 2.0.0" - fi - - local new_remote_write - new_remote_write="" - - local remote_timeout - local remote_write - - local len - readonly len="$(yq r --length - "remoteWrite" <<< "${prometheus_spec}")" - for (( i=0; i - ``` - - to - - ```yaml - spec: - selector: - matchLabels: - app.kubernetes.io/name: fluent-bit - app.kubernetes.io/instance: - ``` - -- Persistence for Fluentd is enabled by default. - -- Gzip compression is enabled by default. - If you already had Fluentd persistence enabled, but gzip compression was disabled, - after the upgrade Fluentd will not be able to read the non-compressed chunks written before the upgrade. - To fix this, see [Troubleshooting - Gzip compression errors](#gzip-compression-errors). - -### How to upgrade - -#### Requirements - -- `helm3` -- `yq` in version: `3.4.0` <= `x` < `4.0.0` -- `bash` 4.0 or higher - -**Note: The below steps are using Helm 3. Helm 2 is not supported.** - -#### 1. Upgrade to helm chart version `v1.3.5` - -If you're running a newer version than `v1.3.5`, instructions from this document -will also work for you. - -##### Ensure you have sumologic helm repo added - -Before running commands shown below please make sure that you have -sumologic helm repo configured. -One can check that using: - -```bash -helm repo list -NAME URL -... -sumologic https://sumologic.github.io/sumologic-kubernetes-collection -... -``` - -If sumologic helm repo is not configured use the following command to add it: - -```bash -helm repo add sumologic https://sumologic.github.io/sumologic-kubernetes-collection -``` - -##### Update - -Run the command shown below to fetch the latest helm chart: - -```bash -helm repo update -``` - -For users who are not already on `v1.3.5` of the helm chart, please upgrade -to that version first by running the below command: - -```bash -helm upgrade sumologic/sumologic --reuse-values --version=1.3.5 -``` - -#### 2. Upgrade Prometheus CRDs - -Due to changes in `kube-prometheus-stack` which this chart depends on, one will -need to run the following commands in order to update Prometheus related CRDs: - -```bash -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml -kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.43.2/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml -``` - -If you have a separate Prometheus operator installation, you need to make sure its version -is [v0.43.2](https://github.com/prometheus-operator/prometheus-operator/releases/tag/v0.43.2) -or higher but compatible before proceeding with the next steps of the collection upgrade. - -#### 3. Prepare Fluent Bit instance - -As `spec.selector` in Fluent Bit Helm chart was modified, it is required to manually recreate -or delete existing DaemonSet with old version of `spec.selector` before upgrade. - -One of the following two strategies can be used: - -- ##### Recreating Fluent Bit DaemonSet - - Recreating Fluent Bit DaemonSet with new `spec.selector` may cause that - applications' logs and Fluent Bit metrics will not be available in the time of recreation. - It usually shouldn't take more than several seconds. - - To recreate the Fluent Bit DaemonSet with new `spec.selector` one can run the following command: - - ```bash - kubectl get daemonset --namespace --selector "app=fluent-bit,release=" --output yaml | \ - yq w - "items[*].spec.selector.matchLabels[app.kubernetes.io/name]" "fluent-bit" | \ - yq w - "items[*].spec.selector.matchLabels[app.kubernetes.io/instance]" "" | \ - yq w - "items[*].spec.template.metadata.labels[app.kubernetes.io/name]" "fluent-bit" | \ - yq w - "items[*].spec.template.metadata.labels[app.kubernetes.io/instance]" "" | \ - yq d - "items[*].spec.selector.matchLabels[app]" | \ - yq d - "items[*].spec.selector.matchLabels[release]" | \ - kubectl apply --namespace --force --filename - - ``` - - **Notice** When DaemonSet managed by helm is modified by the command specified above, - one might expect a warning similar to the one below: - `Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply` - -- ##### Preparing temporary instance of Fluent Bit - - Create temporary instance of Fluent Bit and delete DaemonSet with old version of `spec.selector`. - This will cause application' logs to be duplicated until temporary instance of Fluent Bit is deleted - after the upgrade is complete. As temporary instance of Fluent Bit creates additional Pods - which are selected by the same Fluent Bit Service you may observe changes in Fluent Bit metrics. - - Copy of database, in which Fluent Bit keeps track of monitored files and offsets, - is used by temporary instance of Fluent Bit (Fluent Bit database is copied in initContainer). - Temporary instance of Fluent Bit will start reading logs with offsets saved in database. - - To create a temporary copy of Fluent Bit DaemonSet: - - ```bash - INIT_CONTAINER=$(cat <<-"EOF" - name: init-tmp-fluent-bit - image: busybox:latest - command: ['sh', '-c', 'mkdir -p /tail-db/tmp; cp /tail-db/*.db /tail-db/tmp'] - volumeMounts: - - mountPath: /tail-db - name: tail-db - EOF - ) && \ - TMP_VOLUME=$(cat <<-"EOF" - hostPath: - path: /var/lib/fluent-bit/tmp - type: DirectoryOrCreate - name: tmp-tail-db - EOF - ) && \ - kubectl get daemonset --namespace --selector "app=fluent-bit,release=" --output yaml | \ - yq w - "items[*].metadata.name" "tmp-fluent-bit" | \ - yq w - "items[*].metadata.labels[heritage]" "tmp" | \ - yq w - "items[*].spec.template.metadata.labels[app.kubernetes.io/name]" "fluent-bit" | \ - yq w - "items[*].spec.template.metadata.labels[app.kubernetes.io/instance]" "" | \ - yq w - "items[*].spec.template.spec.initContainers[+]" --from <(echo "${INIT_CONTAINER}") | \ - yq w - "items[*].spec.template.spec.volumes[+]" --from <(echo "${TMP_VOLUME}") | \ - yq w - "items[*].spec.template.spec.containers[*].volumeMounts[*].(.==tail-db)" "tmp-tail-db" | \ - kubectl create --filename - - ``` - - Please make sure that Pods related to new DaemonSet are running: - - ```bash - kubectl get pod \ - --namespace \ - --selector "app=fluent-bit,release=,app.kubernetes.io/name=fluent-bit,app.kubernetes.io/instance=" - ``` - - Please check that the latest logs are duplicated in Sumo. - - To delete Fluent Bit DaemonSet with old version of `spec.selector`: - - ```bash - kubectl delete daemonset \ - --namespace \ - --selector "app=fluent-bit,heritage=Helm,release=" - ``` - - **Notice:** When collection upgrade creates new DaemonSet for Fluent Bit, - logs will be duplicated. - In order to stop data duplication it is required to remove the temporary copy - of Fluent Bit DaemonSet after the upgrade has finished. - - After collection upgrade is done, in order to remove the temporary Fluent Bit - DaemonSet run the following commands: - - ```bash - kubectl wait --for=condition=ready pod \ - --namespace \ - --selector "app.kubernetes.io/name=fluent-bit,app.kubernetes.io/instance=,app!=fluent-bit,release!=" && \ - kubectl delete daemonset \ - --namespace \ - --selector "app=fluent-bit,release=,heritage=tmp" - ``` - -#### 4. Configure Fluentd persistence - -Starting with `v2.0.0` we're using file-based buffer for Fluentd instead of less -reliable in-memory buffer (`fluentd.persistence.enabled=true`) by default. - -When Fluentd persistence is enabled then no action is required in order to upgrade. - -When Fluentd persistence is disabled (default setting in `1.3.5` release) -it is required to either go through persistence enabling procedure before upgrade (recommended) -or preserve existing setting and modify default setting for Fluentd persistence in `2.0.0` release. - -**In order to enable persistence in existing collection** please follow one -of persistence enabling procedures described in -[Enabling Fluentd Persistence guide](fluentd-persistence.md#enabling-fluentd-persistence) -before upgrade. - -If Fluentd persistence is disabled and it is desired to preserve this setting, -modify defaults and disable persistence either by adding `--set fluentd.persistence.enabled=false` -to `helm upgrade` command or in the `user-values.yaml` file under the `fluentd` key as follows: - -```yaml -fluentd: - persistence: - enabled: false -``` - -#### 5. Run upgrade script - -For Helm users, the only breaking changes are the renamed config parameters. -For users who use a `user-values.yaml` file, we provide a script that users can run -to convert their existing `user-values.yaml` file into one that is compatible with the major release. - -- Get the existing values for the helm chart and store it as `current_user-values.yaml` - with the below command: - - ```bash - helm get values --output yaml > current_user-values.yaml - ``` - -- Run the upgrade script. You can run it: - - - On your the host. Please refer to the [requirements](#requirements) so that you have - all the required software packages installed. - - ```bash - curl -LJO https://raw.githubusercontent.com/SumoLogic/sumologic-kubernetes-collection/release-v2.0/deploy/helm/sumologic/upgrade-2.0.0.sh \ - && chmod +x upgrade-2.0.0.sh \ - && ./upgrade-2.0.0.sh current_user-values.yaml - ``` - - - In a docker container: - - ```bash - cat current_user-values.yaml | \ - docker run \ - --rm \ - -i sumologic/kubernetes-tools:2.13.0 upgrade-2.0 | \ - tee new_user-values.yaml - ``` - - Note that this will output both migration script logs and new values file but - only the values file contents will be put into `new_user-values.yaml` due to `tee`. - - - In a container on your cluster: - - ```bash - cat current_user-values.yaml | \ - kubectl run kubernetes-tools -i \ - --quiet \ - --rm \ - --restart=Never \ - --image sumologic/kubernetes-tools:2.13.0 -- upgrade-2.0 | \ - tee new_user-values.yaml - ``` - - Note that this will output both migration script logs and new values file but - only the values file contents will be put into `new_user-values.yaml` due to `tee`. - -- At this point you should have `new_user-values.yaml` in your working directory which - can be used for the upgrade: - - ```bash - helm upgrade sumologic/sumologic --version=2.0.0 -f new_user-values.yaml - ``` - -#### 6. Troubleshooting - -##### Gzip compression errors - -If after the upgrade you can see the following errors from Fluentd logs or metrics pods: - -```console -2021-01-18 15:47:23 +0000 [warn]: #0 [sumologic.endpoint.logs.gc] failed to flush the buffer. retry_time=3 next_retry_seconds=2021-01-18 15:47:27 +0000 chunk="5b92e97a5ee3cbd7e59859644d9686e3" error_class=Zlib::GzipFile::Error error="not in gzip format" -``` - -This means Fluentd persistence has already been enabled before migration, but gzip compression was not. -Fluentd is unable to read the non-compressed chunks persisted before the upgrade. - -To fix this, delete the Fluentd pods that emit this error, -deleting their PVC-s at the same time to drop the non-compressed chunks. - -For example, if the namespace where the collection is installed is `collection` -and the pod that emits the error is named `sumologic-fluentd-logs-1`, -run the following set of commands: - -```bash -NAMESPACE_NAME=collection -POD_NAME=sumologic-fluentd-logs-1 -kubectl -n ${NAMESPACE_NAME} delete pvc "buffer-${POD_NAME}" & -kubectl -n ${NAMESPACE_NAME} delete pod ${POD_NAME} -kubectl -n ${NAMESPACE_NAME} delete pod ${POD_NAME} -``` - -The duplicated pod deletion command is there to make sure the pod is not stuck in `Pending` state -with event `persistentvolumeclaim "buffer-sumologic-fluentd-logs-1" not found`. - -## Non-Helm Users - -### Breaking Changes - -- From `v2.0.0` we recommend to use helm3 template as replacement for pre-generated - kubernetes templates. - Because of that, all custom changes made to the templates should be moved to `user-values.yaml`. - This will simplify and improve experience for non-helm installation. - -### How to upgrade for Non-helm Users - -#### 1. Tear down existing Fluentd, Prometheus, Fluent Bit and Falco resources - -You will need the YAML files you created when you first installed collection. -Run the following commands to remove Falco, Fluent-bit, Prometheus Operator and FluentD. -You do not need to delete the Namespace and Secret you originally created as they will still be used. - -```sh -kubectl delete -f falco.yaml -kubectl delete -f fluent-bit.yaml -kubectl delete -f prometheus.yaml -kubectl delete -f fluentd-sumologic.yaml -``` - -#### 2. Deploy collection with new approach - -- Follow steps mentioned [here][non_helm_installation_customizing_installation] - to deploy new collection. - -[non_helm_installation_customizing_installation]: https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/release-v2.0/deploy/docs/Non_Helm_Installation.md#customizing-installation diff --git a/tests/helm/upgrade_v2_script/run.sh b/tests/helm/upgrade_v2_script/run.sh deleted file mode 100755 index 0afb92780d..0000000000 --- a/tests/helm/upgrade_v2_script/run.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Test generation: -# export test_name=example_test; \ -# bash deploy/helm/sumologic/upgrade-2.0.0.sh \ -# tests/upgrade_v2_script/static/${test_name}.input.yaml \ -# 1> tests/upgrade_v2_script/static/${test_name}.log 2>&1 \ -# && cp new_values.yaml tests/upgrade_v2_script/static/${test_name}.output.yaml - -SCRIPT_PATH="$( dirname "$(realpath "${0}")" )" - -# shellcheck disable=SC1090 -# shellcheck source=tests/helm/functions.sh -source "${SCRIPT_PATH}/../functions.sh" -readonly TEST_TMP_OUT="${SCRIPT_PATH}/tmp/out.log" - -set_variables "${SCRIPT_PATH}" - -prepare_tests - -TEST_SUCCESS=true -for input_file in ${TEST_INPUT_FILES}; do - test_name="${input_file//.input.yaml/}" - output_file="${test_name}.output.yaml" - log_file="${test_name}.log" - - test_start "${test_name}" - bash "${TEST_SCRIPT_PATH}/../../../deploy/helm/sumologic/upgrade-2.0.0.sh" "${TEST_STATICS_PATH}/${input_file}" 1>"${TEST_TMP_OUT}" 2>&1 - mv new_values.yaml "${TEST_OUT}" - - test_output=$(diff "${TEST_STATICS_PATH}/${output_file}" "${TEST_OUT}") - test_log=$(diff "${TEST_STATICS_PATH}/${log_file}" "${TEST_TMP_OUT}") - rm "${TEST_TMP_OUT}" "${TEST_OUT}" - - if [[ -n "${test_output}" || -n "${test_log}" ]]; then - if [[ -n "${test_output}" ]]; then - echo -e "\tOutput diff (${TEST_STATICS_PATH}/${output_file}):\n${test_output}" - fi - if [[ -n "${test_log}" ]]; then - echo -e "\tLog diff (${TEST_STATICS_PATH}/${log_file}):\n${test_log}" - fi - test_failed "${test_name}" - TEST_SUCCESS=false - else - test_passed "${test_name}" - fi -done - -cleanup_tests - -if [[ "${TEST_SUCCESS}" = "true" ]]; then - exit 0 -else - exit 1 -fi diff --git a/tests/helm/upgrade_v2_script/static/fluent_bit.input.yaml b/tests/helm/upgrade_v2_script/static/fluent_bit.input.yaml deleted file mode 100644 index 7aea75f186..0000000000 --- a/tests/helm/upgrade_v2_script/static/fluent_bit.input.yaml +++ /dev/null @@ -1,160 +0,0 @@ -fluent-bit: - podLabels: {} - podAnnotations: {} - env: - - name: CHART - valueFrom: - configMapKeyRef: - name: sumologic-configmap - key: fluentdLogs - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - extraVolumes: - - name: project2-db - emptyDir: {} - - name: project-db - emptyDir: {} - - name: project2-alerts - hostPath: - path: /var/project2/logs/alerts/alerts.json - type: File - - name: company-db - emptyDir: {} - extraVolumeMounts: - - name: project2-db - mountPath: /fluent-bit/project2 - - name: project-db - mountPath: /fluent-bit/project - - name: project2-alerts - mountPath: /var/project2/logs/alerts/alerts.json - readOnly: true - - name: company-db - mountPath: /fluent-bit/company - tolerations: - - effect: NoSchedule - operator: Exists - image: - fluent_bit: - repository: public.ecr.aws/sumologic/fluent-bit - tag: 1.6.10 - pullPolicy: Always - service: - flush: 5 - metrics: - enabled: true - trackOffsets: true - - backend: - type: forward - forward: - ## NOTE: Requires trailing "." for fully-qualified name resolution - host: ${CHART}.${NAMESPACE}.svc.cluster.local. - port: 24321 - tls: "off" - tls_verify: "on" - tls_debug: 1 - shared_key: - input: - systemd: - enabled: true - parsers: - enabled: true - ## This regex matches the first line of a multiline log starting with a date of the format : "2019-11-17 07:14:12" or "2019-11-17T07:14:12" - regex: - - name: multi_line - regex: (?^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*) - - name: crio - regex: ^(?