diff --git a/.circleci/build_and_push.sh b/.circleci/build_and_push.sh index 1dbedda..de4006f 100755 --- a/.circleci/build_and_push.sh +++ b/.circleci/build_and_push.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash set -e +FLUENTD_HEC_GEM_VERSION=`cat docker/FLUENTD_HEC_GEM_VERSION` aws ecr get-login --region $AWS_REGION --no-include-email | bash echo "Building docker image..." cp /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem docker -docker build --build-arg VERSION=$FLUENT_SPLUNK_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:metrics ./docker +docker build --build-arg VERSION=$FLUENTD_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:metrics ./docker docker tag splunk/fluent-plugin-kubernetes-metrics:metrics $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest echo "Push docker image to ecr..." docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' diff --git a/.circleci/build_and_push_to_dockerhub.sh b/.circleci/build_and_push_to_dockerhub.sh index b6b065b..72c3e52 100755 --- a/.circleci/build_and_push_to_dockerhub.sh +++ b/.circleci/build_and_push_to_dockerhub.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash set -e +FLUENTD_HEC_GEM_VERSION=`cat docker/FLUENTD_HEC_GEM_VERSION` echo "Building docker image..." cp /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem docker VERSION=`cat VERSION` -docker build --build-arg VERSION=$FLUENT_SPLUNK_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker +docker build --build-arg VERSION=$FLUENTD_HEC_GEM_VERSION --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker docker tag splunk/fluent-plugin-kubernetes-metrics:ci splunk/${DOCKERHUB_REPO_NAME}:${VERSION} echo "Push docker image to splunk dockerhub..." docker login --username=$DOCKERHUB_ACCOUNT_ID --password=$DOCKERHUB_ACCOUNT_PASS diff --git a/.circleci/check_version_trigger_release.sh b/.circleci/check_version_trigger_release.sh new file mode 100755 index 0000000..bc08e6a --- /dev/null +++ b/.circleci/check_version_trigger_release.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e + +#!/usr/bin/env bash +LATEST_COMMIT=$(git rev-parse HEAD) +VERSION_COMMIT=$(git log -1 --format=format:%H VERSION) +if [ $VERSION_COMMIT = $LATEST_COMMIT ]; + then + if [ -s VERSION ] # Check if content is empty + then + VERSION=`cat VERSION` + echo "VERSION is changed to $VERSION" + else + echo "[ERROR] VERSION file is empty." + exit 1 + fi + git config user.email "splunk-oss-admin@splunk.com" + git config user.name "splunk-oss-admin" + git checkout develop + git pull origin develop + git checkout -b release/$VERSION origin/develop + git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git release/$VERSION + git checkout master + git merge --no-edit release/$VERSION + git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/fluent-plugin-kubernetes-metrics.git master +fi diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c97704..58405b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,12 +82,34 @@ jobs: name: Upload gem to Github command: | .circleci/build_and_push_to_github_release.sh + - run: + name: Upload gem to Ruby Gem + command: | + .circleci/push_gems_to_rubygems.sh + + check_version: + docker: + - image: circleci/ruby:2.6.1-node-browsers + working_directory: ~/repo + steps: + - attach_workspace: + at: /tmp + - checkout + - setup_remote_docker: + reusable: true + - run: + name: Check VERSION file for change + command: | + .circleci/check_version_trigger_release.sh workflows: version: 2 build_test_push: jobs: - - build + - build: + filters: + branches: + ignore: /^release\/.*/ - test: requires: - build @@ -97,9 +119,15 @@ workflows: filters: branches: only: develop - - release: + - check_version: requires: - - test + - push + release: + jobs: + - build: filters: branches: - only: master \ No newline at end of file + only: master + - release: + requires: + - build \ No newline at end of file diff --git a/.circleci/push_gem.sh b/.circleci/push_gem.sh index 62743c4..73697e3 100755 --- a/.circleci/push_gem.sh +++ b/.circleci/push_gem.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +sudo apt-get update sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 diff --git a/.circleci/push_gems_to_rubygems.sh b/.circleci/push_gems_to_rubygems.sh new file mode 100644 index 0000000..6c82a60 --- /dev/null +++ b/.circleci/push_gems_to_rubygems.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +echo "Pushing metrics gem to rubygems.org..." +echo "gem `gem --version`" +cat .circleci/gem_credentials | sed -e "s/__RUBYGEMS_API_KEY__/${RUBYGEMS_API_KEY}/" > ~/.gem/credentials +chmod 0600 ~/.gem/credentials +gem push /tmp/pkg/fluent-plugin-kubernetes-metrics-*.gem \ No newline at end of file diff --git a/VERSION b/VERSION index 8cfbc90..8428158 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 \ No newline at end of file +1.1.2 \ No newline at end of file diff --git a/docker/FLUENTD_HEC_GEM_VERSION b/docker/FLUENTD_HEC_GEM_VERSION new file mode 100644 index 0000000..8428158 --- /dev/null +++ b/docker/FLUENTD_HEC_GEM_VERSION @@ -0,0 +1 @@ +1.1.2 \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 2e654ed..8b6d32a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1 +1,4 @@ # Docker Image for Splunk Connect for Kubernetes +Fluentd with input plugin for k8s metrics +# To Build +`docker build --build-arg VERSION=$(cat docker/FLUENTD_HEC_GEM_VERSION) --no-cache -t splunk/fluent-plugin-kubernetes-metrics:ci ./docker` \ No newline at end of file diff --git a/lib/fluent/plugin/in_kubernetes_metrics.rb b/lib/fluent/plugin/in_kubernetes_metrics.rb index 13118de..acb5a96 100644 --- a/lib/fluent/plugin/in_kubernetes_metrics.rb +++ b/lib/fluent/plugin/in_kubernetes_metrics.rb @@ -300,96 +300,112 @@ def underscore(camlcase) end def emit_uptime(tag:, start_time:, labels:) - uptime = @scraped_at - Time.iso8601(start_time) - router.emit generate_tag("#{tag}.uptime"), Fluent::EventTime.from_time(@scraped_at), labels.merge('value' => uptime) + unless start_time.nil? + uptime = @scraped_at - Time.iso8601(start_time) + router.emit generate_tag("#{tag}.uptime"), Fluent::EventTime.from_time(@scraped_at), labels.merge('value' => uptime) + end end def emit_cpu_metrics(tag:, metrics:, labels:) - time = parse_time metrics['time'] - if usage_rate = metrics['usageNanoCores'] - router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000) - end - if usage = metrics['usageNanoCores'] - router.emit generate_tag("#{tag}.cpu.usage"), time, labels.merge('value' => usage) + unless metrics['time'].nil? + time = parse_time metrics['time'] + if usage_rate = metrics['usageNanoCores'] + router.emit generate_tag("#{tag}.cpu.usage_rate"), time, labels.merge('value' => usage_rate / 1_000_000) + end + if usage = metrics['usageNanoCores'] + router.emit generate_tag("#{tag}.cpu.usage"), time, labels.merge('value' => usage) + end end end def emit_memory_metrics(tag:, metrics:, labels:) - time = parse_time metrics['time'] - %w[availableBytes usageBytes workingSetBytes rssBytes pageFaults majorPageFaults].each do |name| - if value = metrics[name] - router.emit generate_tag("#{tag}.memory.#{underscore name}"), time, labels.merge('value' => value) + unless metrics['time'].nil? + time = parse_time metrics['time'] + %w[availableBytes usageBytes workingSetBytes rssBytes pageFaults majorPageFaults].each do |name| + if value = metrics[name] + router.emit generate_tag("#{tag}.memory.#{underscore name}"), time, labels.merge('value' => value) + end end end end def emit_network_metrics(tag:, metrics:, labels:) - time = parse_time metrics['time'] - Array(metrics['interfaces']).each do |it| - it_name = it['name'] - %w[rxBytes rxErrors txBytes txErrors].each do |metric_name| - if value = it[metric_name] - router.emit generate_tag("#{tag}.network.#{underscore metric_name}"), time, labels.merge('value' => value, 'interface' => it_name) + unless metrics['time'].nil? + time = parse_time metrics['time'] + Array(metrics['interfaces']).each do |it| + it_name = it['name'] + %w[rxBytes rxErrors txBytes txErrors].each do |metric_name| + if value = it[metric_name] + router.emit generate_tag("#{tag}.network.#{underscore metric_name}"), time, labels.merge('value' => value, 'interface' => it_name) + end end end end end def emit_fs_metrics(tag:, metrics:, labels:) - time = parse_time metrics['time'] - %w[availableBytes capacityBytes usedBytes inodesFree inodes inodesUsed].each do |metric_name| - if value = metrics[metric_name] - router.emit generate_tag("#{tag}.#{underscore metric_name}"), time, labels.merge('value' => value) + unless metrics['time'].nil? + time = parse_time metrics['time'] + %w[availableBytes capacityBytes usedBytes inodesFree inodes inodesUsed].each do |metric_name| + if value = metrics[metric_name] + router.emit generate_tag("#{tag}.#{underscore metric_name}"), time, labels.merge('value' => value) + end end end end def emit_node_rlimit_metrics(node_name, rlimit) - time = parse_time rlimit['time'] - %w[maxpid curproc].each do |metric_name| - next unless value = rlimit[metric_name] - - router.emit(generate_tag("node.runtime.imagefs.#{metric_name}"), time, - 'value' => value, - 'node' => node_name) + unless rlimit['time'].nil? + time = parse_time rlimit['time'] + %w[maxpid curproc].each do |metric_name| + next unless value = rlimit[metric_name] + + router.emit(generate_tag("node.runtime.imagefs.#{metric_name}"), time, + 'value' => value, + 'node' => node_name) + end end end def emit_system_container_metrics(node_name, container) tag = 'sys-container' labels = { 'node' => node_name, 'name' => container['name'] } - emit_uptime tag: tag, start_time: container['startTime'], labels: labels - emit_cpu_metrics tag: tag, metrics: container['cpu'], labels: labels - emit_memory_metrics tag: tag, metrics: container['memory'], labels: labels + unless container['startTime'].nil? + emit_uptime tag: tag, start_time: container['startTime'], labels: labels + emit_cpu_metrics tag: tag, metrics: container['cpu'], labels: labels unless container['cpu'].nil? + emit_memory_metrics tag: tag, metrics: container['memory'], labels: labels unless container['memory'].nil? + end end def emit_stats_breakdown(stats) stats_latest = stats[-1] tag = 'node' labels = { 'node' => @node_name } - stats_timestamp = parse_time stats_latest['timestamp'] - unless stats_latest['cpu'].nil? - emit_cpu_metrics_stats tag: tag, metrics: stats_latest['cpu'], labels: labels, time: stats_timestamp - end + unless stats_latest['timestamp'].nil? + stats_timestamp = parse_time stats_latest['timestamp'] + unless stats_latest['cpu'].nil? + emit_cpu_metrics_stats tag: tag, metrics: stats_latest['cpu'], labels: labels, time: stats_timestamp + end - unless stats_latest['diskio'].nil? - emit_diskio_metrics_stats tag: tag, metrics: stats_latest['diskio'], labels: labels, time: stats_timestamp - end + unless stats_latest['diskio'].nil? + emit_diskio_metrics_stats tag: tag, metrics: stats_latest['diskio'], labels: labels, time: stats_timestamp + end - unless stats_latest['memory'].nil? - emit_memory_metrics_stats tag: tag, metrics: stats_latest['memory'], labels: labels, time: stats_timestamp - end + unless stats_latest['memory'].nil? + emit_memory_metrics_stats tag: tag, metrics: stats_latest['memory'], labels: labels, time: stats_timestamp + end - unless stats_latest['network'].nil? - emit_network_metrics_stats tag: tag, metrics: stats_latest['network'], labels: labels, time: stats_timestamp - end + unless stats_latest['network'].nil? + emit_network_metrics_stats tag: tag, metrics: stats_latest['network'], labels: labels, time: stats_timestamp + end - unless stats_latest['filesystem'].nil? - emit_filesystem_metrics_stats tag: tag, metrics: stats_latest['filesystem'], labels: labels, time: stats_timestamp - end + unless stats_latest['filesystem'].nil? + emit_filesystem_metrics_stats tag: tag, metrics: stats_latest['filesystem'], labels: labels, time: stats_timestamp + end - unless stats_latest['task_stats'].nil? - emit_tasks_stats_metrics_stats tag: tag, metrics: stats_latest['task_stats'], labels: labels, time: stats_timestamp + unless stats_latest['task_stats'].nil? + emit_tasks_stats_metrics_stats tag: tag, metrics: stats_latest['task_stats'], labels: labels, time: stats_timestamp + end end end @@ -429,9 +445,9 @@ def emit_diskio_metrics_stats(tag:, metrics:, labels:, time:) if diskio_io_service_bytes_minor = device['minor'] router.emit generate_tag("#{tag}.diskio".concat(metric_name).concat('.minor.')), time, labels.merge('device' => device['device'], 'value' => diskio_io_service_bytes_minor) end - device_stats = device['stats'] + device_stats = device['stats'] unless device['stats'].nil? device_stats.each do |device_stat| - device_key, device_value = device_stat + device_key, device_value = device_stat unless device_stat.nil? router.emit generate_tag("#{tag}.diskio.".concat(metric_name).concat('.stats.').concat(device_key)), time, labels.merge('device' => device['device'], 'value' => device_value) end end @@ -448,7 +464,7 @@ def emit_memory_metrics_stats(tag:, metrics:, labels:, time:) next unless current_memory_metric_group = metrics[metric_name_group] current_memory_metric_group.each do |metric_name| - metric_key, metric_value = metric_name + metric_key, metric_value = metric_name unless metric_name.nil? router.emit generate_tag("#{tag}.memory.".concat(metric_name_group).concat('.').concat(metric_key)), time, labels.merge('value' => metric_value) end end @@ -477,7 +493,7 @@ def emit_network_metrics_stats(tag:, metrics:, labels:, time:) next unless metric_group = metrics[metric_name_group] metric_group.each do |current_metric| - metric_key, metric_value = current_metric + metric_key, metric_value = current_metric unless current_metric.nil? router.emit generate_tag("#{tag}.network.".concat(metric_name_group).concat('.').concat(metric_key)), time, labels.merge('value' => metric_value) end end @@ -488,7 +504,7 @@ def emit_filesystem_metrics_stats(tag:, metrics:, labels:, time:) device = file_system['device'] type = file_system['type'] file_system.each do |file_metric| - file_key, file_value = file_metric + file_key, file_value = file_metric unless file_metric.nil? unless %w[device type has_inodes].include? file_key router.emit generate_tag("#{tag}.filesystem.".concat(file_key)), time, labels.merge('device' => device, 'type' => type, 'value' => file_value) end @@ -498,7 +514,7 @@ def emit_filesystem_metrics_stats(tag:, metrics:, labels:, time:) def emit_tasks_stats_metrics_stats(tag:, metrics:, labels:, time:) metrics.each do |task_stats| - task_key, task_value = task_stats + task_key, task_value = task_stats unless task_stats.nil? router.emit generate_tag("#{tag}.tasks_stats.".concat(task_key)), time, labels.merge('value' => task_value) end end @@ -510,28 +526,28 @@ def emit_node_metrics(node) unless node['startTime'].nil? emit_uptime tag: tag, start_time: node['startTime'], labels: labels - end - unless node['cpu'].nil? - emit_cpu_metrics tag: tag, metrics: node['cpu'], labels: labels - end - unless node['memory'].nil? - emit_memory_metrics tag: tag, metrics: node['memory'], labels: labels - end - unless node['network'].nil? - emit_network_metrics tag: tag, metrics: node['network'], labels: labels - end - unless node['fs'].nil? - emit_fs_metrics tag: "#{tag}.fs", metrics: node['fs'], labels: labels - end - unless node['runtime']['imageFs'].nil? - emit_fs_metrics tag: "#{tag}.imagefs", metrics: node['runtime']['imageFs'], labels: labels - end - unless node['rlimit'].nil? - emit_node_rlimit_metrics node_name, node['rlimit'] - end - unless node['systemContainers'].nil? - node['systemContainers'].each do |c| - emit_system_container_metrics node_name, c + unless node['cpu'].nil? + emit_cpu_metrics tag: tag, metrics: node['cpu'], labels: labels + end + unless node['memory'].nil? + emit_memory_metrics tag: tag, metrics: node['memory'], labels: labels + end + unless node['network'].nil? + emit_network_metrics tag: tag, metrics: node['network'], labels: labels + end + unless node['fs'].nil? + emit_fs_metrics tag: "#{tag}.fs", metrics: node['fs'], labels: labels + end + unless node['runtime']['imageFs'].nil? + emit_fs_metrics tag: "#{tag}.imagefs", metrics: node['runtime']['imageFs'], labels: labels + end + unless node['rlimit'].nil? + emit_node_rlimit_metrics node_name, node['rlimit'] + end + unless node['systemContainers'].nil? + node['systemContainers'].each do |c| + emit_system_container_metrics node_name, c unless c.nil? + end end end end @@ -539,11 +555,13 @@ def emit_node_metrics(node) def emit_container_metrics(pod_labels, container) tag = 'container' labels = pod_labels.merge 'container-name' => container['name'] - emit_uptime tag: tag, start_time: container['startTime'], labels: labels - emit_cpu_metrics tag: tag, metrics: container['cpu'], labels: labels - emit_memory_metrics tag: tag, metrics: container['memory'], labels: labels - emit_fs_metrics tag: "#{tag}.rootfs", metrics: container['rootfs'], labels: labels - emit_fs_metrics tag: "#{tag}.logs", metrics: container['logs'], labels: labels + unless container['startTime'].nil? + emit_uptime tag: tag, start_time: container['startTime'], labels: labels + emit_cpu_metrics tag: tag, metrics: container['cpu'], labels: labels unless container['cpu'].nil? + emit_memory_metrics tag: tag, metrics: container['memory'], labels: labels unless container['memory'].nil? + emit_fs_metrics tag: "#{tag}.rootfs", metrics: container['rootfs'], labels: labels unless container['rootfs'].nil? + emit_fs_metrics tag: "#{tag}.logs", metrics: container['logs'], labels: labels unless container['logs'].nil? + end end def emit_pod_metrics(node_name, pod) @@ -551,16 +569,22 @@ def emit_pod_metrics(node_name, pod) labels = pod['podRef'].transform_keys &'pod-'.method(:+) labels['node'] = node_name - emit_uptime tag: tag, start_time: pod['startTime'], labels: labels - emit_cpu_metrics tag: tag, metrics: pod['cpu'], labels: labels if pod['cpu'] - emit_memory_metrics tag: tag, metrics: pod['memory'], labels: labels if pod['memory'] - emit_network_metrics tag: tag, metrics: pod['network'], labels: labels - emit_fs_metrics tag: "#{tag}.ephemeral-storage", metrics: pod['ephemeral-storage'], labels: labels - Array(pod['volume']).each do |volume| - emit_fs_metrics tag: "#{tag}.volume", metrics: volume, labels: labels.merge('name' => volume['name']) - end - Array(pod['containers']).each do |container| - emit_container_metrics labels, container + unless pod['startTime'].nil? + emit_uptime tag: tag, start_time: pod['startTime'], labels: labels + emit_cpu_metrics tag: tag, metrics: pod['cpu'], labels: labels if pod['cpu'] unless pod['cpu'].nil? + emit_memory_metrics tag: tag, metrics: pod['memory'], labels: labels if pod['memory'] unless pod['memory'].nil? + emit_network_metrics tag: tag, metrics: pod['network'], labels: labels unless pod['network'].nil? + emit_fs_metrics tag: "#{tag}.ephemeral-storage", metrics: pod['ephemeral-storage'], labels: labels unless pod['ephemeral-storage'].nil? + unless pod['volume'].nil? + Array(pod['volume']).each do |volume| + emit_fs_metrics tag: "#{tag}.volume", metrics: volume, labels: labels.merge('name' => volume['name']) unless volume.nil? + end + end + unless pod['containers'].nil? + Array(pod['containers']).each do |container| + emit_container_metrics labels, container unless container.nil? + end + end end end diff --git a/metrics-information.md b/metrics-information.md index 7af7631..a12530c 100644 --- a/metrics-information.md +++ b/metrics-information.md @@ -4,7 +4,7 @@ | Terminology | Definition | | --- | --- | -| Node | A single machine in a kubernetes cluster | +| Node | A single machine in a kubernetes cluster. | | imagefs | Stats for the underlying filesystem where container images are stored. Usage here refers to the total number of bytes occupied by images on the filesystem. | | fs | Stats pertaining to total usage of filesystem resources on the rootfs used by node Kubernetes components. | | sys-container | Stats for system daemons tracked as raw containers. | @@ -39,7 +39,7 @@ Metrics based on values extracted from aggregated files and field data. | kube.cluster.memory.limit | float | megabyte | The cluster's memory limit. | | kube.cluster.memory.request | float | megabyte | The cluster's memory request. | | kube.node.cpu.capacity| float | millicpu | The node's CPU capacity. | -| kube.node.cpu.allocatable | float | millicpu | The node's CPU allocatable. | +| kube.node.cpu.allocatable | float | millicpu | The node's allocatable CPU . | | kube.node.memory.capacity | float | megabyte | The node's memory capacity. | | kube.node.memory.allocatable | float | megabyte | The node's allocatable memory. | | kube.node.cpu.reservation | float | millicpu | The node's CPU reservation. | @@ -49,12 +49,12 @@ Metrics based on values extracted from aggregated files and field data. | kube.namespace.cpu.usage | float | millicpu | The namespace's CPU usage. | | kube.namespace.memory.usage | float | megabyte | The namespace's memory usage. | | kube.cluster.cpu.usage | float | millicpu | The cluster's CPU usage. | -| kube.cluster.memory.usage | float | bytes | The luster's memory usage. | +| kube.cluster.memory.usage | float | bytes | The cluster's memory usage. | | kube.node.runtime | | | | ## Summary Metrics -Provides a total count of observations and as a sum of all observed values. +Provides metrics as a sum of all observed values. | Metric Name | Metric Type | Metric Format | Metric Description | | --- | --- | --- | --- | @@ -71,7 +71,7 @@ Provides a total count of observations and as a sum of all observed values. | kube.node.network.tx_bytes | float | bytes | The cumulative count of bytes transmitted in the node. | | kube.node.network.tx_errors | integer | | The cumulative count of transmit errors encountered in the node. | | kube.node.fs.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem on the node. | -| kube.node.fs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystems underlying storage on the node. | +| kube.node.fs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage on the node. | | kube.node.fs.used_bytes | float | bytes | The bytes used for a specific task on the filesystem on the node. | | kube.node.fs.inodes_free | integer | | The free inodes in the filesystem on the node. | | kube.node.fs.inodes | integer | | The total inodes in the filesystem on the node. | @@ -88,30 +88,30 @@ Provides a total count of observations and as a sum of all observed values. | kube.sys-container.cpu.usage_rate | float | millicpu | Total CPU usage (sum of all cores) averaged over the sample window for sys-container. | | kube.sys-container.cpu.usage | float | millicpu | Total CPU usage rate (sum of all cores) averaged over the sample window for sys-container. | | kube.sys-container.memory.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem for sys-container. | -| kube.sys-container.memory.usage_bytes |float | bytes | The node memory, in bytes, in use for the sys-container. | +| kube.sys-container.memory.usage_bytes | float | bytes | The node memory, in bytes, in use for the sys-container. | | kube.sys-container.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory, in bytes, for the sys-container. | | kube.sys-container.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory, in bytes, for the sys-container. | -| kube.sys-container.memory.page_faults | integer | | The cumulative number of minor page faults for the sys-container. | -| kube.sys-container.memory.major_page_faults | integer | | The cumulative number of major page faults for the sys-container. | +| kube.sys-container.memory.page_faults | integer | | The cumulative number of minor page faults for the sys-container. | +| kube.sys-container.memory.major_page_faults | integer | | The cumulative number of major page faults for the sys-container. | | kube.pod.uptime | float | seconds | The pod's uptime. | | kube.pod.cpu.usage_rate | float | millicpu | Total CPU usage (sum of all cores), averaged over the sample window for the pod. | | kube.pod.cpu.usage | float | nanocpu | Total CPU usage rate (sum of all cores), averaged over the sample window for the pod. | -| kube.pod.memory.available_bytes | float | bytes | Represents the storage space available, in bytes, for the filesystem for the pod. | -| kube.pod.memory.usage_bytes | float | bytes | The bytes used for a specific task on the filesystem for the pod. | -| kube.pod.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory, in bytes, for the pod. | -| kube.pod.memory.page_faults | integer | | The cumulative number of minor page faults for the pod. | -| kube.pod.memory.major_page_faults | integer | | The cumulative number of minor page faults for the pod. | -| kube.pod.network.rx_bytes | float | bytes | The cumulative count of bytes received for the pod. | -| kube.pod.network.rx_errors | integer | | The cumulative count of receive errors encountered for the pod. | -| kube.pod.network.tx_bytes | float | bytes | The cumulative count of bytes transmitted for the pod. | -| kube.pod.network.tx_errors | integer | | The cumulative count of transmit errors encountered for the pod. | +| kube.pod.memory.available_bytes | float | bytes | Represents the storage space available, in bytes, for the filesystem on the pod. | +| kube.pod.memory.usage_bytes | float | bytes | The bytes used for a specific task in the filesystem on the pod. | +| kube.pod.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory, in bytes, in the pod. | +| kube.pod.memory.page_faults | integer | | The cumulative number of minor page faults in the pod. | +| kube.pod.memory.major_page_faults | integer | | The cumulative number of minor page faults in the pod. | +| kube.pod.network.rx_bytes | float | bytes | The cumulative count of bytes received in the pod. | +| kube.pod.network.rx_errors | integer | | The cumulative count of receive errors encountered in the pod. | +| kube.pod.network.tx_bytes | float | bytes | The cumulative count of bytes transmitted in the pod. | +| kube.pod.network.tx_errors | integer | | The cumulative count of transmit errors in the pod. | | kube.pod.ephemeral-storage.available_bytes | float | bytes | The node's available memory for use, in bytes, in the pod's ephemeral storage. | -| kube.pod.ephemeral-storage.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage for the pod's ephemeral storage. | -| kube.pod.ephemeral-storage.used_bytes | float | bytes | The bytes used for a specific task on the filesystem for the pod's ephemeral storage. | +| kube.pod.ephemeral-storage.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage in the pod's ephemeral storage. | +| kube.pod.ephemeral-storage.used_bytes | float | bytes | The bytes used for a specific task on the filesystem in the pod's ephemeral storage. | | kube.pod.ephemeral-storage.inodes_free |integer | | The free inodes in the filesystem in the pod's ephemeral storage. | -| kube.pod.ephemeral-storage.inodes | integer | | The inodes in the filesystem for the pod's ephemeral storage. | -| kube.pod.ephemeral-storage.inodes_used | integer | | The inodes used in the filesystem for the pod's ephemeral storage. | -| kube.pod.volume.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem for the pod's volume. | +| kube.pod.ephemeral-storage.inodes | integer | | The inodes in the filesystem in the pod's ephemeral storage. | +| kube.pod.ephemeral-storage.inodes_used | integer | | The inodes used in the filesystem in the pod's ephemeral storage. | +| kube.pod.volume.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem in the pod's volume. | | kube.pod.volume.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystems underlying storage for the pod's volume. | | kube.pod.volume.used_bytes | float | bytes | The bytes used for a specific task on the filesystem for the pod's volume. | | kube.pod.volume.inodes_free | integer | | The free inodes in the filesystem for the pod's volume. | @@ -121,23 +121,23 @@ Provides a total count of observations and as a sum of all observed values. | kube.container.cpu.usage_rate | float | millicpu | Total CPU usage rate (sum of all cores) averaged over the sample window for the container. | | kube.container.cpu.usage | float | nanocpu | Total CPU usage rate (sum of all cores) averaged over the sample window for the container. | -| kube.container.memory.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem for the container. | -| kube.container.memory.usage_bytes | float | bytes | The bytes used for a specific task on the filesystem for the container. | -| kube.container.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory in bytes for the container. | -| kube.container.memory.page_faults | integer | | The cumulative number of minor page faults for the container. | -| kube.container.memory.major_page_faults | integer | | The cumulative number of major page faults for the container. | -| kube.container.rootfs.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem for the container's rootfs. | -| kube.container.rootfs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystems underlying storage for the container's rootfs. | -| kube.container.rootfs.used_bytes | float | bytes | The bytes used for a specific task on the filesystem for the container's rootfs. | -| kube.container.rootfs.inodes_free | integer | | The free inodes in the filesystem for the container's rootfs. | -| kube.container.rootfs.inodes | integer | | The inodes in the filesystem for the container's rootfs. | -| kube.container.rootfs.inodes_used | integer | | Represents the used inodes in the filesystem for the container's rootfs. | -| kube.container.logs.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem for the container's logs. | -| kube.container.logs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage for the container's logs. | -| kube.container.logs.used_bytes | float | bytes | The bytes used for a specific task on the filesystem for the container's logs. | -| kube.container.logs.inodes_free | integer | | The free inodes in the filesystem for the container's logs. | -| kube.container.logs.inodes | integer | | The inodes in the filesystem for the container's logs. | -| kube.container.logs.inodes_used | integer | | The used inodes in the filesystem for the container's logs. | +| kube.container.memory.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem in the container. | +| kube.container.memory.usage_bytes | float | bytes | The bytes used for a specific task on the filesystem in the container. | +| kube.container.memory.rss_bytes | float | bytes | The amount of anonymous and swap cache memory in bytes in the container. | +| kube.container.memory.page_faults | integer | | The cumulative number of minor page faults in the container. | +| kube.container.memory.major_page_faults | integer | | The cumulative number of major page faults in the container. | +| kube.container.rootfs.available_bytes | float | bytes | The storage space available, in bytes, in the filesystem in the container's rootfs. | +| kube.container.rootfs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage in the container's rootfs. | +| kube.container.rootfs.used_bytes | float | bytes | The bytes used for a specific task in the filesystem on the container's rootfs. | +| kube.container.rootfs.inodes_free | integer | | The free inodes in the filesystem on the container's rootfs. | +| kube.container.rootfs.inodes | integer | | The inodes in the filesystem on the container's rootfs. | +| kube.container.rootfs.inodes_used | integer | | Represents the used inodes in the filesystem on the container's rootfs. | +| kube.container.logs.available_bytes | float | bytes | The storage space available, in bytes, for the filesystem on the container's logs. | +| kube.container.logs.capacity_bytes | float | bytes | The total capacity, in bytes, of the filesystem's underlying storage in the container's logs. | +| kube.container.logs.used_bytes | float | bytes | The bytes used for a specific task on the filesystem in the container's logs. | +| kube.container.logs.inodes_free | integer | | The free inodes in the filesystem in the container's logs. | +| kube.container.logs.inodes | integer | | The inodes in the filesystem in the container's logs. | +| kube.container.logs.inodes_used | integer | | The used inodes in the filesystem in the container's logs. | ## Stats Metrics @@ -166,22 +166,22 @@ Metrics based on measures and recording of data points and views. | kube.node.diskio..minor | | | | | kube.node.diskio..major | | | | | kube.node.filesystem.available | | bytes | The bytes available for non-root user. | -| kube.node.filesystem.base_usage | | integer | Base usage consumed by the container's writable layer. At this time, this value is only applicable for Docker containers. | +| kube.node.filesystem.base_usage | integer | | Base usage consumed by the container's writable layer. At this time, this value is only applicable for Docker containers. | | kube.node.filesystem.capacity | | bytes | Bytes that can be consumed by the container on this filesystem. | -| kube.node.filesystem.inodes | | integer | Number of inodes. | -| kube.node.filesystem.inodes_free | | integer | Number of available inodes. | -| kube.node.filesystem.io_in_progress | | integer | Number of I/Os currently in progress. It is the only field that should go to zero. Value is incremented as requests are given to appropriate struct request_queue and decremented as they finish. | +| kube.node.filesystem.inodes | integer | | Number of inodes. | +| kube.node.filesystem.inodes_free | integer | | Number of available inodes. | +| kube.node.filesystem.io_in_progress | integer | | Number of I/Os currently in progress. It is the only field that should go to zero. Value is incremented as requests are given to appropriate struct request_queue and decremented as they finish. | | kube.node.filesystem.io_time | | | Number of milliseconds spent doing I/Os. This field increases so long as field 9 is nonzero. | | kube.node.filesystem.read_time | | | Number of milliseconds spent reading, total number of milliseconds spent by all reads (as measured from make_request() to end_that_request_last()). | | kube.node.filesystem.reads_completed | | integer | The total number of reads completed successfully. | | kube.node.filesystem.reads_merged | | integer | Number of reads merged. Reads and writes which are adjacent to each other and may be merged for efficiency. For example, two 4K reads may become one 8K read before it is ultimately handed to the disk, and so it will be counted (and queued) as only one I/O. This field lets you know how often this was done. | -| kube.node.filesystem.sectors_read | | integer | Total number of sectors successfully read. | -| kube.node.filesystem.sectors_written | | integer| The total number of sectors successfully written. | +| kube.node.filesystem.sectors_read | | integer | Total number of successfully read sectors. | +| kube.node.filesystem.sectors_written | | integer| The total number of successfully written sectors. | | kube.node.filesystem.usage | | bytes | Number of bytes consumed by the container on this filesystem. | | kube.node.filesystem.weighted_io_time | | | Weighted number of milliseconds spent doing I/Os. This field is incremented at each I/O start, I/O completion, I/O merge. Or the field can read these stats by the number of I/Os in progress (field 9) times the number of milliseconds spent doing I/O since the last update of this field. This can provide an easy measure of both I/O completion time and of the backlog that may be accumulating. | | kube.node.filesystem.write_time | | | Total number of milliseconds spent by all writes, as measured from make_request() to end_that_request_last(). | -| kube.node.filesystem.writes_completed | | integer | Total number of writes completed successfully. | -| kube.node.filesystem.writes_merged | | integer | Number of writes merged. | +| kube.node.filesystem.writes_completed | integer | | Total number of writes completed successfully. | +| kube.node.filesystem.writes_merged | integer | | Number of writes merged. | | kube.node.memory.cache | | bytes | Number of bytes of page cache memory in bytes. | | kube.node.memory.container_data.pgfault | | | | | kube.node.memory.container_data.pgmajfault | | | | @@ -194,11 +194,11 @@ Metrics based on measures and recording of data points and views. | kube.node.memory.usage | | bytes | Current memory usage, in bytes. This includes all memory regardless of when it was accessed. | | kube.node.memory.working_set | | | The amount of working set memory, in bytes. This includes recently accessed memory, dirty memory, and kernel memory. Working set is <= "usage". | | kube.node.memory.mapped_file | | | | -| kube.node.tasks_stats.nr_io_wait | | integer | Number of tasks waiting on IO. | -| kube.node.tasks_stats.nr_running | | integer | Number of running tasks. | -| kube.node.tasks_stats.nr_sleeping | | integer | Number of sleeping tasks. | -| kube.node.tasks_stats.nr_stopped | | integer | Number of tasks in stopped state. | -| kube.node.tasks_stats.nr_uninterruptible | | integer | Number of tasks in uninterruptible state. | +| kube.node.tasks_stats.nr_io_wait | integer | | Number of tasks waiting on IO. | +| kube.node.tasks_stats.nr_running | integer | | Number of running tasks. | +| kube.node.tasks_stats.nr_sleeping | integer | | Number of sleeping tasks. | +| kube.node.tasks_stats.nr_stopped | integer | | Number of tasks in stopped state. | +| kube.node.tasks_stats.nr_uninterruptible | integer | | Number of tasks in uninterruptible state. | | kube.node.network..rx_bytes | | bytes | Cumulative count of bytes received. | | kube.node.network..rx_dropped | | | Cumulative count of packets dropped while receiving. | | kube.node.network..rx_errors | | | Cumulative count of receive errors encountered. | @@ -208,20 +208,20 @@ Metrics based on measures and recording of data points and views. | kube.node.network..tx_errors | | | Cumulative count of transmit errors encountered. | | kube.node.network..tx_packets | | | Cumulative count of packets transmitted. | - ## Cadvisor Metrics + ## cAdvisor Metrics - cAdvisor provides performance and usage information for running containers. + Performance and usage information for running containers. | Metric Name | Metric Description | | --- | --- | -| cadvisor_version_info | A metric with a constant '1' value labeled by kernel version, OS version, docker version, cadvisor version & cadvisor revision. | -| container_cpu_load_average_10s | Value of container cpu load average over the last 10 seconds. | -| container_cpu_system_seconds_total | Cumulative system cpu time consumed in seconds. | -| container_cpu_usage_seconds_total | Cumulative cpu time consumed in seconds. | -| container_cpu_user_seconds_total | Cumulative user cpu time consumed in seconds. | -| container_fs_inodes_free | Number of available Inodes. | -| container_fs_inodes_total | Number of Inodes in use. | -| container_fs_io_current | Number of I/Os currently in progress. | +| cadvisor_version_info | A metric with a constant "1" value labeled by kernel version, OS version, Docker version, cAdvisor version and cAdvisor revision. | +| container_cpu_load_average_10s | Value of container CPU load average over the last 10 seconds. | +| container_cpu_system_seconds_total | Cumulative system CPU time consumed in seconds. | +| container_cpu_usage_seconds_total | Cumulative CPU time consumed in seconds. | +| container_cpu_user_seconds_total | Cumulative user CPU time consumed in seconds. | +| container_fs_inodes_free | Number of available inodes. | +| container_fs_inodes_total | Number of inodes in use. | +| container_fs_io_current | Number of in progress I/Os. | | container_fs_io_time_seconds_total | Cumulative count of seconds spent doing I/Os. | | container_fs_io_time_weighted_seconds_total | Cumulative weighted I/O time in seconds. | | container_fs_limit_bytes | Number of bytes that can be consumed by the container on this filesystem. | @@ -254,16 +254,16 @@ Metrics based on measures and recording of data points and views. | container_network_transmit_errors_total | Cumulative count of errors encountered while transmitting. | | container_network_transmit_packets_dropped_total | Cumulative count of packets dropped while transmitting. | | container_network_transmit_packets_total | Cumulative count of packets transmitted. | -| container_network_udp_usage_total | udp connection usage statistic for container. | -| container_scrape_error | 1 if there was an error while getting container metrics, 0 otherwise. | +| container_network_udp_usage_total | UDP connection usage statistic for container. | +| container_scrape_error | "1" indicates an error while getting container metrics, "0" indicates no error. | | container_spec_cpu_period | CPU period of the container. | | container_spec_cpu_shares | CPU share of the container. | | container_spec_memory_limit_bytes | Memory limit for the container. | | container_spec_memory_reservation_limit_bytes | Memory reservation limit for the container. | | container_spec_memory_swap_limit_bytes | Memory swap limit for the container. | -| container_start_time_seconds | Start time of the container since unix epoch in seconds. | +| container_start_time_seconds | Start time of the container since Unix epoch in seconds. | | container_tasks_state | Number of tasks in given state. | | machine_cpu_cores | Number of CPU cores on the machine. | | machine_memory_bytes | Amount of memory installed on the machine. | -| container_cpu_cfs_throttled_seconds_total | Total time duration the container has been throttled. | +| container_cpu_cfs_throttled_seconds_total | Total time that the container has been throttled. | diff --git a/test/helper.rb b/test/helper.rb index 54985c2..dd820c8 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -83,6 +83,13 @@ def stub_kubelet_summary_api end.close end + def stub_kubelet_summary_api_missing_timestamps + open(File.expand_path('unit_without_time.json', __dir__)).tap do |f| + stub_request(:get, kubelet_summary_api_url.to_s) + .to_return(body: f.read) + end.close + end + def stub_metrics_cadvisor open(File.expand_path('metrics_cadvisor.txt', __dir__)).tap do |f| stub_request(:get, kubelet_cadvisor_api_url.to_s) @@ -119,6 +126,14 @@ def get_unit_parsed_string parsed_string end + def get_unit_parsed_string_missing_timestamps + parsed_string = nil + open(File.expand_path('unit_without_time.json', __dir__)).tap do |f| + parsed_string = f.read + end.close + parsed_string + end + def get_stats_parsed_string get_stats_parsed_string = nil open(File.expand_path('stats.json', __dir__)).tap do |f| diff --git a/test/plugin/test_missing_timestamps.rb b/test/plugin/test_missing_timestamps.rb new file mode 100644 index 0000000..5320d54 --- /dev/null +++ b/test/plugin/test_missing_timestamps.rb @@ -0,0 +1,44 @@ +require 'helper' +require 'fluent/plugin/in_kubernetes_metrics.rb' + +class KubernetesMetricsInputTest < Test::Unit::TestCase + include Fluent::Test::Helpers + include PluginTestHelper + + @@driver = nil + + CONFIG = %( + type kubernetes_metrics + node_name generics-aws-node-name + tag kube.* + insecure_ssl true + interval 10s + use_rest_client true + use_rest_client_ssl false + kubelet_port 10_255 + kubelet_address generics-aws-node-name + ).freeze + + setup do + Fluent::Test.setup + end + + def create_driver(conf = CONFIG) + Fluent::Test::Driver::Input.new(Fluent::Plugin::KubernetesMetricsInput).configure(conf) + end + + test 'timestamps_missing' do + @@parsed_unit_string = JSON.parse(get_unit_parsed_string_missing_timestamps) + ENV['KUBERNETES_SERVICE_HOST'] = k8s_host + ENV['KUBERNETES_SERVICE_PORT'] = k8s_port + # all stub response bodies are from real k8s 1.8 API calls + stub_k8s_api + stub_kubelet_summary_api_missing_timestamps + stub_k8s_v1 + stub_kubelet_summary_api_missing_timestamps + stub_metrics_cadvisor + stub_metrics_stats + @@driver = create_driver + @@driver.run timeout: 20, expect_emits: 1, shutdown: true + end +end diff --git a/test/unit_without_time.json b/test/unit_without_time.json new file mode 100644 index 0000000..2f77229 --- /dev/null +++ b/test/unit_without_time.json @@ -0,0 +1,178 @@ +{ + "node": { + "nodeName": "ip-XXX-XX-XX-XXX.us-west-2.compute.internal", + "systemContainers": [ + { + "name": "pods", + "cpu": { + "usageNanoCores": 69568807, + "usageCoreNanoSeconds": 134724371173 + }, + "memory": { + "availableBytes": 3570184192, + "usageBytes": 379932672, + "workingSetBytes": 379658240, + "rssBytes": 369627136, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "userDefinedMetrics": null + } + ], + "cpu": { + "usageNanoCores": 71501818, + "usageCoreNanoSeconds": 68824254091816 + }, + "memory": { + "availableBytes": 30164504576, + "usageBytes": 10132713472, + "workingSetBytes": 3576983552, + "rssBytes": 517582848, + "pageFaults": 498847843, + "majorPageFaults": 1999 + }, + "network": { + "name": "eth0", + "rxBytes": 8480164820, + "rxErrors": 0, + "txBytes": 5416875722, + "txErrors": 0, + "interfaces": [ + { + "name": "cbr0", + "rxBytes": 4677862821, + "rxErrors": 0, + "txBytes": 5067375443, + "txErrors": 0 + }, + { + "name": "eth0", + "rxBytes": 8480164820, + "rxErrors": 0, + "txBytes": 5416875722, + "txErrors": 0 + } + ] + }, + "fs": { + "availableBytes": 112741339136, + "capacityBytes": 128701009920, + "usedBytes": 10359861248, + "inodesFree": 33124287, + "inodes": 33554432, + "inodesUsed": 430145 + }, + "rlimit": { + "maxpid": 32768, + "curproc": 389 + }, + "runtime": { + "imageFs": { + "availableBytes": 112741339136, + "capacityBytes": 128701009920, + "usedBytes": 6439798904, + "inodesFree": 33124287, + "inodes": 33554432, + "inodesUsed": 430145 + } + } + }, + "pods": [ + { + "podRef": { + "name": "kube-system-logging-splunk-kubernetes-logging-4kbsb", + "namespace": "generic_namespace", + "uid": "6a83cdc1-c055-11e8-bbee-02dfc7c54d24" + }, + "startTime": "2018-09-24T23:56:13Z", + "containers": [ + { + "name": "splunk-fluentd-k8s-logs", + "cpu": { + "usageNanoCores": 3117227, + "usageCoreNanoSeconds": 856491349907 + }, + "memory": { + "usageBytes": 71749632, + "workingSetBytes": 71749632, + "rssBytes": 71430144, + "pageFaults": 204802, + "majorPageFaults": 0 + }, + "rootfs": { + "availableBytes": 112741339136, + "capacityBytes": 128701009920, + "usedBytes": 263282688, + "inodesFree": 33124287, + "inodes": 33554432, + "inodesUsed": 32 + }, + "logs": { + "availableBytes": 112741339136, + "capacityBytes": 128701009920, + "usedBytes": 61440, + "inodesFree": 33124287, + "inodes": 33554432, + "inodesUsed": 430145 + }, + "userDefinedMetrics": null + } + ], + "cpu": { + "usageNanoCores": 3126718, + "usageCoreNanoSeconds": 856492340119 + }, + "memory": { + "usageBytes": 71917568, + "workingSetBytes": 71917568, + "rssBytes": 71471104, + "pageFaults": 0, + "majorPageFaults": 0 + }, + "network": { + "name": "eth0", + "rxBytes": 22482186, + "rxErrors": 0, + "txBytes": 357450663, + "txErrors": 0, + "interfaces": [ + { + "name": "eth0", + "rxBytes": 22482186, + "rxErrors": 0, + "txBytes": 357450663, + "txErrors": 0 + } + ] + }, + "volume": [ + { + "availableBytes": 16870731776, + "capacityBytes": 16870744064, + "usedBytes": 12288, + "inodesFree": 4118825, + "inodes": 4118834, + "inodesUsed": 9, + "name": "default-token-phqbm" + }, + { + "availableBytes": 16870739968, + "capacityBytes": 16870744064, + "usedBytes": 4096, + "inodesFree": 4118829, + "inodes": 4118834, + "inodesUsed": 5, + "name": "secrets" + } + ], + "ephemeral-storage": { + "availableBytes": 112741339136, + "capacityBytes": 128701009920, + "usedBytes": 263344128, + "inodesFree": 33124287, + "inodes": 33554432, + "inodesUsed": 32 + } + } + ] +} \ No newline at end of file