Merge pull request #18 from paulfantom/os-release-1.8

Bump to release 1.8

.gitignore

@@ -1,6 +1,6 @@
 kube-state-metrics
 documented_metrics
-tested_metrics
+code_metrics
 
 # Created by https://www.gitignore.io/api/go
 

.golangci.yml

@@ -7,7 +7,7 @@ linters:
     - gofmt
     - goimports
     - golint
-    - gosimple
+    - staticcheck
     - gocyclo
     - ineffassign
     - misspell
@@ -16,4 +16,4 @@ linters:
 
 linters-settings:
   goimports:
-local-prefixes: k8s.io/kube-state-metrics
+    local-prefixes: k8s.io/kube-state-metrics

Dockerfile

@@ -1,14 +1,8 @@
-FROM alpine:3.9
-
-RUN adduser -D kube-state-metrics
-
 FROM gcr.io/distroless/static
 
 COPY kube-state-metrics /
 
-COPY --from=0 /etc/passwd /etc/passwd
-
-USER kube-state-metrics
+USER nobody
 
 ENTRYPOINT ["/kube-state-metrics", "--port=8080", "--telemetry-port=8081"]
 

Makefile

@@ -3,16 +3,16 @@ TESTENVVAR =
 REGISTRY = quay.io/coreos
 TAG_PREFIX = v
 TAG = $(TAG_PREFIX)$(shell cat VERSION)
-LATEST_RELEASE_BRANCH:=release-$(shell cat VERSION | grep -ohE "[0-9]+.[0-9]+")
+LATEST_RELEASE_BRANCH := release-$(shell grep -ohE "[0-9]+.[0-9]+" VERSION)
 PKGS = $(shell go list ./... | grep -v /vendor/)
 ARCH ?= $(shell go env GOARCH)
 BuildDate = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 Commit = $(shell git rev-parse --short HEAD)
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
-PKG=k8s.io/kube-state-metrics/pkg
-GO_VERSION=1.12
-FIRST_GOPATH:=$(firstword $(subst :, ,$(shell go env GOPATH)))
-BENCHCMP_BINARY:=$(FIRST_GOPATH)/bin/benchcmp
+PKG = k8s.io/kube-state-metrics/pkg
+GO_VERSION = 1.12
+FIRST_GOPATH := $(firstword $(subst :, ,$(shell go env GOPATH)))
+BENCHCMP_BINARY := $(FIRST_GOPATH)/bin/benchcmp
 GOLANGCI_VERSION := v1.17.1
 HAS_GOLANGCI := $(shell which golangci-lint)
 
@@ -29,7 +29,14 @@ validate-modules:
 	@git diff --exit-code -- go.sum go.mod vendor/
 
 licensecheck:
-	./tests/check_license.sh
+	@echo ">> checking license header"
+	@licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \
+               awk 'NR<=5' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
+       done); \
+       if [[ -n "$${licRes}" ]]; then \
+               echo "license header checking failed:"; echo "$${licRes}"; \
+               exit 1; \
+       fi
 
 lint: shellcheck licensecheck
 ifndef HAS_GOLANGCI
@@ -42,10 +49,10 @@ doccheck: generate
 	@git diff --exit-code
 	@echo "- Checking if the documentation is in sync with the code..."
 	@grep -hoE '(kube_[^ |]+)' docs/* --exclude=README.md| sort -u > documented_metrics
-	@sed -n 's/.*# TYPE \(kube_[^ ]\+\).*/\1/p' internal/store/*_test.go | sort -u > tested_metrics
-	@diff -u0 tested_metrics documented_metrics || (echo "ERROR: Metrics with - are present in tests but missing in documentation, metrics with + are documented but not tested."; exit 1)
+	@find internal/store -type f -not -name '*_test.go' -exec sed -nE 's/.*"(kube_[^"]+)"/\1/p' {} \; | sed -E 's/,//g' | sort -u > code_metrics
+	@diff -u0 code_metrics documented_metrics || (echo "ERROR: Metrics with - are present in code but missing in documentation, metrics with + are documented but not found in code."; exit 1)
 	@echo OK
-	@rm -f tested_metrics documented_metrics
+	@rm -f code_metrics documented_metrics
 	@echo "- Checking for orphan documentation files"
 	@cd docs; for doc in *.md; do if [ "$$doc" != "README.md" ] && ! grep -q "$$doc" *.md; then echo "ERROR: No link to documentation file $${doc} detected"; exit 1; fi; done
 	@echo OK
@@ -54,7 +61,7 @@ build-local: clean
 	GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) CGO_ENABLED=0 go build -ldflags "-s -w -X ${PKG}/version.Release=${TAG} -X ${PKG}/version.Commit=${Commit} -X ${PKG}/version.BuildDate=${BuildDate}" -o kube-state-metrics
 
 build: clean
-	docker run --rm -v "$$PWD":/go/src/k8s.io/kube-state-metrics -w /go/src/k8s.io/kube-state-metrics golang:${GO_VERSION} make build-local
+	docker run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics golang:${GO_VERSION} make build-local
 
 test-unit: clean build
 	GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) $(TESTENVVAR) go test --race $(FLAGS) $(PKGS)
@@ -84,10 +91,11 @@ all-push: $(addprefix sub-push-,$(ALL_ARCH))
 
 container: .container-$(ARCH)
 .container-$(ARCH):
-	docker run --rm -v "$$PWD":/go/src/k8s.io/kube-state-metrics -w /go/src/k8s.io/kube-state-metrics -e GOOS=linux -e GOARCH=$(ARCH) -e CGO_ENABLED=0 golang:${GO_VERSION} go build -ldflags "-s -w -X ${PKG}/version.Release=${TAG} -X ${PKG}/version.Commit=${Commit} -X ${PKG}/version.BuildDate=${BuildDate}" -o kube-state-metrics
-	cp -r * $(TEMP_DIR)
-	docker build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)
+	docker run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics -e GOOS=linux -e GOARCH=$(ARCH) -e CGO_ENABLED=0 golang:${GO_VERSION} go build -ldflags "-s -w -X ${PKG}/version.Release=${TAG} -X ${PKG}/version.Commit=${Commit} -X ${PKG}/version.BuildDate=${BuildDate}" -o kube-state-metrics
+	cp -r * "${TEMP_DIR}"
+	docker build -t $(MULTI_ARCH_IMG):$(TAG) "${TEMP_DIR}"
 	docker tag $(MULTI_ARCH_IMG):$(TAG) $(MULTI_ARCH_IMG):latest
+	rm -rf "${TEMP_DIR}"
 
 ifeq ($(ARCH), amd64)
 	# Adding check for amd64

README.md

@@ -35,6 +35,10 @@ the raw metrics.
 - [Resource recommendation](#resource-recommendation)
 - [A note on costing](#a-note-on-costing)
 - [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server)
+- [Scaling kube-state-metrics](#scaling-kube-state-metrics)
+  - [Resource recommendation](#resource-recommendation)
+  - [Horizontal scaling (sharding)](#horizontal-scaling-sharding)
+    - [Automated sharding](#automated-sharding)
 - [Setup](#setup)
   - [Building the Docker container](#building-the-docker-container)
 - [Usage](#usage)
@@ -91,10 +95,10 @@ additional metrics!
 > For now, the following metrics and collectors
 >
 > **metrics**
->	* kube_pod_container_resource_requests_nvidia_gpu_devices
->	* kube_pod_container_resource_limits_nvidia_gpu_devices
->	* kube_node_status_capacity_nvidia_gpu_cards
->	* kube_node_status_allocatable_nvidia_gpu_cards
+>	* `kube_pod_container_resource_requests_nvidia_gpu_devices`
+>	* `kube_pod_container_resource_limits_nvidia_gpu_devices`
+>	* `kube_node_status_capacity_nvidia_gpu_cards`
+>	* `kube_node_status_allocatable_nvidia_gpu_cards`
 >
 >	are removed in kube-state-metrics v1.4.0.
 >
@@ -104,11 +108,27 @@ additional metrics!
 See the [`docs`](docs) directory for more information on the exposed metrics.
 
 ### Kube-state-metrics self metrics
+
 kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 81).
 
-### Resource recommendation
+kube-state-metrics also exposes list and watch success and error metrics. These can be used to calculate the error rate of list or watch resources.
+If you encounter those errors in the metrics, it is most likely a configuration or permission issue, and the next thing to investigate would be looking
+at the logs of kube-state-metrics.
+
+Example of the above mentioned metrics:
+```
+kube_state_metrics_list_total{resource="*v1.Node",result="success"} 1
+kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52
+kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
+```
+
+### Scaling kube-state-metrics
 
-Resource usage for kube-state-metrics changes with the Kubernetes objects(Pods/Nodes/Deployments/Secrets etc.) size of the cluster.
+#### Resource recommendation
+
+> Note: These recommendations are based on scalability tests done over a year ago. They may differ significantly today.
+
+Resource usage for kube-state-metrics changes with the Kubernetes objects (Pods/Nodes/Deployments/Secrets etc.) size of the cluster.
 To some extent, the Kubernetes objects in a cluster are in direct proportion to the node number of the cluster.
 
 As a general rule, you should allocate
@@ -126,7 +146,8 @@ These numbers are based on [scalability tests](https://github.com/kubernetes/kub
 Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation, try increasing the CPU limits.
 
 ### A note on costing
-By default, kube-state-metrics exposes several metrics for events across your cluster. If you have a large number of frequently-updating resources on your cluster, you may find that a lot of data is ingested into these metrics. This can incur high costs on some cloud providers. Please take a moment to [configure what metrics you'd like to expose](docs/cli-arguments.md), as well as consult the documentation for your Kubernetes environment in order to avoid unexpectedly high costs.  
+
+By default, kube-state-metrics exposes several metrics for events across your cluster. If you have a large number of frequently-updating resources on your cluster, you may find that a lot of data is ingested into these metrics. This can incur high costs on some cloud providers. Please take a moment to [configure what metrics you'd like to expose](docs/cli-arguments.md), as well as consult the documentation for your Kubernetes environment in order to avoid unexpectedly high costs.
 
 ### kube-state-metrics vs. metrics-server
 
@@ -151,6 +172,25 @@ metric-server it too is not responsibile for exporting its metrics anywhere.
 Having kube-state-metrics as a separate project also enables access to these
 metrics from monitoring systems such as Prometheus.
 
+#### Horizontal scaling (sharding)
+
+In order to scale kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags:
+
+* `--shard` (zero indexed)
+* `--total-shards`
+
+Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it, with the total number of shards. The configured shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics even if sharded will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones it is responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. Overall memory consumption should be 1/n th of each shard compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus.
+
+Sharding should be used carefully, and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured).
+
+##### Automated sharding
+
+There is also an experimental feature, that allows kube-state-metrics to auto discover its nominal position if it is deployed in a StatefulSet, in order to automatically configure sharding. This is an experimental feature and may be broken or removed without notice.
+
+To enable automated sharding kube-state-metrics must be run by a `StatefulSet` and the pod names and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags.
+
+There are example manifests demonstrating the autosharding functionality in [`/kubernetes/autosharding`](/kubernetes/audosharding).
+
 ### Setup
 
 Install this project to your `$GOPATH` using `go get`:

RELEASE.md

@@ -13,9 +13,10 @@ If a bug fix got accidentally merged into master, cherry-pick commits have to be
 Maintaining the release branches for older minor releases happens on a best effort basis.
 
 ## Prepare your release
+
 * Bump the version in the `VERSION` file in the root of the repository.
 * Make a PR to update:
-  * Kube-state-metrics image tag for both `quay.io` and `staging-k8s.gcr.io`
+  * kube-state-metrics image tag for both `quay.io` and `staging-k8s.gcr.io`.
   * [Compatibility matrix](README.md#compatibility-matrix)
   * Changelog entry
     * Only include user relevant changes
@@ -27,8 +28,12 @@ Maintaining the release branches for older minor releases happens on a best effo
     [BUGFIX]
     ```
     * All lines should be full sentences
-  * Kube-state-metrics image tag used in Kubernetes deployment yaml config
+  * kube-state-metrics image tag used in Kubernetes deployment yaml config.
 * Cut the new release branch, i.e., `release-1.2`, or merge/cherry-pick changes onto the minor release branch you intend to tag the release on
 * Cut the new release tag, i.e., `v1.2.0-rc.0`
 * Ping Googlers(@loburm/@piosz) to build and push newest image to `k8s.gcr.io` (or to `staging-k8s.gcr.io` in case of release candidates)
 * Build and push newest image to `quay.io`(@brancz)
+
+## Stable release
+
+First a release candidate (e.g. `v1.2.0-rc.0`) should be cut. If after a period of 7 days no bugs or issues were reported after publishing the release candidate, a stable release (e.g. `v1.2.0`) can be cut.

SECURITY_CONTACTS

@@ -12,3 +12,5 @@
 
 brancz
 andyxning
+tariq1890
+LiliC

docs/cli-arguments.md

@@ -25,6 +25,7 @@ spec:
 ```txt
 $ kube-state-metrics -h
 Usage of ./kube-state-metrics:
+      --add_dir_header                              If true, adds the file directory to the header
       --alsologtostderr                             log to standard error as well as files
       --apiserver string                            The URL of the apiserver to use as a master
       --collectors string                           Comma-separated list of collectors to be enabled. Defaults to "certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,limitranges,namespaces,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses"
@@ -42,12 +43,16 @@ Usage of ./kube-state-metrics:
       --metric-blacklist string                     Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive.
       --metric-whitelist string                     Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive.
       --namespace string                            Comma-separated list of namespaces to be enabled. Defaults to ""
+      --pod string                                  Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
+      --pod-namespace string                        Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
       --port int                                    Port to expose metrics on. (default 80)
+      --shard int32                                 The instances shard nominal (zero indexed) within the total number of shards. (default 0)
       --skip_headers                                If true, avoid header prefixes in the log messages
       --skip_log_headers                            If true, avoid headers when opening log files
       --stderrthreshold severity                    logs at or above this threshold go to stderr (default 2)
       --telemetry-host string                       Host to expose kube-state-metrics self metrics on. (default "0.0.0.0")
       --telemetry-port int                          Port to expose kube-state-metrics self metrics on. (default 81)
+      --total-shards int                            The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
   -v, --v Level                                     number for the log level verbosity
       --version                                     kube-state-metrics build version information
       --vmodule moduleSpec                          comma-separated list of pattern=N settings for file-filtered logging

docs/ingress-metrics.md

@@ -7,3 +7,4 @@
 | kube_ingress_created  | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; | STABLE |
 | kube_ingress_metadata_resource_version  | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `resource_version`=&lt;ingress-resource-version&gt; | STABLE |
 | kube_ingress_path | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `host`=&lt;ingress-host&gt; <br> `path`=&lt;ingress-path&gt; <br> `service_name`=&lt;service name for the path&gt; <br> `service_port`=&lt;service port for hte path&gt; | STABLE |
+| kube_ingress_tls | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `tls_host`=&lt;tls hostname&gt; <br> `secret`=&lt;tls secret name&gt;| STABLE |