discovery: add metrics to endpoints controller workqueue

[zaharidichev]

This PR adds metrics to the work queue that is used in the external workload endpoints controller.

To test:

1. Apply the following diff to this branch:

diff --git a/controller/cmd/destination/main.go b/controller/cmd/destination/main.go
index 4edddbde5..b7e170819 100644
--- a/controller/cmd/destination/main.go
+++ b/controller/cmd/destination/main.go
@@ -9,6 +9,7 @@ import (
        "syscall"

        "github.com/linkerd/linkerd2/controller/api/destination"
+       externalworkload "github.com/linkerd/linkerd2/controller/api/destination/external-workload"
        "github.com/linkerd/linkerd2/controller/api/destination/watcher"
        "github.com/linkerd/linkerd2/controller/k8s"
        "github.com/linkerd/linkerd2/pkg/admin"
@@ -159,6 +160,9 @@ func Main(args []string) {
        metadataAPI.Sync(nil)
        clusterStore.Sync(nil)

+       ec, _ := externalworkload.NewEndpointsController(k8sAPI, "my-hostname", "default", done)
+       ec.Start()
+
        go func() {
                log.Infof("starting gRPC server on %s", *addr)
                if err := server.Serve(lis); err != nil {

2. Create the following resources:

apiVersion: v1
kind: Service
metadata:
  name: nginx-svc
spec:
  selector:
    app: nginx
  ports:
    - name: http
      protocol: TCP
      port: 80
---
 apiVersion: workload.linkerd.io/v1alpha1
 kind: ExternalWorkload
 metadata:
  name: test
  labels:
    app: nginx
 spec:
  meshTls:
    serverName: "foo"
    identity: "foo"
  workloadIPs:
  - ip: "192.0.2.0"
  ports:
  - port: 80

3. Hit the /metrics endpoint of the destination controller:
4. Observe that queue metrics are present:

# HELP workqueue_adds_total Total number of adds handled by workqueue
# TYPE workqueue_adds_total counter
workqueue_adds_total{name="endpoints_controller_workqueue"} 9
# HELP workqueue_depth Current depth of workqueue
# TYPE workqueue_depth gauge
workqueue_depth{name="endpoints_controller_workqueue"} 0
# HELP workqueue_longest_running_processor_seconds How many seconds has the longest running processor for workqueue been running.
# TYPE workqueue_longest_running_processor_seconds gauge
workqueue_longest_running_processor_seconds{name="endpoints_controller_workqueue"} 0
# HELP workqueue_queue_duration_seconds How long in seconds an item stays in workqueue before being requested.
# TYPE workqueue_queue_duration_seconds histogram
workqueue_queue_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-08"} 0
workqueue_queue_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-07"} 0
workqueue_queue_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-06"} 0

Signed-off-by: Zahari Dichev zaharidichev@gmail.com

[mateiidavid]

Nice 👍🏻 LGTM

workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-08"} 0
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-07"} 0
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1e-06"} 0
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="9.999999999999999e-06"} 6
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="9.999999999999999e-05"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="0.001"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="0.01"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="0.1"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="1"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="10"} 10
workqueue_work_duration_seconds_bucket{name="endpoints_controller_workqueue",le="+Inf"} 10
workqueue_work_duration_seconds_sum{name="endpoints_controller_workqueue"} 0.00010890500000000002
workqueue_work_duration_seconds_count{name="endpoints_controller_workqueue"} 10