feat(arc): add Prometheus monitoring and Grafana dashboard

.github/CODEOWNERS

@@ -1 +1 @@
-* @xnoto
+* @makeitworkcloud/admins

operators/arc/arcsystem.yaml

@@ -37,6 +37,11 @@ spec:
     targetRevision: 0.13.1
     helm:
       releaseName: arc
+      valuesObject:
+        metrics:
+          controllerManagerAddr: ":8080"
+          listenerAddr: ":8080"
+          listenerEndpoint: /metrics
   destination:
     server: "https://kubernetes.default.svc"
     namespace: arc-systems

workloads/arc/arc-tf-application.yaml

@@ -28,7 +28,77 @@ spec:
         githubConfigSecret: arc-runner-github-token
         githubConfigUrl: https://github.com/makeitworkcloud
         maxRunners: 3
-        minRunners: 0
+        minRunners: 1
+        listenerTemplate:
+          spec:
+            containers:
+              - name: listener
+                ports:
+                  - name: metrics
+                    containerPort: 8080
+                    protocol: TCP
+        listenerMetrics:
+          counters:
+            gha_started_jobs_total:
+              labels:
+                - repository
+                - organization
+                - enterprise
+                - job_name
+                - event_name
+                - job_workflow_ref
+                - job_workflow_name
+                - job_workflow_target
+            gha_completed_jobs_total:
+              labels:
+                - repository
+                - organization
+                - enterprise
+                - job_name
+                - event_name
+                - job_result
+                - job_workflow_ref
+                - job_workflow_name
+                - job_workflow_target
+          gauges:
+            gha_assigned_jobs:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_running_jobs:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_registered_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_busy_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_min_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_max_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_desired_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+            gha_idle_runners:
+              labels: [name, namespace, repository, organization, enterprise]
+          histograms:
+            gha_job_startup_duration_seconds:
+              labels:
+                - repository
+                - organization
+                - enterprise
+                - job_name
+                - event_name
+                - job_workflow_ref
+                - job_workflow_name
+                - job_workflow_target
+            gha_job_execution_duration_seconds:
+              labels:
+                - repository
+                - organization
+                - enterprise
+                - job_name
+                - event_name
+                - job_result
+                - job_workflow_ref
+                - job_workflow_name
+                - job_workflow_target
         template:
           spec:
             serviceAccountName: arc-tf-runner

workloads/arc/kustomization.yaml

@@ -5,5 +5,6 @@ resources:
   - namespace.yaml
   - runner-rbac.yaml
   - arc-tf-application.yaml
+  - metrics-monitoring.yaml
 generators:
   - ksops-arc-secrets.yaml

workloads/arc/metrics-monitoring.yaml

@@ -0,0 +1,85 @@
+---
+# ARC publishes Prometheus metrics from the controller-manager and scale-set
+# listener pods. Ephemeral runner pods do not expose separate ARC metrics;
+# their health and resource usage are covered by the cluster's standard pod
+# metrics and kube-state-metrics.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: arc-controller-metrics
+  namespace: arc-systems
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+    ignore-check.kube-linter.io/dangling-service: "Targets ARC chart-managed controller pods not declared in this repo"
+  labels:
+    app.kubernetes.io/component: controller-manager
+    app.kubernetes.io/name: gha-rs-controller
+spec:
+  selector:
+    app.kubernetes.io/component: controller-manager
+    app.kubernetes.io/name: gha-rs-controller
+  ports:
+    - name: metrics
+      port: 8080
+      targetPort: 8080
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: arc-controller
+  namespace: arc-systems
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  namespaceSelector:
+    matchNames:
+      - arc-systems
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller-manager
+      app.kubernetes.io/name: gha-rs-controller
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: arc-tf-listener-metrics
+  namespace: arc-runners
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+    ignore-check.kube-linter.io/dangling-service: "Targets ARC listener pods created dynamically by the scale-set controller"
+  labels:
+    actions.github.com/scale-set-name: arc-tf
+    app.kubernetes.io/component: runner-scale-set-listener
+spec:
+  selector:
+    actions.github.com/scale-set-name: arc-tf
+    app.kubernetes.io/component: runner-scale-set-listener
+  ports:
+    - name: metrics
+      port: 8080
+      targetPort: 8080
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: arc-tf-listener
+  namespace: arc-runners
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  namespaceSelector:
+    matchNames:
+      - arc-runners
+  selector:
+    matchLabels:
+      actions.github.com/scale-set-name: arc-tf
+      app.kubernetes.io/component: runner-scale-set-listener
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s