Missing Grafana Operator on OpenShift cluster

1. Verify OLM setup on your OpenShift cluster

   Example output:

   [root@helper ~]# oc get olm -A
   NAMESPACE                              NAME                                                   AGE
   openshift-operator-lifecycle-manager   operatorcondition.operators.coreos.com/packageserver   160m
   
   NAMESPACE   NAME                                     AGE
   olmconfig.operators.coreos.com/cluster   163m
   
   NAMESPACE                              NAME                                                              AGE
   openshift-monitoring                   operatorgroup.operators.coreos.com/openshift-cluster-monitoring   163m
   openshift-operator-lifecycle-manager   operatorgroup.operators.coreos.com/olm-operators                  163m
   openshift-operators                    operatorgroup.operators.coreos.com/global-operators               163m
   
   NAMESPACE               NAME                                                     DISPLAY               TYPE   PUBLISHER   AGE
   openshift-marketplace   catalogsource.operators.coreos.com/certified-operators   Certified Operators   grpc   Red Hat     159m
   openshift-marketplace   catalogsource.operators.coreos.com/community-operators   Community Operators   grpc   Red Hat     159m
   openshift-marketplace   catalogsource.operators.coreos.com/redhat-marketplace    Red Hat Marketplace   grpc   Red Hat     159m
   openshift-marketplace   catalogsource.operators.coreos.com/redhat-operators      Red Hat Operators     grpc   Red Hat     159m
   
   NAMESPACE                              NAME                                                       DISPLAY          VERSION   REPLACES                                     PHASE
   openshift-operator-lifecycle-manager   clusterserviceversion.operators.coreos.com/packageserver   Package Server   0.19.0                                                 Succeeded
   [root@helper ~]#

   Make note of the namespace name where the Community Operators are deployed (default namespace: "openshift-marketplace")

   Note: If you don't see OLM resources on your cluster, please check RedHat OpenShift documentation about installing cluster operators.

2. List resources of the namespace where the Community Operators are deployed

   Example output:

   [root@helper ~]# oc project openshift-marketplace
   Now using project "openshift-marketplace" on server "https://api.ocp1.kvmlocal:6443".
   
   [root@helper ~]# oc get all
   NAME                                       READY   STATUS             RESTARTS       AGE
   pod/certified-operators-g7xgr              0/1     ImagePullBackOff   0              165m
   pod/certified-operators-gv6xs              0/1     ErrImagePull       0              154m
   pod/community-operators-bb2sr              0/1     ImagePullBackOff   0              153m
   pod/community-operators-tpzb7              0/1     ImagePullBackOff   0              165m
   pod/marketplace-operator-68ddcd9c6-57m7v   1/1     Running            1 (158m ago)   169m
   pod/redhat-marketplace-4mf5l               0/1     ImagePullBackOff   0              153m
   pod/redhat-marketplace-hvfzz               0/1     ImagePullBackOff   0              165m
   pod/redhat-operators-7q4p7                 0/1     ImagePullBackOff   0              165m
   pod/redhat-operators-bd4dc                 0/1     ImagePullBackOff   0              154m
   
   NAME                                   TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
   service/certified-operators            ClusterIP   172.30.144.63    <none>        50051/TCP           165m
   service/community-operators            ClusterIP   172.30.194.181   <none>        50051/TCP           165m
   service/marketplace-operator-metrics   ClusterIP   172.30.62.173    <none>        8383/TCP,8081/TCP   169m
   service/redhat-marketplace             ClusterIP   172.30.2.123     <none>        50051/TCP           165m
   service/redhat-operators               ClusterIP   172.30.38.189    <none>        50051/TCP           165m
   
   NAME                                   READY   UP-TO-DATE   AVAILABLE   AGE
   deployment.apps/marketplace-operator   1/1     1            1           169m
   
   NAME                                             DESIRED   CURRENT   READY   AGE
   replicaset.apps/marketplace-operator-68ddcd9c6   1         1         1       169m
   

3. Check status and events of the community-operator pods

   Example output:

   [root@helper ~]# oc get pod/community-operators-tpzb7 -o jsonpath='{}' | jq '.status.phase'
   "Pending"
   [root@helper ~]# oc get pod/community-operators-tpzb7 -o jsonpath='{}' | jq '.status.containerStatuses[]'
   {
   	"image": "registry.redhat.io/redhat/community-operator-index:v4.11",
   	"imageID": "",
   	"lastState": {},
   	"name": "registry-server",
   	"ready": false,
   	"restartCount": 0,
   	"started": false,
   	"state": {
   		"waiting": {
     			"message": "Back-off pulling image \"registry.redhat.io/redhat/community-operator-index:v4.11\"",
     			"reason": "ImagePullBackOff"
   		}
   	}
   }
   
   [root@helper ~]# oc get event --field-selector involvedObject.name=community-operators-tpzb7,involvedObject.kind=Pod
   LAST SEEN   TYPE      REASON           OBJECT                          MESSAGE
   3h7m        Normal    Scheduled        pod/community-operators-tpzb7   Successfully assigned openshift-marketplace/community-operators-tpzb7 to master2.ocp1.kvmlocal by bootstrap.ocp1.kvmlocal
   3h7m        Normal    AddedInterface   pod/community-operators-tpzb7   Add eth0 [10.254.1.15/24] from openshift-sdn
   3h6m        Normal    Pulling          pod/community-operators-tpzb7   Pulling image "registry.redhat.io/redhat/community-operator-index:v4.11"
   3h6m        Warning   Failed           pod/community-operators-tpzb7   Failed to pull image "registry.redhat.io/redhat/community-operator-index:v4.11": rpc error: code = Unknown desc = unable to retrieve auth token: invalid username/password: unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/articles/3399531
   3h6m        Warning   Failed           pod/community-operators-tpzb7   Error: ErrImagePull
   2m36s       Normal    BackOff          pod/community-operators-tpzb7   Back-off pulling image "registry.redhat.io/redhat/community-operator-index:v4.11"
   167m        Warning   Failed           pod/community-operators-tpzb7   Error: ImagePullBackOff
   [root@helper ~]#
   

4. Follow instructions described in https://access.redhat.com/articles/3399531 and make sure your credentials for accessing "registry.redhat.io" are stored in global pull secret

   Example output:

   [root@helper ~]# oc get secret/pull-secret -n openshift-config -ojson | jq -r '.data[".dockerconfigjson"]' | base64 -d - | jq '.auths | with_entries(select(.key |contains("redhat.io")))'
   
   "registry.redhat.io": {
     "username": "my-user-name",
     "password": "eyJhbGciOiJSUzUxMiJ9.eyJzdWIiOiJkOGYzODJmNjQ4ODU0ZmQ5YmI0N2Y0MDM4NjRkYjYwNiJ9.p_MxSU7cWtJ0-G1xHxHp2usvnLdv-XrOEikG_yO1L_WEdEcPxLQt55_oQwCscE5veQsXcjBTjziRj1iWXFuw0bzbGNcAkEjomB3-PhsKX-b3saltcfmcFQscDER4ZSc6veo9-_f_EyHVXG1FUFOk3XwR8SGSw9xMEo9NjsQRnaAKFHQ7CWOMN4kVIDzS8SO3wbC9kbLI6ufLrjxI1DvgT8UV_QH9MD3XYelVhfqu5Pv2jX3WeFBmR_az6PPXM_25Qdk5nfF799NOfj7uv2ksc5eeEu_-ADwsxzSicer4OqiqdIV85V6E5IyOrONRQzHZogYxE3DL2o_qFbdpAeCOZQOrbS8dJYtp8bjeGto2VSd3rnG1hdWxYl3JxbbDANInS_qiNO6SvBapQgk9O_LPb-xhJJiFQimbjnVrNrgkRp41lBnzen2FwPyWh_Va5DPX__DYTYEliXL0tdXmRmpLQa3dxSBp5XhJMEXN6SdQoFFkox3R53rbPCWeU9lLnyGGy0E_WFbLa0-F2Ss0pm7tr7aVGWOjkwU4a7nemDqLWU1ocgwHyFMZeIi7m6Ans9Rho6EdSD7TxyhxRx6bAk5-ByLj0AHuaxvF63r2Yf82oYpHmv2VHDRKA5F1CF2N4Q1myq5CGFNDpy7IlaG6wHqQ9b_VRGvjdoxqdOZGGYuZtS0",
     "auth": "NTI4MzcxMDB8dWhjLTFRTEdMSm1NNzlzRmlmMUUyb2lLWWdCcFhlSTpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSmtPR1l6T0RKbU5qUTRPRFUwWm1RNVltSTBOMlkwTURNNE5qUmtZall3TmlKOS5wX014U1U3Y1d0SjAtRzF4SHhIcDJ1c3ZuTGR2LVhyT0Vpa0dfeU8xTF9XRWRFY1B4TFF0NTVfb1F3Q3NjRTV2ZVFzWGNqQlRqemlSajFpV1hGdXcwYnpiR05jQWtFam9tQjMtUGhzS1gtYjNzYWx0Y2ZtY0ZRc2NERVI0WlNjNnZlbzktX2ZfRXlIVlhHMUZVRk9rM1h3UjhTR1N3OXhNRW85TmpzUVJuYUFLRkhRN0NXT01ONGtWSUR6UzhTTzN3YkM5a2JMSTZ1ZkxyanhJMUR2Z1Q4VVZfUUg5TUQzWFllbFZoZnF1NVB2MmpYM1dlRkJtUl9hejZQUFhNXzI1UWRrNW5mRjc5OU5PZmo3dXYya3NjNWVlRXVfLUFEd3N4elNpY2VyNE9xaXFkSVY4NVY2RTVJeU9yT05SUXpIWm9nWXhFM0RMMm9fcUZiZHBBZUNPWlFPcmJTOGRKWXRwOGJqZUd0bzJWU2Qzcm5HMWhkV3hZbDNKeGJiREFOSW5TX3FpTk82U3ZCYXBRZ2s5T19MUGIteGhKSmlGUWltYmpuVnJOcmdrUnA0MWxCbnplbjJGd1B5V2hfVmE1RFBYX19EWVRZRWxpWEwwdGRYbVJtcExRYTNkeFNCcDVYaEpNRVhONlNkUW9GRmtveDNSNTNyYlBDV2VVOWxMbnlHR3kwRV9XRmJMYTAtRjJTczBwbTd0cjdhVkdXT2prd1U0YTduZW1EcUxXVTFvY2d3SHlGTVplSWk3bTZBbnM5UmhvNkVkU0Q3VHh5aHhSeDZiQWs1LUJ5TGowQUh1YXh2RjYzcjJZZjgyb1lwSG12MlZIRFJLQTVGMUNGMk40UTFteXE1Q0dGTkRweTdJbGFHNndIcVE5Yl9WUkd2amRveHFkT1pHR1l1WnRTMA==",
     "email": "my-user-name@example.com"
   }
   

Related reference material:

• https://docs.openshift.com/container-platform/4.10/operators/understanding/olm/olm-understanding-olm.html
• https://docs.openshift.com/container-platform/4.10/operators/index.html#operators-overview-administrator-tasks
• https://access.redhat.com/RegistryAuthentication#creating-registry-service-accounts-6
• https://access.redhat.com/solutions/4902871