Issue with attaching volume to container

[allanian]

hello, i installed openebs cstor, all working fine (i test it with busybox container and its mounted to the pod), trying install prometheus, can you help me to find the correct way to solve it?

helm upgrade --install prometheus -n monitoring prometheus-community/kube-prometheus-stack --version 23.3.1 \
  --set prometheus.enabled=false \
  --set grafana.enabled=false \
  --set nodeExporter.enabled=false \
  --set alertmanager.enabled=true \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.storageClassName=cstor-csi-disk2 \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.accessModes[0]=ReadWriteOnce \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.resources.requests.storage=5Gi

[root@k8s-master1 prometheus]# kubectl -n monitoring get po
NAME                                                     READY   STATUS              RESTARTS   AGE
alertmanager-prometheus-kube-prometheus-alertmanager-0   0/2     ContainerCreating   0          2m27s
prometheus-kube-prometheus-operator-54d7d5c88c-cmktj     1/1     Running             0          10m
prometheus-kube-state-metrics-5f76df6f9-6mgbk            1/1     Running             0          10m

[root@k8s-master1 prometheus]# kubectl get pvc -n monitoring
NAME                                                                                                             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
alertmanager-prometheus-kube-prometheus-alertmanager-db-alertmanager-prometheus-kube-prometheus-alertmanager-0   Bound    pvc-cb144ccc-0d7f-4896-b661-6c3b5a049641   5Gi        RWO            cstor-csi-disk2   2m57s

Events:
  Type     Reason            Age                 From               Message
  ----     ------            ----                ----               -------
  Warning  FailedScheduling  3m19s               default-scheduler  0/9 nodes are available: 9 pod has unbound immediate PersistentVolumeClaims.
  Normal   Scheduled         3m17s               default-scheduler  Successfully assigned monitoring/alertmanager-prometheus-kube-prometheus-alertmanager-0 to k8s-node6
  Warning  FailedMount       75s                 kubelet            Unable to attach or mount volumes: unmounted volumes=[alertmanager-prometheus-kube-prometheus-alertmanager-db], unattached volumes=[tls-assets alertmanager-prometheus-kube-prometheus-alertmanager-db kube-api-access-7vzwc config-volume]: timed out waiting for the condition
  Warning  FailedMount       54s (x8 over 3m8s)  kubelet            MountVolume.MountDevice failed for volume "pvc-cb144ccc-0d7f-4896-b661-6c3b5a049641" : rpc error: code = Internal desc = Waiting for pvc-cb144ccc-0d7f-4896-b661-6c3b5a049641's CVC to be bound

other checks

### iscsiadm 
[root@k8s-node6 ~]# sudo iscsiadm -m session
iscsiadm: No active sessions.
[root@k8s-node6 ~]# sudo iscsiadm -m session -P 3
iscsiadm: No active sessions.

### FDISK
[root@k8s-node6 ~]# fdisk -l
Disk /dev/nvme1n1: 2.9 TiB, 3200631791616 bytes, 6251233968 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: A99024C5-5D57-3341-B002-F902C0EE4910

Device              Start        End    Sectors  Size Type
/dev/nvme1n1p1       2048 6251216895 6251214848  2.9T Solaris /usr & Apple ZFS
/dev/nvme1n1p9 6251216896 6251233279      16384    8M Solaris reserved 1


Disk /dev/nvme0n1: 2.9 TiB, 3200631791616 bytes, 6251233968 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: EC496BF5-CF45-BC48-9B89-573651605625

Device              Start        End    Sectors  Size Type
/dev/nvme0n1p1       2048 6251216895 6251214848  2.9T Solaris /usr & Apple ZFS
/dev/nvme0n1p9 6251216896 6251233279      16384    8M Solaris reserved 1

Disk /dev/sda: 446.6 GiB, 479559942144 bytes, 936640512 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 262144 bytes / 262144 bytes
Disklabel type: gpt
Disk identifier: 7A1BEB4B-705B-4A6D-8A07-EA1A155A5081

Device       Start       End   Sectors  Size Type
/dev/sda1     2048   1230847   1228800  600M EFI System
/dev/sda2  1230848   3327999   2097152    1G Linux filesystem
/dev/sda3  3328000 936638463 933310464  445G Linux LVM

Disk /dev/mapper/cs-root: 70 GiB, 75161927680 bytes, 146800640 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 262144 bytes / 262144 bytes

Disk /dev/mapper/cs-swap: 2 GiB, 2147483648 bytes, 4194304 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 262144 bytes / 262144 bytes

Disk /dev/mapper/cs-home: 50 GiB, 53687091200 bytes, 104857600 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 262144 bytes / 262144 bytes

Disk /dev/mapper/cs-var: 323 GiB, 346852163584 bytes, 677445632 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 262144 bytes / 262144 bytes

#### LSBLK
[root@k8s-node6 ~]# lsblk
NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sda           8:0    0 446.6G  0 disk 
├─sda1        8:1    0   600M  0 part /boot/efi
├─sda2        8:2    0     1G  0 part /boot
└─sda3        8:3    0   445G  0 part 
  ├─cs-root 253:0    0    70G  0 lvm  /
  ├─cs-swap 253:1    0     2G  0 lvm  
  ├─cs-home 253:2    0    50G  0 lvm  /home
  └─cs-var  253:3    0   323G  0 lvm  /var
nvme1n1     259:0    0   2.9T  0 disk 
├─nvme1n1p1 259:6    0   2.9T  0 part 
└─nvme1n1p9 259:7    0     8M  0 part 
nvme0n1     259:1    0   2.9T  0 disk 
├─nvme0n1p1 259:8    0   2.9T  0 part 
└─nvme0n1p9 259:9    0     8M  0 part 
[root@k8s-node6 ~]# 
[root@k8s-node6 ~]# cat /etc/os-release
NAME="CentOS Stream"
VERSION="8"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="8"
PLATFORM_ID="platform:el8"
PRETTY_NAME="CentOS Stream 8"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:8"
HOME_URL="https://centos.org/"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8"
REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"