ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist

[willzhang]

env

[root@ceph ~]# cat /etc/redhat-release 
CentOS Linux release 7.8.2003 (Core)

[root@ceph ~]# lsblk
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda               8:0    0   70G  0 disk 
├─sda1            8:1    0    1G  0 part /boot
└─sda2            8:2    0   69G  0 part 
  ├─centos-root 253:0    0 60.1G  0 lvm  /
  ├─centos-swap 253:1    0  3.9G  0 lvm  [SWAP]
  └─centos-home 253:2    0    5G  0 lvm  /home
sdb               8:16   0   20G  0 disk 
sdc               8:32   0   20G  0 disk 
sr0              11:0    1    1G  0 rom  

what i do

# run mon
docker run -d --net=host \
  --restart always \
  --name monitor \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  -e MON_IP=192.168.93.31 \
  -e CEPH_PUBLIC_NETWORK=192.168.93.0/24 \
  ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7 mon


# run mgr
docker run -d --net=host \
  --restart always \
  --name manager \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7 mgr


# run osd
docker run -d --net=host \
  --restart always \
  --name osd \
  --privileged=true \
  --pid=host \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  -v /dev/:/dev/ \
  -v /run/udev/:/run/udev/ \
  -e OSD_DEVICE=/dev/sdb \
  -e OSD_TYPE=disk \
  ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7 osd

what happend

osd not start

[root@ceph ~]# docker ps -a
CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS                          PORTS               NAMES
c7ae0a91aeec        ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7   "/opt/ceph-container…"   3 minutes ago       Restarting (1) 40 seconds ago                       osd
a76f27996860        ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7   "/opt/ceph-container…"   3 minutes ago       Up 3 minutes                                        manager
b1fd21dd3f14        ceph/daemon:v4.0.12-stable-4.0-nautilus-centos-7   "/opt/ceph-container…"   3 minutes ago       Up 3 minutes                                        monitor

logs

[root@ceph ~]# docker logs osd
2020-06-15 17:07:50  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:50  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:50  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:50  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:51  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:51  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:51  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:51  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:52  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:52  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:54  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:54  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:07:57  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:07:57  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:08:04  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:08:04  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:08:17  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:08:17  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-15 17:08:43  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-15 17:08:43  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'

then

[root@ceph ~]# docker exec -it monitor sh
sh-4.2# ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
exported keyring for client.bootstrap-osd

error

[root@ceph ~]# docker logs osd
2020-06-16 17:05:43  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:05:43  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
2020-06-16 17:05:43  /opt/ceph-container/bin/entrypoint.sh: ERROR- /var/lib/ceph/bootstrap-osd/ceph.keyring must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring'
2020-06-16 17:05:56  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:05:56  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:05:56  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:19  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:20  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:20  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:20  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:21  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:21  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:21  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:23  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:23  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:23  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:25  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:25  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:25  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
2020-06-16 17:06:29  /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2020-06-16 17:06:29  /opt/ceph-container/bin/entrypoint.sh: Device detected, assuming ceph-disk scenario is desired
2020-06-16 17:06:29  /opt/ceph-container/bin/entrypoint.sh: Preparing and activating /dev/sdb
HEALTH_WARN OSD count 0 < osd_pool_default_size 3
/opt/ceph-container/bin/osd_disk_prepare.sh: line 40: ceph-disk: command not found
[root@ceph ~]# 

[willzhang]

only ceph/daemon:latest-mimic tag is ok, why?

docker run -d --net=host \
  --restart always \
  --name monitor \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  -e MON_IP=192.168.93.31 \
  -e CEPH_PUBLIC_NETWORK=192.168.93.0/24 \
  ceph/daemon:latest-mimic mon


启动manager
docker run -d --net=host \
  --restart always \
  --name manager \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  ceph/daemon:latest-mimic mgr


启动osd，同时需要指定硬盘
docker run -d --net=host \
  --restart always \
  --name osd \
  --privileged=true \
  --pid=host \
  -v /etc/ceph:/etc/ceph \
  -v /var/lib/ceph/:/var/lib/ceph/ \
  -v /dev/:/dev/ \
  -v /run/udev/:/run/udev/ \
  -e OSD_DEVICE=/dev/sdb \
  -e OSD_TYPE=disk \
  ceph/daemon:latest-mimic osd

container status

[root@ceph ~]# docker ps
CONTAINER ID        IMAGE                      COMMAND                  CREATED              STATUS              PORTS               NAMES
9add9602f2c0        ceph/daemon:latest-mimic   "/opt/ceph-container…"   About a minute ago   Up About a minute                       osd
3e6cacc346cd        ceph/daemon:latest-mimic   "/opt/ceph-container…"   About a minute ago   Up About a minute                       manager
be0ca55f54e6        ceph/daemon:latest-mimic   "/opt/ceph-container…"   About a minute ago   Up About a minute                       monitor

[willzhang]

ok ，t's a long-standing problem，not fixed now，but way？

[stale]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[snowolfhawk]

I meet with the same problem, do anyone fixed it?