Skip to content

Commit

Permalink
vhost-user-scsi: support reconnect to backend
Browse files Browse the repository at this point in the history 
If the backend crashes and restarts, the device is broken.
This patch adds reconnect for vhost-user-scsi.

This patch also improves the error messages, and reports some silent errors.

Tested with spdk backend.

Signed-off-by: Li Feng <fengli@smartx.com>
Message-Id: <20231009044735.941655-4-fengli@smartx.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Reviewed-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
  • Loading branch information
@mstsirkin
Li Feng authored and mstsirkin committed Oct 22, 2023
1 parent 4dfcc09 commit 7962e43
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 30 deletions.
16 changes: 9 additions & 7 deletions hw/scsi/vhost-scsi-common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/

#include "qemu/osdep.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "qemu/module.h"
#include "hw/virtio/vhost.h"
Expand All @@ -25,7 +26,7 @@
#include "hw/virtio/virtio-access.h"
#include "hw/fw-path-provider.h"

int vhost_scsi_common_start(VHostSCSICommon *vsc)
int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp)
{
int ret, i;
VirtIODevice *vdev = VIRTIO_DEVICE(vsc);
Expand All @@ -35,26 +36,27 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
VirtIOSCSICommon *vs = (VirtIOSCSICommon *)vsc;

if (!k->set_guest_notifiers) {
error_report("binding does not support guest notifiers");
error_setg(errp, "binding does not support guest notifiers");
return -ENOSYS;
}

ret = vhost_dev_enable_notifiers(&vsc->dev, vdev);
if (ret < 0) {
error_setg_errno(errp, -ret, "Error enabling host notifiers");
return ret;
}

ret = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, true);
if (ret < 0) {
error_report("Error binding guest notifier");
error_setg_errno(errp, -ret, "Error binding guest notifier");
goto err_host_notifiers;
}

vsc->dev.acked_features = vdev->guest_features;

ret = vhost_dev_prepare_inflight(&vsc->dev, vdev);
if (ret < 0) {
error_report("Error setting inflight format: %d", -ret);
error_setg_errno(errp, -ret, "Error setting inflight format");
goto err_guest_notifiers;
}

Expand All @@ -64,21 +66,21 @@ int vhost_scsi_common_start(VHostSCSICommon *vsc)
vs->conf.virtqueue_size,
vsc->inflight);
if (ret < 0) {
error_report("Error getting inflight: %d", -ret);
error_setg_errno(errp, -ret, "Error getting inflight");
goto err_guest_notifiers;
}
}

ret = vhost_dev_set_inflight(&vsc->dev, vsc->inflight);
if (ret < 0) {
error_report("Error setting inflight: %d", -ret);
error_setg_errno(errp, -ret, "Error setting inflight");
goto err_guest_notifiers;
}
}

ret = vhost_dev_start(&vsc->dev, vdev, true);
if (ret < 0) {
error_report("Error start vhost dev");
error_setg_errno(errp, -ret, "Error starting vhost dev");
goto err_guest_notifiers;
}

Expand Down
6 changes: 4 additions & 2 deletions hw/scsi/vhost-scsi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ static int vhost_scsi_start(VHostSCSI *s)
int ret, abi_version;
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
const VhostOps *vhost_ops = vsc->dev.vhost_ops;
Error *local_err = NULL;

ret = vhost_ops->vhost_scsi_get_abi_version(&vsc->dev, &abi_version);
if (ret < 0) {
Expand All @@ -88,14 +89,15 @@ static int vhost_scsi_start(VHostSCSI *s)
return -ENOSYS;
}

ret = vhost_scsi_common_start(vsc);
ret = vhost_scsi_common_start(vsc, &local_err);
if (ret < 0) {
error_reportf_err(local_err, "Error starting vhost-scsi");
return ret;
}

ret = vhost_scsi_set_endpoint(s);
if (ret < 0) {
error_report("Error setting vhost-scsi endpoint");
error_reportf_err(local_err, "Error setting vhost-scsi endpoint");
vhost_scsi_common_stop(vsc);
}

Expand Down
201 changes: 181 additions & 20 deletions hw/scsi/vhost-user-scsi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,41 +39,181 @@ static const int user_feature_bits[] = {
VHOST_INVALID_FEATURE_BIT
};

static int vhost_user_scsi_start(VHostUserSCSI *s, Error **errp)
{
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
int ret;

ret = vhost_scsi_common_start(vsc, errp);
s->started_vu = !(ret < 0);

return ret;
}

static void vhost_user_scsi_stop(VHostUserSCSI *s)
{
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);

if (!s->started_vu) {
return;
}
s->started_vu = false;

vhost_scsi_common_stop(vsc);
}

static void vhost_user_scsi_set_status(VirtIODevice *vdev, uint8_t status)
{
VHostUserSCSI *s = (VHostUserSCSI *)vdev;
DeviceState *dev = DEVICE(vdev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
bool start = (status & VIRTIO_CONFIG_S_DRIVER_OK) && vdev->vm_running;
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
bool should_start = virtio_device_should_start(vdev, status);
Error *local_err = NULL;
int ret;

if (vhost_dev_is_started(&vsc->dev) == start) {
if (!s->connected) {
return;
}

if (start) {
int ret;
if (vhost_dev_is_started(&vsc->dev) == should_start) {
return;
}

ret = vhost_scsi_common_start(vsc);
if (should_start) {
ret = vhost_user_scsi_start(s, &local_err);
if (ret < 0) {
error_report("unable to start vhost-user-scsi: %s", strerror(-ret));
exit(1);
error_reportf_err(local_err, "unable to start vhost-user-scsi: %s",
strerror(-ret));
qemu_chr_fe_disconnect(&vs->conf.chardev);
}
} else {
vhost_scsi_common_stop(vsc);
vhost_user_scsi_stop(s);
}
}

static void vhost_dummy_handle_output(VirtIODevice *vdev, VirtQueue *vq)
{
}

static int vhost_user_scsi_connect(DeviceState *dev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
int ret = 0;

if (s->connected) {
return 0;
}
s->connected = true;

vsc->dev.num_queues = vs->conf.num_queues;
vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
vsc->dev.vqs = s->vhost_vqs;
vsc->dev.vq_index = 0;
vsc->dev.backend_features = 0;

ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
errp);
if (ret < 0) {
return ret;
}

/* restore vhost state */
if (virtio_device_started(vdev, vdev->status)) {
ret = vhost_user_scsi_start(s, errp);
}

return ret;
}

static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event);

static void vhost_user_scsi_disconnect(DeviceState *dev)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);

if (!s->connected) {
return;
}
s->connected = false;

vhost_user_scsi_stop(s);

vhost_dev_cleanup(&vsc->dev);

/* Re-instate the event handler for new connections */
qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
vhost_user_scsi_event, NULL, dev, NULL, true);
}

static void vhost_user_scsi_event(void *opaque, QEMUChrEvent event)
{
DeviceState *dev = opaque;
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(vdev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
Error *local_err = NULL;

switch (event) {
case CHR_EVENT_OPENED:
if (vhost_user_scsi_connect(dev, &local_err) < 0) {
error_report_err(local_err);
qemu_chr_fe_disconnect(&vs->conf.chardev);
return;
}
break;
case CHR_EVENT_CLOSED:
/* defer close until later to avoid circular close */
vhost_user_async_close(dev, &vs->conf.chardev, &vsc->dev,
vhost_user_scsi_disconnect);
break;
case CHR_EVENT_BREAK:
case CHR_EVENT_MUX_IN:
case CHR_EVENT_MUX_OUT:
/* Ignore */
break;
}
}

static int vhost_user_scsi_realize_connect(VHostUserSCSI *s, Error **errp)
{
DeviceState *dev = DEVICE(s);
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
int ret;

s->connected = false;

ret = qemu_chr_fe_wait_connected(&vs->conf.chardev, errp);
if (ret < 0) {
return ret;
}

ret = vhost_user_scsi_connect(dev, errp);
if (ret < 0) {
qemu_chr_fe_disconnect(&vs->conf.chardev);
return ret;
}
assert(s->connected);

return 0;
}

static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
{
ERRP_GUARD();
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(dev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
struct vhost_virtqueue *vqs = NULL;
Error *err = NULL;
int ret;
int retries = VU_REALIZE_CONN_RETRIES;

if (!vs->conf.chardev.chr) {
error_setg(errp, "vhost-user-scsi: missing chardev");
Expand All @@ -92,18 +232,28 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
goto free_virtio;
}

vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues;
vsc->dev.vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs);
vsc->dev.vq_index = 0;
vsc->dev.backend_features = 0;
vqs = vsc->dev.vqs;
vsc->inflight = g_new0(struct vhost_inflight, 1);
s->vhost_vqs = g_new0(struct vhost_virtqueue,
VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues);

assert(!*errp);
do {
if (*errp) {
error_prepend(errp, "Reconnecting after error: ");
error_report_err(*errp);
*errp = NULL;
}
ret = vhost_user_scsi_realize_connect(s, errp);
} while (ret < 0 && retries--);

ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
VHOST_BACKEND_TYPE_USER, 0, errp);
if (ret < 0) {
goto free_vhost;
}

/* we're fully initialized, now we can operate, so add the handler */
qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL,
vhost_user_scsi_event, NULL, (void *)dev,
NULL, true);
/* Channel and lun both are 0 for bootable vhost-user-scsi disk */
vsc->channel = 0;
vsc->lun = 0;
Expand All @@ -112,8 +262,12 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
return;

free_vhost:
g_free(s->vhost_vqs);
s->vhost_vqs = NULL;
g_free(vsc->inflight);
vsc->inflight = NULL;
vhost_user_cleanup(&s->vhost_user);
g_free(vqs);

free_virtio:
virtio_scsi_common_unrealize(dev);
}
Expand All @@ -123,16 +277,23 @@ static void vhost_user_scsi_unrealize(DeviceState *dev)
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VHostUserSCSI *s = VHOST_USER_SCSI(dev);
VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
struct vhost_virtqueue *vqs = vsc->dev.vqs;
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);

/* This will stop the vhost backend. */
vhost_user_scsi_set_status(vdev, 0);
qemu_chr_fe_set_handlers(&vs->conf.chardev, NULL, NULL, NULL, NULL, NULL,
NULL, false);

vhost_dev_cleanup(&vsc->dev);
g_free(vqs);
g_free(s->vhost_vqs);
s->vhost_vqs = NULL;

vhost_dev_free_inflight(vsc->inflight);
g_free(vsc->inflight);
vsc->inflight = NULL;

virtio_scsi_common_unrealize(dev);
vhost_user_cleanup(&s->vhost_user);
virtio_scsi_common_unrealize(dev);
}

static Property vhost_user_scsi_properties[] = {
Expand Down
2 changes: 1 addition & 1 deletion include/hw/virtio/vhost-scsi-common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ struct VHostSCSICommon {
struct vhost_inflight *inflight;
};

int vhost_scsi_common_start(VHostSCSICommon *vsc);
int vhost_scsi_common_start(VHostSCSICommon *vsc, Error **errp);
void vhost_scsi_common_stop(VHostSCSICommon *vsc);
char *vhost_scsi_common_get_fw_dev_path(FWPathProvider *p, BusState *bus,
DeviceState *dev);
Expand Down
6 changes: 6 additions & 0 deletions include/hw/virtio/vhost-user-scsi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,13 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserSCSI, VHOST_USER_SCSI)

struct VHostUserSCSI {
VHostSCSICommon parent_obj;

/* Properties */
bool connected;
bool started_vu;

VhostUserState vhost_user;
struct vhost_virtqueue *vhost_vqs;
};

#endif /* VHOST_USER_SCSI_H */

0 comments on commit 7962e43

Please sign in to comment.