Projects
home:dongzhihai
qemu
_service:tar_scm:vhost-implement-vhost_vdpa_dev...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:vhost-implement-vhost_vdpa_device_suspend-resume.patch of Package qemu
From 4c5a9a0703e227186639124f09cdf7214e40ea7d Mon Sep 17 00:00:00 2001 From: libai <libai12@huawei.com> Date: Mon, 4 Dec 2023 15:27:34 +0800 Subject: [PATCH] vhost: implement vhost_vdpa_device_suspend/resume Implement vhost device suspend & resume interface Signed-off-by: jiangdongxu <jiangdongxu1@huawei.com> Signed-off-by: fangyi <eric.fangyi@huawei.com> Signed-off-by: libai <libai12@huawei.com> --- hw/virtio/meson.build | 2 +- hw/virtio/vdpa-dev-mig.c | 178 +++++++++++++++++++++++++++++++ hw/virtio/vhost.c | 138 ++++++++++++++++++++++++ include/hw/virtio/vdpa-dev-mig.h | 16 +++ include/hw/virtio/vdpa-dev.h | 1 + include/hw/virtio/vhost.h | 3 + migration/migration.c | 3 +- migration/migration.h | 2 + 8 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 hw/virtio/vdpa-dev-mig.c create mode 100644 include/hw/virtio/vdpa-dev-mig.h diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build index c0055a7832..596651d113 100644 --- a/hw/virtio/meson.build +++ b/hw/virtio/meson.build @@ -5,7 +5,7 @@ system_virtio_ss.add(when: 'CONFIG_VIRTIO_MMIO', if_true: files('virtio-mmio.c') system_virtio_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('virtio-crypto.c')) system_virtio_ss.add(when: 'CONFIG_VHOST_VSOCK_COMMON', if_true: files('vhost-vsock-common.c')) system_virtio_ss.add(when: 'CONFIG_VIRTIO_IOMMU', if_true: files('virtio-iommu.c')) -system_virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) +system_virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c', 'vdpa-dev-mig.c')) specific_virtio_ss = ss.source_set() specific_virtio_ss.add(files('virtio.c')) diff --git a/hw/virtio/vdpa-dev-mig.c b/hw/virtio/vdpa-dev-mig.c new file mode 100644 index 0000000000..1d2bed2571 --- /dev/null +++ b/hw/virtio/vdpa-dev-mig.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qemu/osdep.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/vdpa-dev.h" +#include "hw/virtio/virtio-bus.h" +#include "migration/migration.h" +#include "qemu/error-report.h" +#include "hw/virtio/vdpa-dev-mig.h" + +static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request, + void *arg) +{ + struct vhost_vdpa *v = dev->opaque; + int fd = v->device_fd; + + if (dev->vhost_ops->backend_type != VHOST_BACKEND_TYPE_VDPA) { + error_report("backend type isn't VDPA. Operation not permitted!\n"); + return -EPERM; + } + + return ioctl(fd, request, arg); +} + +static int vhost_vdpa_device_suspend(VhostVdpaDevice *vdpa) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(vdpa); + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + int ret; + + if (!vdpa->started) { + return -EFAULT; + } + + if (!k->set_guest_notifiers) { + return -EFAULT; + } + + vdpa->started = false; + + ret = vhost_dev_suspend(&vdpa->dev, vdev, false); + if (ret) { + goto suspend_fail; + } + + ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, false); + if (ret < 0) { + error_report("vhost guest notifier cleanup failed: %d\n", ret); + goto set_guest_notifiers_fail; + } + + vhost_dev_disable_notifiers(&vdpa->dev, vdev); + return ret; + +set_guest_notifiers_fail: + ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, true); + if (ret) { + error_report("vhost guest notifier restore failed: %d\n", ret); + } + +suspend_fail: + vdpa->started = true; + return ret; +} + +static int vhost_vdpa_device_resume(VhostVdpaDevice *vdpa) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(vdpa); + BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); + VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); + int i, ret; + + if (!k->set_guest_notifiers) { + error_report("binding does not support guest notifiers\n"); + return -ENOSYS; + } + + ret = vhost_dev_enable_notifiers(&vdpa->dev, vdev); + if (ret < 0) { + error_report("Error enabling host notifiers: %d\n", ret); + return ret; + } + + ret = k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, true); + if (ret < 0) { + error_report("Error binding guest notifier: %d\n", ret); + goto err_host_notifiers; + } + + vdpa->dev.acked_features = vdev->guest_features; + + ret = vhost_dev_resume(&vdpa->dev, vdev, false); + if (ret < 0) { + error_report("Error starting vhost: %d\n", ret); + goto err_guest_notifiers; + } + vdpa->started = true; + + /* + * guest_notifier_mask/pending not used yet, so just unmask + * everything here. virtio-pci will do the right thing by + * enabling/disabling irqfd. + */ + for (i = 0; i < vdpa->dev.nvqs; i++) { + vhost_virtqueue_mask(&vdpa->dev, vdev, i, false); + } + + return ret; + +err_guest_notifiers: + k->set_guest_notifiers(qbus->parent, vdpa->dev.nvqs, false); +err_host_notifiers: + vhost_dev_disable_notifiers(&vdpa->dev, vdev); + return ret; +} + +static void vdpa_dev_vmstate_change(void *opaque, bool running, RunState state) +{ + VhostVdpaDevice *vdpa = VHOST_VDPA_DEVICE(opaque); + struct vhost_dev *hdev = &vdpa->dev; + int ret; + MigrationState *ms = migrate_get_current(); + MigrationIncomingState *mis = migration_incoming_get_current(); + + if (!running) { + if (ms->state == RUN_STATE_PAUSED) { + ret = vhost_vdpa_device_suspend(vdpa); + if (ret) { + error_report("suspend vdpa device failed: %d\n", ret); + if (ms->migration_thread_running) { + migrate_fd_cancel(ms); + } + } + } + } else { + if (ms->state == RUN_STATE_RESTORE_VM) { + ret = vhost_vdpa_device_resume(vdpa); + if (ret) { + error_report("migration dest resume device failed, abort!\n"); + exit(EXIT_FAILURE); + } + } + + if (mis->state == RUN_STATE_RESTORE_VM) { + vhost_vdpa_call(hdev, VHOST_VDPA_RESUME, NULL); + } + } +} + +void vdpa_migration_register(VhostVdpaDevice *vdev) +{ + vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), + vdpa_dev_vmstate_change, + DEVICE(vdev)); +} + +void vdpa_migration_unregister(VhostVdpaDevice *vdev) +{ + qemu_del_vm_change_state_handler(vdev->vmstate); +} diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 438182d850..d073a6d5a5 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -2492,3 +2492,141 @@ bool used_memslots_is_exceeded(void) { return used_memslots_exceeded; } + +int vhost_dev_resume(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings) +{ + int i, r; + EventNotifier *e = &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier; + + /* should only be called after backend is connected */ + if (!hdev->vhost_ops) { + error_report("Missing vhost_ops! Operation not permitted!\n"); + return -EPERM; + } + + vdev->vhost_started = true; + hdev->started = true; + hdev->vdev = vdev; + + if (vhost_dev_has_iommu(hdev)) { + memory_listener_register(&hdev->iommu_listener, vdev->dma_as); + } + + r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem); + if (r < 0) { + VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed"); + goto fail_mem; + } + for (i = 0; i < hdev->nvqs; ++i) { + r = vhost_virtqueue_start(hdev, + vdev, + hdev->vqs + i, + hdev->vq_index + i); + if (r < 0) { + goto fail_vq; + } + } + + r = event_notifier_init(e, 0); + if (r < 0) { + return r; + } + event_notifier_test_and_clear(e); + if (!vdev->use_guest_notifier_mask) { + vhost_config_mask(hdev, vdev, true); + } + if (vrings) { + r = vhost_dev_set_vring_enable(hdev, true); + if (r) { + goto fail_vq; + } + } + if (hdev->vhost_ops->vhost_dev_resume) { + r = hdev->vhost_ops->vhost_dev_resume(hdev); + if (r) { + goto fail_start; + } + } + if (vhost_dev_has_iommu(hdev)) { + hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true); + + /* + * Update used ring information for IOTLB to work correctly, + * vhost-kernel code requires for this. + */ + for (i = 0; i < hdev->nvqs; ++i) { + struct vhost_virtqueue *vq = hdev->vqs + i; + vhost_device_iotlb_miss(hdev, vq->used_phys, true); + } + } + vhost_start_config_intr(hdev); + return 0; +fail_start: + if (vrings) { + vhost_dev_set_vring_enable(hdev, false); + } +fail_vq: + while (--i >= 0) { + vhost_virtqueue_stop(hdev, + vdev, + hdev->vqs + i, + hdev->vq_index + i); + } + +fail_mem: + vdev->vhost_started = false; + hdev->started = false; + return r; +} + +int vhost_dev_suspend(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings) +{ + int i; + int ret = 0; + EventNotifier *e = &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier; + + /* should only be called after backend is connected */ + if (!hdev->vhost_ops) { + error_report("Missing vhost_ops! Operation not permitted!\n"); + return -EPERM; + } + + event_notifier_test_and_clear(e); + event_notifier_test_and_clear(&vdev->config_notifier); + + if (hdev->vhost_ops->vhost_dev_suspend) { + ret = hdev->vhost_ops->vhost_dev_suspend(hdev); + if (ret) { + goto fail_suspend; + } + } + if (vrings) { + ret = vhost_dev_set_vring_enable(hdev, false); + if (ret) { + goto fail_suspend; + } + } + for (i = 0; i < hdev->nvqs; ++i) { + vhost_virtqueue_stop(hdev, + vdev, + hdev->vqs + i, + hdev->vq_index + i); + } + + if (vhost_dev_has_iommu(hdev)) { + hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false); + memory_listener_unregister(&hdev->iommu_listener); + } + vhost_stop_config_intr(hdev); + vhost_log_put(hdev, true); + hdev->started = false; + vdev->vhost_started = false; + hdev->vdev = NULL; + + return ret; + +fail_suspend: + event_notifier_test_and_clear(e); + + return ret; +} diff --git a/include/hw/virtio/vdpa-dev-mig.h b/include/hw/virtio/vdpa-dev-mig.h new file mode 100644 index 0000000000..89665ca747 --- /dev/null +++ b/include/hw/virtio/vdpa-dev-mig.h @@ -0,0 +1,16 @@ +/* + * Vhost Vdpa Device Migration Header + * + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All Rights Reserved. + */ + +#ifndef _VHOST_VDPA_MIGRATION_H +#define _VHOST_VDPA_MIGRATION_H + +#include "hw/virtio/vdpa-dev.h" + +void vdpa_migration_register(VhostVdpaDevice *vdev); + +void vdpa_migration_unregister(VhostVdpaDevice *vdev); + +#endif /* _VHOST_VDPA_MIGRATION_H */ diff --git a/include/hw/virtio/vdpa-dev.h b/include/hw/virtio/vdpa-dev.h index 4dbf98195c..43cbcef81b 100644 --- a/include/hw/virtio/vdpa-dev.h +++ b/include/hw/virtio/vdpa-dev.h @@ -38,6 +38,7 @@ struct VhostVdpaDevice { uint16_t queue_size; bool started; int (*post_init)(VhostVdpaDevice *v, Error **errp); + VMChangeStateEntry *vmstate; }; #endif diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h index 6ae86833e3..9ca5819deb 100644 --- a/include/hw/virtio/vhost.h +++ b/include/hw/virtio/vhost.h @@ -466,4 +466,7 @@ int vhost_save_backend_state(struct vhost_dev *dev, QEMUFile *f, Error **errp); */ int vhost_load_backend_state(struct vhost_dev *dev, QEMUFile *f, Error **errp); +int vhost_dev_resume(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings); +int vhost_dev_suspend(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings); + #endif diff --git a/migration/migration.c b/migration/migration.c index 23d9233bbe..dce22c2da5 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -99,7 +99,6 @@ static bool migration_object_check(MigrationState *ms, Error **errp); static int migration_maybe_pause(MigrationState *s, int *current_active_state, int new_state); -static void migrate_fd_cancel(MigrationState *s); static bool close_return_path_on_source(MigrationState *s); static void migration_downtime_start(MigrationState *s) @@ -1386,7 +1385,7 @@ void migrate_fd_error(MigrationState *s, const Error *error) migrate_set_error(s, error); } -static void migrate_fd_cancel(MigrationState *s) +void migrate_fd_cancel(MigrationState *s) { int old_state ; diff --git a/migration/migration.h b/migration/migration.h index 6aafa04314..2f26c9509b 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -551,4 +551,6 @@ void migration_rp_kick(MigrationState *s); int migration_stop_vm(RunState state); +void migrate_fd_cancel(MigrationState *s); + #endif -- 2.27.0
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2