From patchwork Fri May 25 11:04:37 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 920429 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 40sk5G33bTz9s1b for ; Fri, 25 May 2018 21:08:34 +1000 (AEST) Received: from localhost ([::1]:42992 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAaF-0006C5-KS for incoming@patchwork.ozlabs.org; Fri, 25 May 2018 07:08:31 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:48820) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAWQ-0003iq-4c for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fMAWM-0000VD-Vm for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:34 -0400 Received: from mga04.intel.com ([192.55.52.120]:54812) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fMAWM-0000Pd-NI for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:30 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 May 2018 04:04:26 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,439,1520924400"; d="scan'208";a="231598570" Received: from debian.sh.intel.com ([10.67.104.203]) by fmsmga005.fm.intel.com with ESMTP; 25 May 2018 04:04:25 -0700 From: Tiwei Bie To: mst@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org Date: Fri, 25 May 2018 19:04:37 +0800 Message-Id: <20180525110441.7185-2-tiwei.bie@intel.com> X-Mailer: git-send-email 2.17.0 In-Reply-To: <20180525110441.7185-1-tiwei.bie@intel.com> References: <20180525110441.7185-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.120 Subject: [Qemu-devel] [PATCH v2 1/5] vhost: allow backends to filter memory sections X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch introduces a vhost op for vhost backends to allow them to filter the memory sections that they can handle. Signed-off-by: Tiwei Bie --- hw/virtio/vhost-user.c | 11 +++++++++++ hw/virtio/vhost.c | 9 +++++++-- include/hw/virtio/vhost-backend.h | 4 ++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index ca554d4ff1..da0756effe 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -1620,6 +1620,16 @@ vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id) return 0; } +static bool vhost_user_mem_section_filter(struct vhost_dev *dev, + MemoryRegionSection *section) +{ + bool result; + + result = memory_region_get_fd(section->mr) >= 0; + + return result; +} + const VhostOps user_ops = { .backend_type = VHOST_BACKEND_TYPE_USER, .vhost_backend_init = vhost_user_init, @@ -1650,4 +1660,5 @@ const VhostOps user_ops = { .vhost_set_config = vhost_user_set_config, .vhost_crypto_create_session = vhost_user_crypto_create_session, .vhost_crypto_close_session = vhost_user_crypto_close_session, + .vhost_backend_mem_section_filter = vhost_user_mem_section_filter, }; diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index b08290036d..624ade9682 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -386,7 +386,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev, return r; } -static bool vhost_section(MemoryRegionSection *section) +static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section) { bool result; bool log_dirty = memory_region_get_dirty_log_mask(section->mr) & @@ -399,6 +399,11 @@ static bool vhost_section(MemoryRegionSection *section) */ result &= !log_dirty; + if (result && dev->vhost_ops->vhost_backend_mem_section_filter) { + result &= + dev->vhost_ops->vhost_backend_mem_section_filter(dev, section); + } + trace_vhost_section(section->mr->name, result); return result; } @@ -632,7 +637,7 @@ static void vhost_region_addnop(MemoryListener *listener, struct vhost_dev *dev = container_of(listener, struct vhost_dev, memory_listener); - if (!vhost_section(section)) { + if (!vhost_section(dev, section)) { return; } vhost_region_add_section(dev, section); diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h index 5dac61f9ea..81283ec50f 100644 --- a/include/hw/virtio/vhost-backend.h +++ b/include/hw/virtio/vhost-backend.h @@ -101,6 +101,9 @@ typedef int (*vhost_crypto_create_session_op)(struct vhost_dev *dev, typedef int (*vhost_crypto_close_session_op)(struct vhost_dev *dev, uint64_t session_id); +typedef bool (*vhost_backend_mem_section_filter_op)(struct vhost_dev *dev, + MemoryRegionSection *section); + typedef struct VhostOps { VhostBackendType backend_type; vhost_backend_init vhost_backend_init; @@ -138,6 +141,7 @@ typedef struct VhostOps { vhost_set_config_op vhost_set_config; vhost_crypto_create_session_op vhost_crypto_create_session; vhost_crypto_close_session_op vhost_crypto_close_session; + vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter; } VhostOps; extern const VhostOps user_ops; From patchwork Fri May 25 11:04:38 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 920424 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 40sk1h3pNsz9s1d for ; Fri, 25 May 2018 21:05:28 +1000 (AEST) Received: from localhost ([::1]:42976 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAXG-0003jj-20 for incoming@patchwork.ozlabs.org; Fri, 25 May 2018 07:05:26 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:48818) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAWQ-0003io-4Q for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:36 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fMAWN-0000VI-06 for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:34 -0400 Received: from mga04.intel.com ([192.55.52.120]:54815) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fMAWM-0000Rp-Lw for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:30 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 May 2018 04:04:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,439,1520924400"; d="scan'208";a="231598578" Received: from debian.sh.intel.com ([10.67.104.203]) by fmsmga005.fm.intel.com with ESMTP; 25 May 2018 04:04:26 -0700 From: Tiwei Bie To: mst@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org Date: Fri, 25 May 2018 19:04:38 +0800 Message-Id: <20180525110441.7185-3-tiwei.bie@intel.com> X-Mailer: git-send-email 2.17.0 In-Reply-To: <20180525110441.7185-1-tiwei.bie@intel.com> References: <20180525110441.7185-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.120 Subject: [Qemu-devel] [PATCH v2 2/5] vhost-user: introduce shared vhost-user state X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" When multi queue is enabled e.g. for a virtio-net device, each queue pair will have a vhost_dev, and the only thing shared between vhost devs currently is the chardev. This patch introduces a vhost-user state structure which will be shared by all vhost devs of the same virtio device. Signed-off-by: Tiwei Bie Signed-off-by: Michael S. Tsirkin --- backends/cryptodev-vhost-user.c | 20 +++++++++- hw/block/vhost-user-blk.c | 22 ++++++++++- hw/scsi/vhost-user-scsi.c | 20 +++++++++- hw/virtio/Makefile.objs | 2 +- hw/virtio/vhost-stub.c | 10 +++++ hw/virtio/vhost-user.c | 31 +++++++++++----- include/hw/virtio/vhost-user-blk.h | 2 + include/hw/virtio/vhost-user-scsi.h | 2 + include/hw/virtio/vhost-user.h | 20 ++++++++++ net/vhost-user.c | 57 +++++++++++++++++++++-------- 10 files changed, 158 insertions(+), 28 deletions(-) create mode 100644 include/hw/virtio/vhost-user.h diff --git a/backends/cryptodev-vhost-user.c b/backends/cryptodev-vhost-user.c index 862d4f2580..d52daccfcd 100644 --- a/backends/cryptodev-vhost-user.c +++ b/backends/cryptodev-vhost-user.c @@ -26,6 +26,7 @@ #include "qapi/error.h" #include "qapi/qmp/qerror.h" #include "qemu/error-report.h" +#include "hw/virtio/vhost-user.h" #include "standard-headers/linux/virtio_crypto.h" #include "sysemu/cryptodev-vhost.h" #include "chardev/char-fe.h" @@ -46,6 +47,7 @@ typedef struct CryptoDevBackendVhostUser { CryptoDevBackend parent_obj; + VhostUserState *vhost_user; CharBackend chr; char *chr_name; bool opened; @@ -102,7 +104,7 @@ cryptodev_vhost_user_start(int queues, continue; } - options.opaque = &s->chr; + options.opaque = s->vhost_user; options.backend_type = VHOST_BACKEND_TYPE_USER; options.cc = b->conf.peers.ccs[i]; s->vhost_crypto[i] = cryptodev_vhost_init(&options); @@ -185,6 +187,7 @@ static void cryptodev_vhost_user_init( size_t i; Error *local_err = NULL; Chardev *chr; + VhostUserState *user; CryptoDevBackendClient *cc; CryptoDevBackendVhostUser *s = CRYPTODEV_BACKEND_VHOST_USER(backend); @@ -215,6 +218,15 @@ static void cryptodev_vhost_user_init( } } + user = vhost_user_init(); + if (!user) { + error_setg(errp, "Failed to init vhost_user"); + return; + } + + user->chr = &s->chr; + s->vhost_user = user; + qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, cryptodev_vhost_user_event, NULL, s, NULL, true); @@ -299,6 +311,12 @@ static void cryptodev_vhost_user_cleanup( backend->conf.peers.ccs[i] = NULL; } } + + if (s->vhost_user) { + vhost_user_cleanup(s->vhost_user); + g_free(s->vhost_user); + s->vhost_user = NULL; + } } static void cryptodev_vhost_user_set_chardev(Object *obj, diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c index 975eae6211..7c3fa8bb1c 100644 --- a/hw/block/vhost-user-blk.c +++ b/hw/block/vhost-user-blk.c @@ -226,6 +226,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) { VirtIODevice *vdev = VIRTIO_DEVICE(dev); VHostUserBlk *s = VHOST_USER_BLK(vdev); + VhostUserState *user; int i, ret; if (!s->chardev.chr) { @@ -243,6 +244,15 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) return; } + user = vhost_user_init(); + if (!user) { + error_setg(errp, "vhost-user-blk: failed to init vhost_user"); + return; + } + + user->chr = &s->chardev; + s->vhost_user = user; + virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK, sizeof(struct virtio_blk_config)); @@ -258,7 +268,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) vhost_dev_set_config_notifier(&s->dev, &blk_ops); - ret = vhost_dev_init(&s->dev, &s->chardev, VHOST_BACKEND_TYPE_USER, 0); + ret = vhost_dev_init(&s->dev, s->vhost_user, VHOST_BACKEND_TYPE_USER, 0); if (ret < 0) { error_setg(errp, "vhost-user-blk: vhost initialization failed: %s", strerror(-ret)); @@ -283,6 +293,10 @@ vhost_err: virtio_err: g_free(s->dev.vqs); virtio_cleanup(vdev); + + vhost_user_cleanup(user); + g_free(user); + s->vhost_user = NULL; } static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp) @@ -294,6 +308,12 @@ static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp) vhost_dev_cleanup(&s->dev); g_free(s->dev.vqs); virtio_cleanup(vdev); + + if (s->vhost_user) { + vhost_user_cleanup(s->vhost_user); + g_free(s->vhost_user); + s->vhost_user = NULL; + } } static void vhost_user_blk_instance_init(Object *obj) diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c index 9389ed48e0..9355cfdf07 100644 --- a/hw/scsi/vhost-user-scsi.c +++ b/hw/scsi/vhost-user-scsi.c @@ -69,6 +69,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp) VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev); VHostUserSCSI *s = VHOST_USER_SCSI(dev); VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s); + VhostUserState *user; Error *err = NULL; int ret; @@ -85,19 +86,30 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp) return; } + user = vhost_user_init(); + if (!user) { + error_setg(errp, "vhost-user-scsi: failed to init vhost_user"); + return; + } + user->chr = &vs->conf.chardev; + vsc->dev.nvqs = 2 + vs->conf.num_queues; vsc->dev.vqs = g_new(struct vhost_virtqueue, vsc->dev.nvqs); vsc->dev.vq_index = 0; vsc->dev.backend_features = 0; - ret = vhost_dev_init(&vsc->dev, (void *)&vs->conf.chardev, + ret = vhost_dev_init(&vsc->dev, user, VHOST_BACKEND_TYPE_USER, 0); if (ret < 0) { error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s", strerror(-ret)); + vhost_user_cleanup(user); + g_free(user); return; } + s->vhost_user = user; + /* Channel and lun both are 0 for bootable vhost-user-scsi disk */ vsc->channel = 0; vsc->lun = 0; @@ -117,6 +129,12 @@ static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp) g_free(vsc->dev.vqs); virtio_scsi_common_unrealize(dev, errp); + + if (s->vhost_user) { + vhost_user_cleanup(s->vhost_user); + g_free(s->vhost_user); + s->vhost_user = NULL; + } } static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev, diff --git a/hw/virtio/Makefile.objs b/hw/virtio/Makefile.objs index 765d363c1f..030969e28c 100644 --- a/hw/virtio/Makefile.objs +++ b/hw/virtio/Makefile.objs @@ -11,5 +11,5 @@ obj-y += virtio-crypto.o obj-$(CONFIG_VIRTIO_PCI) += virtio-crypto-pci.o endif -common-obj-$(call lnot,$(CONFIG_LINUX)) += vhost-stub.o +common-obj-$(call lnot,$(call land,$(CONFIG_VIRTIO),$(CONFIG_LINUX))) += vhost-stub.o common-obj-$(CONFIG_ALL) += vhost-stub.o diff --git a/hw/virtio/vhost-stub.c b/hw/virtio/vhost-stub.c index 2d76cdebdc..049089b5e2 100644 --- a/hw/virtio/vhost-stub.c +++ b/hw/virtio/vhost-stub.c @@ -1,7 +1,17 @@ #include "qemu/osdep.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" bool vhost_has_free_slot(void) { return true; } + +VhostUserState *vhost_user_init(void) +{ + return NULL; +} + +void vhost_user_cleanup(VhostUserState *user) +{ +} diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index da0756effe..5f72ed68da 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -11,6 +11,7 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-backend.h" #include "hw/virtio/virtio-net.h" #include "chardev/char-fe.h" @@ -173,7 +174,8 @@ static VhostUserMsg m __attribute__ ((unused)); struct vhost_user { struct vhost_dev *dev; - CharBackend *chr; + /* Shared between vhost devs of the same virtio device */ + VhostUserState *user; int slave_fd; NotifierWithReturn postcopy_notifier; struct PostCopyFD postcopy_fd; @@ -199,7 +201,7 @@ static bool ioeventfd_enabled(void) static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg) { struct vhost_user *u = dev->opaque; - CharBackend *chr = u->chr; + CharBackend *chr = u->user->chr; uint8_t *p = (uint8_t *) msg; int r, size = VHOST_USER_HDR_SIZE; @@ -285,7 +287,7 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, int *fds, int fd_num) { struct vhost_user *u = dev->opaque; - CharBackend *chr = u->chr; + CharBackend *chr = u->user->chr; int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size; /* @@ -1083,7 +1085,7 @@ static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb, static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp) { struct vhost_user *u = dev->opaque; - CharBackend *chr = u->chr; + CharBackend *chr = u->user->chr; int ufd; VhostUserMsg msg = { .hdr.request = VHOST_USER_POSTCOPY_ADVISE, @@ -1221,7 +1223,7 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier, return 0; } -static int vhost_user_init(struct vhost_dev *dev, void *opaque) +static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque) { uint64_t features, protocol_features; struct vhost_user *u; @@ -1230,7 +1232,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque) assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); u = g_new0(struct vhost_user, 1); - u->chr = opaque; + u->user = opaque; u->slave_fd = -1; u->dev = dev; dev->opaque = u; @@ -1306,7 +1308,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque) return 0; } -static int vhost_user_cleanup(struct vhost_dev *dev) +static int vhost_user_backend_cleanup(struct vhost_dev *dev) { struct vhost_user *u; @@ -1630,10 +1632,21 @@ static bool vhost_user_mem_section_filter(struct vhost_dev *dev, return result; } +VhostUserState *vhost_user_init(void) +{ + VhostUserState *user = g_new0(struct VhostUserState, 1); + + return user; +} + +void vhost_user_cleanup(VhostUserState *user) +{ +} + const VhostOps user_ops = { .backend_type = VHOST_BACKEND_TYPE_USER, - .vhost_backend_init = vhost_user_init, - .vhost_backend_cleanup = vhost_user_cleanup, + .vhost_backend_init = vhost_user_backend_init, + .vhost_backend_cleanup = vhost_user_backend_cleanup, .vhost_backend_memslots_limit = vhost_user_memslots_limit, .vhost_set_log_base = vhost_user_set_log_base, .vhost_set_mem_table = vhost_user_set_mem_table, diff --git a/include/hw/virtio/vhost-user-blk.h b/include/hw/virtio/vhost-user-blk.h index 5804cc904a..f1258ae545 100644 --- a/include/hw/virtio/vhost-user-blk.h +++ b/include/hw/virtio/vhost-user-blk.h @@ -21,6 +21,7 @@ #include "hw/block/block.h" #include "chardev/char-fe.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #define TYPE_VHOST_USER_BLK "vhost-user-blk" #define VHOST_USER_BLK(obj) \ @@ -36,6 +37,7 @@ typedef struct VHostUserBlk { uint32_t config_wce; uint32_t config_ro; struct vhost_dev dev; + VhostUserState *vhost_user; } VHostUserBlk; #endif diff --git a/include/hw/virtio/vhost-user-scsi.h b/include/hw/virtio/vhost-user-scsi.h index 01861f78d0..3ec34ae867 100644 --- a/include/hw/virtio/vhost-user-scsi.h +++ b/include/hw/virtio/vhost-user-scsi.h @@ -21,6 +21,7 @@ #include "hw/qdev.h" #include "hw/virtio/virtio-scsi.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-scsi-common.h" #define TYPE_VHOST_USER_SCSI "vhost-user-scsi" @@ -30,6 +31,7 @@ typedef struct VHostUserSCSI { VHostSCSICommon parent_obj; uint64_t host_features; + VhostUserState *vhost_user; } VHostUserSCSI; #endif /* VHOST_USER_SCSI_H */ diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h new file mode 100644 index 0000000000..eb8bc0d90d --- /dev/null +++ b/include/hw/virtio/vhost-user.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2017-2018 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2. + * See the COPYING file in the top-level directory. + */ + +#ifndef HW_VIRTIO_VHOST_USER_H +#define HW_VIRTIO_VHOST_USER_H + +#include "chardev/char-fe.h" + +typedef struct VhostUserState { + CharBackend *chr; +} VhostUserState; + +VhostUserState *vhost_user_init(void); +void vhost_user_cleanup(VhostUserState *user); + +#endif diff --git a/net/vhost-user.c b/net/vhost-user.c index fa28aad12d..6b926c1f46 100644 --- a/net/vhost-user.c +++ b/net/vhost-user.c @@ -12,6 +12,7 @@ #include "clients.h" #include "net/vhost_net.h" #include "net/vhost-user.h" +#include "hw/virtio/vhost-user.h" #include "chardev/char-fe.h" #include "qapi/error.h" #include "qapi/qapi-commands-net.h" @@ -23,6 +24,7 @@ typedef struct NetVhostUserState { NetClientState nc; CharBackend chr; /* only queue index 0 */ + VhostUserState *vhost_user; /* only queue index 0 */ VHostNetState *vhost_net; guint watch; uint64_t acked_features; @@ -64,7 +66,8 @@ static void vhost_user_stop(int queues, NetClientState *ncs[]) } } -static int vhost_user_start(int queues, NetClientState *ncs[], CharBackend *be) +static int vhost_user_start(int queues, NetClientState *ncs[], + VhostUserState *be) { VhostNetOptions options; struct vhost_net *net = NULL; @@ -144,7 +147,7 @@ static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf, return size; } -static void vhost_user_cleanup(NetClientState *nc) +static void net_vhost_user_cleanup(NetClientState *nc) { NetVhostUserState *s = DO_UPCAST(NetVhostUserState, nc, nc); @@ -159,6 +162,11 @@ static void vhost_user_cleanup(NetClientState *nc) s->watch = 0; } qemu_chr_fe_deinit(&s->chr, true); + if (s->vhost_user) { + vhost_user_cleanup(s->vhost_user); + g_free(s->vhost_user); + s->vhost_user = NULL; + } } qemu_purge_queued_packets(nc); @@ -182,7 +190,7 @@ static NetClientInfo net_vhost_user_info = { .type = NET_CLIENT_DRIVER_VHOST_USER, .size = sizeof(NetVhostUserState), .receive = vhost_user_receive, - .cleanup = vhost_user_cleanup, + .cleanup = net_vhost_user_cleanup, .has_vnet_hdr = vhost_user_has_vnet_hdr, .has_ufo = vhost_user_has_ufo, }; @@ -244,7 +252,7 @@ static void net_vhost_user_event(void *opaque, int event) trace_vhost_user_event(chr->label, event); switch (event) { case CHR_EVENT_OPENED: - if (vhost_user_start(queues, ncs, &s->chr) < 0) { + if (vhost_user_start(queues, ncs, s->vhost_user) < 0) { qemu_chr_fe_disconnect(&s->chr); return; } @@ -283,12 +291,19 @@ static int net_vhost_user_init(NetClientState *peer, const char *device, { Error *err = NULL; NetClientState *nc, *nc0 = NULL; - NetVhostUserState *s; + NetVhostUserState *s0 = NULL; + VhostUserState *user = NULL; int i; assert(name); assert(queues > 0); + user = vhost_user_init(); + if (!user) { + error_report("failed to init vhost_user"); + goto err; + } + for (i = 0; i < queues; i++) { nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name); snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user%d to %s", @@ -296,29 +311,41 @@ static int net_vhost_user_init(NetClientState *peer, const char *device, nc->queue_index = i; if (!nc0) { nc0 = nc; - s = DO_UPCAST(NetVhostUserState, nc, nc); - if (!qemu_chr_fe_init(&s->chr, chr, &err)) { + s0 = DO_UPCAST(NetVhostUserState, nc, nc); + if (!qemu_chr_fe_init(&s0->chr, chr, &err)) { error_report_err(err); - return -1; + goto err; } + user->chr = &s0->chr; + s0->vhost_user = user; } - } - s = DO_UPCAST(NetVhostUserState, nc, nc0); + /* Both of nc0 and s0 are valid now. */ do { - if (qemu_chr_fe_wait_connected(&s->chr, &err) < 0) { + if (qemu_chr_fe_wait_connected(&s0->chr, &err) < 0) { error_report_err(err); - return -1; + goto err; } - qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, + qemu_chr_fe_set_handlers(&s0->chr, NULL, NULL, net_vhost_user_event, NULL, nc0->name, NULL, true); - } while (!s->started); + } while (!s0->started); - assert(s->vhost_net); + assert(s0->vhost_net); return 0; + +err: + if (user) { + vhost_user_cleanup(user); + g_free(user); + if (s0) { + s0->vhost_user = NULL; + } + } + + return -1; } static Chardev *net_vhost_claim_chardev( From patchwork Fri May 25 11:04:39 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 920428 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 40sk5C6fBtz9s0q for ; Fri, 25 May 2018 21:08:31 +1000 (AEST) Received: from localhost ([::1]:42989 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAaD-000696-Ff for incoming@patchwork.ozlabs.org; Fri, 25 May 2018 07:08:29 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:48819) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAWQ-0003ip-4Z for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fMAWN-0000VZ-E7 for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:34 -0400 Received: from mga04.intel.com ([192.55.52.120]:54812) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fMAWN-0000Pd-6A for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:31 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 May 2018 04:04:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,439,1520924400"; d="scan'208";a="231598582" Received: from debian.sh.intel.com ([10.67.104.203]) by fmsmga005.fm.intel.com with ESMTP; 25 May 2018 04:04:27 -0700 From: Tiwei Bie To: mst@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org Date: Fri, 25 May 2018 19:04:39 +0800 Message-Id: <20180525110441.7185-4-tiwei.bie@intel.com> X-Mailer: git-send-email 2.17.0 In-Reply-To: <20180525110441.7185-1-tiwei.bie@intel.com> References: <20180525110441.7185-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.120 Subject: [Qemu-devel] [PATCH v2 3/5] vhost-user: support registering external host notifiers X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER. With this feature negotiated, vhost-user backend can register memory region based host notifiers. And it will allow the guest driver in the VM to notify the hardware accelerator at the vhost-user backend directly. Signed-off-by: Tiwei Bie --- docs/interop/vhost-user.txt | 33 +++++++++ hw/virtio/vhost-user.c | 123 +++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-user.h | 8 +++ 3 files changed, 164 insertions(+) diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt index 534caab18a..9e57b36b20 100644 --- a/docs/interop/vhost-user.txt +++ b/docs/interop/vhost-user.txt @@ -132,6 +132,16 @@ Depending on the request type, payload can be: Payload: Size bytes array holding the contents of the virtio device's configuration space + * Vring area description + ----------------------- + | u64 | size | offset | + ----------------------- + + u64: a 64-bit integer contains vring index and flags + Size: a 64-bit size of this area + Offset: a 64-bit offset of this area from the start of the + supplied file descriptor + In QEMU the vhost-user message is implemented with the following struct: typedef struct VhostUserMsg { @@ -146,6 +156,7 @@ typedef struct VhostUserMsg { VhostUserLog log; struct vhost_iotlb_msg iotlb; VhostUserConfig config; + VhostUserVringArea area; }; } QEMU_PACKED VhostUserMsg; @@ -380,6 +391,7 @@ Protocol features #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7 #define VHOST_USER_PROTOCOL_F_PAGEFAULT 8 #define VHOST_USER_PROTOCOL_F_CONFIG 9 +#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 10 Master message types -------------------- @@ -777,6 +789,27 @@ Slave message types the VHOST_USER_NEED_REPLY flag, master must respond with zero when operation is successfully completed, or non-zero otherwise. + * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG + + Id: 3 + Equivalent ioctl: N/A + Slave payload: vring area description + Master payload: N/A + + Sets host notifier for a specified queue. The queue index is contained + in the u64 field of the vring area description. The host notifier is + described by the file descriptor (typically it's a VFIO device fd) which + is passed as ancillary data and the size (which is mmap size and should + be the same as host page size) and offset (which is mmap offset) carried + in the vring area description. QEMU can mmap the file descriptor based + on the size and offset to get a memory range. Registering a host notifier + means mapping this memory range to the VM as the specified queue's notify + MMIO region. Slave sends this request to tell QEMU to de-register the + existing notifier if any and register the new notifier if the request is + sent with a file descriptor. + This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER + protocol feature has been successfully negotiated. + VHOST_USER_PROTOCOL_F_REPLY_ACK: ------------------------------- The original vhost-user specification only demands replies for certain diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index 5f72ed68da..259a841af9 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -13,6 +13,7 @@ #include "hw/virtio/vhost.h" #include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-backend.h" +#include "hw/virtio/virtio.h" #include "hw/virtio/virtio-net.h" #include "chardev/char-fe.h" #include "sysemu/kvm.h" @@ -48,6 +49,7 @@ enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, VHOST_USER_PROTOCOL_F_CONFIG = 9, + VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 10, VHOST_USER_PROTOCOL_F_MAX }; @@ -92,6 +94,7 @@ typedef enum VhostUserSlaveRequest { VHOST_USER_SLAVE_NONE = 0, VHOST_USER_SLAVE_IOTLB_MSG = 1, VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, + VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3, VHOST_USER_SLAVE_MAX } VhostUserSlaveRequest; @@ -136,6 +139,12 @@ static VhostUserConfig c __attribute__ ((unused)); + sizeof(c.size) \ + sizeof(c.flags)) +typedef struct VhostUserVringArea { + uint64_t u64; + uint64_t size; + uint64_t offset; +} VhostUserVringArea; + typedef struct { VhostUserRequest request; @@ -157,6 +166,7 @@ typedef union { struct vhost_iotlb_msg iotlb; VhostUserConfig config; VhostUserCryptoSession session; + VhostUserVringArea area; } VhostUserPayload; typedef struct VhostUserMsg { @@ -638,9 +648,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev, return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring); } +static void vhost_user_host_notifier_restore(struct vhost_dev *dev, + int queue_idx) +{ + struct vhost_user *u = dev->opaque; + VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; + VirtIODevice *vdev = dev->vdev; + + if (n->addr && !n->set) { + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true); + n->set = true; + } +} + +static void vhost_user_host_notifier_remove(struct vhost_dev *dev, + int queue_idx) +{ + struct vhost_user *u = dev->opaque; + VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; + VirtIODevice *vdev = dev->vdev; + + if (n->addr && n->set) { + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); + n->set = false; + } +} + static int vhost_user_set_vring_base(struct vhost_dev *dev, struct vhost_vring_state *ring) { + vhost_user_host_notifier_restore(dev, ring->index); + return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring); } @@ -674,6 +712,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.state), }; + vhost_user_host_notifier_remove(dev, ring->index); + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { return -1; } @@ -847,6 +887,76 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) return ret; } +static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev, + VhostUserVringArea *area, + int fd) +{ + int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK; + size_t page_size = qemu_real_host_page_size; + struct vhost_user *u = dev->opaque; + VhostUserState *user = u->user; + VirtIODevice *vdev = dev->vdev; + VhostUserHostNotifier *n; + int ret = 0; + void *addr; + char *name; + + if (!virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) || + vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) { + ret = -1; + goto out; + } + + n = &user->notifier[queue_idx]; + + if (n->addr) { + virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); + object_unparent(OBJECT(&n->mr)); + munmap(n->addr, page_size); + n->addr = NULL; + } + + if (area->u64 & VHOST_USER_VRING_NOFD_MASK) { + goto out; + } + + /* Sanity check. */ + if (area->size != page_size) { + ret = -1; + goto out; + } + + addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, area->offset); + if (addr == MAP_FAILED) { + ret = -1; + goto out; + } + + name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]", + user, queue_idx); + memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, + page_size, addr); + g_free(name); + + if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) { + munmap(addr, page_size); + ret = -1; + goto out; + } + + n->addr = addr; + n->set = true; + +out: + /* Always close the fd. */ + if (fd != -1) { + close(fd); + } + return ret; +} + static void slave_read(void *opaque) { struct vhost_dev *dev = opaque; @@ -913,6 +1023,10 @@ static void slave_read(void *opaque) case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : ret = vhost_user_slave_handle_config_change(dev); break; + case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: + ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area, + fd); + break; default: error_report("Received unexpected msg type."); if (fd != -1) { @@ -1641,6 +1755,15 @@ VhostUserState *vhost_user_init(void) void vhost_user_cleanup(VhostUserState *user) { + int i; + + for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { + if (user->notifier[i].addr) { + object_unparent(OBJECT(&user->notifier[i].mr)); + munmap(user->notifier[i].addr, qemu_real_host_page_size); + user->notifier[i].addr = NULL; + } + } } const VhostOps user_ops = { diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h index eb8bc0d90d..fd660393a0 100644 --- a/include/hw/virtio/vhost-user.h +++ b/include/hw/virtio/vhost-user.h @@ -9,9 +9,17 @@ #define HW_VIRTIO_VHOST_USER_H #include "chardev/char-fe.h" +#include "hw/virtio/virtio.h" + +typedef struct VhostUserHostNotifier { + MemoryRegion mr; + void *addr; + bool set; +} VhostUserHostNotifier; typedef struct VhostUserState { CharBackend *chr; + VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX]; } VhostUserState; VhostUserState *vhost_user_init(void); From patchwork Fri May 25 11:04:40 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 920427 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 40sk1h3lhMz9s0q for ; Fri, 25 May 2018 21:05:27 +1000 (AEST) Received: from localhost ([::1]:42973 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAXF-0003jN-H8 for incoming@patchwork.ozlabs.org; Fri, 25 May 2018 07:05:25 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:48822) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAWQ-0003is-4r for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fMAWN-0000Vf-Ek for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:34 -0400 Received: from mga04.intel.com ([192.55.52.120]:54815) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fMAWN-0000Rp-5n for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:31 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 May 2018 04:04:29 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,439,1520924400"; d="scan'208";a="231598588" Received: from debian.sh.intel.com ([10.67.104.203]) by fmsmga005.fm.intel.com with ESMTP; 25 May 2018 04:04:28 -0700 From: Tiwei Bie To: mst@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org Date: Fri, 25 May 2018 19:04:40 +0800 Message-Id: <20180525110441.7185-5-tiwei.bie@intel.com> X-Mailer: git-send-email 2.17.0 In-Reply-To: <20180525110441.7185-1-tiwei.bie@intel.com> References: <20180525110441.7185-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.120 Subject: [Qemu-devel] [PATCH v2 4/5] libvhost-user: support host notifier X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch introduces the host notifier support in libvhost-user. A new API is added to support setting host notifier for each queue. Signed-off-by: Tiwei Bie --- contrib/libvhost-user/libvhost-user.c | 80 ++++++++++++++++++++++++--- contrib/libvhost-user/libvhost-user.h | 31 +++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/contrib/libvhost-user/libvhost-user.c b/contrib/libvhost-user/libvhost-user.c index 54e643d871..6b0cdfbe8e 100644 --- a/contrib/libvhost-user/libvhost-user.c +++ b/contrib/libvhost-user/libvhost-user.c @@ -314,11 +314,6 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg) msg.msg_controllen = 0; } - /* Set the version in the flags when sending the reply */ - vmsg->flags &= ~VHOST_USER_VERSION_MASK; - vmsg->flags |= VHOST_USER_VERSION; - vmsg->flags |= VHOST_USER_REPLY_MASK; - do { rc = sendmsg(conn_fd, &msg, 0); } while (rc < 0 && (errno == EINTR || errno == EAGAIN)); @@ -341,6 +336,39 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg) return true; } +static bool +vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg) +{ + /* Set the version in the flags when sending the reply */ + vmsg->flags &= ~VHOST_USER_VERSION_MASK; + vmsg->flags |= VHOST_USER_VERSION; + vmsg->flags |= VHOST_USER_REPLY_MASK; + + return vu_message_write(dev, conn_fd, vmsg); +} + +static bool +vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg) +{ + VhostUserMsg msg_reply; + + if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) { + return true; + } + + if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) { + return false; + } + + if (msg_reply.request != vmsg->request) { + DPRINT("Received unexpected msg type. Expected %d received %d", + vmsg->request, msg_reply.request); + return false; + } + + return msg_reply.payload.u64 == 0; +} + /* Kick the log_call_fd if required. */ static void vu_log_kick(VuDev *dev) @@ -536,7 +564,7 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg) /* Send the message back to qemu with the addresses filled in */ vmsg->fd_num = 0; - if (!vu_message_write(dev, dev->sock, vmsg)) { + if (!vu_send_reply(dev, dev->sock, vmsg)) { vu_panic(dev, "failed to respond to set-mem-table for postcopy"); return false; } @@ -916,6 +944,41 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq, } } +bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd, + int size, int offset) +{ + int qidx = vq - dev->vq; + int fd_num = 0; + VhostUserMsg vmsg = { + .request = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG, + .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, + .size = sizeof(vmsg.payload.area), + .payload.area = { + .u64 = qidx & VHOST_USER_VRING_IDX_MASK, + .size = size, + .offset = offset, + }, + }; + + if ((dev->protocol_features & VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) == 0) { + return false; + } + + if (fd == -1) { + vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK; + } else { + vmsg.fds[fd_num++] = fd; + } + + vmsg.fd_num = fd_num; + + if (!vu_message_write(dev, dev->slave_fd, &vmsg)) { + return false; + } + + return vu_process_message_reply(dev, &vmsg); +} + static bool vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg) { @@ -968,7 +1031,8 @@ static bool vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg) { uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD | - 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ; + 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ | + 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER; if (have_userfault()) { features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT; @@ -1252,7 +1316,7 @@ vu_dispatch(VuDev *dev) goto end; } - if (!vu_message_write(dev, dev->sock, &vmsg)) { + if (!vu_send_reply(dev, dev->sock, &vmsg)) { goto end; } diff --git a/contrib/libvhost-user/libvhost-user.h b/contrib/libvhost-user/libvhost-user.h index b27075ea3b..90806a25f9 100644 --- a/contrib/libvhost-user/libvhost-user.h +++ b/contrib/libvhost-user/libvhost-user.h @@ -51,6 +51,7 @@ enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, VHOST_USER_PROTOCOL_F_CONFIG = 9, + VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 10, VHOST_USER_PROTOCOL_F_MAX }; @@ -92,6 +93,14 @@ typedef enum VhostUserRequest { VHOST_USER_MAX } VhostUserRequest; +typedef enum VhostUserSlaveRequest { + VHOST_USER_SLAVE_NONE = 0, + VHOST_USER_SLAVE_IOTLB_MSG = 1, + VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, + VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3, + VHOST_USER_SLAVE_MAX +} VhostUserSlaveRequest; + typedef struct VhostUserMemoryRegion { uint64_t guest_phys_addr; uint64_t memory_size; @@ -122,6 +131,12 @@ static VhostUserConfig c __attribute__ ((unused)); + sizeof(c.size) \ + sizeof(c.flags)) +typedef struct VhostUserVringArea { + uint64_t u64; + uint64_t size; + uint64_t offset; +} VhostUserVringArea; + #if defined(_WIN32) # define VU_PACKED __attribute__((gcc_struct, packed)) #else @@ -133,6 +148,7 @@ typedef struct VhostUserMsg { #define VHOST_USER_VERSION_MASK (0x3) #define VHOST_USER_REPLY_MASK (0x1 << 2) +#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3) uint32_t flags; uint32_t size; /* the following payload size */ @@ -145,6 +161,7 @@ typedef struct VhostUserMsg { VhostUserMemory memory; VhostUserLog log; VhostUserConfig config; + VhostUserVringArea area; } payload; int fds[VHOST_MEMORY_MAX_NREGIONS]; @@ -368,6 +385,20 @@ VuVirtq *vu_get_queue(VuDev *dev, int qidx); void vu_set_queue_handler(VuDev *dev, VuVirtq *vq, vu_queue_handler_cb handler); +/** + * vu_set_queue_host_notifier: + * @dev: a VuDev context + * @vq: a VuVirtq queue + * @fd: a file descriptor + * @size: host page size + * @offset: notifier offset in @fd file + * + * Set queue's host notifier. This function may be called several + * times for the same queue. If called with -1 @fd, the notifier + * is removed. + */ +bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd, + int size, int offset); /** * vu_queue_set_notification: From patchwork Fri May 25 11:04:41 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 920426 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 40sk1h3lyhz9s15 for ; Fri, 25 May 2018 21:05:27 +1000 (AEST) Received: from localhost ([::1]:42975 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAXF-0003jh-Hu for incoming@patchwork.ozlabs.org; Fri, 25 May 2018 07:05:25 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:48821) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fMAWQ-0003ir-4h for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fMAWN-0000Vy-SS for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:34 -0400 Received: from mga04.intel.com ([192.55.52.120]:54812) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fMAWN-0000Pd-KQ for qemu-devel@nongnu.org; Fri, 25 May 2018 07:04:31 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 May 2018 04:04:30 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,439,1520924400"; d="scan'208";a="231598601" Received: from debian.sh.intel.com ([10.67.104.203]) by fmsmga005.fm.intel.com with ESMTP; 25 May 2018 04:04:29 -0700 From: Tiwei Bie To: mst@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org Date: Fri, 25 May 2018 19:04:41 +0800 Message-Id: <20180525110441.7185-6-tiwei.bie@intel.com> X-Mailer: git-send-email 2.17.0 In-Reply-To: <20180525110441.7185-1-tiwei.bie@intel.com> References: <20180525110441.7185-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.120 Subject: [Qemu-devel] [PATCH v2 5/5] vhost-user-bridge: support host notifier X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch introduces the host notifier support in vhost-user-bridge. A new option (-H) is added to use the host notifier. This is mainly used to test the host notifier implementation in vhost user. Signed-off-by: Tiwei Bie --- tests/vhost-user-bridge.c | 98 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/tests/vhost-user-bridge.c b/tests/vhost-user-bridge.c index e0605a529e..0884294141 100644 --- a/tests/vhost-user-bridge.c +++ b/tests/vhost-user-bridge.c @@ -29,6 +29,7 @@ #define _FILE_OFFSET_BITS 64 +#include "qemu/atomic.h" #include "qemu/osdep.h" #include "qemu/iov.h" #include "standard-headers/linux/virtio_net.h" @@ -65,6 +66,11 @@ typedef struct VubrDev { int sock; int ready; int quit; + struct { + int fd; + void *addr; + pthread_t thread; + } notifier; } VubrDev; static void @@ -445,14 +451,22 @@ static uint64_t vubr_get_features(VuDev *dev) { return 1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE | - 1ULL << VIRTIO_NET_F_MRG_RXBUF; + 1ULL << VIRTIO_NET_F_MRG_RXBUF | + 1ULL << VIRTIO_F_VERSION_1; } static void vubr_queue_set_started(VuDev *dev, int qidx, bool started) { + VubrDev *vubr = container_of(dev, VubrDev, vudev); VuVirtq *vq = vu_get_queue(dev, qidx); + if (started && vubr->notifier.fd >= 0) { + vu_set_queue_host_notifier(dev, vq, vubr->notifier.fd, + getpagesize(), + qidx * getpagesize()); + } + if (qidx % 2 == 1) { vu_set_queue_handler(dev, vq, started ? vubr_handle_tx : NULL); } @@ -522,6 +536,8 @@ vubr_new(const char *path, bool client) vubr_die("socket"); } + dev->notifier.fd = -1; + un.sun_family = AF_UNIX; strcpy(un.sun_path, path); len = sizeof(un.sun_family) + strlen(path); @@ -559,6 +575,73 @@ vubr_new(const char *path, bool client) return dev; } +static void *notifier_thread(void *arg) +{ + VuDev *dev = (VuDev *)arg; + VubrDev *vubr = container_of(dev, VubrDev, vudev); + int pagesize = getpagesize(); + int qidx; + + while (true) { + for (qidx = 0; qidx < VHOST_MAX_NR_VIRTQUEUE; qidx++) { + uint16_t *n = vubr->notifier.addr + pagesize * qidx; + + if (*n == qidx) { + *n = 0xffff; + /* We won't miss notifications if we reset + * the memory first. */ + smp_mb(); + + DPRINT("Got a notification for queue%d via host notifier.\n", + qidx); + + if (qidx % 2 == 1) { + vubr_handle_tx(dev, qidx); + } + } + usleep(1000); + } + } + + return NULL; +} + +static void +vubr_host_notifier_setup(VubrDev *dev) +{ + char template[] = "/tmp/vubr-XXXXXX"; + pthread_t thread; + size_t length; + void *addr; + int fd; + + length = getpagesize() * VHOST_MAX_NR_VIRTQUEUE; + + fd = mkstemp(template); + if (fd < 0) { + vubr_die("mkstemp()"); + } + + if (posix_fallocate(fd, 0, length) != 0) { + vubr_die("posix_fallocate()"); + } + + addr = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + vubr_die("mmap()"); + } + + memset(addr, 0xff, length); + + if (pthread_create(&thread, NULL, notifier_thread, &dev->vudev) != 0) { + vubr_die("pthread_create()"); + } + + dev->notifier.fd = fd; + dev->notifier.addr = addr; + dev->notifier.thread = thread; +} + static void vubr_set_host(struct sockaddr_in *saddr, const char *host) { @@ -673,8 +756,9 @@ main(int argc, char *argv[]) VubrDev *dev; int opt; bool client = false; + bool host_notifier = false; - while ((opt = getopt(argc, argv, "l:r:u:c")) != -1) { + while ((opt = getopt(argc, argv, "l:r:u:cH")) != -1) { switch (opt) { case 'l': @@ -693,6 +777,9 @@ main(int argc, char *argv[]) case 'c': client = true; break; + case 'H': + host_notifier = true; + break; default: goto out; } @@ -708,6 +795,10 @@ main(int argc, char *argv[]) return 1; } + if (host_notifier) { + vubr_host_notifier_setup(dev); + } + vubr_backend_udp_setup(dev, lhost, lport, rhost, rport); vubr_run(dev); @@ -717,7 +808,7 @@ main(int argc, char *argv[]) out: fprintf(stderr, "Usage: %s ", argv[0]); - fprintf(stderr, "[-c] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n"); + fprintf(stderr, "[-c] [-H] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n"); fprintf(stderr, "\t-u path to unix doman socket. default: %s\n", DEFAULT_UD_SOCKET); fprintf(stderr, "\t-l local host and port. default: %s:%s\n", @@ -725,6 +816,7 @@ out: fprintf(stderr, "\t-r remote host and port. default: %s:%s\n", DEFAULT_RHOST, DEFAULT_RPORT); fprintf(stderr, "\t-c client mode\n"); + fprintf(stderr, "\t-H use host notifier\n"); return 1; }