From patchwork Mon Jul 23 04:59:54 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 947607 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 41Yq7x555wz9s1x for ; Mon, 23 Jul 2018 15:01:00 +1000 (AEST) Received: from localhost ([::1]:32778 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhSxt-0003ft-3h for incoming@patchwork.ozlabs.org; Mon, 23 Jul 2018 01:00:57 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:43704) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhSxM-0003fj-4q for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:25 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fhSxJ-0004mu-3i for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:24 -0400 Received: from mga03.intel.com ([134.134.136.65]:61814) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fhSxI-0004mc-Qv for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:21 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 22 Jul 2018 22:00:19 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,392,1526367600"; d="scan'208";a="59837271" Received: from debian.sh.intel.com ([10.67.104.228]) by orsmga006.jf.intel.com with ESMTP; 22 Jul 2018 22:00:17 -0700 From: Tiwei Bie To: mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org Date: Mon, 23 Jul 2018 12:59:54 +0800 Message-Id: <20180723045956.27521-2-tiwei.bie@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20180723045956.27521-1-tiwei.bie@intel.com> References: <20180723045956.27521-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.65 Subject: [Qemu-devel] [RFC 1/3] vfio: split vfio_get_group() into small functions X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch splits vfio_get_group() into small functions. It makes it easier to implement other vfio_get_group*() functions in the future. Signed-off-by: Tiwei Bie --- hw/vfio/common.c | 83 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index fb396cf00a..52a05532cd 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -1279,37 +1279,20 @@ static void vfio_disconnect_container(VFIOGroup *group) } } -VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) +static VFIOGroup *vfio_init_group(int groupfd, int groupid, AddressSpace *as, + Error **errp) { VFIOGroup *group; - char path[32]; struct vfio_group_status status = { .argsz = sizeof(status) }; - QLIST_FOREACH(group, &vfio_group_list, next) { - if (group->groupid == groupid) { - /* Found it. Now is it already in the right context? */ - if (group->container->space->as == as) { - return group; - } else { - error_setg(errp, "group %d used in multiple address spaces", - group->groupid); - return NULL; - } - } - } - group = g_malloc0(sizeof(*group)); - snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); - group->fd = qemu_open(path, O_RDWR); - if (group->fd < 0) { - error_setg_errno(errp, errno, "failed to open %s", path); - goto free_group_exit; - } + group->fd = groupfd; + group->groupid = groupid; if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { error_setg_errno(errp, errno, "failed to get group %d status", groupid); - goto close_fd_exit; + goto free_group_exit; } if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { @@ -1317,16 +1300,15 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) error_append_hint(errp, "Please ensure all devices within the iommu_group " "are bound to their vfio bus driver.\n"); - goto close_fd_exit; + goto free_group_exit; } - group->groupid = groupid; QLIST_INIT(&group->device_list); if (vfio_connect_container(group, as, errp)) { error_prepend(errp, "failed to setup container for group %d: ", groupid); - goto close_fd_exit; + goto free_group_exit; } if (QLIST_EMPTY(&vfio_group_list)) { @@ -1337,15 +1319,60 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) return group; -close_fd_exit: - close(group->fd); - free_group_exit: g_free(group); return NULL; } +static VFIOGroup *vfio_find_group(int groupid, AddressSpace *as, + Error **errp) +{ + VFIOGroup *group; + + QLIST_FOREACH(group, &vfio_group_list, next) { + if (group->groupid == groupid) { + /* Found it. Now is it already in the right context? */ + if (group->container->space->as == as) { + return group; + } else { + error_setg(errp, "group %d used in multiple address spaces", + group->groupid); + return NULL; + } + } + } + + return NULL; +} + +VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) +{ + VFIOGroup *group; + char path[32]; + int groupfd; + + group = vfio_find_group(groupid, as, errp); + if (group) { + return group; + } + + snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); + groupfd = qemu_open(path, O_RDWR); + if (groupfd < 0) { + error_setg_errno(errp, errno, "failed to open %s", path); + return NULL; + } + + group = vfio_init_group(groupfd, groupid, as, errp); + if (!group) { + close(groupfd); + return NULL; + } + + return group; +} + void vfio_put_group(VFIOGroup *group) { if (!group || !QLIST_EMPTY(&group->device_list)) { From patchwork Mon Jul 23 04:59:55 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 947608 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 41Yq8F5G7Rz9s1x for ; Mon, 23 Jul 2018 15:01:17 +1000 (AEST) Received: from localhost ([::1]:32779 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhSyB-0003rd-C2 for incoming@patchwork.ozlabs.org; Mon, 23 Jul 2018 01:01:15 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:43748) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhSxa-0003lL-Hv for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:39 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fhSxX-00050j-8q for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:38 -0400 Received: from mga07.intel.com ([134.134.136.100]:45043) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fhSxW-0004yY-V6 for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:35 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 22 Jul 2018 22:00:33 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,392,1526367600"; d="scan'208";a="59837285" Received: from debian.sh.intel.com ([10.67.104.228]) by orsmga006.jf.intel.com with ESMTP; 22 Jul 2018 22:00:19 -0700 From: Tiwei Bie To: mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org Date: Mon, 23 Jul 2018 12:59:55 +0800 Message-Id: <20180723045956.27521-3-tiwei.bie@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20180723045956.27521-1-tiwei.bie@intel.com> References: <20180723045956.27521-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.100 Subject: [Qemu-devel] [RFC 2/3] vfio: support getting VFIOGroup from groupfd X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" This patch introduces an API to support getting VFIOGroup from groupfd. This is useful when the groupfd is opened and shared by another process via UNIX socket. Signed-off-by: Tiwei Bie --- hw/vfio/common.c | 44 +++++++++++++++++++++++++++++++++++ include/hw/vfio/vfio-common.h | 1 + 2 files changed, 45 insertions(+) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 52a05532cd..4c19a33dd5 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -1279,6 +1279,30 @@ static void vfio_disconnect_container(VFIOGroup *group) } } +static int vfio_groupfd_to_groupid(int groupfd) +{ + char *tmp, group_path[PATH_MAX], *group_name; + int groupid, len; + + tmp = g_strdup_printf("/proc/self/fd/%d", groupfd); + len = readlink(tmp, group_path, sizeof(group_path)); + g_free(tmp); + + if (len <= 0 || len >= sizeof(group_path)) { + return -1; + } + + group_path[len] = '\0'; + + group_name = g_path_get_basename(group_path); + if (sscanf(group_name, "%d", &groupid) != 1) { + groupid = -1; + } + g_free(group_name); + + return groupid; +} + static VFIOGroup *vfio_init_group(int groupfd, int groupid, AddressSpace *as, Error **errp) { @@ -1373,6 +1397,26 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) return group; } +VFIOGroup *vfio_get_group_from_fd(int groupfd, AddressSpace *as, Error **errp) +{ + VFIOGroup *group; + int groupid; + + groupid = vfio_groupfd_to_groupid(groupfd); + if (groupid < 0) { + error_setg(errp, "failed to get group id from group fd %d", + groupfd); + return NULL; + } + + group = vfio_find_group(groupid, as, errp); + if (group) { + return group; + } + + return vfio_init_group(groupfd, groupid, as, errp); +} + void vfio_put_group(VFIOGroup *group) { if (!group || !QLIST_EMPTY(&group->device_list)) { diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index a9036929b2..9bb1068a36 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -170,6 +170,7 @@ void vfio_region_exit(VFIORegion *region); void vfio_region_finalize(VFIORegion *region); void vfio_reset_handler(void *opaque); VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp); +VFIOGroup *vfio_get_group_from_fd(int groupfd, AddressSpace *as, Error **errp); void vfio_put_group(VFIOGroup *group); int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vbasedev, Error **errp); From patchwork Mon Jul 23 04:59:56 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tiwei Bie X-Patchwork-Id: 947610 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=nongnu.org (client-ip=2001:4830:134:3::11; helo=lists.gnu.org; envelope-from=qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=fail (p=none dis=none) header.from=intel.com Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 41YqBk44HRz9s2M for ; Mon, 23 Jul 2018 15:03:26 +1000 (AEST) Received: from localhost ([::1]:32789 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhT0G-00057M-8m for incoming@patchwork.ozlabs.org; Mon, 23 Jul 2018 01:03:24 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:43749) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fhSxa-0003lN-I6 for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:39 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fhSxX-00051W-NZ for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:38 -0400 Received: from mga07.intel.com ([134.134.136.100]:45043) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fhSxX-0004yY-Ef for qemu-devel@nongnu.org; Mon, 23 Jul 2018 01:00:35 -0400 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 22 Jul 2018 22:00:33 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,392,1526367600"; d="scan'208";a="59837300" Received: from debian.sh.intel.com ([10.67.104.228]) by orsmga006.jf.intel.com with ESMTP; 22 Jul 2018 22:00:20 -0700 From: Tiwei Bie To: mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, qemu-devel@nongnu.org Date: Mon, 23 Jul 2018 12:59:56 +0800 Message-Id: <20180723045956.27521-4-tiwei.bie@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20180723045956.27521-1-tiwei.bie@intel.com> References: <20180723045956.27521-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 134.134.136.100 Subject: [Qemu-devel] [RFC 3/3] vhost-user: support programming VFIO group in master X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: tiwei.bie@intel.com Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: "Qemu-devel" Introduce a slave message to allow slave to share its VFIO group fd to master and do the IOMMU programming based on virtio device's DMA address space for this group in QEMU. For the vhost backends which support vDPA, they could leverage this message to ask master to do the IOMMU programming in QEMU for the vDPA device in backend. Signed-off-by: Tiwei Bie --- docs/interop/vhost-user.txt | 16 ++++++++++++++ hw/virtio/vhost-user.c | 40 ++++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-user.h | 2 ++ 3 files changed, 58 insertions(+) diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt index f59667f498..a57a8f9451 100644 --- a/docs/interop/vhost-user.txt +++ b/docs/interop/vhost-user.txt @@ -397,6 +397,7 @@ Protocol features #define VHOST_USER_PROTOCOL_F_CONFIG 9 #define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10 #define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11 +#define VHOST_USER_PROTOCOL_F_VFIO_GROUP 12 Master message types -------------------- @@ -815,6 +816,21 @@ Slave message types This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER protocol feature has been successfully negotiated. + * VHOST_USER_SLAVE_VFIO_GROUP_MSG + + Id: 4 + Equivalent ioctl: N/A + Slave payload: N/A + Master payload: N/A + + When VHOST_USER_PROTOCOL_F_VFIO_GROUP is negotiated, vhost-user slave + could send this request to share its VFIO group fd via ancillary data + to master. After receiving this request from slave, master will close + the existing VFIO group if any and do the DMA programming based on the + virtio device's DMA address space for the new group if the request is + sent with a file descriptor. + + VHOST_USER_PROTOCOL_F_REPLY_ACK: ------------------------------- The original vhost-user specification only demands replies for certain diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index b041343632..db958e24c7 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -52,6 +52,7 @@ enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_CONFIG = 9, VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, + VHOST_USER_PROTOCOL_F_VFIO_GROUP = 12, VHOST_USER_PROTOCOL_F_MAX }; @@ -97,6 +98,7 @@ typedef enum VhostUserSlaveRequest { VHOST_USER_SLAVE_IOTLB_MSG = 1, VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3, + VHOST_USER_SLAVE_VFIO_GROUP_MSG = 4, VHOST_USER_SLAVE_MAX } VhostUserSlaveRequest; @@ -949,6 +951,41 @@ static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev, return 0; } +static int vhost_user_slave_handle_vfio_group(struct vhost_dev *dev, + int *fd) +{ + struct vhost_user *u = dev->opaque; + VhostUserState *user = u->user; + VirtIODevice *vdev = dev->vdev; + int groupfd = fd[0]; + VFIOGroup *group; + + if (!virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_VFIO_GROUP) || + vdev == NULL) { + return -1; + } + + if (user->vfio_group) { + vfio_put_group(user->vfio_group); + user->vfio_group = NULL; + } + + group = vfio_get_group_from_fd(groupfd, vdev->dma_as, NULL); + if (group == NULL) { + return -1; + } + + if (group->fd != groupfd) { + close(groupfd); + } + + user->vfio_group = group; + fd[0] = -1; + + return 0; +} + static void slave_read(void *opaque) { struct vhost_dev *dev = opaque; @@ -1021,6 +1058,9 @@ static void slave_read(void *opaque) ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area, fd[0]); break; + case VHOST_USER_SLAVE_VFIO_GROUP_MSG: + ret = vhost_user_slave_handle_vfio_group(dev, fd); + break; default: error_report("Received unexpected msg type."); ret = -EINVAL; diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h index fd660393a0..9e11473274 100644 --- a/include/hw/virtio/vhost-user.h +++ b/include/hw/virtio/vhost-user.h @@ -10,6 +10,7 @@ #include "chardev/char-fe.h" #include "hw/virtio/virtio.h" +#include "hw/vfio/vfio-common.h" typedef struct VhostUserHostNotifier { MemoryRegion mr; @@ -20,6 +21,7 @@ typedef struct VhostUserHostNotifier { typedef struct VhostUserState { CharBackend *chr; VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX]; + VFIOGroup *vfio_group; } VhostUserState; VhostUserState *vhost_user_init(void);