From patchwork Thu May 17 13:32:39 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jan Kiszka X-Patchwork-Id: 159918 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (Client did not present a certificate) by ozlabs.org (Postfix) with ESMTPS id B69B8B6FC2 for ; Fri, 18 May 2012 00:36:40 +1000 (EST) Received: from localhost ([::1]:48096 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1SV0pp-0002JE-2q for incoming@patchwork.ozlabs.org; Thu, 17 May 2012 09:33:41 -0400 Received: from eggs.gnu.org ([208.118.235.92]:58922) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1SV0pW-0001tB-Py for qemu-devel@nongnu.org; Thu, 17 May 2012 09:33:28 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1SV0pP-0004cK-I0 for qemu-devel@nongnu.org; Thu, 17 May 2012 09:33:22 -0400 Received: from fmmailgate02.web.de ([217.72.192.227]:49861) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1SV0pP-0004bp-3G for qemu-devel@nongnu.org; Thu, 17 May 2012 09:33:15 -0400 Received: from moweb001.kundenserver.de (moweb001.kundenserver.de [172.19.20.114]) by fmmailgate02.web.de (Postfix) with ESMTP id CF4D11C4B9009 for ; Thu, 17 May 2012 15:33:13 +0200 (CEST) Received: from localhost.localdomain ([187.105.13.240]) by smtp.web.de (mrweb001) with ESMTPA (Nemesis) id 0Lsy7e-1S775u0YQM-012c6S; Thu, 17 May 2012 15:33:13 +0200 From: Jan Kiszka To: Avi Kivity , Marcelo Tosatti Date: Thu, 17 May 2012 10:32:39 -0300 Message-Id: <9aefc5dc51fc018d0dc1d5ec0be4035a8e61893e.1337261556.git.jan.kiszka@web.de> X-Mailer: git-send-email 1.7.3.4 In-Reply-To: References: In-Reply-To: References: X-Provags-ID: V02:K0:PMJVeqx9MBVKv4/4RooWT3v2mwu1YSMNPoGmR6ByyJU 9ofjL6JPP/oMjy6EqNeHuV+QBbnuSg5pV5yYvO8b5HXo3DdhxB AKhqNxc1Y4nUezPd8OJUMMIMu4BjwLiCB7Y1amyoYR51RM2Beq nIarx0R6yB1HjRjzlxEJNIe7Uz9tbLjBnuWncjK0dEK5cCN99q 4ykxUPAi7t/VhgXBByBSw== X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.4-2.6 X-Received-From: 217.72.192.227 Cc: qemu-devel , kvm@vger.kernel.org, "Michael S. Tsirkin" Subject: [Qemu-devel] [RFC][PATCH v2 11/11] virtio/vhost: Add support for KVM in-kernel MSI injection X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org From: Jan Kiszka Make use of the new vector notifier to track changes of the MSI-X configuration of virtio PCI devices. On enabling events, we establish the required virtual IRQ to MSI-X message route and link the signaling eventfd file descriptor to this vIRQ line. That way, vhost-generated interrupts can be directly delivered to an in-kernel MSI-X consumer like the x86 APIC. Signed-off-by: Jan Kiszka --- hw/virtio-pci.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ hw/virtio-pci.h | 6 +++ 2 files changed, 132 insertions(+), 0 deletions(-) diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c index 4a4413d..01f5b92 100644 --- a/hw/virtio-pci.c +++ b/hw/virtio-pci.c @@ -24,6 +24,7 @@ #include "virtio-scsi.h" #include "pci.h" #include "qemu-error.h" +#include "msi.h" #include "msix.h" #include "net.h" #include "loader.h" @@ -539,6 +540,107 @@ static void virtio_pci_guest_notifier_read(void *opaque) } } +static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy, + unsigned int queue_no, + unsigned int vector, + MSIMessage msg) +{ + VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no); + VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; + int fd, ret; + + fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vq)); + + if (irqfd->users == 0) { + ret = kvm_irqchip_add_msi_route(kvm_state, msg); + if (ret < 0) { + return ret; + } + irqfd->virq = ret; + } + irqfd->users++; + + ret = kvm_irqchip_add_irqfd(kvm_state, fd, irqfd->virq); + if (ret < 0) { + if (--irqfd->users == 0) { + kvm_irqchip_release_virq(kvm_state, irqfd->virq); + } + return ret; + } + + qemu_set_fd_handler(fd, NULL, NULL, NULL); + + return 0; +} + +static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy, + unsigned int queue_no, + unsigned int vector) +{ + VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no); + VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector]; + int fd, ret; + + fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vq)); + + ret = kvm_irqchip_remove_irqfd(kvm_state, fd, irqfd->virq); + assert(ret == 0); + + if (--irqfd->users == 0) { + kvm_irqchip_release_virq(kvm_state, irqfd->virq); + } + + qemu_set_fd_handler(fd, virtio_pci_guest_notifier_read, NULL, vq); +} + +static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector, + MSIMessage msg) +{ + VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); + VirtIODevice *vdev = proxy->vdev; + int ret, queue_no; + + for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) { + if (!virtio_queue_get_num(vdev, queue_no)) { + break; + } + if (virtio_queue_vector(vdev, queue_no) != vector) { + continue; + } + ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg); + if (ret < 0) { + goto undo; + } + } + return 0; + +undo: + while (--queue_no >= 0) { + if (virtio_queue_vector(vdev, queue_no) != vector) { + continue; + } + kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector); + } + return ret; +} + +static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector) +{ + VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev); + VirtIODevice *vdev = proxy->vdev; + int queue_no; + + for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) { + if (!virtio_queue_get_num(vdev, queue_no)) { + break; + } + if (virtio_queue_vector(vdev, queue_no) != vector) { + continue; + } + kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector); + } +} + static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign) { VirtIOPCIProxy *proxy = opaque; @@ -555,6 +657,9 @@ static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign) } else { qemu_set_fd_handler(event_notifier_get_fd(notifier), NULL, NULL, NULL); + /* Test and clear notifier before closing it, + * in case poll callback didn't have time to run. */ + virtio_pci_guest_notifier_read(vq); event_notifier_cleanup(notifier); } @@ -573,6 +678,13 @@ static int virtio_pci_set_guest_notifiers(void *opaque, bool assign) VirtIODevice *vdev = proxy->vdev; int r, n; + /* Must unset vector notifier while guest notifier is still assigned */ + if (kvm_irqchip_in_kernel() && !assign) { + msix_unset_vector_notifiers(&proxy->pci_dev); + g_free(proxy->vector_irqfd); + proxy->vector_irqfd = NULL; + } + for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) { if (!virtio_queue_get_num(vdev, n)) { break; @@ -584,10 +696,24 @@ static int virtio_pci_set_guest_notifiers(void *opaque, bool assign) } } + /* Must set vector notifier after guest notifier has been assigned */ + if (kvm_irqchip_in_kernel() && assign) { + proxy->vector_irqfd = + g_malloc0(sizeof(*proxy->vector_irqfd) * + msix_nr_vectors_allocated(&proxy->pci_dev)); + r = msix_set_vector_notifiers(&proxy->pci_dev, + kvm_virtio_pci_vector_use, + kvm_virtio_pci_vector_release); + if (r < 0) { + goto assign_error; + } + } + return 0; assign_error: /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */ + assert(assign); while (--n >= 0) { virtio_pci_set_guest_notifier(opaque, n, !assign); } diff --git a/hw/virtio-pci.h b/hw/virtio-pci.h index e560428..8d28d4b 100644 --- a/hw/virtio-pci.h +++ b/hw/virtio-pci.h @@ -25,6 +25,11 @@ #define VIRTIO_PCI_FLAG_USE_IOEVENTFD (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT) typedef struct { + int virq; + unsigned int users; +} VirtIOIRQFD; + +typedef struct { PCIDevice pci_dev; VirtIODevice *vdev; MemoryRegion bar; @@ -44,6 +49,7 @@ typedef struct { VirtIOSCSIConf scsi; bool ioeventfd_disabled; bool ioeventfd_started; + VirtIOIRQFD *vector_irqfd; } VirtIOPCIProxy; void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev);