From patchwork Tue Apr 9 03:04:38 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: mrhines@linux.vnet.ibm.com X-Patchwork-Id: 234949 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (Client did not present a certificate) by ozlabs.org (Postfix) with ESMTPS id B22A22C0084 for ; Tue, 9 Apr 2013 13:10:42 +1000 (EST) Received: from localhost ([::1]:60221 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UPOxE-00008s-QI for incoming@patchwork.ozlabs.org; Mon, 08 Apr 2013 23:10:40 -0400 Received: from eggs.gnu.org ([208.118.235.92]:37898) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UPOsN-0000ai-DY for qemu-devel@nongnu.org; Mon, 08 Apr 2013 23:05:42 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1UPOsK-0004ox-D7 for qemu-devel@nongnu.org; Mon, 08 Apr 2013 23:05:39 -0400 Received: from e31.co.us.ibm.com ([32.97.110.149]:60469) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1UPOsK-0004oq-4g for qemu-devel@nongnu.org; Mon, 08 Apr 2013 23:05:36 -0400 Received: from /spool/local by e31.co.us.ibm.com with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted for from ; Mon, 8 Apr 2013 21:05:35 -0600 Received: from d03dlp02.boulder.ibm.com (9.17.202.178) by e31.co.us.ibm.com (192.168.1.131) with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted; Mon, 8 Apr 2013 21:05:21 -0600 Received: from d03relay02.boulder.ibm.com (d03relay02.boulder.ibm.com [9.17.195.227]) by d03dlp02.boulder.ibm.com (Postfix) with ESMTP id 509093E40042 for ; Mon, 8 Apr 2013 21:05:08 -0600 (MDT) Received: from d03av06.boulder.ibm.com (d03av06.boulder.ibm.com [9.17.195.245]) by d03relay02.boulder.ibm.com (8.13.8/8.13.8/NCO v10.0) with ESMTP id r3935K4G134088 for ; Mon, 8 Apr 2013 21:05:20 -0600 Received: from d03av06.boulder.ibm.com (loopback [127.0.0.1]) by d03av06.boulder.ibm.com (8.14.4/8.13.1/NCO v10.0 AVout) with ESMTP id r39386J2013380 for ; Mon, 8 Apr 2013 21:08:06 -0600 Received: from mrhinesdev.klabtestbed.com (klinux.watson.ibm.com [9.2.208.21]) by d03av06.boulder.ibm.com (8.14.4/8.13.1/NCO v10.0 AVin) with ESMTP id r3937w2B012821; Mon, 8 Apr 2013 21:08:05 -0600 From: mrhines@linux.vnet.ibm.com To: qemu-devel@nongnu.org Date: Mon, 8 Apr 2013 23:04:38 -0400 Message-Id: <1365476681-31593-10-git-send-email-mrhines@linux.vnet.ibm.com> X-Mailer: git-send-email 1.7.10.4 In-Reply-To: <1365476681-31593-1-git-send-email-mrhines@linux.vnet.ibm.com> References: <1365476681-31593-1-git-send-email-mrhines@linux.vnet.ibm.com> X-TM-AS-MML: No X-Content-Scanned: Fidelis XPS MAILER x-cbid: 13040903-7282-0000-0000-000015FE6D20 X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.4.x-2.6.x [generic] X-Received-From: 32.97.110.149 Cc: aliguori@us.ibm.com, mst@redhat.com, owasserm@redhat.com, abali@us.ibm.com, mrhines@us.ibm.com, gokul@us.ibm.com, pbonzini@redhat.com Subject: [Qemu-devel] [RFC PATCH RDMA support v5: 09/12] transmit pc.ram using RDMA X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org From: "Michael R. Hines" Signed-off-by: Michael R. Hines --- arch_init.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/arch_init.c b/arch_init.c index c2cbc71..5cf7509 100644 --- a/arch_init.c +++ b/arch_init.c @@ -45,6 +45,7 @@ #include "exec/address-spaces.h" #include "hw/pcspk.h" #include "migration/page_cache.h" +#include "migration/rdma.h" #include "qemu/config-file.h" #include "qmp-commands.h" #include "trace.h" @@ -115,6 +116,7 @@ const uint32_t arch_type = QEMU_ARCH; #define RAM_SAVE_FLAG_EOS 0x10 #define RAM_SAVE_FLAG_CONTINUE 0x20 #define RAM_SAVE_FLAG_XBZRLE 0x40 +#define RAM_SAVE_FLAG_RDMA 0x80 /* Do server dynamic RDMA registerations */ static struct defconfig_file { @@ -447,15 +449,23 @@ static int ram_save_block(QEMUFile *f, bool last_stage) ram_bulk_stage = false; } } else { + bool zero; uint8_t *p; int cont = (block == last_sent_block) ? RAM_SAVE_FLAG_CONTINUE : 0; p = memory_region_get_ram_ptr(mr) + offset; + /* use capability now, defaults to true */ + zero = migrate_check_for_zero() ? is_zero_page(p) : false; + /* In doubt sent page as normal */ bytes_sent = -1; - if (is_zero_page(p)) { + if ((bytes_sent = save_rdma_page(f, block->offset, + offset, cont, TARGET_PAGE_SIZE, zero)) >= 0) { + acct_info.norm_pages++; + qemu_file_update_position(f, bytes_sent); + } else if (zero) { acct_info.dup_pages++; if (!ram_bulk_stage) { bytes_sent = save_block_hdr(f, block, offset, cont, @@ -476,7 +486,7 @@ static int ram_save_block(QEMUFile *f, bool last_stage) } /* XBZRLE overflow or normal page */ - if (bytes_sent == -1) { + if (bytes_sent == -1 || bytes_sent == -ENOTSUP) { bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE); qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE); bytes_sent += TARGET_PAGE_SIZE; @@ -603,6 +613,33 @@ static int ram_save_setup(QEMUFile *f, void *opaque) return 0; } +/* + * Inform server to begin handling dynamic page registrations + */ +static void ram_registration_start(QEMUFile *f) +{ + if(qemu_file_ops_are(f, &rdma_write_ops)) { + qemu_put_be64(f, RAM_SAVE_FLAG_RDMA); + } +} + +/* + * Inform server that dynamic registrations are done for now. + * First, flush writes, if any. + */ +static int ram_registration_stop(QEMUFile *f) +{ + int ret = 0; + + if (qemu_file_ops_are(f, &rdma_write_ops)) { + ret = qemu_rdma_drain_cq(f); + if(ret >= 0) + ret = qemu_rdma_finish_registrations(f); + } + + return ret; +} + static int ram_save_iterate(QEMUFile *f, void *opaque) { int ret; @@ -616,6 +653,8 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) reset_ram_globals(); } + ram_registration_start(f); + t0 = qemu_get_clock_ns(rt_clock); i = 0; while ((ret = qemu_file_rate_limit(f)) == 0) { @@ -646,6 +685,9 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) qemu_mutex_unlock_ramlist(); + if(ret >= 0) + ret = ram_registration_stop(f); + if (ret < 0) { bytes_transferred += total_sent; return ret; @@ -660,8 +702,11 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) static int ram_save_complete(QEMUFile *f, void *opaque) { + int ret = 0; + qemu_mutex_lock_ramlist(); migration_bitmap_sync(); + ram_registration_start(f); /* try transferring iterative blocks of memory */ @@ -676,12 +721,15 @@ static int ram_save_complete(QEMUFile *f, void *opaque) } bytes_transferred += bytes_sent; } + + ret = ram_registration_stop(f); + migration_end(); qemu_mutex_unlock_ramlist(); qemu_put_be64(f, RAM_SAVE_FLAG_EOS); - return 0; + return ret; } static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size) @@ -864,6 +912,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) ret = -EINVAL; goto done; } + } else if ((flags & RAM_SAVE_FLAG_RDMA) && + qemu_file_ops_are(f, &rdma_read_ops)) { + ret = qemu_rdma_handle_registrations(f); + if(ret < 0) + goto done; } error = qemu_file_get_error(f); if (error) {