@@ -390,9 +390,9 @@ static uint16_t nvme_dma_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
return status;
}
-static uint16_t nvme_map(NvmeCtrl *n, NvmeCmd *cmd, size_t len,
- NvmeRequest *req)
+static uint16_t nvme_map(NvmeCtrl *n, size_t len, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
@@ -562,7 +562,7 @@ static void nvme_rw_cb(void *opaque, int ret)
nvme_enqueue_req_completion(cq, req);
}
-static uint16_t nvme_flush(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
{
block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
BLOCK_ACCT_FLUSH);
@@ -571,9 +571,9 @@ static uint16_t nvme_flush(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
return NVME_NO_COMPLETE;
}
-static uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
{
- NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
+ NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
NvmeNamespace *ns = req->ns;
const uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
const uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
@@ -598,9 +598,9 @@ static uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
return NVME_NO_COMPLETE;
}
-static uint16_t nvme_rw(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_rw(NvmeCtrl *n, NvmeRequest *req)
{
- NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
+ NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
NvmeNamespace *ns = req->ns;
uint32_t nlb = le32_to_cpu(rw->nlb) + 1;
uint64_t slba = le64_to_cpu(rw->slba);
@@ -629,7 +629,7 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
return status;
}
- if (nvme_map(n, cmd, data_size, req)) {
+ if (nvme_map(n, data_size, req)) {
block_acct_invalid(blk_get_stats(n->conf.blk), acct);
return NVME_INVALID_FIELD | NVME_DNR;
}
@@ -655,11 +655,12 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
return NVME_NO_COMPLETE;
}
-static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
{
- uint32_t nsid = le32_to_cpu(cmd->nsid);
+ uint32_t nsid = le32_to_cpu(req->cmd.nsid);
- trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req), cmd->opcode);
+ trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
+ req->cmd.opcode);
if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
trace_pci_nvme_err_invalid_ns(nsid, n->num_namespaces);
@@ -667,16 +668,16 @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
}
req->ns = &n->namespaces[nsid - 1];
- switch (cmd->opcode) {
+ switch (req->cmd.opcode) {
case NVME_CMD_FLUSH:
- return nvme_flush(n, cmd, req);
+ return nvme_flush(n, req);
case NVME_CMD_WRITE_ZEROES:
- return nvme_write_zeroes(n, cmd, req);
+ return nvme_write_zeroes(n, req);
case NVME_CMD_WRITE:
case NVME_CMD_READ:
- return nvme_rw(n, cmd, req);
+ return nvme_rw(n, req);
default:
- trace_pci_nvme_err_invalid_opc(cmd->opcode);
+ trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
return NVME_INVALID_OPCODE | NVME_DNR;
}
}
@@ -692,10 +693,10 @@ static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
}
}
-static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
+static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
{
- NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
- NvmeRequest *req, *next;
+ NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
+ NvmeRequest *r, *next;
NvmeSQueue *sq;
NvmeCQueue *cq;
uint16_t qid = le16_to_cpu(c->qid);
@@ -709,19 +710,19 @@ static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
sq = n->sq[qid];
while (!QTAILQ_EMPTY(&sq->out_req_list)) {
- req = QTAILQ_FIRST(&sq->out_req_list);
- assert(req->aiocb);
- blk_aio_cancel(req->aiocb);
+ r = QTAILQ_FIRST(&sq->out_req_list);
+ assert(r->aiocb);
+ blk_aio_cancel(r->aiocb);
}
if (!nvme_check_cqid(n, sq->cqid)) {
cq = n->cq[sq->cqid];
QTAILQ_REMOVE(&cq->sq_list, sq, entry);
nvme_post_cqes(cq);
- QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
- if (req->sq == sq) {
- QTAILQ_REMOVE(&cq->req_list, req, entry);
- QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
+ QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
+ if (r->sq == sq) {
+ QTAILQ_REMOVE(&cq->req_list, r, entry);
+ QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
}
}
}
@@ -758,10 +759,10 @@ static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
n->sq[sqid] = sq;
}
-static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
+static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
{
NvmeSQueue *sq;
- NvmeCreateSq *c = (NvmeCreateSq *)cmd;
+ NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
uint16_t cqid = le16_to_cpu(c->cqid);
uint16_t sqid = le16_to_cpu(c->sqid);
@@ -796,10 +797,10 @@ static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
return NVME_SUCCESS;
}
-static uint16_t nvme_smart_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
- uint32_t buf_len, uint64_t off,
- NvmeRequest *req)
+static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
+ uint64_t off, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
uint32_t nsid = le32_to_cpu(cmd->nsid);
@@ -855,10 +856,11 @@ static uint16_t nvme_smart_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_fw_log_info(NvmeCtrl *n, NvmeCmd *cmd, uint32_t buf_len,
- uint64_t off, NvmeRequest *req)
+static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
+ NvmeRequest *req)
{
uint32_t trans_len;
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
NvmeFwSlotInfoLog fw_log = {
@@ -877,11 +879,11 @@ static uint16_t nvme_fw_log_info(NvmeCtrl *n, NvmeCmd *cmd, uint32_t buf_len,
DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_error_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
- uint32_t buf_len, uint64_t off,
- NvmeRequest *req)
+static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
+ uint64_t off, NvmeRequest *req)
{
uint32_t trans_len;
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
NvmeErrorLog errlog;
@@ -902,8 +904,10 @@ static uint16_t nvme_error_info(NvmeCtrl *n, NvmeCmd *cmd, uint8_t rae,
DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_get_log(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
+
uint32_t dw10 = le32_to_cpu(cmd->cdw10);
uint32_t dw11 = le32_to_cpu(cmd->cdw11);
uint32_t dw12 = le32_to_cpu(cmd->cdw12);
@@ -938,11 +942,11 @@ static uint16_t nvme_get_log(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
switch (lid) {
case NVME_LOG_ERROR_INFO:
- return nvme_error_info(n, cmd, rae, len, off, req);
+ return nvme_error_info(n, rae, len, off, req);
case NVME_LOG_SMART_INFO:
- return nvme_smart_info(n, cmd, rae, len, off, req);
+ return nvme_smart_info(n, rae, len, off, req);
case NVME_LOG_FW_SLOT_INFO:
- return nvme_fw_log_info(n, cmd, len, off, req);
+ return nvme_fw_log_info(n, len, off, req);
default:
trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
return NVME_INVALID_FIELD | NVME_DNR;
@@ -960,9 +964,9 @@ static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
}
}
-static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
+static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
{
- NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
+ NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
NvmeCQueue *cq;
uint16_t qid = le16_to_cpu(c->qid);
@@ -1003,10 +1007,10 @@ static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
}
-static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
+static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
{
NvmeCQueue *cq;
- NvmeCreateCq *c = (NvmeCreateCq *)cmd;
+ NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
uint16_t cqid = le16_to_cpu(c->cqid);
uint16_t vector = le16_to_cpu(c->irq_vector);
uint16_t qsize = le16_to_cpu(c->qsize);
@@ -1054,9 +1058,9 @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
return NVME_SUCCESS;
}
-static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c,
- NvmeRequest *req)
+static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
uint64_t prp1 = le64_to_cpu(c->prp1);
uint64_t prp2 = le64_to_cpu(c->prp2);
@@ -1066,10 +1070,10 @@ static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c,
prp2, DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c,
- NvmeRequest *req)
+static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req)
{
NvmeNamespace *ns;
+ NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
uint32_t nsid = le32_to_cpu(c->nsid);
uint64_t prp1 = le64_to_cpu(c->prp1);
uint64_t prp2 = le64_to_cpu(c->prp2);
@@ -1087,9 +1091,9 @@ static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c,
prp2, DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c,
- NvmeRequest *req)
+static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
static const int data_len = NVME_IDENTIFY_DATA_SIZE;
uint32_t min_nsid = le32_to_cpu(c->nsid);
uint64_t prp1 = le64_to_cpu(c->prp1);
@@ -1116,9 +1120,9 @@ static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c,
return ret;
}
-static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeIdentify *c,
- NvmeRequest *req)
+static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
uint32_t nsid = le32_to_cpu(c->nsid);
uint64_t prp1 = le64_to_cpu(c->prp1);
uint64_t prp2 = le64_to_cpu(c->prp2);
@@ -1157,28 +1161,28 @@ static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeIdentify *c,
DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
{
- NvmeIdentify *c = (NvmeIdentify *)cmd;
+ NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
switch (le32_to_cpu(c->cns)) {
case NVME_ID_CNS_NS:
- return nvme_identify_ns(n, c, req);
+ return nvme_identify_ns(n, req);
case NVME_ID_CNS_CTRL:
- return nvme_identify_ctrl(n, c, req);
+ return nvme_identify_ctrl(n, req);
case NVME_ID_CNS_NS_ACTIVE_LIST:
- return nvme_identify_nslist(n, c, req);
+ return nvme_identify_nslist(n, req);
case NVME_ID_CNS_NS_DESCR_LIST:
- return nvme_identify_ns_descr_list(n, c, req);
+ return nvme_identify_ns_descr_list(n, req);
default:
trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
return NVME_INVALID_FIELD | NVME_DNR;
}
}
-static uint16_t nvme_abort(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
{
- uint16_t sqid = le32_to_cpu(cmd->cdw10) & 0xffff;
+ uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
req->cqe.result = 1;
if (nvme_check_sqid(n, sqid)) {
@@ -1228,9 +1232,9 @@ static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
return cpu_to_le64(ts.all);
}
-static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd,
- NvmeRequest *req)
+static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
@@ -1240,8 +1244,9 @@ static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd,
prp2, DMA_DIRECTION_FROM_DEVICE, req);
}
-static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
uint32_t dw10 = le32_to_cpu(cmd->cdw10);
uint32_t dw11 = le32_to_cpu(cmd->cdw11);
uint32_t nsid = le32_to_cpu(cmd->nsid);
@@ -1311,7 +1316,7 @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
result = cpu_to_le32(n->features.async_config);
break;
case NVME_TIMESTAMP:
- return nvme_get_feature_timestamp(n, cmd, req);
+ return nvme_get_feature_timestamp(n, req);
default:
break;
}
@@ -1358,11 +1363,11 @@ out:
return NVME_SUCCESS;
}
-static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd,
- NvmeRequest *req)
+static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
{
uint16_t ret;
uint64_t timestamp;
+ NvmeCmd *cmd = &req->cmd;
uint64_t prp1 = le64_to_cpu(cmd->dptr.prp1);
uint64_t prp2 = le64_to_cpu(cmd->dptr.prp2);
@@ -1377,8 +1382,9 @@ static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeCmd *cmd,
return NVME_SUCCESS;
}
-static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
{
+ NvmeCmd *cmd = &req->cmd;
uint32_t dw10 = le32_to_cpu(cmd->cdw10);
uint32_t dw11 = le32_to_cpu(cmd->cdw11);
uint32_t nsid = le32_to_cpu(cmd->nsid);
@@ -1469,14 +1475,14 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
n->features.async_config = dw11;
break;
case NVME_TIMESTAMP:
- return nvme_set_feature_timestamp(n, cmd, req);
+ return nvme_set_feature_timestamp(n, req);
default:
return NVME_FEAT_NOT_CHANGABLE | NVME_DNR;
}
return NVME_SUCCESS;
}
-static uint16_t nvme_aer(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
{
trace_pci_nvme_aer(nvme_cid(req));
@@ -1495,33 +1501,33 @@ static uint16_t nvme_aer(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
return NVME_NO_COMPLETE;
}
-static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
{
- trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), cmd->opcode);
+ trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode);
- switch (cmd->opcode) {
+ switch (req->cmd.opcode) {
case NVME_ADM_CMD_DELETE_SQ:
- return nvme_del_sq(n, cmd);
+ return nvme_del_sq(n, req);
case NVME_ADM_CMD_CREATE_SQ:
- return nvme_create_sq(n, cmd);
+ return nvme_create_sq(n, req);
case NVME_ADM_CMD_GET_LOG_PAGE:
- return nvme_get_log(n, cmd, req);
+ return nvme_get_log(n, req);
case NVME_ADM_CMD_DELETE_CQ:
- return nvme_del_cq(n, cmd);
+ return nvme_del_cq(n, req);
case NVME_ADM_CMD_CREATE_CQ:
- return nvme_create_cq(n, cmd);
+ return nvme_create_cq(n, req);
case NVME_ADM_CMD_IDENTIFY:
- return nvme_identify(n, cmd, req);
+ return nvme_identify(n, req);
case NVME_ADM_CMD_ABORT:
- return nvme_abort(n, cmd, req);
+ return nvme_abort(n, req);
case NVME_ADM_CMD_SET_FEATURES:
- return nvme_set_feature(n, cmd, req);
+ return nvme_set_feature(n, req);
case NVME_ADM_CMD_GET_FEATURES:
- return nvme_get_feature(n, cmd, req);
+ return nvme_get_feature(n, req);
case NVME_ADM_CMD_ASYNC_EV_REQ:
- return nvme_aer(n, cmd, req);
+ return nvme_aer(n, req);
default:
- trace_pci_nvme_err_invalid_admin_opc(cmd->opcode);
+ trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
return NVME_INVALID_OPCODE | NVME_DNR;
}
}
@@ -1546,9 +1552,10 @@ static void nvme_process_sq(void *opaque)
QTAILQ_REMOVE(&sq->req_list, req, entry);
QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
req->cqe.cid = cmd.cid;
+ memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
- status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
- nvme_admin_cmd(n, &cmd, req);
+ status = sq->sqid ? nvme_io_cmd(n, req) :
+ nvme_admin_cmd(n, req);
if (status != NVME_NO_COMPLETE) {
req->status = status;
nvme_enqueue_req_completion(cq, req);
@@ -25,6 +25,7 @@ typedef struct NvmeRequest {
BlockAIOCB *aiocb;
uint16_t status;
NvmeCqe cqe;
+ NvmeCmd cmd;
BlockAcctCookie acct;
QEMUSGList qsg;
QEMUIOVector iov;