@@ -180,8 +180,8 @@ struct fsync_iocb {
struct poll_iocb {
struct file *file;
struct wait_queue_head *head;
+ bool *taken;
__poll_t events;
- bool woken;
bool cancelled;
struct wait_queue_entry wait;
struct work_struct work;
@@ -209,8 +209,6 @@ struct aio_kiocb {
struct list_head ki_list; /* the aio core uses this
* for cancellation */
- refcount_t ki_refcnt;
-
/*
* If the aio_resfd field of the userspace iocb is not zero,
* this is the underlying eventfd context to deliver events to.
@@ -1034,7 +1032,6 @@ static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
percpu_ref_get(&ctx->reqs);
req->ki_ctx = ctx;
INIT_LIST_HEAD(&req->ki_list);
- refcount_set(&req->ki_refcnt, 0);
req->ki_eventfd = NULL;
return req;
}
@@ -1069,13 +1066,10 @@ static struct kioctx *lookup_ioctx(unsigned long ctx_id)
static inline void iocb_put(struct aio_kiocb *iocb)
{
- if (refcount_read(&iocb->ki_refcnt) == 0 ||
- refcount_dec_and_test(&iocb->ki_refcnt)) {
- if (iocb->ki_filp)
- fput(iocb->ki_filp);
- percpu_ref_put(&iocb->ki_ctx->reqs);
- kmem_cache_free(kiocb_cachep, iocb);
- }
+ if (iocb->ki_filp)
+ fput(iocb->ki_filp);
+ percpu_ref_put(&iocb->ki_ctx->reqs);
+ kmem_cache_free(kiocb_cachep, iocb);
}
static void aio_fill_event(struct io_event *ev, struct aio_kiocb *iocb,
@@ -1672,8 +1666,10 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
if (mask && !(mask & req->events))
return 0;
- req->woken = true;
-
+ if (unlikely(req->taken)) {
+ *req->taken = true;
+ req->taken = NULL;
+ }
if (mask) {
/*
* Try to complete the iocb inline if we can. Use
@@ -1698,6 +1694,7 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
struct aio_poll_table {
struct poll_table_struct pt;
+ struct wait_queue_head *head;
struct aio_kiocb *iocb;
int error;
};
@@ -1715,7 +1712,7 @@ aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,
}
pt->error = 0;
- pt->iocb->poll.head = head;
+ pt->head = pt->iocb->poll.head = head;
add_wait_queue(head, &pt->iocb->poll.wait);
}
@@ -1738,7 +1735,7 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
req->head = NULL;
- req->woken = false;
+ req->taken = &async;
req->cancelled = false;
apt.pt._qproc = aio_poll_queue_proc;
@@ -1750,36 +1747,38 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
INIT_LIST_HEAD(&req->wait.entry);
init_waitqueue_func_entry(&req->wait, aio_poll_wake);
- /* one for removal from waitqueue, one for this function */
- refcount_set(&aiocb->ki_refcnt, 2);
-
- mask = vfs_poll(req->file, &apt.pt) & req->events;
- if (unlikely(!req->head)) {
+ mask = req->events;
+ mask &= vfs_poll(req->file, &apt.pt);
+ /*
+ * Careful: we might've been put into waitqueue *and* already
+ * woken up before vfs_poll() returns. The caller is holding
+ * a reference to file, so it couldn't have been killed under
+ * us, no matter what. However, in case of early wakeup, @req
+ * might be already gone by now.
+ */
+ if (unlikely(!apt.head)) {
/* we did not manage to set up a waitqueue, done */
goto out;
}
-
spin_lock_irq(&ctx->ctx_lock);
- spin_lock(&req->head->lock);
- if (req->woken) { /* already taken up by aio_poll_wake() */
- async = true;
+ spin_lock(&apt.head->lock);
+ if (async) { /* already taken up by aio_poll_wake() */
apt.error = 0;
} else if (!mask && !apt.error) { /* actually waiting for an event */
list_add_tail(&aiocb->ki_list, &ctx->active_reqs);
aiocb->ki_cancel = aio_poll_cancel;
+ req->taken = NULL;
async = true;
} else { /* if we get an error or a mask we are done */
WARN_ON_ONCE(list_empty(&req->wait.entry));
list_del_init(&req->wait.entry);
/* no wakeup in the future either; aiocb is ours to dispose of */
}
- spin_unlock(&req->head->lock);
+ spin_unlock(&apt.head->lock);
spin_unlock_irq(&ctx->ctx_lock);
-
out:
- if (async && !apt.error)
+ if (!async && !apt.error)
aio_poll_complete(aiocb, mask);
- iocb_put(aiocb);
return apt.error;
}