@@ -15,12 +15,17 @@ config CXL_EEH
bool
default n
+config CXL_AFU_DRIVER_OPS
+ bool
+ default n
+
config CXL
tristate "Support for IBM Coherent Accelerators (CXL)"
depends on PPC_POWERNV && PCI_MSI && EEH
select CXL_BASE
select CXL_KERNEL_API
select CXL_EEH
+ select CXL_AFU_DRIVER_OPS
default m
help
Select this option to enable driver support for IBM Coherent
@@ -313,6 +313,33 @@ struct cxl_context *cxl_fops_get_context(struct file *file)
}
EXPORT_SYMBOL_GPL(cxl_fops_get_context);
+void cxl_set_driver_ops(struct cxl_context *ctx,
+ struct cxl_afu_driver_ops *ops)
+{
+ WARN_ON(!ops->deliver_event);
+ atomic_set(&ctx->afu_driver_events, 0);
+ ctx->afu_driver_ops = ops;
+}
+EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
+
+int cxl_unset_driver_ops(struct cxl_context *ctx)
+{
+ if (atomic_read(&ctx->afu_driver_events))
+ return -EBUSY;
+
+ ctx->afu_driver_ops = NULL;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cxl_unset_driver_ops);
+
+void cxl_context_events_pending(struct cxl_context *ctx,
+ unsigned int new_events)
+{
+ atomic_add(new_events, &ctx->afu_driver_events);
+ wake_up_all(&ctx->wq);
+}
+EXPORT_SYMBOL_GPL(cxl_context_events_pending);
+
int cxl_start_work(struct cxl_context *ctx,
struct cxl_ioctl_start_work *work)
{
@@ -24,6 +24,7 @@
#include <asm/reg.h>
#include <misc/cxl-base.h>
+#include <misc/cxl.h>
#include <uapi/misc/cxl.h>
extern uint cxl_verbose;
@@ -34,7 +35,7 @@ extern uint cxl_verbose;
* Bump version each time a user API change is made, whether it is
* backwards compatible ot not.
*/
-#define CXL_API_VERSION 2
+#define CXL_API_VERSION 3
#define CXL_API_VERSION_COMPATIBLE 1
/*
@@ -528,6 +529,9 @@ struct cxl_context {
bool pending_fault;
bool pending_afu_err;
+ /* Used by AFU drivers for driver specific event delivery */
+ struct cxl_afu_driver_ops *afu_driver_ops;
+ atomic_t afu_driver_events;
struct rcu_head rcu;
};
@@ -293,6 +293,17 @@ int afu_mmap(struct file *file, struct vm_area_struct *vm)
return cxl_context_iomap(ctx, vm);
}
+static inline bool ctx_event_pending(struct cxl_context *ctx)
+{
+ if (ctx->pending_irq || ctx->pending_fault || ctx->pending_afu_err)
+ return true;
+
+ if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events))
+ return true;
+
+ return false;
+}
+
unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
{
struct cxl_context *ctx = file->private_data;
@@ -305,8 +316,7 @@ unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
pr_devel("afu_poll wait done pe: %i\n", ctx->pe);
spin_lock_irqsave(&ctx->lock, flags);
- if (ctx->pending_irq || ctx->pending_fault ||
- ctx->pending_afu_err)
+ if (ctx_event_pending(ctx))
mask |= POLLIN | POLLRDNORM;
else if (ctx->status == CLOSED)
/* Only error on closed when there are no futher events pending
@@ -319,12 +329,6 @@ unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
return mask;
}
-static inline int ctx_event_pending(struct cxl_context *ctx)
-{
- return (ctx->pending_irq || ctx->pending_fault ||
- ctx->pending_afu_err || (ctx->status == CLOSED));
-}
-
ssize_t afu_read(struct file *file, char __user *buf, size_t count,
loff_t *off)
{
@@ -344,7 +348,7 @@ ssize_t afu_read(struct file *file, char __user *buf, size_t count,
for (;;) {
prepare_to_wait(&ctx->wq, &wait, TASK_INTERRUPTIBLE);
- if (ctx_event_pending(ctx))
+ if (ctx_event_pending(ctx) || (ctx->status == CLOSED))
break;
if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
@@ -374,7 +378,14 @@ ssize_t afu_read(struct file *file, char __user *buf, size_t count,
memset(&event, 0, sizeof(event));
event.header.process_element = ctx->pe;
event.header.size = sizeof(struct cxl_event_header);
- if (ctx->pending_irq) {
+ if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events)) {
+ pr_devel("afu_read delivering AFU driver specific event\n");
+ event.header.type = CXL_EVENT_AFU_DRIVER;
+ ctx->afu_driver_ops->deliver_event(ctx, &event, sizeof(event));
+ WARN_ON(event.header.size > sizeof(event));
+ atomic_dec(&ctx->afu_driver_events);
+
+ } else if (ctx->pending_irq) {
pr_devel("afu_read delivering AFU interrupt\n");
event.header.size += sizeof(struct cxl_event_afu_interrupt);
event.header.type = CXL_EVENT_AFU_INTERRUPT;
@@ -382,6 +393,7 @@ ssize_t afu_read(struct file *file, char __user *buf, size_t count,
clear_bit(event.irq.irq - 1, ctx->irq_bitmap);
if (bitmap_empty(ctx->irq_bitmap, ctx->irq_count))
ctx->pending_irq = false;
+
} else if (ctx->pending_fault) {
pr_devel("afu_read delivering data storage fault\n");
event.header.size += sizeof(struct cxl_event_data_storage);
@@ -389,12 +401,14 @@ ssize_t afu_read(struct file *file, char __user *buf, size_t count,
event.fault.addr = ctx->fault_addr;
event.fault.dsisr = ctx->fault_dsisr;
ctx->pending_fault = false;
+
} else if (ctx->pending_afu_err) {
pr_devel("afu_read delivering afu error\n");
event.header.size += sizeof(struct cxl_event_afu_error);
event.header.type = CXL_EVENT_AFU_ERROR;
event.afu_error.error = ctx->afu_err;
ctx->pending_afu_err = false;
+
} else if (ctx->status == CLOSED) {
pr_devel("afu_read fatal error\n");
spin_unlock_irqrestore(&ctx->lock, flags);
@@ -558,7 +572,7 @@ int __init cxl_file_init(void)
* If these change we really need to update API. Either change some
* flags or update API version number CXL_API_VERSION.
*/
- BUILD_BUG_ON(CXL_API_VERSION != 2);
+ BUILD_BUG_ON(CXL_API_VERSION != 3);
BUILD_BUG_ON(sizeof(struct cxl_ioctl_start_work) != 64);
BUILD_BUG_ON(sizeof(struct cxl_event_header) != 8);
BUILD_BUG_ON(sizeof(struct cxl_event_afu_interrupt) != 8);
@@ -212,4 +212,47 @@ void cxl_perst_reloads_same_image(struct cxl_afu *afu,
*/
ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count);
+/*
+ * AFU driver ops allows an AFU driver to create their own events to pass to
+ * userspace through the file descriptor as a simpler alternative to overriding
+ * the read() and poll() calls that works with the generic cxl events. These
+ * events are given priority over the generic cxl events, so they will be
+ * delivered first if multiple types of events are pending.
+ *
+ * The AFU driver must call cxl_context_events_pending() to notify the cxl
+ * driver that new events are ready to be delivered for a specific context.
+ * cxl_context_events_pending() will adjust the current count of AFU driver
+ * events for this context, and wake up anyone waiting on the context wait
+ * queue.
+ *
+ * The cxl driver will then call deliver_event() to fill out a cxl_event
+ * structure with the driver specific event. The header will already have the
+ * type and process_element fields filled in, and header.size will be set to
+ * sizeof(struct cxl_event_header). The AFU driver can extend that size up to
+ * max_size (if an afu driver requires more space, they should submit a patch
+ * increasing the size in the struct cxl_event_afu_driver_reserved definition).
+ *
+ * deliver_event() is called with a spin lock held, so it must not sleep.
+ */
+struct cxl_afu_driver_ops {
+ void (*deliver_event) (struct cxl_context *ctx,
+ struct cxl_event *event, size_t max_size);
+};
+
+/*
+ * Associate the above driver ops with a specific context.
+ * Reset the current count of AFU driver events.
+ */
+void cxl_set_driver_ops(struct cxl_context *ctx,
+ struct cxl_afu_driver_ops *ops);
+/*
+ * Remove the driver ops from a specific context.
+ * The current count of AFU driver events must be 0.
+ */
+int cxl_unset_driver_ops(struct cxl_context *ctx);
+
+/* Notify cxl driver that new events are ready to be delivered for context */
+void cxl_context_events_pending(struct cxl_context *ctx,
+ unsigned int new_events);
+
#endif /* _MISC_CXL_H */
@@ -93,6 +93,7 @@ enum cxl_event_type {
CXL_EVENT_AFU_INTERRUPT = 1,
CXL_EVENT_DATA_STORAGE = 2,
CXL_EVENT_AFU_ERROR = 3,
+ CXL_EVENT_AFU_DRIVER = 4,
};
struct cxl_event_header {
@@ -124,12 +125,33 @@ struct cxl_event_afu_error {
__u64 error;
};
+struct cxl_event_afu_driver_reserved {
+ /*
+ * Reserves space for AFU driver specific events. Not actually
+ * reserving any more space compared to other events as we can't know
+ * how much an AFU driver will need (but it is likely to be small). If
+ * your AFU driver needs more than this, please submit a patch
+ * increasing it as part of your driver submission.
+ *
+ * This is not ABI since the event header.size passed to the user for
+ * existing events is set in the read call to sizeof(cxl_event_header)
+ * + sizeof(whatever event is being dispatched) and will not increase
+ * just because this is, and the user is already required to use a 4K
+ * buffer on the read call. This is merely the size of the buffer
+ * passed between the cxl and AFU drivers.
+ *
+ * Of course the contents will be ABI, but that's up the AFU driver.
+ */
+ __u64 reserved[4];
+};
+
struct cxl_event {
struct cxl_event_header header;
union {
struct cxl_event_afu_interrupt irq;
struct cxl_event_data_storage fault;
struct cxl_event_afu_error afu_error;
+ struct cxl_event_afu_driver_reserved afu_driver_event;
};
};