@@ -19,6 +19,7 @@
#include <linux/types.h>
#include <linux/pm_runtime.h>
#include <linux/pci.h>
+#include <ras/ras_event.h>
#include "pciehp.h"
/* The following routines constitute the bulk of the
@@ -245,6 +246,8 @@ void pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events)
if (events & PCI_EXP_SLTSTA_PDC)
ctrl_info(ctrl, "Slot(%s): Card not present\n",
slot_name(ctrl));
+ trace_pciehp_event(dev_name(&ctrl->pcie->port->dev),
+ slot_name(ctrl), ON_STATE, events);
pciehp_disable_slot(ctrl, SURPRISE_REMOVAL);
break;
default:
@@ -282,6 +285,8 @@ void pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events)
if (link_active)
ctrl_info(ctrl, "Slot(%s): Link Up\n",
slot_name(ctrl));
+ trace_pciehp_event(dev_name(&ctrl->pcie->port->dev),
+ slot_name(ctrl), OFF_STATE, events);
ctrl->request_result = pciehp_enable_slot(ctrl);
break;
default:
@@ -338,6 +338,35 @@ TRACE_EVENT(aer_event,
"Not available")
);
+TRACE_EVENT(pciehp_event,
+ TP_PROTO(const char *port_name,
+ const char *slot,
+ const u8 state,
+ const u32 events),
+
+ TP_ARGS(port_name, slot, state, events),
+
+ TP_STRUCT__entry(
+ __string( port_name, port_name )
+ __string( slot, slot )
+ __field( u8, state )
+ __field( u32, events )
+ ),
+
+ TP_fast_assign(
+ __assign_str(port_name);
+ __assign_str(slot);
+ __entry->state = state;
+ __entry->events = events;
+ ),
+
+ TP_printk("%s slot:%s, state:%d, events:%d\n",
+ __get_str(port_name),
+ __get_str(slot),
+ __entry->state,
+ __entry->events)
+);
+
/*
* memory-failure recovery action result event
*
Hotplug events are critical indicators for analyzing hardware health, particularly in AI supercomputers where surprise link downs can significantly impact system performance and reliability. The failure characterization analysis illustrates the significance of failures caused by the Infiniband link errors. Meta observe that 2% in a machine learning cluster and 6% in a vision application cluster of Infiniband failures co-occur with GPU failures, such as falling off the bus, which may indicate a correlation with PCIe.[1] Generate a RAS tracepoint for hotplug event to help healthy check. The output like below: $ echo 1 > /sys/kernel/debug/tracing/events/ras/pciehp_event/enable $ cat /sys/kernel/debug/tracing/trace_pipe <...>-213 [001] ..... 43.762740: pciehp_event: 0000:00:02.0 slot:10, state:5, events:65792 [1]https://arxiv.org/abs/2410.21680 Signed-off-by: Shuai Xue <xueshuai@linux.alibaba.com> --- drivers/pci/hotplug/pciehp_ctrl.c | 5 +++++ include/ras/ras_event.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+)