From patchwork Mon Mar 26 20:57:32 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891258 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096YH681Pz9s1B for ; Tue, 27 Mar 2018 08:22:27 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752036AbeCZVWZ (ORCPT ); Mon, 26 Mar 2018 17:22:25 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49894 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752307AbeCZVWX (ORCPT ); Mon, 26 Mar 2018 17:22:23 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id 908E82B8D7; Mon, 26 Mar 2018 16:22:22 -0500 (CDT) Message-Id: <326e72eaa0a6ed3d42aa857f13c80232a2550c77.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:32 -0700 Subject: [PATCH v3 iproute2-next 1/8] rdma: include rdma-core To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org This avoids requiring rdma-core be installed on systems. Signed-off-by: Steve Wise --- rdma/include/rdma/rdma_cma.h | 728 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 rdma/include/rdma/rdma_cma.h diff --git a/rdma/include/rdma/rdma_cma.h b/rdma/include/rdma/rdma_cma.h new file mode 100644 index 0000000..456aa4e --- /dev/null +++ b/rdma/include/rdma/rdma_cma.h @@ -0,0 +1,728 @@ +/* + * Copyright (c) 2005 Voltaire Inc. All rights reserved. + * Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if !defined(_RDMA_CMA_H) +#define _RDMA_CMA_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Upon receiving a device removal event, users must destroy the associated + * RDMA identifier and release all resources allocated with the device. + */ +enum rdma_cm_event_type { + RDMA_CM_EVENT_ADDR_RESOLVED, + RDMA_CM_EVENT_ADDR_ERROR, + RDMA_CM_EVENT_ROUTE_RESOLVED, + RDMA_CM_EVENT_ROUTE_ERROR, + RDMA_CM_EVENT_CONNECT_REQUEST, + RDMA_CM_EVENT_CONNECT_RESPONSE, + RDMA_CM_EVENT_CONNECT_ERROR, + RDMA_CM_EVENT_UNREACHABLE, + RDMA_CM_EVENT_REJECTED, + RDMA_CM_EVENT_ESTABLISHED, + RDMA_CM_EVENT_DISCONNECTED, + RDMA_CM_EVENT_DEVICE_REMOVAL, + RDMA_CM_EVENT_MULTICAST_JOIN, + RDMA_CM_EVENT_MULTICAST_ERROR, + RDMA_CM_EVENT_ADDR_CHANGE, + RDMA_CM_EVENT_TIMEWAIT_EXIT +}; + +enum rdma_port_space { + RDMA_PS_IPOIB = 0x0002, + RDMA_PS_TCP = 0x0106, + RDMA_PS_UDP = 0x0111, + RDMA_PS_IB = 0x013F, +}; + +#define RDMA_IB_IP_PS_MASK 0xFFFFFFFFFFFF0000ULL +#define RDMA_IB_IP_PORT_MASK 0x000000000000FFFFULL +#define RDMA_IB_IP_PS_TCP 0x0000000001060000ULL +#define RDMA_IB_IP_PS_UDP 0x0000000001110000ULL +#define RDMA_IB_PS_IB 0x00000000013F0000ULL + +/* + * Global qkey value for UDP QPs and multicast groups created via the + * RDMA CM. + */ +#define RDMA_UDP_QKEY 0x01234567 + +struct rdma_ib_addr { + union ibv_gid sgid; + union ibv_gid dgid; + __be16 pkey; +}; + +struct rdma_addr { + union { + struct sockaddr src_addr; + struct sockaddr_in src_sin; + struct sockaddr_in6 src_sin6; + struct sockaddr_storage src_storage; + }; + union { + struct sockaddr dst_addr; + struct sockaddr_in dst_sin; + struct sockaddr_in6 dst_sin6; + struct sockaddr_storage dst_storage; + }; + union { + struct rdma_ib_addr ibaddr; + } addr; +}; + +struct rdma_route { + struct rdma_addr addr; + struct ibv_sa_path_rec *path_rec; + int num_paths; +}; + +struct rdma_event_channel { + int fd; +}; + +struct rdma_cm_id { + struct ibv_context *verbs; + struct rdma_event_channel *channel; + void *context; + struct ibv_qp *qp; + struct rdma_route route; + enum rdma_port_space ps; + uint8_t port_num; + struct rdma_cm_event *event; + struct ibv_comp_channel *send_cq_channel; + struct ibv_cq *send_cq; + struct ibv_comp_channel *recv_cq_channel; + struct ibv_cq *recv_cq; + struct ibv_srq *srq; + struct ibv_pd *pd; + enum ibv_qp_type qp_type; +}; + +enum { + RDMA_MAX_RESP_RES = 0xFF, + RDMA_MAX_INIT_DEPTH = 0xFF +}; + +struct rdma_conn_param { + const void *private_data; + uint8_t private_data_len; + uint8_t responder_resources; + uint8_t initiator_depth; + uint8_t flow_control; + uint8_t retry_count; /* ignored when accepting */ + uint8_t rnr_retry_count; + /* Fields below ignored if a QP is created on the rdma_cm_id. */ + uint8_t srq; + uint32_t qp_num; +}; + +struct rdma_ud_param { + const void *private_data; + uint8_t private_data_len; + struct ibv_ah_attr ah_attr; + uint32_t qp_num; + uint32_t qkey; +}; + +struct rdma_cm_event { + struct rdma_cm_id *id; + struct rdma_cm_id *listen_id; + enum rdma_cm_event_type event; + int status; + union { + struct rdma_conn_param conn; + struct rdma_ud_param ud; + } param; +}; + +#define RAI_PASSIVE 0x00000001 +#define RAI_NUMERICHOST 0x00000002 +#define RAI_NOROUTE 0x00000004 +#define RAI_FAMILY 0x00000008 + +struct rdma_addrinfo { + int ai_flags; + int ai_family; + int ai_qp_type; + int ai_port_space; + socklen_t ai_src_len; + socklen_t ai_dst_len; + struct sockaddr *ai_src_addr; + struct sockaddr *ai_dst_addr; + char *ai_src_canonname; + char *ai_dst_canonname; + size_t ai_route_len; + void *ai_route; + size_t ai_connect_len; + void *ai_connect; + struct rdma_addrinfo *ai_next; +}; + +/* Multicast join compatibility mask attributes */ +enum rdma_cm_join_mc_attr_mask { + RDMA_CM_JOIN_MC_ATTR_ADDRESS = 1 << 0, + RDMA_CM_JOIN_MC_ATTR_JOIN_FLAGS = 1 << 1, + RDMA_CM_JOIN_MC_ATTR_RESERVED = 1 << 2, +}; + +/* Multicast join flags */ +enum rdma_cm_mc_join_flags { + RDMA_MC_JOIN_FLAG_FULLMEMBER, + RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER, + RDMA_MC_JOIN_FLAG_RESERVED, +}; + +struct rdma_cm_join_mc_attr_ex { + /* Bitwise OR between "rdma_cm_join_mc_attr_mask" enum */ + uint32_t comp_mask; + /* Use a flag from "rdma_cm_mc_join_flags" enum */ + uint32_t join_flags; + /* Multicast address identifying the group to join */ + struct sockaddr *addr; +}; + +/** + * rdma_create_event_channel - Open a channel used to report communication events. + * Description: + * Asynchronous events are reported to users through event channels. Each + * event channel maps to a file descriptor. + * Notes: + * All created event channels must be destroyed by calling + * rdma_destroy_event_channel. Users should call rdma_get_cm_event to + * retrieve events on an event channel. + * See also: + * rdma_get_cm_event, rdma_destroy_event_channel + */ +struct rdma_event_channel *rdma_create_event_channel(void); + +/** + * rdma_destroy_event_channel - Close an event communication channel. + * @channel: The communication channel to destroy. + * Description: + * Release all resources associated with an event channel and closes the + * associated file descriptor. + * Notes: + * All rdma_cm_id's associated with the event channel must be destroyed, + * and all returned events must be acked before calling this function. + * See also: + * rdma_create_event_channel, rdma_get_cm_event, rdma_ack_cm_event + */ +void rdma_destroy_event_channel(struct rdma_event_channel *channel); + +/** + * rdma_create_id - Allocate a communication identifier. + * @channel: The communication channel that events associated with the + * allocated rdma_cm_id will be reported on. + * @id: A reference where the allocated communication identifier will be + * returned. + * @context: User specified context associated with the rdma_cm_id. + * @ps: RDMA port space. + * Description: + * Creates an identifier that is used to track communication information. + * Notes: + * Rdma_cm_id's are conceptually equivalent to a socket for RDMA + * communication. The difference is that RDMA communication requires + * explicitly binding to a specified RDMA device before communication + * can occur, and most operations are asynchronous in nature. Communication + * events on an rdma_cm_id are reported through the associated event + * channel. Users must release the rdma_cm_id by calling rdma_destroy_id. + * See also: + * rdma_create_event_channel, rdma_destroy_id, rdma_get_devices, + * rdma_bind_addr, rdma_resolve_addr, rdma_connect, rdma_listen, + */ +int rdma_create_id(struct rdma_event_channel *channel, + struct rdma_cm_id **id, void *context, + enum rdma_port_space ps); + +/** + * rdma_create_ep - Allocate a communication identifier and qp. + * @id: A reference where the allocated communication identifier will be + * returned. + * @res: Result from rdma_getaddrinfo, which specifies the source and + * destination addresses, plus optional routing and connection information. + * @pd: Optional protection domain. This parameter is ignored if qp_init_attr + * is NULL. + * @qp_init_attr: Optional attributes for a QP created on the rdma_cm_id. + * Description: + * Create an identifier and option QP used for communication. + * Notes: + * If qp_init_attr is provided, then a queue pair will be allocated and + * associated with the rdma_cm_id. If a pd is provided, the QP will be + * created on that PD. Otherwise, the QP will be allocated on a default + * PD. + * The rdma_cm_id will be set to use synchronous operations (connect, + * listen, and get_request). To convert to asynchronous operation, the + * rdma_cm_id should be migrated to a user allocated event channel. + * See also: + * rdma_create_id, rdma_create_qp, rdma_migrate_id, rdma_connect, + * rdma_listen + */ +int rdma_create_ep(struct rdma_cm_id **id, struct rdma_addrinfo *res, + struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr); + +/** + * rdma_destroy_ep - Deallocates a communication identifier and qp. + * @id: The communication identifier to destroy. + * Description: + * Destroys the specified rdma_cm_id and any associated QP created + * on that id. + * See also: + * rdma_create_ep + */ +void rdma_destroy_ep(struct rdma_cm_id *id); + +/** + * rdma_destroy_id - Release a communication identifier. + * @id: The communication identifier to destroy. + * Description: + * Destroys the specified rdma_cm_id and cancels any outstanding + * asynchronous operation. + * Notes: + * Users must free any associated QP with the rdma_cm_id before + * calling this routine and ack an related events. + * See also: + * rdma_create_id, rdma_destroy_qp, rdma_ack_cm_event + */ +int rdma_destroy_id(struct rdma_cm_id *id); + +/** + * rdma_bind_addr - Bind an RDMA identifier to a source address. + * @id: RDMA identifier. + * @addr: Local address information. Wildcard values are permitted. + * Description: + * Associates a source address with an rdma_cm_id. The address may be + * wildcarded. If binding to a specific local address, the rdma_cm_id + * will also be bound to a local RDMA device. + * Notes: + * Typically, this routine is called before calling rdma_listen to bind + * to a specific port number, but it may also be called on the active side + * of a connection before calling rdma_resolve_addr to bind to a specific + * address. + * See also: + * rdma_create_id, rdma_listen, rdma_resolve_addr, rdma_create_qp + */ +int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr); + +/** + * rdma_resolve_addr - Resolve destination and optional source addresses. + * @id: RDMA identifier. + * @src_addr: Source address information. This parameter may be NULL. + * @dst_addr: Destination address information. + * @timeout_ms: Time to wait for resolution to complete. + * Description: + * Resolve destination and optional source addresses from IP addresses + * to an RDMA address. If successful, the specified rdma_cm_id will + * be bound to a local device. + * Notes: + * This call is used to map a given destination IP address to a usable RDMA + * address. If a source address is given, the rdma_cm_id is bound to that + * address, the same as if rdma_bind_addr were called. If no source + * address is given, and the rdma_cm_id has not yet been bound to a device, + * then the rdma_cm_id will be bound to a source address based on the + * local routing tables. After this call, the rdma_cm_id will be bound to + * an RDMA device. This call is typically made from the active side of a + * connection before calling rdma_resolve_route and rdma_connect. + * See also: + * rdma_create_id, rdma_resolve_route, rdma_connect, rdma_create_qp, + * rdma_get_cm_event, rdma_bind_addr + */ +int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, + struct sockaddr *dst_addr, int timeout_ms); + +/** + * rdma_resolve_route - Resolve the route information needed to establish a connection. + * @id: RDMA identifier. + * @timeout_ms: Time to wait for resolution to complete. + * Description: + * Resolves an RDMA route to the destination address in order to establish + * a connection. The destination address must have already been resolved + * by calling rdma_resolve_addr. + * Notes: + * This is called on the client side of a connection after calling + * rdma_resolve_addr, but before calling rdma_connect. + * See also: + * rdma_resolve_addr, rdma_connect, rdma_get_cm_event + */ +int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms); + +/** + * rdma_create_qp - Allocate a QP. + * @id: RDMA identifier. + * @pd: Optional protection domain for the QP. + * @qp_init_attr: initial QP attributes. + * Description: + * Allocate a QP associated with the specified rdma_cm_id and transition it + * for sending and receiving. + * Notes: + * The rdma_cm_id must be bound to a local RDMA device before calling this + * function, and the protection domain must be for that same device. + * QPs allocated to an rdma_cm_id are automatically transitioned by the + * librdmacm through their states. After being allocated, the QP will be + * ready to handle posting of receives. If the QP is unconnected, it will + * be ready to post sends. + * If pd is NULL, then the QP will be allocated using a default protection + * domain associated with the underlying RDMA device. + * See also: + * rdma_bind_addr, rdma_resolve_addr, rdma_destroy_qp, ibv_create_qp, + * ibv_modify_qp + */ +int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr); +int rdma_create_qp_ex(struct rdma_cm_id *id, + struct ibv_qp_init_attr_ex *qp_init_attr); + +/** + * rdma_destroy_qp - Deallocate a QP. + * @id: RDMA identifier. + * Description: + * Destroy a QP allocated on the rdma_cm_id. + * Notes: + * Users must destroy any QP associated with an rdma_cm_id before + * destroying the ID. + * See also: + * rdma_create_qp, rdma_destroy_id, ibv_destroy_qp + */ +void rdma_destroy_qp(struct rdma_cm_id *id); + +/** + * rdma_connect - Initiate an active connection request. + * @id: RDMA identifier. + * @conn_param: optional connection parameters. + * Description: + * For a connected rdma_cm_id, this call initiates a connection request + * to a remote destination. For an unconnected rdma_cm_id, it initiates + * a lookup of the remote QP providing the datagram service. + * Notes: + * Users must have resolved a route to the destination address + * by having called rdma_resolve_route before calling this routine. + * A user may override the default connection parameters and exchange + * private data as part of the connection by using the conn_param parameter. + * See also: + * rdma_resolve_route, rdma_disconnect, rdma_listen, rdma_get_cm_event + */ +int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param); + +/** + * rdma_listen - Listen for incoming connection requests. + * @id: RDMA identifier. + * @backlog: backlog of incoming connection requests. + * Description: + * Initiates a listen for incoming connection requests or datagram service + * lookup. The listen will be restricted to the locally bound source + * address. + * Notes: + * Users must have bound the rdma_cm_id to a local address by calling + * rdma_bind_addr before calling this routine. If the rdma_cm_id is + * bound to a specific IP address, the listen will be restricted to that + * address and the associated RDMA device. If the rdma_cm_id is bound + * to an RDMA port number only, the listen will occur across all RDMA + * devices. + * See also: + * rdma_bind_addr, rdma_connect, rdma_accept, rdma_reject, rdma_get_cm_event + */ +int rdma_listen(struct rdma_cm_id *id, int backlog); + +/** + * rdma_get_request + */ +int rdma_get_request(struct rdma_cm_id *listen, struct rdma_cm_id **id); + +/** + * rdma_accept - Called to accept a connection request. + * @id: Connection identifier associated with the request. + * @conn_param: Optional information needed to establish the connection. + * Description: + * Called from the listening side to accept a connection or datagram + * service lookup request. + * Notes: + * Unlike the socket accept routine, rdma_accept is not called on a + * listening rdma_cm_id. Instead, after calling rdma_listen, the user + * waits for a connection request event to occur. Connection request + * events give the user a newly created rdma_cm_id, similar to a new + * socket, but the rdma_cm_id is bound to a specific RDMA device. + * rdma_accept is called on the new rdma_cm_id. + * A user may override the default connection parameters and exchange + * private data as part of the connection by using the conn_param parameter. + * See also: + * rdma_listen, rdma_reject, rdma_get_cm_event + */ +int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param); + +/** + * rdma_reject - Called to reject a connection request. + * @id: Connection identifier associated with the request. + * @private_data: Optional private data to send with the reject message. + * @private_data_len: Size of the private_data to send, in bytes. + * Description: + * Called from the listening side to reject a connection or datagram + * service lookup request. + * Notes: + * After receiving a connection request event, a user may call rdma_reject + * to reject the request. If the underlying RDMA transport supports + * private data in the reject message, the specified data will be passed to + * the remote side. + * See also: + * rdma_listen, rdma_accept, rdma_get_cm_event + */ +int rdma_reject(struct rdma_cm_id *id, const void *private_data, + uint8_t private_data_len); + +/** + * rdma_notify - Notifies the librdmacm of an asynchronous event. + * @id: RDMA identifier. + * @event: Asynchronous event. + * Description: + * Used to notify the librdmacm of asynchronous events that have occurred + * on a QP associated with the rdma_cm_id. + * Notes: + * Asynchronous events that occur on a QP are reported through the user's + * device event handler. This routine is used to notify the librdmacm of + * communication events. In most cases, use of this routine is not + * necessary, however if connection establishment is done out of band + * (such as done through Infiniband), it's possible to receive data on a + * QP that is not yet considered connected. This routine forces the + * connection into an established state in this case in order to handle + * the rare situation where the connection never forms on its own. + * Events that should be reported to the CM are: IB_EVENT_COMM_EST. + * See also: + * rdma_connect, rdma_accept, rdma_listen + */ +int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event); + +/** + * rdma_disconnect - This function disconnects a connection. + * @id: RDMA identifier. + * Description: + * Disconnects a connection and transitions any associated QP to the + * error state. + * See also: + * rdma_connect, rdma_listen, rdma_accept + */ +int rdma_disconnect(struct rdma_cm_id *id); + +/** + * rdma_join_multicast - Joins a multicast group. + * @id: Communication identifier associated with the request. + * @addr: Multicast address identifying the group to join. + * @context: User-defined context associated with the join request. + * Description: + * Joins a multicast group and attaches an associated QP to the group. + * Notes: + * Before joining a multicast group, the rdma_cm_id must be bound to + * an RDMA device by calling rdma_bind_addr or rdma_resolve_addr. Use of + * rdma_resolve_addr requires the local routing tables to resolve the + * multicast address to an RDMA device. The user must call + * rdma_leave_multicast to leave the multicast group and release any + * multicast resources. The context is returned to the user through + * the private_data field in the rdma_cm_event. + * See also: + * rdma_leave_multicast, rdma_bind_addr, rdma_resolve_addr, rdma_create_qp + */ +int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, + void *context); + +/** + * rdma_leave_multicast - Leaves a multicast group. + * @id: Communication identifier associated with the request. + * @addr: Multicast address identifying the group to leave. + * Description: + * Leaves a multicast group and detaches an associated QP from the group. + * Notes: + * Calling this function before a group has been fully joined results in + * canceling the join operation. Users should be aware that messages + * received from the multicast group may stilled be queued for + * completion processing immediately after leaving a multicast group. + * Destroying an rdma_cm_id will automatically leave all multicast groups. + * See also: + * rdma_join_multicast, rdma_destroy_qp + */ +int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr); + +/** + * rdma_multicast_ex - Joins a multicast group with options. + * @id: Communication identifier associated with the request. + * @mc_join_attr: Extensive struct containing multicast join parameters. + * @context: User-defined context associated with the join request. + * Description: + * Joins a multicast group with options. Currently supporting MC join flags. + * The QP will be attached based on the given join flag. + * Join message will be sent according to the join flag. + * Notes: + * Before joining a multicast group, the rdma_cm_id must be bound to + * an RDMA device by calling rdma_bind_addr or rdma_resolve_addr. Use of + * rdma_resolve_addr requires the local routing tables to resolve the + * multicast address to an RDMA device. The user must call + * rdma_leave_multicast to leave the multicast group and release any + * multicast resources. The context is returned to the user through + * the private_data field in the rdma_cm_event. + * See also: + * rdma_leave_multicast, rdma_bind_addr, rdma_resolve_addr, rdma_create_qp + */ +int rdma_join_multicast_ex(struct rdma_cm_id *id, + struct rdma_cm_join_mc_attr_ex *mc_join_attr, + void *context); + +/** + * rdma_get_cm_event - Retrieves the next pending communication event. + * @channel: Event channel to check for events. + * @event: Allocated information about the next communication event. + * Description: + * Retrieves a communication event. If no events are pending, by default, + * the call will block until an event is received. + * Notes: + * The default synchronous behavior of this routine can be changed by + * modifying the file descriptor associated with the given channel. All + * events that are reported must be acknowledged by calling rdma_ack_cm_event. + * Destruction of an rdma_cm_id will block until related events have been + * acknowledged. + * See also: + * rdma_ack_cm_event, rdma_create_event_channel, rdma_event_str + */ +int rdma_get_cm_event(struct rdma_event_channel *channel, + struct rdma_cm_event **event); + +/** + * rdma_ack_cm_event - Free a communication event. + * @event: Event to be released. + * Description: + * All events which are allocated by rdma_get_cm_event must be released, + * there should be a one-to-one correspondence between successful gets + * and acks. + * See also: + * rdma_get_cm_event, rdma_destroy_id + */ +int rdma_ack_cm_event(struct rdma_cm_event *event); + +__be16 rdma_get_src_port(struct rdma_cm_id *id); +__be16 rdma_get_dst_port(struct rdma_cm_id *id); + +static inline struct sockaddr *rdma_get_local_addr(struct rdma_cm_id *id) +{ + return &id->route.addr.src_addr; +} + +static inline struct sockaddr *rdma_get_peer_addr(struct rdma_cm_id *id) +{ + return &id->route.addr.dst_addr; +} + +/** + * rdma_get_devices - Get list of RDMA devices currently available. + * @num_devices: If non-NULL, set to the number of devices returned. + * Description: + * Return a NULL-terminated array of opened RDMA devices. Callers can use + * this routine to allocate resources on specific RDMA devices that will be + * shared across multiple rdma_cm_id's. + * Notes: + * The returned array must be released by calling rdma_free_devices. Devices + * remain opened while the librdmacm is loaded. + * See also: + * rdma_free_devices + */ +struct ibv_context **rdma_get_devices(int *num_devices); + +/** + * rdma_free_devices - Frees the list of devices returned by rdma_get_devices. + * @list: List of devices returned from rdma_get_devices. + * Description: + * Frees the device array returned by rdma_get_devices. + * See also: + * rdma_get_devices + */ +void rdma_free_devices(struct ibv_context **list); + +/** + * rdma_event_str - Returns a string representation of an rdma cm event. + * @event: Asynchronous event. + * Description: + * Returns a string representation of an asynchronous event. + * See also: + * rdma_get_cm_event + */ +const char *rdma_event_str(enum rdma_cm_event_type event); + +/* Option levels */ +enum { + RDMA_OPTION_ID = 0, + RDMA_OPTION_IB = 1 +}; + +/* Option details */ +enum { + RDMA_OPTION_ID_TOS = 0, /* uint8_t: RFC 2474 */ + RDMA_OPTION_ID_REUSEADDR = 1, /* int: ~SO_REUSEADDR */ + RDMA_OPTION_ID_AFONLY = 2, /* int: ~IPV6_V6ONLY */ + RDMA_OPTION_IB_PATH = 1 /* struct ibv_path_data[] */ +}; + +/** + * rdma_set_option - Set options for an rdma_cm_id. + * @id: Communication identifier to set option for. + * @level: Protocol level of the option to set. + * @optname: Name of the option to set. + * @optval: Reference to the option data. + * @optlen: The size of the %optval buffer. + */ +int rdma_set_option(struct rdma_cm_id *id, int level, int optname, + void *optval, size_t optlen); + +/** + * rdma_migrate_id - Move an rdma_cm_id to a new event channel. + * @id: Communication identifier to migrate. + * @channel: New event channel for rdma_cm_id events. + */ +int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel); + +/** + * rdma_getaddrinfo - RDMA address and route resolution service. + */ +int rdma_getaddrinfo(const char *node, const char *service, + const struct rdma_addrinfo *hints, + struct rdma_addrinfo **res); + +void rdma_freeaddrinfo(struct rdma_addrinfo *res); + +#ifdef __cplusplus +} +#endif + +#endif /* _RDMA_CMA_H */ From patchwork Mon Mar 26 20:57:33 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891259 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096YS4rThz9s1B for ; Tue, 27 Mar 2018 08:22:36 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752342AbeCZVWd (ORCPT ); Mon, 26 Mar 2018 17:22:33 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49898 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752040AbeCZVW2 (ORCPT ); Mon, 26 Mar 2018 17:22:28 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id 9A6182B8D7; Mon, 26 Mar 2018 16:22:27 -0500 (CDT) Message-Id: In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:33 -0700 Subject: [PATCH v3 iproute2-next 2/8] rdma: Makefile change to find To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Signed-off-by: Steve Wise --- rdma/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdma/Makefile b/rdma/Makefile index 819fcbe..2cb46b6 100644 --- a/rdma/Makefile +++ b/rdma/Makefile @@ -4,7 +4,7 @@ include ../config.mk TARGETS := ifeq ($(HAVE_MNL),y) -CFLAGS += -I./include/uapi/ +CFLAGS += -I./include/uapi/ -I./include/ RDMA_OBJ = rdma.o utils.o dev.o link.o res.o From patchwork Mon Mar 26 20:57:34 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891260 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096YY4kK7z9s1B for ; Tue, 27 Mar 2018 08:22:41 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752360AbeCZVWi (ORCPT ); Mon, 26 Mar 2018 17:22:38 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49902 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752307AbeCZVWd (ORCPT ); Mon, 26 Mar 2018 17:22:33 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id A44D42B8D7; Mon, 26 Mar 2018 16:22:32 -0500 (CDT) Message-Id: <01a0fdf37e80e5df5baaf461d283da5306c0ce57.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:34 -0700 Subject: [PATCH v3 iproute2-next 3/8] rdma: update rdma_netlink.h To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org From: Steve Wise Pull in the latest rdma_netlink.h which has support for the rdma nldev resource tracking objects being added with this patch series. Signed-off-by: Steve Wise --- rdma/include/uapi/rdma/rdma_netlink.h | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h index dbac3b8..9446a72 100644 --- a/rdma/include/uapi/rdma/rdma_netlink.h +++ b/rdma/include/uapi/rdma/rdma_netlink.h @@ -238,6 +238,14 @@ enum rdma_nldev_command { RDMA_NLDEV_CMD_RES_QP_GET, /* can dump */ + RDMA_NLDEV_CMD_RES_CM_ID_GET, /* can dump */ + + RDMA_NLDEV_CMD_RES_CQ_GET, /* can dump */ + + RDMA_NLDEV_CMD_RES_MR_GET, /* can dump */ + + RDMA_NLDEV_CMD_RES_PD_GET, /* can dump */ + RDMA_NLDEV_NUM_OPS }; @@ -350,6 +358,36 @@ enum rdma_nldev_attr { */ RDMA_NLDEV_ATTR_RES_KERN_NAME, /* string */ + RDMA_NLDEV_ATTR_RES_CM_ID, /* nested table */ + RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY, /* nested table */ + /* + * rdma_cm_id port space. + */ + RDMA_NLDEV_ATTR_RES_PS, /* u32 */ + /* + * Source and destination socket addresses + */ + RDMA_NLDEV_ATTR_RES_SRC_ADDR, /* __kernel_sockaddr_storage */ + RDMA_NLDEV_ATTR_RES_DST_ADDR, /* __kernel_sockaddr_storage */ + + RDMA_NLDEV_ATTR_RES_CQ, /* nested table */ + RDMA_NLDEV_ATTR_RES_CQ_ENTRY, /* nested table */ + RDMA_NLDEV_ATTR_RES_CQE, /* u32 */ + RDMA_NLDEV_ATTR_RES_USECNT, /* u64 */ + RDMA_NLDEV_ATTR_RES_POLL_CTX, /* u8 */ + + RDMA_NLDEV_ATTR_RES_MR, /* nested table */ + RDMA_NLDEV_ATTR_RES_MR_ENTRY, /* nested table */ + RDMA_NLDEV_ATTR_RES_RKEY, /* u32 */ + RDMA_NLDEV_ATTR_RES_LKEY, /* u32 */ + RDMA_NLDEV_ATTR_RES_IOVA, /* u64 */ + RDMA_NLDEV_ATTR_RES_MRLEN, /* u64 */ + + RDMA_NLDEV_ATTR_RES_PD, /* nested table */ + RDMA_NLDEV_ATTR_RES_PD_ENTRY, /* nested table */ + RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY, /* u32 */ + RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY, /* u32 */ + RDMA_NLDEV_ATTR_MAX }; #endif /* _RDMA_NETLINK_H */ From patchwork Mon Mar 26 20:57:36 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891261 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096Yg4Z2Tz9s1B for ; Tue, 27 Mar 2018 08:22:47 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752387AbeCZVWp (ORCPT ); Mon, 26 Mar 2018 17:22:45 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49907 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752348AbeCZVWi (ORCPT ); Mon, 26 Mar 2018 17:22:38 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id B02EA2B8D7; Mon, 26 Mar 2018 16:22:37 -0500 (CDT) Message-Id: <2de3f506d81317bab93103874147c5eb90b56798.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:36 -0700 Subject: [PATCH v3 iproute2-next 4/8] rdma: initialize the rd struct To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Initialize the rd struct so port_idx is 0 unless set otherwise. Otherwise, strict_port queries end up passing an uninitialized PORT nlattr. Signed-off-by: Steve Wise Reviewed-by: Leon Romanovsky --- rdma/rdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdma/rdma.c b/rdma/rdma.c index ab2c960..b43e538 100644 --- a/rdma/rdma.c +++ b/rdma/rdma.c @@ -135,7 +135,7 @@ int main(int argc, char **argv) bool json_output = false; bool force = false; char *filename; - struct rd rd; + struct rd rd = {}; int opt; int err; From patchwork Mon Mar 26 20:57:37 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891265 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096ZV710qz9s1b for ; Tue, 27 Mar 2018 08:23:30 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752380AbeCZVWo (ORCPT ); Mon, 26 Mar 2018 17:22:44 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49917 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752124AbeCZVWn (ORCPT ); Mon, 26 Mar 2018 17:22:43 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id BC45F2B8F3; Mon, 26 Mar 2018 16:22:42 -0500 (CDT) Message-Id: <5a2d6eb9b5767e1691d12f1fe206d7a24c9d7ef0.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:37 -0700 Subject: [PATCH v3 iproute2-next 5/8] rdma: Add CM_ID resource tracking information To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Sample output: # rdma resource 2: cxgb4_0: pd 5 cq 2 qp 2 cm_id 3 mr 7 3: mlx4_0: pd 7 cq 3 qp 3 cm_id 3 mr 7 # rdma resource show cm_id link cxgb4_0/- lqpn 0 qp-type RC state LISTEN ps TCP pid 30485 comm rping src-addr 0.0.0.0:7174 link cxgb4_0/2 lqpn 1048 qp-type RC state CONNECT ps TCP pid 30503 comm rping src-addr 172.16.2.1:7174 dst-addr 172.16.2.1:38246 link cxgb4_0/2 lqpn 1040 qp-type RC state CONNECT ps TCP pid 30498 comm rping src-addr 172.16.2.1:38246 dst-addr 172.16.2.1:7174 link mlx4_0/- lqpn 0 qp-type RC state LISTEN ps TCP pid 30485 comm rping src-addr 0.0.0.0:7174 link mlx4_0/1 lqpn 539 qp-type RC state CONNECT ps TCP pid 30494 comm rping src-addr 172.16.99.1:7174 dst-addr 172.16.99.1:43670 link mlx4_0/1 lqpn 538 qp-type RC state CONNECT ps TCP pid 30492 comm rping src-addr 172.16.99.1:43670 dst-addr 172.16.99.1:7174 # rdma resource show cm_id dst-port 7174 link cxgb4_0/2 lqpn 1040 qp-type RC state CONNECT ps TCP pid 30498 comm rping src-addr 172.16.2.1:38246 dst-addr 172.16.2.1:7174 link mlx4_0/1 lqpn 538 qp-type RC state CONNECT ps TCP pid 30492 comm rping src-addr 172.16.99.1:43670 dst-addr 172.16.99.1:7174 Signed-off-by: Steve Wise Reviewed-by: Leon Romanovsky --- rdma/rdma.h | 2 + rdma/res.c | 262 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- rdma/utils.c | 5 ++ 3 files changed, 268 insertions(+), 1 deletion(-) diff --git a/rdma/rdma.h b/rdma/rdma.h index 5809f70..e55205b 100644 --- a/rdma/rdma.h +++ b/rdma/rdma.h @@ -18,10 +18,12 @@ #include #include #include +#include #include "list.h" #include "utils.h" #include "json_writer.h" +#include #define pr_err(args...) fprintf(stderr, ##args) #define pr_out(args...) fprintf(stdout, ##args) diff --git a/rdma/res.c b/rdma/res.c index 62f5c54..5506cf3 100644 --- a/rdma/res.c +++ b/rdma/res.c @@ -16,9 +16,11 @@ static int res_help(struct rd *rd) { pr_out("Usage: %s resource\n", rd->filename); pr_out(" resource show [DEV]\n"); - pr_out(" resource show [qp]\n"); + pr_out(" resource show [qp|cm_id]\n"); pr_out(" resource show qp link [DEV/PORT]\n"); pr_out(" resource show qp link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n"); + pr_out(" resource show cm_id link [DEV/PORT]\n"); + pr_out(" resource show cm_id link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n"); return 0; } @@ -433,6 +435,245 @@ static int res_qp_parse_cb(const struct nlmsghdr *nlh, void *data) return MNL_CB_OK; } +static void print_qp_type(struct rd *rd, uint32_t val) +{ + if (rd->json_output) + jsonw_string_field(rd->jw, "qp-type", + qp_types_to_str(val)); + else + pr_out("qp-type %s ", qp_types_to_str(val)); +} + +static const char *cm_id_state_to_str(uint8_t idx) +{ + static const char * const cm_id_states_str[] = { + "IDLE", "ADDR_QUERY", "ADDR_RESOLVED", "ROUTE_QUERY", + "ROUTE_RESOLVED", "CONNECT", "DISCONNECT", "ADDR_BOUND", + "LISTEN", "DEVICE_REMOVAL", "DESTROYING" }; + + if (idx < ARRAY_SIZE(cm_id_states_str)) + return cm_id_states_str[idx]; + return "UNKNOWN"; +} + +static const char *cm_id_ps_to_str(uint32_t ps) +{ + switch (ps) { + case RDMA_PS_IPOIB: + return "IPoIB"; + case RDMA_PS_IB: + return "IPoIB"; + case RDMA_PS_TCP: + return "TCP"; + case RDMA_PS_UDP: + return "UDP"; + default: + return "---"; + } +} + +static void print_cm_id_state(struct rd *rd, uint8_t state) +{ + if (rd->json_output) { + jsonw_string_field(rd->jw, "state", cm_id_state_to_str(state)); + return; + } + pr_out("state %s ", cm_id_state_to_str(state)); +} + +static void print_ps(struct rd *rd, uint32_t ps) +{ + if (rd->json_output) { + jsonw_string_field(rd->jw, "ps", cm_id_ps_to_str(ps)); + return; + } + pr_out("ps %s ", cm_id_ps_to_str(ps)); +} + +static void print_ipaddr(struct rd *rd, const char *key, char *addrstr, + uint16_t port) +{ + if (rd->json_output) { + int name_size = INET6_ADDRSTRLEN+strlen(":65535"); + char json_name[name_size]; + + snprintf(json_name, name_size, "%s:%u", addrstr, port); + jsonw_string_field(rd->jw, key, json_name); + return; + } + pr_out("%s %s:%u ", key, addrstr, port); +} + +static int ss_ntop(struct nlattr *nla_line, char *addr_str, uint16_t *port) +{ + struct __kernel_sockaddr_storage *addr; + + addr = (struct __kernel_sockaddr_storage *) + mnl_attr_get_payload(nla_line); + switch (addr->ss_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + + if (!inet_ntop(AF_INET, (const void *)&sin->sin_addr, addr_str, + INET6_ADDRSTRLEN)) + return -EINVAL; + *port = ntohs(sin->sin_port); + break; + } + case AF_INET6: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + + if (!inet_ntop(AF_INET6, (const void *)&sin6->sin6_addr, + addr_str, INET6_ADDRSTRLEN)) + return -EINVAL; + *port = ntohs(sin6->sin6_port); + break; + } + default: + return -EINVAL; + } + return 0; +} + +static int res_cm_id_parse_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {}; + struct nlattr *nla_table, *nla_entry; + struct rd *rd = data; + const char *name; + int idx; + + mnl_attr_parse(nlh, 0, rd_attr_cb, tb); + if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || + !tb[RDMA_NLDEV_ATTR_DEV_NAME] || + !tb[RDMA_NLDEV_ATTR_RES_CM_ID]) + return MNL_CB_ERROR; + + name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]); + idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); + nla_table = tb[RDMA_NLDEV_ATTR_RES_CM_ID]; + mnl_attr_for_each_nested(nla_entry, nla_table) { + struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {}; + char src_addr_str[INET6_ADDRSTRLEN]; + char dst_addr_str[INET6_ADDRSTRLEN]; + uint16_t src_port, dst_port; + uint32_t port = 0, pid = 0; + uint8_t type = 0, state; + uint32_t lqpn = 0, ps; + char *comm = NULL; + int err; + + err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line); + if (err != MNL_CB_OK) + return -EINVAL; + + if (!nla_line[RDMA_NLDEV_ATTR_RES_STATE] || + !nla_line[RDMA_NLDEV_ATTR_RES_PS] || + (!nla_line[RDMA_NLDEV_ATTR_RES_PID] && + !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) { + return MNL_CB_ERROR; + } + + if (nla_line[RDMA_NLDEV_ATTR_PORT_INDEX]) + port = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_PORT_INDEX]); + + if (port && port != rd->port_idx) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_LQPN]) { + lqpn = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_LQPN]); + if (rd_check_is_filtered(rd, "lqpn", lqpn)) + continue; + } + if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE]) { + type = mnl_attr_get_u8( + nla_line[RDMA_NLDEV_ATTR_RES_TYPE]); + if (rd_check_is_string_filtered(rd, "qp-type", + qp_types_to_str(type))) + continue; + } + + ps = mnl_attr_get_u32(nla_line[RDMA_NLDEV_ATTR_RES_PS]); + if (rd_check_is_string_filtered(rd, "ps", cm_id_ps_to_str(ps))) + continue; + + state = mnl_attr_get_u8(nla_line[RDMA_NLDEV_ATTR_RES_STATE]); + if (rd_check_is_string_filtered(rd, "state", + cm_id_state_to_str(state))) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR]) { + if (ss_ntop(nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR], + src_addr_str, &src_port)) + continue; + if (rd_check_is_string_filtered(rd, "src-addr", + src_addr_str)) + continue; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR]) { + if (ss_ntop(nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR], + dst_addr_str, &dst_port)) + continue; + if (rd_check_is_string_filtered(rd, "dst-addr", + dst_addr_str)) + continue; + } + + if (rd_check_is_filtered(rd, "src-port", src_port)) + continue; + + if (rd_check_is_filtered(rd, "dst-port", dst_port)) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) { + pid = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_PID]); + comm = get_task_name(pid); + } + + if (rd_check_is_filtered(rd, "pid", pid)) { + free(comm); + continue; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]) { + /* discard const from mnl_attr_get_str */ + comm = (char *)mnl_attr_get_str( + nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]); + } + + if (rd->json_output) + jsonw_start_array(rd->jw); + + print_link(rd, idx, name, port, nla_line); + if (nla_line[RDMA_NLDEV_ATTR_RES_LQPN]) + print_lqpn(rd, lqpn); + if (nla_line[RDMA_NLDEV_ATTR_RES_TYPE]) + print_qp_type(rd, type); + print_cm_id_state(rd, state); + print_ps(rd, ps); + print_pid(rd, pid); + print_comm(rd, comm, nla_line); + + if (nla_line[RDMA_NLDEV_ATTR_RES_SRC_ADDR]) + print_ipaddr(rd, "src-addr", src_addr_str, src_port); + if (nla_line[RDMA_NLDEV_ATTR_RES_DST_ADDR]) + print_ipaddr(rd, "dst-addr", dst_addr_str, dst_port); + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) + free(comm); + + if (rd->json_output) + jsonw_end_array(rd->jw); + else + pr_out("\n"); + } + return MNL_CB_OK; +} + RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true); static const struct @@ -457,11 +698,30 @@ filters qp_valid_filters[MAX_NUMBER_OF_FILTERS] = {{ .name = "link", RES_FUNC(res_qp, RDMA_NLDEV_CMD_RES_QP_GET, qp_valid_filters, false); +static const +struct filters cm_id_valid_filters[MAX_NUMBER_OF_FILTERS] = { + { .name = "link", .is_number = false }, + { .name = "lqpn", .is_number = true }, + { .name = "qp-type", .is_number = false }, + { .name = "state", .is_number = false }, + { .name = "ps", .is_number = false }, + { .name = "dev-type", .is_number = false }, + { .name = "transport-type", .is_number = false }, + { .name = "pid", .is_number = true }, + { .name = "src-addr", .is_number = false }, + { .name = "src-port", .is_number = true }, + { .name = "dst-addr", .is_number = false }, + { .name = "dst-port", .is_number = true } +}; + +RES_FUNC(res_cm_id, RDMA_NLDEV_CMD_RES_CM_ID_GET, cm_id_valid_filters, false); + static int res_show(struct rd *rd) { const struct rd_cmd cmds[] = { { NULL, res_no_args }, { "qp", res_qp }, + { "cm_id", res_cm_id }, { 0 } }; diff --git a/rdma/utils.c b/rdma/utils.c index f946016..ec81737 100644 --- a/rdma/utils.c +++ b/rdma/utils.c @@ -375,6 +375,11 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_STATE] = MNL_TYPE_U8, [RDMA_NLDEV_ATTR_RES_PID] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_RES_KERN_NAME] = MNL_TYPE_NUL_STRING, + [RDMA_NLDEV_ATTR_RES_CM_ID] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_PS] = MNL_TYPE_U32, + [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = MNL_TYPE_UNSPEC, + [RDMA_NLDEV_ATTR_RES_DST_ADDR] = MNL_TYPE_UNSPEC, }; int rd_attr_cb(const struct nlattr *attr, void *data) From patchwork Mon Mar 26 20:57:38 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891262 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096Ym2K9bz9s1B for ; Tue, 27 Mar 2018 08:22:52 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752416AbeCZVWu (ORCPT ); Mon, 26 Mar 2018 17:22:50 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49930 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752392AbeCZVWt (ORCPT ); Mon, 26 Mar 2018 17:22:49 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id C85C82B900; Mon, 26 Mar 2018 16:22:47 -0500 (CDT) Message-Id: <6898c7a0ed89932630ff2b87a74849ba25d7c323.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:38 -0700 Subject: [PATCH v3 iproute2-next 6/8] rdma: Add CQ resource tracking information To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Sample output: # rdma resource show cq link cxgb4_0/- cqe 46 users 2 pid 30503 comm rping link cxgb4_0/- cqe 46 users 2 pid 30498 comm rping link mlx4_0/- cqe 63 users 2 pid 30494 comm rping link mlx4_0/- cqe 63 users 2 pid 30489 comm rping link mlx4_0/- cqe 1023 users 2 poll_ctx WORKQUEUE pid 0 comm [ib_core] # rdma resource show cq pid 30489 link mlx4_0/- cqe 63 users 2 pid 30489 comm rping Signed-off-by: Steve Wise Reviewed-by: Leon Romanovsky --- rdma/res.c | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ rdma/utils.c | 5 +++ 2 files changed, 144 insertions(+) diff --git a/rdma/res.c b/rdma/res.c index 5506cf3..f1a90d3 100644 --- a/rdma/res.c +++ b/rdma/res.c @@ -21,6 +21,8 @@ static int res_help(struct rd *rd) pr_out(" resource show qp link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n"); pr_out(" resource show cm_id link [DEV/PORT]\n"); pr_out(" resource show cm_id link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n"); + pr_out(" resource show cq link [DEV/PORT]\n"); + pr_out(" resource show cq link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n"); return 0; } @@ -674,6 +676,132 @@ static int res_cm_id_parse_cb(const struct nlmsghdr *nlh, void *data) return MNL_CB_OK; } +static void print_cqe(struct rd *rd, uint32_t val) +{ + if (rd->json_output) + jsonw_uint_field(rd->jw, "cqe", val); + else + pr_out("cqe %u ", val); +} + +static void print_users(struct rd *rd, uint64_t val) +{ + if (rd->json_output) + jsonw_uint_field(rd->jw, "users", val); + else + pr_out("users %" PRIu64 " ", val); +} + +static const char *poll_ctx_to_str(uint8_t idx) +{ + static const char * const cm_id_states_str[] = { + "DIRECT", "SOFTIRQ", "WORKQUEUE"}; + + if (idx < ARRAY_SIZE(cm_id_states_str)) + return cm_id_states_str[idx]; + return "UNKNOWN"; +} + +static void print_poll_ctx(struct rd *rd, uint8_t poll_ctx) +{ + if (rd->json_output) { + jsonw_string_field(rd->jw, "poll-ctx", + poll_ctx_to_str(poll_ctx)); + return; + } + pr_out("poll-ctx %s ", poll_ctx_to_str(poll_ctx)); +} + +static int res_cq_parse_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {}; + struct nlattr *nla_table, *nla_entry; + struct rd *rd = data; + const char *name; + uint32_t idx; + + mnl_attr_parse(nlh, 0, rd_attr_cb, tb); + if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || + !tb[RDMA_NLDEV_ATTR_DEV_NAME] || + !tb[RDMA_NLDEV_ATTR_RES_CQ]) + return MNL_CB_ERROR; + + name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]); + idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); + nla_table = tb[RDMA_NLDEV_ATTR_RES_CQ]; + + mnl_attr_for_each_nested(nla_entry, nla_table) { + struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {}; + char *comm = NULL; + uint32_t pid = 0; + uint8_t poll_ctx = 0; + uint64_t users; + uint32_t cqe; + int err; + + err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line); + if (err != MNL_CB_OK) + return MNL_CB_ERROR; + + if (!nla_line[RDMA_NLDEV_ATTR_RES_CQE] || + !nla_line[RDMA_NLDEV_ATTR_RES_USECNT] || + (!nla_line[RDMA_NLDEV_ATTR_RES_PID] && + !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) { + return MNL_CB_ERROR; + } + + cqe = mnl_attr_get_u32(nla_line[RDMA_NLDEV_ATTR_RES_CQE]); + + users = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_USECNT]); + if (rd_check_is_filtered(rd, "users", users)) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]) { + poll_ctx = mnl_attr_get_u8( + nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]); + if (rd_check_is_string_filtered(rd, "poll-ctx", + poll_ctx_to_str(poll_ctx))) + continue; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) { + pid = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_PID]); + comm = get_task_name(pid); + } + + if (rd_check_is_filtered(rd, "pid", pid)) { + free(comm); + continue; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]) + /* discard const from mnl_attr_get_str */ + comm = (char *)mnl_attr_get_str( + nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]); + + if (rd->json_output) + jsonw_start_array(rd->jw); + + print_link(rd, idx, name, 0, nla_line); + print_cqe(rd, cqe); + print_users(rd, users); + if (nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX]) + print_poll_ctx(rd, poll_ctx); + print_pid(rd, pid); + print_comm(rd, comm, nla_line); + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) + free(comm); + + if (rd->json_output) + jsonw_end_array(rd->jw); + else + pr_out("\n"); + } + return MNL_CB_OK; +} + RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true); static const struct @@ -716,12 +844,23 @@ struct filters cm_id_valid_filters[MAX_NUMBER_OF_FILTERS] = { RES_FUNC(res_cm_id, RDMA_NLDEV_CMD_RES_CM_ID_GET, cm_id_valid_filters, false); +static const +struct filters cq_valid_filters[MAX_NUMBER_OF_FILTERS] = { + { .name = "link", .is_number = false }, + { .name = "users", .is_number = true }, + { .name = "poll-ctx", .is_number = false }, + { .name = "pid", .is_number = true } +}; + +RES_FUNC(res_cq, RDMA_NLDEV_CMD_RES_CQ_GET, cq_valid_filters, true); + static int res_show(struct rd *rd) { const struct rd_cmd cmds[] = { { NULL, res_no_args }, { "qp", res_qp }, { "cm_id", res_cm_id }, + { "cq", res_cq }, { 0 } }; diff --git a/rdma/utils.c b/rdma/utils.c index ec81737..5e79b62 100644 --- a/rdma/utils.c +++ b/rdma/utils.c @@ -380,6 +380,11 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_PS] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = MNL_TYPE_UNSPEC, [RDMA_NLDEV_ATTR_RES_DST_ADDR] = MNL_TYPE_UNSPEC, + [RDMA_NLDEV_ATTR_RES_CQ] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_CQE] = MNL_TYPE_U32, + [RDMA_NLDEV_ATTR_RES_USECNT] = MNL_TYPE_U64, + [RDMA_NLDEV_ATTR_RES_POLL_CTX] = MNL_TYPE_U8, }; int rd_attr_cb(const struct nlattr *attr, void *data) From patchwork Mon Mar 26 20:57:39 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891263 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096Yt2klBz9s1B for ; Tue, 27 Mar 2018 08:22:58 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752454AbeCZVWz (ORCPT ); Mon, 26 Mar 2018 17:22:55 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49939 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752425AbeCZVWx (ORCPT ); Mon, 26 Mar 2018 17:22:53 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id D487B2B900; Mon, 26 Mar 2018 16:22:52 -0500 (CDT) Message-Id: <4f1876e2a31e9b6d0784131bda2140d421f25ffb.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:39 -0700 Subject: [PATCH v3 iproute2-next 7/8] rdma: Add MR resource tracking information To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Sample output: Without CAP_NET_ADMIN: $ rdma resource show mr mrlen 65536 link mlx4_0/- mrlen 65536 pid 0 comm [nvme_rdma] link cxgb4_0/- mrlen 65536 pid 0 comm [nvme_rdma] With CAP_NET_ADMIN: # rdma resource show mr mrlen 65536 link mlx4_0/- rkey 0x12702 lkey 0x12702 iova 0x85724a000 mrlen 65536 pid 0 comm [nvme_rdma] link cxgb4_0/- rkey 0x68fe4e9 lkey 0x68fe4e9 iova 0x835b91000 mrlen 65536 pid 0 comm [nvme_rdma] Signed-off-by: Steve Wise Reviewed-by: Leon Romanovsky --- include/json_writer.h | 2 + lib/json_writer.c | 11 +++++ rdma/res.c | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++ rdma/utils.c | 6 +++ 4 files changed, 146 insertions(+) diff --git a/include/json_writer.h b/include/json_writer.h index 45459fa..4b4dec2 100644 --- a/include/json_writer.h +++ b/include/json_writer.h @@ -35,6 +35,7 @@ void jsonw_bool(json_writer_t *self, bool value); void jsonw_float(json_writer_t *self, double number); void jsonw_float_fmt(json_writer_t *self, const char *fmt, double num); void jsonw_uint(json_writer_t *self, uint64_t number); +void jsonw_xint(json_writer_t *self, uint64_t number); void jsonw_hu(json_writer_t *self, unsigned short number); void jsonw_int(json_writer_t *self, int64_t number); void jsonw_null(json_writer_t *self); @@ -45,6 +46,7 @@ void jsonw_string_field(json_writer_t *self, const char *prop, const char *val); void jsonw_bool_field(json_writer_t *self, const char *prop, bool value); void jsonw_float_field(json_writer_t *self, const char *prop, double num); void jsonw_uint_field(json_writer_t *self, const char *prop, uint64_t num); +void jsonw_xint_field(json_writer_t *self, const char *prop, uint64_t num); void jsonw_hu_field(json_writer_t *self, const char *prop, unsigned short num); void jsonw_int_field(json_writer_t *self, const char *prop, int64_t num); void jsonw_null_field(json_writer_t *self, const char *prop); diff --git a/lib/json_writer.c b/lib/json_writer.c index 68401ae..0ad0421 100644 --- a/lib/json_writer.c +++ b/lib/json_writer.c @@ -225,6 +225,11 @@ void jsonw_uint(json_writer_t *self, uint64_t num) jsonw_printf(self, "%"PRIu64, num); } +void jsonw_xint(json_writer_t *self, uint64_t num) +{ + jsonw_printf(self, "%"PRIx64, num); +} + void jsonw_lluint(json_writer_t *self, unsigned long long int num) { jsonw_printf(self, "%llu", num); @@ -269,6 +274,12 @@ void jsonw_uint_field(json_writer_t *self, const char *prop, uint64_t num) jsonw_uint(self, num); } +void jsonw_xint_field(json_writer_t *self, const char *prop, uint64_t num) +{ + jsonw_name(self, prop); + jsonw_xint(self, num); +} + void jsonw_hu_field(json_writer_t *self, const char *prop, unsigned short num) { jsonw_name(self, prop); diff --git a/rdma/res.c b/rdma/res.c index f1a90d3..aa61e73 100644 --- a/rdma/res.c +++ b/rdma/res.c @@ -802,6 +802,121 @@ static int res_cq_parse_cb(const struct nlmsghdr *nlh, void *data) return MNL_CB_OK; } +static void print_key(struct rd *rd, const char *name, uint32_t val) +{ + if (rd->json_output) + jsonw_xint_field(rd->jw, name, val); + else + pr_out("%s 0x%x ", name, val); +} + +static void print_iova(struct rd *rd, uint64_t val) +{ + if (rd->json_output) + jsonw_xint_field(rd->jw, "iova", val); + else + pr_out("iova 0x%" PRIx64 " ", val); +} + +static void print_mrlen(struct rd *rd, uint64_t val) +{ + if (rd->json_output) + jsonw_uint_field(rd->jw, "mrlen", val); + else + pr_out("mrlen %" PRIu64 " ", val); +} + +static int res_mr_parse_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {}; + struct nlattr *nla_table, *nla_entry; + struct rd *rd = data; + const char *name; + uint32_t idx; + + mnl_attr_parse(nlh, 0, rd_attr_cb, tb); + if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || + !tb[RDMA_NLDEV_ATTR_DEV_NAME] || + !tb[RDMA_NLDEV_ATTR_RES_MR]) + return MNL_CB_ERROR; + + name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]); + idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); + nla_table = tb[RDMA_NLDEV_ATTR_RES_MR]; + + mnl_attr_for_each_nested(nla_entry, nla_table) { + struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {}; + uint32_t rkey = 0, lkey = 0; + uint64_t iova = 0, mrlen; + char *comm = NULL; + uint32_t pid = 0; + int err; + + err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line); + if (err != MNL_CB_OK) + return MNL_CB_ERROR; + + if (!nla_line[RDMA_NLDEV_ATTR_RES_MRLEN] || + (!nla_line[RDMA_NLDEV_ATTR_RES_PID] && + !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) { + return MNL_CB_ERROR; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_RKEY]) + rkey = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_RKEY]); + if (nla_line[RDMA_NLDEV_ATTR_RES_LKEY]) + lkey = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_LKEY]); + if (nla_line[RDMA_NLDEV_ATTR_RES_IOVA]) + iova = mnl_attr_get_u64( + nla_line[RDMA_NLDEV_ATTR_RES_IOVA]); + + mrlen = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_MRLEN]); + if (rd_check_is_filtered(rd, "mrlen", mrlen)) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) { + pid = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_PID]); + comm = get_task_name(pid); + } + + if (rd_check_is_filtered(rd, "pid", pid)) { + free(comm); + continue; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]) + /* discard const from mnl_attr_get_str */ + comm = (char *)mnl_attr_get_str( + nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]); + + if (rd->json_output) + jsonw_start_array(rd->jw); + + print_link(rd, idx, name, 0, nla_line); + if (nla_line[RDMA_NLDEV_ATTR_RES_RKEY]) + print_key(rd, "rkey", rkey); + if (nla_line[RDMA_NLDEV_ATTR_RES_LKEY]) + print_key(rd, "lkey", lkey); + if (nla_line[RDMA_NLDEV_ATTR_RES_IOVA]) + print_iova(rd, iova); + print_mrlen(rd, mrlen); + print_pid(rd, pid); + print_comm(rd, comm, nla_line); + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) + free(comm); + + if (rd->json_output) + jsonw_end_array(rd->jw); + else + pr_out("\n"); + } + return MNL_CB_OK; +} + RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true); static const struct @@ -854,6 +969,17 @@ struct filters cq_valid_filters[MAX_NUMBER_OF_FILTERS] = { RES_FUNC(res_cq, RDMA_NLDEV_CMD_RES_CQ_GET, cq_valid_filters, true); +static const +struct filters mr_valid_filters[MAX_NUMBER_OF_FILTERS] = { + { .name = "link", .is_number = false }, + { .name = "rkey", .is_number = true }, + { .name = "lkey", .is_number = true }, + { .name = "mrlen", .is_number = true }, + { .name = "pid", .is_number = true } +}; + +RES_FUNC(res_mr, RDMA_NLDEV_CMD_RES_MR_GET, mr_valid_filters, true); + static int res_show(struct rd *rd) { const struct rd_cmd cmds[] = { @@ -861,6 +987,7 @@ static int res_show(struct rd *rd) { "qp", res_qp }, { "cm_id", res_cm_id }, { "cq", res_cq }, + { "mr", res_mr }, { 0 } }; diff --git a/rdma/utils.c b/rdma/utils.c index 5e79b62..a2e08e9 100644 --- a/rdma/utils.c +++ b/rdma/utils.c @@ -385,6 +385,12 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_CQE] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_RES_USECNT] = MNL_TYPE_U64, [RDMA_NLDEV_ATTR_RES_POLL_CTX] = MNL_TYPE_U8, + [RDMA_NLDEV_ATTR_RES_MR] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = MNL_TYPE_NESTED, + [RDMA_NLDEV_ATTR_RES_RKEY] = MNL_TYPE_U32, + [RDMA_NLDEV_ATTR_RES_LKEY] = MNL_TYPE_U32, + [RDMA_NLDEV_ATTR_RES_IOVA] = MNL_TYPE_U64, + [RDMA_NLDEV_ATTR_RES_MRLEN] = MNL_TYPE_U64, }; int rd_attr_cb(const struct nlattr *attr, void *data) From patchwork Mon Mar 26 20:57:41 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 891264 X-Patchwork-Delegate: dsahern@gmail.com Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=none (mailfrom) smtp.mailfrom=vger.kernel.org (client-ip=209.132.180.67; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=opengridcomputing.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by ozlabs.org (Postfix) with ESMTP id 4096ZM3hZVz9s1b for ; Tue, 27 Mar 2018 08:23:23 +1100 (AEDT) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752496AbeCZVXV (ORCPT ); Mon, 26 Mar 2018 17:23:21 -0400 Received: from opengridcomputing.com ([70.118.0.34]:49948 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752456AbeCZVW7 (ORCPT ); Mon, 26 Mar 2018 17:22:59 -0400 Received: by smtp.opengridcomputing.com (Postfix, from userid 503) id DC90E2B900; Mon, 26 Mar 2018 16:22:57 -0500 (CDT) Message-Id: <959caaae7bdf6f635bcdd43aa62f1bd070c49c57.1522097991.git.swise@opengridcomputing.com> In-Reply-To: References: From: Steve Wise Date: Mon, 26 Mar 2018 13:57:41 -0700 Subject: [PATCH v3 iproute2-next 8/8] rdma: Add PD resource tracking information To: dsahern@gmail.com Cc: leon@kernel.org, stephen@networkplumber.org, netdev@vger.kernel.org, linux-rdma@vger.kernel.org Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Sample output: Without CAP_NET_ADMIN capability: link mlx4_0/- users 0 pid 0 comm [ib_srpt] link mlx4_0/- users 0 pid 0 comm [ib_srp] link mlx4_0/- users 1 pid 0 comm [ib_core] link cxgb4_0/- users 0 pid 0 comm [ib_srp] With CAP_NET_ADMIN capability: link mlx4_0/- local_dma_lkey 0x8000 users 0 pid 0 comm [ib_srpt] link mlx4_0/- local_dma_lkey 0x8000 users 0 pid 0 comm [ib_srp] link mlx4_0/- local_dma_lkey 0x8000 users 1 pid 0 comm [ib_core] link cxgb4_0/- local_dma_lkey 0x0 users 0 pid 0 comm [ib_srp] Signed-off-by: Steve Wise Reviewed-by: Leon Romanovsky --- rdma/res.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/rdma/res.c b/rdma/res.c index aa61e73..6177e46 100644 --- a/rdma/res.c +++ b/rdma/res.c @@ -917,6 +917,91 @@ static int res_mr_parse_cb(const struct nlmsghdr *nlh, void *data) return MNL_CB_OK; } +static int res_pd_parse_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {}; + struct nlattr *nla_table, *nla_entry; + struct rd *rd = data; + const char *name; + uint32_t idx; + + mnl_attr_parse(nlh, 0, rd_attr_cb, tb); + if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || + !tb[RDMA_NLDEV_ATTR_DEV_NAME] || + !tb[RDMA_NLDEV_ATTR_RES_PD]) + return MNL_CB_ERROR; + + name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]); + idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); + nla_table = tb[RDMA_NLDEV_ATTR_RES_PD]; + + mnl_attr_for_each_nested(nla_entry, nla_table) { + uint32_t local_dma_lkey = 0, unsafe_global_rkey = 0; + struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {}; + char *comm = NULL; + uint32_t pid = 0; + uint64_t users; + int err; + + err = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line); + if (err != MNL_CB_OK) + return MNL_CB_ERROR; + + if (!nla_line[RDMA_NLDEV_ATTR_RES_USECNT] || + (!nla_line[RDMA_NLDEV_ATTR_RES_PID] && + !nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME])) { + return MNL_CB_ERROR; + } + + if (nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]) + local_dma_lkey = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]); + + users = mnl_attr_get_u64(nla_line[RDMA_NLDEV_ATTR_RES_USECNT]); + if (rd_check_is_filtered(rd, "users", users)) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]) + unsafe_global_rkey = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]); + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) { + pid = mnl_attr_get_u32( + nla_line[RDMA_NLDEV_ATTR_RES_PID]); + comm = get_task_name(pid); + } + + if (rd_check_is_filtered(rd, "pid", pid)) + continue; + + if (nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]) + /* discard const from mnl_attr_get_str */ + comm = (char *)mnl_attr_get_str( + nla_line[RDMA_NLDEV_ATTR_RES_KERN_NAME]); + + if (rd->json_output) + jsonw_start_array(rd->jw); + + print_link(rd, idx, name, 0, nla_line); + if (nla_line[RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]) + print_key(rd, "local_dma_lkey", local_dma_lkey); + print_users(rd, users); + if (nla_line[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]) + print_key(rd, "unsafe_global_rkey", unsafe_global_rkey); + print_pid(rd, pid); + print_comm(rd, comm, nla_line); + + if (nla_line[RDMA_NLDEV_ATTR_RES_PID]) + free(comm); + + if (rd->json_output) + jsonw_end_array(rd->jw); + else + pr_out("\n"); + } + return MNL_CB_OK; +} + RES_FUNC(res_no_args, RDMA_NLDEV_CMD_RES_GET, NULL, true); static const struct @@ -980,6 +1065,15 @@ struct filters mr_valid_filters[MAX_NUMBER_OF_FILTERS] = { RES_FUNC(res_mr, RDMA_NLDEV_CMD_RES_MR_GET, mr_valid_filters, true); +static const +struct filters pd_valid_filters[MAX_NUMBER_OF_FILTERS] = { + { .name = "link", .is_number = false }, + { .name = "users", .is_number = true }, + { .name = "pid", .is_number = true } +}; + +RES_FUNC(res_pd, RDMA_NLDEV_CMD_RES_PD_GET, pd_valid_filters, true); + static int res_show(struct rd *rd) { const struct rd_cmd cmds[] = { @@ -988,6 +1082,7 @@ static int res_show(struct rd *rd) { "cm_id", res_cm_id }, { "cq", res_cq }, { "mr", res_mr }, + { "pd", res_pd }, { 0 } };