@@ -2146,6 +2146,7 @@ static int efx_ef10_irq_test_generate(struct efx_nic *efx)
static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue)
{
+ tx_queue->type = tx_queue->label & EFX_TXQ_TYPE_OFFLOAD;
return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf,
(tx_queue->ptr_mask + 1) *
sizeof(efx_qword_t),
@@ -2254,7 +2255,7 @@ static u32 efx_ef10_tso_versions(struct efx_nic *efx)
static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)
{
- bool csum_offload = tx_queue->label & EFX_TXQ_TYPE_OFFLOAD;
+ bool csum_offload = tx_queue->type & EFX_TXQ_TYPE_OFFLOAD;
struct efx_channel *channel = tx_queue->channel;
struct efx_nic *efx = tx_queue->efx;
struct efx_ef10_nic_data *nic_data;
@@ -2880,7 +2881,7 @@ efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
/* Get the transmit queue */
tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL);
tx_queue = efx_channel_get_tx_queue(channel,
- tx_ev_q_label % EFX_TXQ_TYPES);
+ tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);
if (!tx_queue->timestamping) {
/* Transmit completion */
@@ -151,7 +151,7 @@ static int efx_allocate_msix_channels(struct efx_nic *efx,
*/
n_xdp_tx = num_possible_cpus();
- n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES);
+ n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_MAX_TXQ_PER_CHANNEL);
vec_count = pci_msix_vec_count(efx->pci_dev);
if (vec_count < 0)
@@ -179,7 +179,7 @@ static int efx_allocate_msix_channels(struct efx_nic *efx,
efx->xdp_tx_queue_count = 0;
} else {
efx->n_xdp_channels = n_xdp_ev;
- efx->xdp_tx_per_channel = EFX_TXQ_TYPES;
+ efx->xdp_tx_per_channel = EFX_MAX_TXQ_PER_CHANNEL;
efx->xdp_tx_queue_count = n_xdp_tx;
n_channels += n_xdp_ev;
netif_dbg(efx, drv, efx->net_dev,
@@ -521,7 +521,7 @@ efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
channel->channel = i;
channel->type = &efx_default_channel_type;
- for (j = 0; j < EFX_TXQ_TYPES; j++) {
+ for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
tx_queue = &channel->tx_queue[j];
tx_queue->efx = efx;
tx_queue->queue = -1;
@@ -595,7 +595,7 @@ struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
channel->napi_str.state = 0;
memset(&channel->eventq, 0, sizeof(channel->eventq));
- for (j = 0; j < EFX_TXQ_TYPES; j++) {
+ for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
tx_queue = &channel->tx_queue[j];
if (tx_queue->channel)
tx_queue->channel = channel;
@@ -895,7 +895,7 @@ int efx_set_channels(struct efx_nic *efx)
xdp_queue_number, tx_queue->queue);
/* We may have a few left-over XDP TX
* queues owing to xdp_tx_queue_count
- * not dividing evenly by EFX_TXQ_TYPES.
+ * not dividing evenly by EFX_MAX_TXQ_PER_CHANNEL.
* We still allocate and probe those
* TXQs, but never use them.
*/
@@ -407,7 +407,7 @@ static size_t efx_describe_per_queue_stats(struct efx_nic *efx, u8 *strings)
snprintf(strings, ETH_GSTRING_LEN,
"tx-%u.tx_packets",
channel->tx_queue[0].queue /
- EFX_TXQ_TYPES);
+ EFX_MAX_TXQ_PER_CHANNEL);
strings += ETH_GSTRING_LEN;
}
@@ -372,6 +372,8 @@ int efx_farch_tx_probe(struct efx_tx_queue *tx_queue)
struct efx_nic *efx = tx_queue->efx;
unsigned entries;
+ tx_queue->type = ((tx_queue->label & 1) ? EFX_TXQ_TYPE_OFFLOAD : 0) |
+ ((tx_queue->label & 2) ? EFX_TXQ_TYPE_HIGHPRI : 0);
entries = tx_queue->ptr_mask + 1;
return efx_alloc_special_buffer(efx, &tx_queue->txd,
entries * sizeof(efx_qword_t));
@@ -379,7 +381,7 @@ int efx_farch_tx_probe(struct efx_tx_queue *tx_queue)
void efx_farch_tx_init(struct efx_tx_queue *tx_queue)
{
- int csum = tx_queue->label & EFX_TXQ_TYPE_OFFLOAD;
+ int csum = tx_queue->type & EFX_TXQ_TYPE_OFFLOAD;
struct efx_nic *efx = tx_queue->efx;
efx_oword_t reg;
@@ -409,7 +411,7 @@ void efx_farch_tx_init(struct efx_tx_queue *tx_queue)
EFX_POPULATE_OWORD_1(reg,
FRF_BZ_TX_PACE,
- (tx_queue->label & EFX_TXQ_TYPE_HIGHPRI) ?
+ (tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ?
FFE_BZ_TX_PACE_OFF :
FFE_BZ_TX_PACE_RESERVED);
efx_writeo_table(efx, ®, FR_BZ_TX_PACE_TBL, tx_queue->queue);
@@ -832,13 +834,13 @@ efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
tx_queue = efx_channel_get_tx_queue(
- channel, tx_ev_q_label % EFX_TXQ_TYPES);
+ channel, tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);
efx_xmit_done(tx_queue, tx_ev_desc_ptr);
} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
/* Rewrite the FIFO write pointer */
tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
tx_queue = efx_channel_get_tx_queue(
- channel, tx_ev_q_label % EFX_TXQ_TYPES);
+ channel, tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);
netif_tx_lock(efx->net_dev);
efx_farch_notify_tx_desc(tx_queue);
@@ -1080,9 +1082,9 @@ efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
int qid;
qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
- if (qid < EFX_TXQ_TYPES * (efx->n_tx_channels + efx->n_extra_tx_channels)) {
- tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,
- qid % EFX_TXQ_TYPES);
+ if (qid < EFX_MAX_TXQ_PER_CHANNEL * (efx->n_tx_channels + efx->n_extra_tx_channels)) {
+ tx_queue = efx_get_tx_queue(efx, qid / EFX_MAX_TXQ_PER_CHANNEL,
+ qid % EFX_MAX_TXQ_PER_CHANNEL);
if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0)) {
efx_farch_magic_event(tx_queue->channel,
EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue));
@@ -1675,10 +1677,10 @@ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
* and the descriptor caches for those channels.
*/
buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
- total_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
+ total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_DMAQ_SIZE +
efx->n_channels * EFX_MAX_EVQ_SIZE)
* sizeof(efx_qword_t) / EFX_BUF_SIZE);
- vi_count = max(efx->n_channels, total_tx_channels * EFX_TXQ_TYPES);
+ vi_count = max(efx->n_channels, total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL);
#ifdef CONFIG_SFC_SRIOV
if (efx->type->sriov_wanted) {
@@ -164,7 +164,7 @@ int efx_mcdi_tx_init(struct efx_tx_queue *tx_queue, bool tso_v2)
{
MCDI_DECLARE_BUF(inbuf, MC_CMD_INIT_TXQ_IN_LEN(EFX_MAX_DMAQ_SIZE * 8 /
EFX_BUF_SIZE));
- bool csum_offload = tx_queue->label & EFX_TXQ_TYPE_OFFLOAD;
+ bool csum_offload = tx_queue->type & EFX_TXQ_TYPE_OFFLOAD;
size_t entries = tx_queue->txd.buf.len / EFX_BUF_SIZE;
struct efx_channel *channel = tx_queue->channel;
struct efx_nic *efx = tx_queue->efx;
@@ -66,7 +66,8 @@
#define EFX_TXQ_TYPE_OFFLOAD 1 /* flag */
#define EFX_TXQ_TYPE_HIGHPRI 2 /* flag */
#define EFX_TXQ_TYPES 4
-#define EFX_MAX_TX_QUEUES (EFX_TXQ_TYPES * EFX_MAX_CHANNELS)
+#define EFX_MAX_TXQ_PER_CHANNEL 4
+#define EFX_MAX_TX_QUEUES (EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_CHANNELS)
/* Maximum possible MTU the driver supports */
#define EFX_MAX_MTU (9 * 1024)
@@ -190,6 +191,7 @@ struct efx_tx_buffer {
* @queue: DMA queue number
* @label: Label for TX completion events.
* Is our index within @channel->tx_queue array.
+ * @type: configuration type of this TX queue. A bitmask of %EFX_TXQ_TYPE_* flags.
* @tso_version: Version of TSO in use for this queue.
* @channel: The associated channel
* @core_txq: The networking core TX queue structure
@@ -254,6 +256,7 @@ struct efx_tx_queue {
struct efx_nic *efx ____cacheline_aligned_in_smp;
unsigned int queue;
unsigned int label;
+ unsigned int type;
unsigned int tso_version;
struct efx_channel *channel;
struct netdev_queue *core_txq;
@@ -479,6 +482,7 @@ enum efx_sync_events_state {
* @rx_list: list of SKBs from current RX, awaiting processing
* @rx_queue: RX queue for this channel
* @tx_queue: TX queues for this channel
+ * @tx_queue_by_type: pointers into @tx_queue, or %NULL, indexed by txq type
* @sync_events_state: Current state of sync events on this channel
* @sync_timestamp_major: Major part of the last ptp sync event
* @sync_timestamp_minor: Minor part of the last ptp sync event
@@ -540,7 +544,8 @@ struct efx_channel {
struct list_head *rx_list;
struct efx_rx_queue rx_queue;
- struct efx_tx_queue tx_queue[EFX_TXQ_TYPES];
+ struct efx_tx_queue tx_queue[EFX_MAX_TXQ_PER_CHANNEL];
+ struct efx_tx_queue *tx_queue_by_type[EFX_TXQ_TYPES];
enum efx_sync_events_state sync_events_state;
u32 sync_timestamp_major;
@@ -1200,7 +1205,7 @@ struct efx_udp_tunnel {
* a pointer to the &struct efx_msi_context for the channel.
* @irq_handle_legacy: Handle legacy interrupt. The @dev_id argument
* is a pointer to the &struct efx_nic.
- * @tx_probe: Allocate resources for TX queue
+ * @tx_probe: Allocate resources for TX queue (and select TXQ type)
* @tx_init: Initialise TX queue on the NIC
* @tx_remove: Free resources for TX queue
* @tx_write: Write TX descriptors and doorbell
@@ -1495,14 +1500,6 @@ efx_get_tx_channel(struct efx_nic *efx, unsigned int index)
return efx->channel[efx->tx_channel_offset + index];
}
-static inline struct efx_tx_queue *
-efx_get_tx_queue(struct efx_nic *efx, unsigned index, unsigned type)
-{
- EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels ||
- type >= efx->tx_queues_per_channel);
- return &efx->channel[efx->tx_channel_offset + index]->tx_queue[type];
-}
-
static inline struct efx_channel *
efx_get_xdp_channel(struct efx_nic *efx, unsigned int index)
{
@@ -1529,10 +1526,18 @@ static inline unsigned int efx_channel_num_tx_queues(struct efx_channel *channel
}
static inline struct efx_tx_queue *
-efx_channel_get_tx_queue(struct efx_channel *channel, unsigned type)
+efx_channel_get_tx_queue(struct efx_channel *channel, unsigned int type)
{
- EFX_WARN_ON_ONCE_PARANOID(type >= efx_channel_num_tx_queues(channel));
- return &channel->tx_queue[type];
+ EFX_WARN_ON_ONCE_PARANOID(type >= EFX_TXQ_TYPES);
+ return channel->tx_queue_by_type[type];
+}
+
+static inline struct efx_tx_queue *
+efx_get_tx_queue(struct efx_nic *efx, unsigned int index, unsigned int type)
+{
+ struct efx_channel *channel = efx_get_tx_channel(efx, index);
+
+ return efx_channel_get_tx_queue(channel, type);
}
/* Iterate over all TX queues belonging to a channel */
@@ -1085,7 +1085,7 @@ static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb)
struct efx_tx_queue *tx_queue;
u8 type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0;
- tx_queue = &ptp_data->channel->tx_queue[type];
+ tx_queue = efx_channel_get_tx_queue(ptp_data->channel, type);
if (tx_queue && tx_queue->timestamping) {
efx_enqueue_skb(tx_queue, skb);
} else {
@@ -656,7 +656,7 @@ static int efx_test_loopbacks(struct efx_nic *efx, struct efx_self_tests *tests,
/* Test all enabled types of TX queue */
efx_for_each_channel_tx_queue(tx_queue, channel) {
- state->offload_csum = (tx_queue->label &
+ state->offload_csum = (tx_queue->type &
EFX_TXQ_TYPE_OFFLOAD);
rc = efx_test_loopback(tx_queue,
&tests->loopback[mode]);
@@ -15,8 +15,8 @@
*/
struct efx_loopback_self_tests {
- int tx_sent[EFX_TXQ_TYPES];
- int tx_done[EFX_TXQ_TYPES];
+ int tx_sent[EFX_MAX_TXQ_PER_CHANNEL];
+ int tx_done[EFX_MAX_TXQ_PER_CHANNEL];
int rx_good;
int rx_bad;
};
@@ -527,6 +527,12 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
}
tx_queue = efx_get_tx_queue(efx, index, type);
+ if (WARN_ON(!tx_queue))
+ /* We don't have a TXQ of the right type.
+ * This should never happen, as we don't advertise offload
+ * features unless we can support them.
+ */
+ return NETDEV_TX_BUSY;
return __efx_enqueue_skb(tx_queue, skb);
}
@@ -577,7 +583,7 @@ void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
tx_queue->core_txq =
netdev_get_tx_queue(efx->net_dev,
tx_queue->channel->channel +
- ((tx_queue->label & EFX_TXQ_TYPE_HIGHPRI) ?
+ ((tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ?
efx->n_tx_channels : 0));
}
@@ -47,11 +47,12 @@ int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
goto fail1;
}
- /* Allocate hardware ring */
+ /* Allocate hardware ring, determine TXQ type */
rc = efx_nic_probe_tx(tx_queue);
if (rc)
goto fail2;
+ tx_queue->channel->tx_queue_by_type[tx_queue->type] = tx_queue;
return 0;
fail2:
@@ -141,6 +142,7 @@ void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
kfree(tx_queue->buffer);
tx_queue->buffer = NULL;
+ tx_queue->channel->tx_queue_by_type[tx_queue->type] = NULL;
}
void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
Make it possible to have an arbitrary mapping from types to labels, because when we add inner-csum-offload TXQs there will no longer be a convenient nesting hierarchy of NIC types (EF10 will have inner-csum TXQs, while Siena will have HIGHPRI). Signed-off-by: Edward Cree <ecree@solarflare.com> --- drivers/net/ethernet/sfc/ef10.c | 5 ++-- drivers/net/ethernet/sfc/efx_channels.c | 10 +++---- drivers/net/ethernet/sfc/ethtool_common.c | 2 +- drivers/net/ethernet/sfc/farch.c | 20 +++++++------- drivers/net/ethernet/sfc/mcdi_functions.c | 2 +- drivers/net/ethernet/sfc/net_driver.h | 33 +++++++++++++---------- drivers/net/ethernet/sfc/ptp.c | 2 +- drivers/net/ethernet/sfc/selftest.c | 2 +- drivers/net/ethernet/sfc/selftest.h | 4 +-- drivers/net/ethernet/sfc/tx.c | 8 +++++- drivers/net/ethernet/sfc/tx_common.c | 4 ++- 11 files changed, 54 insertions(+), 38 deletions(-)