@@ -71,7 +71,12 @@ static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
}
int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
- struct tcf_result *res, bool compat_mode);
+ const struct tcf_proto *orig_tp, struct tcf_result *res,
+ bool compat_mode);
+int tcf_classify_ingress(struct sk_buff *skb,
+ const struct tcf_block *ingress_block,
+ const struct tcf_proto *tp, struct tcf_result *res,
+ bool compat_mode);
#else
static inline bool tcf_block_shared(struct tcf_block *block)
@@ -129,10 +134,20 @@ void tc_setup_cb_block_unregister(struct tcf_block *block, flow_setup_cb_t *cb,
}
static inline int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
+ const struct tcf_proto *orig_tp,
struct tcf_result *res, bool compat_mode)
{
return TC_ACT_UNSPEC;
}
+
+static inline int tcf_classify_ingress(struct sk_buff *skb,
+ const struct tcf_block *ingress_block,
+ const struct tcf_proto *tp,
+ struct tcf_result *res, bool compat_mode)
+{
+ return TC_ACT_UNSPEC;
+}
+
#endif
static inline unsigned long
@@ -381,7 +381,8 @@ struct qdisc_skb_cb {
unsigned char data[QDISC_CB_PRIV_LEN];
};
-typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
+typedef void tcf_chain_head_change_t(struct tcf_block *block,
+ struct tcf_proto *tp_head, void *priv);
struct tcf_chain {
/* Protects filter_chain. */
@@ -1268,6 +1269,7 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res,
*/
struct mini_Qdisc {
struct tcf_proto *filter_list;
+ struct tcf_block *block;
struct gnet_stats_basic_cpu __percpu *cpu_bstats;
struct gnet_stats_queue __percpu *cpu_qstats;
struct rcu_head rcu;
@@ -1291,7 +1293,7 @@ struct mini_Qdisc_pair {
};
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
- struct tcf_proto *tp_head);
+ struct tcf_block *block, struct tcf_proto *tp_head);
void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
struct mini_Qdisc __rcu **p_miniq);
@@ -3743,7 +3743,8 @@ int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
mini_qdisc_bstats_cpu_update(miniq, skb);
- switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
+ switch (tcf_classify(skb, miniq->filter_list, miniq->filter_list,
+ &cl_res, false)) {
case TC_ACT_OK:
case TC_ACT_RECLASSIFY:
skb->tc_index = TC_H_MIN(cl_res.classid);
@@ -4810,7 +4811,8 @@ static __latent_entropy void net_tx_action(struct softirq_action *h)
skb->tc_at_ingress = 1;
mini_qdisc_bstats_cpu_update(miniq, skb);
- switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
+ switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
+ &cl_res, false)) {
case TC_ACT_OK:
case TC_ACT_RECLASSIFY:
skb->tc_index = TC_H_MIN(cl_res.classid);
@@ -22,6 +22,7 @@
#include <linux/idr.h>
#include <linux/rhashtable.h>
#include <linux/jhash.h>
+#include <linux/rculist.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/netlink.h>
@@ -354,7 +355,7 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
chain = kzalloc(sizeof(*chain), GFP_KERNEL);
if (!chain)
return NULL;
- list_add_tail(&chain->list, &block->chain_list);
+ list_add_tail_rcu(&chain->list, &block->chain_list);
mutex_init(&chain->filter_chain_lock);
chain->block = block;
chain->index = chain_index;
@@ -365,10 +366,12 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
}
static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
+ struct tcf_block *block,
struct tcf_proto *tp_head)
{
if (item->chain_head_change)
- item->chain_head_change(tp_head, item->chain_head_change_priv);
+ item->chain_head_change(block, tp_head,
+ item->chain_head_change_priv);
}
static void tcf_chain0_head_change(struct tcf_chain *chain,
@@ -382,7 +385,7 @@ static void tcf_chain0_head_change(struct tcf_chain *chain,
mutex_lock(&block->lock);
list_for_each_entry(item, &block->chain0.filter_chain_list, list)
- tcf_chain_head_change_item(item, tp_head);
+ tcf_chain_head_change_item(item, block, tp_head);
mutex_unlock(&block->lock);
}
@@ -394,7 +397,7 @@ static bool tcf_chain_detach(struct tcf_chain *chain)
ASSERT_BLOCK_LOCKED(block);
- list_del(&chain->list);
+ list_del_rcu(&chain->list);
if (!chain->index)
block->chain0.chain = NULL;
@@ -453,6 +456,20 @@ static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
return NULL;
}
+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
+static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
+ u32 chain_index)
+{
+ struct tcf_chain *chain;
+
+ list_for_each_entry_rcu(chain, &block->chain_list, list) {
+ if (chain->index == chain_index)
+ return chain;
+ }
+ return NULL;
+}
+#endif
+
static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
u32 seq, u16 flags, int event, bool unicast);
@@ -822,7 +839,7 @@ static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
if (tp_head)
- tcf_chain_head_change_item(item, tp_head);
+ tcf_chain_head_change_item(item, block, tp_head);
mutex_lock(&block->lock);
list_add(&item->list, &block->chain0.filter_chain_list);
@@ -847,7 +864,7 @@ static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
(item->chain_head_change == ei->chain_head_change &&
item->chain_head_change_priv == ei->chain_head_change_priv)) {
if (block->chain0.chain)
- tcf_chain_head_change_item(item, NULL);
+ tcf_chain_head_change_item(item, NULL, NULL);
list_del(&item->list);
mutex_unlock(&block->lock);
@@ -1384,7 +1401,8 @@ int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
}
EXPORT_SYMBOL(tcf_block_get_ext);
-static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
+static void tcf_chain_head_change_dflt(struct tcf_block *block,
+ struct tcf_proto *tp_head, void *priv)
{
struct tcf_proto __rcu **p_filter_chain = priv;
@@ -1560,11 +1578,11 @@ static int tcf_block_setup(struct tcf_block *block,
* specific classifiers.
*/
int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
- struct tcf_result *res, bool compat_mode)
+ const struct tcf_proto *orig_tp, struct tcf_result *res,
+ bool compat_mode)
{
#ifdef CONFIG_NET_CLS_ACT
const int max_reclassify_loop = 4;
- const struct tcf_proto *orig_tp = tp;
const struct tcf_proto *first_tp;
int limit = 0;
@@ -1621,6 +1639,34 @@ int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
}
EXPORT_SYMBOL(tcf_classify);
+int tcf_classify_ingress(struct sk_buff *skb,
+ const struct tcf_block *ingress_block,
+ const struct tcf_proto *tp, struct tcf_result *res,
+ bool compat_mode)
+{
+ const struct tcf_proto *orig_tp = tp;
+
+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
+ {
+ struct tc_skb_ext *ext = skb_ext_find(skb, TC_SKB_EXT);
+
+ if (ext && ext->chain && ingress_block) {
+ struct tcf_chain *fchain;
+
+ fchain = tcf_chain_lookup_rcu(ingress_block,
+ ext->chain);
+ if (!fchain)
+ return TC_ACT_UNSPEC;
+
+ tp = rcu_dereference_bh(fchain->filter_chain);
+ }
+ }
+#endif
+
+ return tcf_classify(skb, tp, orig_tp, res, compat_mode);
+}
+EXPORT_SYMBOL(tcf_classify_ingress);
+
struct tcf_chain_info {
struct tcf_proto __rcu **pprev;
struct tcf_proto __rcu *next;
@@ -393,7 +393,7 @@ static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
list_for_each_entry(flow, &p->flows, list) {
fl = rcu_dereference_bh(flow->filter_list);
if (fl) {
- result = tcf_classify(skb, fl, &res, true);
+ result = tcf_classify(skb, fl, fl, &res, true);
if (result < 0)
continue;
flow = (struct atm_flow_data *)res.class;
@@ -1600,7 +1600,7 @@ static u32 cake_classify(struct Qdisc *sch, struct cake_tin_data **t,
goto hash;
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
- result = tcf_classify(skb, filter, &res, false);
+ result = tcf_classify(skb, filter, filter, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
@@ -228,7 +228,7 @@ struct cbq_sched_data {
/*
* Step 2+n. Apply classifier.
*/
- result = tcf_classify(skb, fl, &res, true);
+ result = tcf_classify(skb, fl, fl, &res, true);
if (!fl || result < 0)
goto fallback;
@@ -316,7 +316,7 @@ static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch,
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
fl = rcu_dereference_bh(q->filter_list);
- result = tcf_classify(skb, fl, &res, false);
+ result = tcf_classify(skb, fl, fl, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
@@ -241,7 +241,7 @@ static int dsmark_enqueue(struct sk_buff *skb, struct Qdisc *sch,
else {
struct tcf_result res;
struct tcf_proto *fl = rcu_dereference_bh(p->filter_list);
- int result = tcf_classify(skb, fl, &res, false);
+ int result = tcf_classify(skb, fl, fl, &res, false);
pr_debug("result %d class 0x%04x\n", result, res.classid);
@@ -91,7 +91,7 @@ static unsigned int fq_codel_classify(struct sk_buff *skb, struct Qdisc *sch,
return fq_codel_hash(q, skb) + 1;
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
- result = tcf_classify(skb, filter, &res, false);
+ result = tcf_classify(skb, filter, filter, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
@@ -1355,7 +1355,7 @@ static void mini_qdisc_rcu_func(struct rcu_head *head)
}
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
- struct tcf_proto *tp_head)
+ struct tcf_block *block, struct tcf_proto *tp_head)
{
/* Protected with chain0->filter_chain_lock.
* Can't access chain directly because tp_head can be NULL.
@@ -1380,6 +1380,7 @@ void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
*/
rcu_barrier();
miniq->filter_list = tp_head;
+ miniq->block = block;
rcu_assign_pointer(*miniqp->p_miniq, miniq);
if (miniq_old)
@@ -1129,7 +1129,8 @@ struct hfsc_sched {
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
head = &q->root;
tcf = rcu_dereference_bh(q->root.filter_list);
- while (tcf && (result = tcf_classify(skb, tcf, &res, false)) >= 0) {
+ while (tcf && (result = tcf_classify(skb, tcf, tcf, &res, false))
+ >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
case TC_ACT_QUEUED:
@@ -232,7 +232,8 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch,
}
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
- while (tcf && (result = tcf_classify(skb, tcf, &res, false)) >= 0) {
+ while (tcf && (result = tcf_classify(skb, tcf, tcf, &res, false))
+ >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
case TC_ACT_QUEUED:
@@ -52,11 +52,12 @@ static struct tcf_block *ingress_tcf_block(struct Qdisc *sch, unsigned long cl,
return q->block;
}
-static void clsact_chain_head_change(struct tcf_proto *tp_head, void *priv)
+static void clsact_chain_head_change(struct tcf_block *block,
+ struct tcf_proto *tp_head, void *priv)
{
struct mini_Qdisc_pair *miniqp = priv;
- mini_qdisc_pair_swap(miniqp, tp_head);
+ mini_qdisc_pair_swap(miniqp, block, tp_head);
};
static void ingress_ingress_block_set(struct Qdisc *sch, u32 block_index)
@@ -36,7 +36,7 @@ struct multiq_sched_data {
int err;
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
- err = tcf_classify(skb, fl, &res, false);
+ err = tcf_classify(skb, fl, fl, &res, false);
#ifdef CONFIG_NET_CLS_ACT
switch (err) {
case TC_ACT_STOLEN:
@@ -39,7 +39,7 @@ struct prio_sched_data {
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
if (TC_H_MAJ(skb->priority) != sch->handle) {
fl = rcu_dereference_bh(q->filter_list);
- err = tcf_classify(skb, fl, &res, false);
+ err = tcf_classify(skb, fl, fl, &res, false);
#ifdef CONFIG_NET_CLS_ACT
switch (err) {
case TC_ACT_STOLEN:
@@ -691,7 +691,7 @@ static struct qfq_class *qfq_classify(struct sk_buff *skb, struct Qdisc *sch,
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
fl = rcu_dereference_bh(q->filter_list);
- result = tcf_classify(skb, fl, &res, false);
+ result = tcf_classify(skb, fl, fl, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
@@ -257,7 +257,7 @@ static bool sfb_classify(struct sk_buff *skb, struct tcf_proto *fl,
struct tcf_result res;
int result;
- result = tcf_classify(skb, fl, &res, false);
+ result = tcf_classify(skb, fl, fl, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
@@ -178,7 +178,7 @@ static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
return sfq_hash(q, skb) + 1;
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
- result = tcf_classify(skb, fl, &res, false);
+ result = tcf_classify(skb, fl, fl, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {