| From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| Date: Wed, 13 Jan 2016 15:55:02 +0100 |
| Subject: net: move xmit_recursion to per-task variable on -RT |
| |
| A softirq on -RT can be preempted. That means one task is in |
| __dev_queue_xmit(), gets preempted and another task may enter |
| __dev_queue_xmit() aw well. netperf together with a bridge device |
| will then trigger the `recursion alert` because each task increments |
| the xmit_recursion variable which is per-CPU. |
| A virtual device like br0 is required to trigger this warning. |
| |
| This patch moves the counter to per task instead per-CPU so it counts |
| the recursion properly on -RT. |
| |
| Cc: stable-rt@vger.kernel.org |
| Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| --- |
| include/linux/netdevice.h | 41 ++++++++++++++++++++++++++++++++++++++++- |
| include/linux/sched.h | 3 +++ |
| net/core/dev.c | 9 +++++---- |
| net/core/filter.c | 6 +++--- |
| 4 files changed, 51 insertions(+), 8 deletions(-) |
| |
| --- a/include/linux/netdevice.h |
| +++ b/include/linux/netdevice.h |
| @@ -2461,14 +2461,53 @@ void netdev_freemem(struct net_device *d |
| void synchronize_net(void); |
| int init_dummy_netdev(struct net_device *dev); |
| |
| -DECLARE_PER_CPU(int, xmit_recursion); |
| #define XMIT_RECURSION_LIMIT 10 |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| +static inline int dev_recursion_level(void) |
| +{ |
| + return current->xmit_recursion; |
| +} |
| + |
| +static inline int xmit_rec_read(void) |
| +{ |
| + return current->xmit_recursion; |
| +} |
| + |
| +static inline void xmit_rec_inc(void) |
| +{ |
| + current->xmit_recursion++; |
| +} |
| + |
| +static inline void xmit_rec_dec(void) |
| +{ |
| + current->xmit_recursion--; |
| +} |
| + |
| +#else |
| + |
| +DECLARE_PER_CPU(int, xmit_recursion); |
| |
| static inline int dev_recursion_level(void) |
| { |
| return this_cpu_read(xmit_recursion); |
| } |
| |
| +static inline int xmit_rec_read(void) |
| +{ |
| + return __this_cpu_read(xmit_recursion); |
| +} |
| + |
| +static inline void xmit_rec_inc(void) |
| +{ |
| + __this_cpu_inc(xmit_recursion); |
| +} |
| + |
| +static inline void xmit_rec_dec(void) |
| +{ |
| + __this_cpu_dec(xmit_recursion); |
| +} |
| +#endif |
| + |
| struct net_device *dev_get_by_index(struct net *net, int ifindex); |
| struct net_device *__dev_get_by_index(struct net *net, int ifindex); |
| struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); |
| --- a/include/linux/sched.h |
| +++ b/include/linux/sched.h |
| @@ -1982,6 +1982,9 @@ struct task_struct { |
| #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
| unsigned long task_state_change; |
| #endif |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + int xmit_recursion; |
| +#endif |
| int pagefault_disabled; |
| #ifdef CONFIG_MMU |
| struct task_struct *oom_reaper_list; |
| --- a/net/core/dev.c |
| +++ b/net/core/dev.c |
| @@ -3143,8 +3143,10 @@ static void skb_update_prio(struct sk_bu |
| #define skb_update_prio(skb) |
| #endif |
| |
| +#ifndef CONFIG_PREEMPT_RT_FULL |
| DEFINE_PER_CPU(int, xmit_recursion); |
| EXPORT_SYMBOL(xmit_recursion); |
| +#endif |
| |
| /** |
| * dev_loopback_xmit - loop back @skb |
| @@ -3378,8 +3380,7 @@ static int __dev_queue_xmit(struct sk_bu |
| int cpu = smp_processor_id(); /* ok because BHs are off */ |
| |
| if (txq->xmit_lock_owner != cpu) { |
| - if (unlikely(__this_cpu_read(xmit_recursion) > |
| - XMIT_RECURSION_LIMIT)) |
| + if (unlikely(xmit_rec_read() > XMIT_RECURSION_LIMIT)) |
| goto recursion_alert; |
| |
| skb = validate_xmit_skb(skb, dev); |
| @@ -3389,9 +3390,9 @@ static int __dev_queue_xmit(struct sk_bu |
| HARD_TX_LOCK(dev, txq, cpu); |
| |
| if (!netif_xmit_stopped(txq)) { |
| - __this_cpu_inc(xmit_recursion); |
| + xmit_rec_inc(); |
| skb = dev_hard_start_xmit(skb, dev, txq, &rc); |
| - __this_cpu_dec(xmit_recursion); |
| + xmit_rec_dec(); |
| if (dev_xmit_complete(rc)) { |
| HARD_TX_UNLOCK(dev, txq); |
| goto out; |
| --- a/net/core/filter.c |
| +++ b/net/core/filter.c |
| @@ -1645,7 +1645,7 @@ static inline int __bpf_tx_skb(struct ne |
| { |
| int ret; |
| |
| - if (unlikely(__this_cpu_read(xmit_recursion) > XMIT_RECURSION_LIMIT)) { |
| + if (unlikely(xmit_rec_read() > XMIT_RECURSION_LIMIT)) { |
| net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); |
| kfree_skb(skb); |
| return -ENETDOWN; |
| @@ -1653,9 +1653,9 @@ static inline int __bpf_tx_skb(struct ne |
| |
| skb->dev = dev; |
| |
| - __this_cpu_inc(xmit_recursion); |
| + xmit_rec_inc(); |
| ret = dev_queue_xmit(skb); |
| - __this_cpu_dec(xmit_recursion); |
| + xmit_rec_dec(); |
| |
| return ret; |
| } |