| From b3d2f640a242c120cdf2c38928b94bc73dff394e Mon Sep 17 00:00:00 2001 |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Tue, 12 Jul 2011 15:38:34 +0200 |
| Subject: [PATCH] net: Use skbufhead with raw lock |
| |
| Use the rps lock as rawlock so we can keep irq-off regions. It looks low |
| latency. However we can't kfree() from this context therefore we defer this |
| to the softirq and use the tofree_queue list for it (similar to process_queue). |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h |
| index 9c23bd2efb56..34d4b6665bb9 100644 |
| --- a/include/linux/netdevice.h |
| +++ b/include/linux/netdevice.h |
| @@ -2785,6 +2785,7 @@ struct softnet_data { |
| unsigned int dropped; |
| struct sk_buff_head input_pkt_queue; |
| struct napi_struct backlog; |
| + struct sk_buff_head tofree_queue; |
| |
| }; |
| |
| diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h |
| index a098d95b3d84..e64ebd1134f3 100644 |
| --- a/include/linux/skbuff.h |
| +++ b/include/linux/skbuff.h |
| @@ -285,6 +285,7 @@ struct sk_buff_head { |
| |
| __u32 qlen; |
| spinlock_t lock; |
| + raw_spinlock_t raw_lock; |
| }; |
| |
| struct sk_buff; |
| @@ -1590,6 +1591,12 @@ static inline void skb_queue_head_init(struct sk_buff_head *list) |
| __skb_queue_head_init(list); |
| } |
| |
| +static inline void skb_queue_head_init_raw(struct sk_buff_head *list) |
| +{ |
| + raw_spin_lock_init(&list->raw_lock); |
| + __skb_queue_head_init(list); |
| +} |
| + |
| static inline void skb_queue_head_init_class(struct sk_buff_head *list, |
| struct lock_class_key *class) |
| { |
| diff --git a/net/core/dev.c b/net/core/dev.c |
| index 963350b05579..f5c305942b80 100644 |
| --- a/net/core/dev.c |
| +++ b/net/core/dev.c |
| @@ -212,14 +212,14 @@ static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) |
| static inline void rps_lock(struct softnet_data *sd) |
| { |
| #ifdef CONFIG_RPS |
| - spin_lock(&sd->input_pkt_queue.lock); |
| + raw_spin_lock(&sd->input_pkt_queue.raw_lock); |
| #endif |
| } |
| |
| static inline void rps_unlock(struct softnet_data *sd) |
| { |
| #ifdef CONFIG_RPS |
| - spin_unlock(&sd->input_pkt_queue.lock); |
| + raw_spin_unlock(&sd->input_pkt_queue.raw_lock); |
| #endif |
| } |
| |
| @@ -4457,7 +4457,7 @@ static void flush_backlog(struct work_struct *work) |
| skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { |
| if (skb->dev->reg_state == NETREG_UNREGISTERING) { |
| __skb_unlink(skb, &sd->input_pkt_queue); |
| - kfree_skb(skb); |
| + __skb_queue_tail(&sd->tofree_queue, skb); |
| input_queue_head_incr(sd); |
| } |
| } |
| @@ -4467,11 +4467,14 @@ static void flush_backlog(struct work_struct *work) |
| skb_queue_walk_safe(&sd->process_queue, skb, tmp) { |
| if (skb->dev->reg_state == NETREG_UNREGISTERING) { |
| __skb_unlink(skb, &sd->process_queue); |
| - kfree_skb(skb); |
| + __skb_queue_tail(&sd->tofree_queue, skb); |
| input_queue_head_incr(sd); |
| } |
| } |
| + if (!skb_queue_empty(&sd->tofree_queue)) |
| + raise_softirq_irqoff(NET_RX_SOFTIRQ); |
| local_bh_enable(); |
| + |
| } |
| |
| static void flush_all_backlogs(void) |
| @@ -5005,7 +5008,9 @@ static int process_backlog(struct napi_struct *napi, int quota) |
| while (again) { |
| struct sk_buff *skb; |
| |
| + local_irq_disable(); |
| while ((skb = __skb_dequeue(&sd->process_queue))) { |
| + local_irq_enable(); |
| rcu_read_lock(); |
| __netif_receive_skb(skb); |
| rcu_read_unlock(); |
| @@ -5013,9 +5018,9 @@ static int process_backlog(struct napi_struct *napi, int quota) |
| if (++work >= quota) |
| return work; |
| |
| + local_irq_disable(); |
| } |
| |
| - local_irq_disable(); |
| rps_lock(sd); |
| if (skb_queue_empty(&sd->input_pkt_queue)) { |
| /* |
| @@ -5454,13 +5459,21 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) |
| unsigned long time_limit = jiffies + |
| usecs_to_jiffies(netdev_budget_usecs); |
| int budget = netdev_budget; |
| + struct sk_buff_head tofree_q; |
| + struct sk_buff *skb; |
| LIST_HEAD(list); |
| LIST_HEAD(repoll); |
| |
| + __skb_queue_head_init(&tofree_q); |
| + |
| local_irq_disable(); |
| + skb_queue_splice_init(&sd->tofree_queue, &tofree_q); |
| list_splice_init(&sd->poll_list, &list); |
| local_irq_enable(); |
| |
| + while ((skb = __skb_dequeue(&tofree_q))) |
| + kfree_skb(skb); |
| + |
| for (;;) { |
| struct napi_struct *n; |
| |
| @@ -8231,6 +8244,9 @@ static int dev_cpu_dead(unsigned int oldcpu) |
| netif_rx_ni(skb); |
| input_queue_head_incr(oldsd); |
| } |
| + while ((skb = __skb_dequeue(&oldsd->tofree_queue))) { |
| + kfree_skb(skb); |
| + } |
| |
| return 0; |
| } |
| @@ -8534,8 +8550,9 @@ static int __init net_dev_init(void) |
| |
| INIT_WORK(flush, flush_backlog); |
| |
| - skb_queue_head_init(&sd->input_pkt_queue); |
| - skb_queue_head_init(&sd->process_queue); |
| + skb_queue_head_init_raw(&sd->input_pkt_queue); |
| + skb_queue_head_init_raw(&sd->process_queue); |
| + skb_queue_head_init_raw(&sd->tofree_queue); |
| INIT_LIST_HEAD(&sd->poll_list); |
| sd->output_queue_tailp = &sd->output_queue; |
| #ifdef CONFIG_RPS |
| -- |
| 2.1.4 |
| |