| From 57a743fec5e1c25d057768bf994c4b055f357d1f Mon Sep 17 00:00:00 2001 |
| From: Eric Dumazet <edumazet@google.com> |
| Date: Thu, 10 Jan 2013 15:26:34 -0800 |
| Subject: [PATCH] softirq: reduce latencies |
| |
| commit c10d73671ad30f54692f7f69f0e09e75d3a8926a upstream. |
| |
| In various network workloads, __do_softirq() latencies can be up |
| to 20 ms if HZ=1000, and 200 ms if HZ=100. |
| |
| This is because we iterate 10 times in the softirq dispatcher, |
| and some actions can consume a lot of cycles. |
| |
| This patch changes the fallback to ksoftirqd condition to : |
| |
| - A time limit of 2 ms. |
| - need_resched() being set on current task |
| |
| When one of this condition is met, we wakeup ksoftirqd for further |
| softirq processing if we still have pending softirqs. |
| |
| Using need_resched() as the only condition can trigger RCU stalls, |
| as we can keep BH disabled for too long. |
| |
| I ran several benchmarks and got no significant difference in |
| throughput, but a very significant reduction of latencies (one order |
| of magnitude) : |
| |
| In following bench, 200 antagonist "netperf -t TCP_RR" are started in |
| background, using all available cpus. |
| |
| Then we start one "netperf -t TCP_RR", bound to the cpu handling the NIC |
| IRQ (hard+soft) |
| |
| Before patch : |
| |
| # netperf -H 7.7.7.84 -t TCP_RR -T2,2 -- -k |
| RT_LATENCY,MIN_LATENCY,MAX_LATENCY,P50_LATENCY,P90_LATENCY,P99_LATENCY,MEAN_LATENCY,STDDEV_LATENCY |
| MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET |
| to 7.7.7.84 () port 0 AF_INET : first burst 0 : cpu bind |
| RT_LATENCY=550110.424 |
| MIN_LATENCY=146858 |
| MAX_LATENCY=997109 |
| P50_LATENCY=305000 |
| P90_LATENCY=550000 |
| P99_LATENCY=710000 |
| MEAN_LATENCY=376989.12 |
| STDDEV_LATENCY=184046.92 |
| |
| After patch : |
| |
| # netperf -H 7.7.7.84 -t TCP_RR -T2,2 -- -k |
| RT_LATENCY,MIN_LATENCY,MAX_LATENCY,P50_LATENCY,P90_LATENCY,P99_LATENCY,MEAN_LATENCY,STDDEV_LATENCY |
| MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET |
| to 7.7.7.84 () port 0 AF_INET : first burst 0 : cpu bind |
| RT_LATENCY=40545.492 |
| MIN_LATENCY=9834 |
| MAX_LATENCY=78366 |
| P50_LATENCY=33583 |
| P90_LATENCY=59000 |
| P99_LATENCY=69000 |
| MEAN_LATENCY=38364.67 |
| STDDEV_LATENCY=12865.26 |
| |
| Signed-off-by: Eric Dumazet <edumazet@google.com> |
| Cc: David Miller <davem@davemloft.net> |
| Cc: Tom Herbert <therbert@google.com> |
| Cc: Ben Hutchings <bhutchings@solarflare.com> |
| Signed-off-by: David S. Miller <davem@davemloft.net> |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| --- |
| kernel/softirq.c | 17 +++++++++-------- |
| 1 file changed, 9 insertions(+), 8 deletions(-) |
| |
| diff --git a/kernel/softirq.c b/kernel/softirq.c |
| index 7c1a67ef0274..0df9a9406271 100644 |
| --- a/kernel/softirq.c |
| +++ b/kernel/softirq.c |
| @@ -178,21 +178,21 @@ void local_bh_enable_ip(unsigned long ip) |
| EXPORT_SYMBOL(local_bh_enable_ip); |
| |
| /* |
| - * We restart softirq processing MAX_SOFTIRQ_RESTART times, |
| - * and we fall back to softirqd after that. |
| + * We restart softirq processing for at most 2 ms, |
| + * and if need_resched() is not set. |
| * |
| - * This number has been established via experimentation. |
| + * These limits have been established via experimentation. |
| * The two things to balance is latency against fairness - |
| * we want to handle softirqs as soon as possible, but they |
| * should not be able to lock up the box. |
| */ |
| -#define MAX_SOFTIRQ_RESTART 10 |
| +#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2) |
| |
| asmlinkage void __do_softirq(void) |
| { |
| struct softirq_action *h; |
| __u32 pending; |
| - int max_restart = MAX_SOFTIRQ_RESTART; |
| + unsigned long end = jiffies + MAX_SOFTIRQ_TIME; |
| int cpu; |
| |
| pending = local_softirq_pending(); |
| @@ -236,11 +236,12 @@ restart: |
| local_irq_disable(); |
| |
| pending = local_softirq_pending(); |
| - if (pending && --max_restart) |
| - goto restart; |
| + if (pending) { |
| + if (time_before(jiffies, end) && !need_resched()) |
| + goto restart; |
| |
| - if (pending) |
| wakeup_softirqd(); |
| + } |
| |
| lockdep_softirq_exit(); |
| |
| -- |
| 1.8.5.2 |
| |