| From fe7e3c8d5c2582d4f67a3a90de36eb7b4f449634 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Mon, 19 Apr 2021 12:53:06 +0000 |
| Subject: gro: fix napi_gro_frags() Fast GRO breakage due to IP alignment check |
| |
| From: Alexander Lobakin <alobakin@pm.me> |
| |
| [ Upstream commit 7ad18ff6449cbd6beb26b53128ddf56d2685aa93 ] |
| |
| Commit 38ec4944b593 ("gro: ensure frag0 meets IP header alignment") |
| did the right thing, but missed the fact that napi_gro_frags() logics |
| calls for skb_gro_reset_offset() *before* pulling Ethernet header |
| to the skb linear space. |
| That said, the introduced check for frag0 address being aligned to 4 |
| always fails for it as Ethernet header is obviously 14 bytes long, |
| and in case with NET_IP_ALIGN its start is not aligned to 4. |
| |
| Fix this by adding @nhoff argument to skb_gro_reset_offset() which |
| tells if an IP header is placed right at the start of frag0 or not. |
| This restores Fast GRO for napi_gro_frags() that became very slow |
| after the mentioned commit, and preserves the introduced check to |
| avoid silent unaligned accesses. |
| |
| From v1 [0]: |
| - inline tiny skb_gro_reset_offset() to let the code be optimized |
| more efficively (esp. for the !NET_IP_ALIGN case) (Eric); |
| - pull in Reviewed-by from Eric. |
| |
| [0] https://lore.kernel.org/netdev/20210418114200.5839-1-alobakin@pm.me |
| |
| Fixes: 38ec4944b593 ("gro: ensure frag0 meets IP header alignment") |
| Reviewed-by: Eric Dumazet <edumazet@google.com> |
| Signed-off-by: Alexander Lobakin <alobakin@pm.me> |
| Signed-off-by: David S. Miller <davem@davemloft.net> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| net/core/dev.c | 8 ++++---- |
| 1 file changed, 4 insertions(+), 4 deletions(-) |
| |
| diff --git a/net/core/dev.c b/net/core/dev.c |
| index 3c0d3b6d674d..633c2d6f1a35 100644 |
| --- a/net/core/dev.c |
| +++ b/net/core/dev.c |
| @@ -5867,7 +5867,7 @@ static struct list_head *gro_list_prepare(struct napi_struct *napi, |
| return head; |
| } |
| |
| -static void skb_gro_reset_offset(struct sk_buff *skb) |
| +static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) |
| { |
| const struct skb_shared_info *pinfo = skb_shinfo(skb); |
| const skb_frag_t *frag0 = &pinfo->frags[0]; |
| @@ -5878,7 +5878,7 @@ static void skb_gro_reset_offset(struct sk_buff *skb) |
| |
| if (!skb_headlen(skb) && pinfo->nr_frags && |
| !PageHighMem(skb_frag_page(frag0)) && |
| - (!NET_IP_ALIGN || !(skb_frag_off(frag0) & 3))) { |
| + (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { |
| NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); |
| NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, |
| skb_frag_size(frag0), |
| @@ -6111,7 +6111,7 @@ gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) |
| skb_mark_napi_id(skb, napi); |
| trace_napi_gro_receive_entry(skb); |
| |
| - skb_gro_reset_offset(skb); |
| + skb_gro_reset_offset(skb, 0); |
| |
| ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); |
| trace_napi_gro_receive_exit(ret); |
| @@ -6204,7 +6204,7 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) |
| napi->skb = NULL; |
| |
| skb_reset_mac_header(skb); |
| - skb_gro_reset_offset(skb); |
| + skb_gro_reset_offset(skb, hlen); |
| |
| if (unlikely(skb_gro_header_hard(skb, hlen))) { |
| eth = skb_gro_header_slow(skb, hlen, 0); |
| -- |
| 2.30.2 |
| |