udp: Add GRO/GSO support Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 1d4737c..5eb77de 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h
@@ -55,8 +55,9 @@ enum { NETIF_F_GSO_TUNNEL_REMCSUM_BIT, /* ... TUNNEL with TSO & REMCSUM */ NETIF_F_GSO_SCTP_BIT, /* ... SCTP fragmentation */ NETIF_F_GSO_ESP_BIT, /* ... ESP with TSO */ + NETIF_F_GSO_UDP_BYFRAGS_BIT, /* ... UDP generic */ /**/NETIF_F_GSO_LAST = /* last bit, see GSO_MASK */ - NETIF_F_GSO_ESP_BIT, + NETIF_F_GSO_UDP_BYFRAGS_BIT, NETIF_F_FCOE_CRC_BIT, /* FCoE CRC32 */ NETIF_F_SCTP_CRC_BIT, /* SCTP checksum offload */ @@ -133,6 +134,7 @@ enum { #define NETIF_F_GSO_TUNNEL_REMCSUM __NETIF_F(GSO_TUNNEL_REMCSUM) #define NETIF_F_GSO_SCTP __NETIF_F(GSO_SCTP) #define NETIF_F_GSO_ESP __NETIF_F(GSO_ESP) +#define NETIF_F_GSO_UDP_BYFRAGS __NETIF_F(GSO_UDP_BYFRAGS) #define NETIF_F_HW_VLAN_STAG_FILTER __NETIF_F(HW_VLAN_STAG_FILTER) #define NETIF_F_HW_VLAN_STAG_RX __NETIF_F(HW_VLAN_STAG_RX) #define NETIF_F_HW_VLAN_STAG_TX __NETIF_F(HW_VLAN_STAG_TX)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d707afd..6dc3221 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h
@@ -4109,6 +4109,7 @@ static inline bool net_gso_ok(netdev_features_t features, int gso_type) BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT)); BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT)); BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT)); + BUILD_BUG_ON(SKB_GSO_UDP_BYFRAGS != (NETIF_F_GSO_UDP_BYFRAGS >> NETIF_F_GSO_SHIFT)); return (features & feature) == feature; }
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a17e235..c39ff4b 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h
@@ -496,6 +496,8 @@ enum { SKB_GSO_SCTP = 1 << 15, SKB_GSO_ESP = 1 << 16, + + SKB_GSO_UDP_BYFRAGS = 1 << 17, }; #if BITS_PER_LONG > 32
diff --git a/net/core/skbuff.c b/net/core/skbuff.c index be26034..93c171d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c
@@ -3217,6 +3217,9 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, if (unlikely(!proto)) return ERR_PTR(-EINVAL); + if (skb_shinfo(head_skb)->gso_type & SKB_GSO_UDP_BYFRAGS) + return skb_segment_fraglist(head_skb, doffset); + sg = !!(features & NETIF_F_SG); csum = !!can_checksum_protocol(features, proto);
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 7812501..fe704fb 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c
@@ -248,10 +248,73 @@ static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, * inet_gso_segment() */ segs = skb_segment(skb, features); + out: return segs; } +static struct sk_buff *udp4_gso_segment(struct sk_buff *skb, + netdev_features_t features) +{ + struct sk_buff *segs = ERR_PTR(-EINVAL); + + if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_BYFRAGS)) + return udp4_ufo_fragment(skb, features); + + if (!pskb_may_pull(skb, sizeof(struct udphdr))) + goto out; + + segs = skb_segment(skb, features); + +out: + return segs; +} + + +static struct sk_buff **udp_gro_ffwd_receive(struct sk_buff **head, struct sk_buff *skb, + struct udphdr *uh) +{ + struct sk_buff *p = NULL; + struct sk_buff **pp = NULL; + struct udphdr *uh2; + unsigned int off = skb_gro_offset(skb); + int flush = 0; + + for (; (p = *head); head = &p->next) { + + if (!NAPI_GRO_CB(p)->same_flow) + continue; + + uh2 = (struct udphdr *)(p->data + off); + + /* Match ports and either checksums are either both zero + * or nonzero. + */ + if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || + (!uh->check ^ !uh2->check)) { + NAPI_GRO_CB(p)->same_flow = 0; + continue; + } + goto found; + } + + goto out; + +found: + p = *head; + + if (skb_gro_receive(head, skb)) { + flush = 1; + } + +out: + if (p && (!NAPI_GRO_CB(skb)->same_flow || flush)) + pp = head; + + NAPI_GRO_CB(skb)->flush |= flush; + return pp; +} + struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, struct udphdr *uh, udp_lookup_t lookup) { @@ -320,6 +383,9 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head, if (NAPI_GRO_CB(skb)->flush) goto skip; + if (NAPI_GRO_CB(skb)->is_ffwd) + return udp_gro_ffwd_receive(head, skb, uh); + if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, inet_gro_compute_pseudo)) goto flush; @@ -335,6 +401,13 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head, return NULL; } +static int udp_gro_ffwd_complete(struct sk_buff *skb, int nhoff) +{ + skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_BYFRAGS; + + return 0; +} + int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup) { @@ -369,6 +442,9 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff) const struct iphdr *iph = ip_hdr(skb); struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); + if (NAPI_GRO_CB(skb)->is_ffwd) + return udp_gro_ffwd_complete(skb, nhoff); + if (uh->check) { skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, @@ -382,7 +458,7 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff) static const struct net_offload udpv4_offload = { .callbacks = { - .gso_segment = udp4_ufo_fragment, + .gso_segment = udp4_gso_segment, .gro_receive = udp4_gro_receive, .gro_complete = udp4_gro_complete, },