udp: Add GRO/GSO support
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index 1d4737c..5eb77de 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -55,8 +55,9 @@ enum {
NETIF_F_GSO_TUNNEL_REMCSUM_BIT, /* ... TUNNEL with TSO & REMCSUM */
NETIF_F_GSO_SCTP_BIT, /* ... SCTP fragmentation */
NETIF_F_GSO_ESP_BIT, /* ... ESP with TSO */
+ NETIF_F_GSO_UDP_BYFRAGS_BIT, /* ... UDP generic */
/**/NETIF_F_GSO_LAST = /* last bit, see GSO_MASK */
- NETIF_F_GSO_ESP_BIT,
+ NETIF_F_GSO_UDP_BYFRAGS_BIT,
NETIF_F_FCOE_CRC_BIT, /* FCoE CRC32 */
NETIF_F_SCTP_CRC_BIT, /* SCTP checksum offload */
@@ -133,6 +134,7 @@ enum {
#define NETIF_F_GSO_TUNNEL_REMCSUM __NETIF_F(GSO_TUNNEL_REMCSUM)
#define NETIF_F_GSO_SCTP __NETIF_F(GSO_SCTP)
#define NETIF_F_GSO_ESP __NETIF_F(GSO_ESP)
+#define NETIF_F_GSO_UDP_BYFRAGS __NETIF_F(GSO_UDP_BYFRAGS)
#define NETIF_F_HW_VLAN_STAG_FILTER __NETIF_F(HW_VLAN_STAG_FILTER)
#define NETIF_F_HW_VLAN_STAG_RX __NETIF_F(HW_VLAN_STAG_RX)
#define NETIF_F_HW_VLAN_STAG_TX __NETIF_F(HW_VLAN_STAG_TX)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index d707afd..6dc3221 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -4109,6 +4109,7 @@ static inline bool net_gso_ok(netdev_features_t features, int gso_type)
BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
+ BUILD_BUG_ON(SKB_GSO_UDP_BYFRAGS != (NETIF_F_GSO_UDP_BYFRAGS >> NETIF_F_GSO_SHIFT));
return (features & feature) == feature;
}
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index a17e235..c39ff4b 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -496,6 +496,8 @@ enum {
SKB_GSO_SCTP = 1 << 15,
SKB_GSO_ESP = 1 << 16,
+
+ SKB_GSO_UDP_BYFRAGS = 1 << 17,
};
#if BITS_PER_LONG > 32
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index be26034..93c171d 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3217,6 +3217,9 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb,
if (unlikely(!proto))
return ERR_PTR(-EINVAL);
+ if (skb_shinfo(head_skb)->gso_type & SKB_GSO_UDP_BYFRAGS)
+ return skb_segment_fraglist(head_skb, doffset);
+
sg = !!(features & NETIF_F_SG);
csum = !!can_checksum_protocol(features, proto);
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index 7812501..fe704fb 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -248,10 +248,73 @@ static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
* inet_gso_segment()
*/
segs = skb_segment(skb, features);
+
out:
return segs;
}
+static struct sk_buff *udp4_gso_segment(struct sk_buff *skb,
+ netdev_features_t features)
+{
+ struct sk_buff *segs = ERR_PTR(-EINVAL);
+
+ if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_BYFRAGS))
+ return udp4_ufo_fragment(skb, features);
+
+ if (!pskb_may_pull(skb, sizeof(struct udphdr)))
+ goto out;
+
+ segs = skb_segment(skb, features);
+
+out:
+ return segs;
+}
+
+
+static struct sk_buff **udp_gro_ffwd_receive(struct sk_buff **head, struct sk_buff *skb,
+ struct udphdr *uh)
+{
+ struct sk_buff *p = NULL;
+ struct sk_buff **pp = NULL;
+ struct udphdr *uh2;
+ unsigned int off = skb_gro_offset(skb);
+ int flush = 0;
+
+ for (; (p = *head); head = &p->next) {
+
+ if (!NAPI_GRO_CB(p)->same_flow)
+ continue;
+
+ uh2 = (struct udphdr *)(p->data + off);
+
+ /* Match ports and either checksums are either both zero
+ * or nonzero.
+ */
+ if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
+ (!uh->check ^ !uh2->check)) {
+ NAPI_GRO_CB(p)->same_flow = 0;
+ continue;
+ }
+ goto found;
+ }
+
+ goto out;
+
+found:
+ p = *head;
+
+ if (skb_gro_receive(head, skb)) {
+ flush = 1;
+ }
+
+out:
+ if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))
+ pp = head;
+
+ NAPI_GRO_CB(skb)->flush |= flush;
+ return pp;
+}
+
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
struct udphdr *uh, udp_lookup_t lookup)
{
@@ -320,6 +383,9 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
if (NAPI_GRO_CB(skb)->flush)
goto skip;
+ if (NAPI_GRO_CB(skb)->is_ffwd)
+ return udp_gro_ffwd_receive(head, skb, uh);
+
if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
inet_gro_compute_pseudo))
goto flush;
@@ -335,6 +401,13 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
return NULL;
}
+static int udp_gro_ffwd_complete(struct sk_buff *skb, int nhoff)
+{
+ skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_BYFRAGS;
+
+ return 0;
+}
+
int udp_gro_complete(struct sk_buff *skb, int nhoff,
udp_lookup_t lookup)
{
@@ -369,6 +442,9 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
const struct iphdr *iph = ip_hdr(skb);
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
+ if (NAPI_GRO_CB(skb)->is_ffwd)
+ return udp_gro_ffwd_complete(skb, nhoff);
+
if (uh->check) {
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
@@ -382,7 +458,7 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
static const struct net_offload udpv4_offload = {
.callbacks = {
- .gso_segment = udp4_ufo_fragment,
+ .gso_segment = udp4_gso_segment,
.gro_receive = udp4_gro_receive,
.gro_complete = udp4_gro_complete,
},
