| From 8b5553ace83cced775eefd0f3f18b5c6214ccf7a Mon Sep 17 00:00:00 2001 |
| From: "Jason A. Donenfeld" <Jason@zx2c4.com> |
| Date: Mon, 22 Feb 2021 17:25:48 +0100 |
| Subject: wireguard: queueing: get rid of per-peer ring buffers |
| MIME-Version: 1.0 |
| Content-Type: text/plain; charset=UTF-8 |
| Content-Transfer-Encoding: 8bit |
| |
| From: Jason A. Donenfeld <Jason@zx2c4.com> |
| |
| commit 8b5553ace83cced775eefd0f3f18b5c6214ccf7a upstream. |
| |
| Having two ring buffers per-peer means that every peer results in two |
| massive ring allocations. On an 8-core x86_64 machine, this commit |
| reduces the per-peer allocation from 18,688 bytes to 1,856 bytes, which |
| is an 90% reduction. Ninety percent! With some single-machine |
| deployments approaching 500,000 peers, we're talking about a reduction |
| from 7 gigs of memory down to 700 megs of memory. |
| |
| In order to get rid of these per-peer allocations, this commit switches |
| to using a list-based queueing approach. Currently GSO fragments are |
| chained together using the skb->next pointer (the skb_list_* singly |
| linked list approach), so we form the per-peer queue around the unused |
| skb->prev pointer (which sort of makes sense because the links are |
| pointing backwards). Use of skb_queue_* is not possible here, because |
| that is based on doubly linked lists and spinlocks. Multiple cores can |
| write into the queue at any given time, because its writes occur in the |
| start_xmit path or in the udp_recv path. But reads happen in a single |
| workqueue item per-peer, amounting to a multi-producer, single-consumer |
| paradigm. |
| |
| The MPSC queue is implemented locklessly and never blocks. However, it |
| is not linearizable (though it is serializable), with a very tight and |
| unlikely race on writes, which, when hit (some tiny fraction of the |
| 0.15% of partial adds on a fully loaded 16-core x86_64 system), causes |
| the queue reader to terminate early. However, because every packet sent |
| queues up the same workqueue item after it is fully added, the worker |
| resumes again, and stopping early isn't actually a problem, since at |
| that point the packet wouldn't have yet been added to the encryption |
| queue. These properties allow us to avoid disabling interrupts or |
| spinning. The design is based on Dmitry Vyukov's algorithm [1]. |
| |
| Performance-wise, ordinarily list-based queues aren't preferable to |
| ringbuffers, because of cache misses when following pointers around. |
| However, we *already* have to follow the adjacent pointers when working |
| through fragments, so there shouldn't actually be any change there. A |
| potential downside is that dequeueing is a bit more complicated, but the |
| ptr_ring structure used prior had a spinlock when dequeueing, so all and |
| all the difference appears to be a wash. |
| |
| Actually, from profiling, the biggest performance hit, by far, of this |
| commit winds up being atomic_add_unless(count, 1, max) and atomic_ |
| dec(count), which account for the majority of CPU time, according to |
| perf. In that sense, the previous ring buffer was superior in that it |
| could check if it was full by head==tail, which the list-based approach |
| cannot do. |
| |
| But all and all, this enables us to get massive memory savings, allowing |
| WireGuard to scale for real world deployments, without taking much of a |
| performance hit. |
| |
| [1] http://www.1024cores.net/home/lock-free-algorithms/queues/intrusive-mpsc-node-based-queue |
| |
| Reviewed-by: Dmitry Vyukov <dvyukov@google.com> |
| Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com> |
| Fixes: e7096c131e51 ("net: WireGuard secure network tunnel") |
| Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com> |
| Signed-off-by: Jakub Kicinski <kuba@kernel.org> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| drivers/net/wireguard/device.c | 12 ++--- |
| drivers/net/wireguard/device.h | 15 +++--- |
| drivers/net/wireguard/peer.c | 28 ++++-------- |
| drivers/net/wireguard/peer.h | 4 - |
| drivers/net/wireguard/queueing.c | 86 +++++++++++++++++++++++++++++++-------- |
| drivers/net/wireguard/queueing.h | 45 ++++++++++++++------ |
| drivers/net/wireguard/receive.c | 16 ++----- |
| drivers/net/wireguard/send.c | 31 ++++---------- |
| 8 files changed, 144 insertions(+), 93 deletions(-) |
| |
| --- a/drivers/net/wireguard/device.c |
| +++ b/drivers/net/wireguard/device.c |
| @@ -235,8 +235,8 @@ static void wg_destruct(struct net_devic |
| destroy_workqueue(wg->handshake_receive_wq); |
| destroy_workqueue(wg->handshake_send_wq); |
| destroy_workqueue(wg->packet_crypt_wq); |
| - wg_packet_queue_free(&wg->decrypt_queue, true); |
| - wg_packet_queue_free(&wg->encrypt_queue, true); |
| + wg_packet_queue_free(&wg->decrypt_queue); |
| + wg_packet_queue_free(&wg->encrypt_queue); |
| rcu_barrier(); /* Wait for all the peers to be actually freed. */ |
| wg_ratelimiter_uninit(); |
| memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); |
| @@ -338,12 +338,12 @@ static int wg_newlink(struct net *src_ne |
| goto err_destroy_handshake_send; |
| |
| ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, |
| - true, MAX_QUEUED_PACKETS); |
| + MAX_QUEUED_PACKETS); |
| if (ret < 0) |
| goto err_destroy_packet_crypt; |
| |
| ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, |
| - true, MAX_QUEUED_PACKETS); |
| + MAX_QUEUED_PACKETS); |
| if (ret < 0) |
| goto err_free_encrypt_queue; |
| |
| @@ -368,9 +368,9 @@ static int wg_newlink(struct net *src_ne |
| err_uninit_ratelimiter: |
| wg_ratelimiter_uninit(); |
| err_free_decrypt_queue: |
| - wg_packet_queue_free(&wg->decrypt_queue, true); |
| + wg_packet_queue_free(&wg->decrypt_queue); |
| err_free_encrypt_queue: |
| - wg_packet_queue_free(&wg->encrypt_queue, true); |
| + wg_packet_queue_free(&wg->encrypt_queue); |
| err_destroy_packet_crypt: |
| destroy_workqueue(wg->packet_crypt_wq); |
| err_destroy_handshake_send: |
| --- a/drivers/net/wireguard/device.h |
| +++ b/drivers/net/wireguard/device.h |
| @@ -27,13 +27,14 @@ struct multicore_worker { |
| |
| struct crypt_queue { |
| struct ptr_ring ring; |
| - union { |
| - struct { |
| - struct multicore_worker __percpu *worker; |
| - int last_cpu; |
| - }; |
| - struct work_struct work; |
| - }; |
| + struct multicore_worker __percpu *worker; |
| + int last_cpu; |
| +}; |
| + |
| +struct prev_queue { |
| + struct sk_buff *head, *tail, *peeked; |
| + struct { struct sk_buff *next, *prev; } empty; // Match first 2 members of struct sk_buff. |
| + atomic_t count; |
| }; |
| |
| struct wg_device { |
| --- a/drivers/net/wireguard/peer.c |
| +++ b/drivers/net/wireguard/peer.c |
| @@ -32,27 +32,22 @@ struct wg_peer *wg_peer_create(struct wg |
| peer = kzalloc(sizeof(*peer), GFP_KERNEL); |
| if (unlikely(!peer)) |
| return ERR_PTR(ret); |
| - peer->device = wg; |
| + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) |
| + goto err; |
| |
| + peer->device = wg; |
| wg_noise_handshake_init(&peer->handshake, &wg->static_identity, |
| public_key, preshared_key, peer); |
| - if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) |
| - goto err_1; |
| - if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, |
| - MAX_QUEUED_PACKETS)) |
| - goto err_2; |
| - if (wg_packet_queue_init(&peer->rx_queue, NULL, false, |
| - MAX_QUEUED_PACKETS)) |
| - goto err_3; |
| - |
| peer->internal_id = atomic64_inc_return(&peer_counter); |
| peer->serial_work_cpu = nr_cpumask_bits; |
| wg_cookie_init(&peer->latest_cookie); |
| wg_timers_init(peer); |
| wg_cookie_checker_precompute_peer_keys(peer); |
| spin_lock_init(&peer->keypairs.keypair_update_lock); |
| - INIT_WORK(&peer->transmit_handshake_work, |
| - wg_packet_handshake_send_worker); |
| + INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker); |
| + INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker); |
| + wg_prev_queue_init(&peer->tx_queue); |
| + wg_prev_queue_init(&peer->rx_queue); |
| rwlock_init(&peer->endpoint_lock); |
| kref_init(&peer->refcount); |
| skb_queue_head_init(&peer->staged_packet_queue); |
| @@ -68,11 +63,7 @@ struct wg_peer *wg_peer_create(struct wg |
| pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); |
| return peer; |
| |
| -err_3: |
| - wg_packet_queue_free(&peer->tx_queue, false); |
| -err_2: |
| - dst_cache_destroy(&peer->endpoint_cache); |
| -err_1: |
| +err: |
| kfree(peer); |
| return ERR_PTR(ret); |
| } |
| @@ -197,8 +188,7 @@ static void rcu_release(struct rcu_head |
| struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); |
| |
| dst_cache_destroy(&peer->endpoint_cache); |
| - wg_packet_queue_free(&peer->rx_queue, false); |
| - wg_packet_queue_free(&peer->tx_queue, false); |
| + WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue)); |
| |
| /* The final zeroing takes care of clearing any remaining handshake key |
| * material and other potentially sensitive information. |
| --- a/drivers/net/wireguard/peer.h |
| +++ b/drivers/net/wireguard/peer.h |
| @@ -36,7 +36,7 @@ struct endpoint { |
| |
| struct wg_peer { |
| struct wg_device *device; |
| - struct crypt_queue tx_queue, rx_queue; |
| + struct prev_queue tx_queue, rx_queue; |
| struct sk_buff_head staged_packet_queue; |
| int serial_work_cpu; |
| struct noise_keypairs keypairs; |
| @@ -45,7 +45,7 @@ struct wg_peer { |
| rwlock_t endpoint_lock; |
| struct noise_handshake handshake; |
| atomic64_t last_sent_handshake; |
| - struct work_struct transmit_handshake_work, clear_peer_work; |
| + struct work_struct transmit_handshake_work, clear_peer_work, transmit_packet_work; |
| struct cookie latest_cookie; |
| struct hlist_node pubkey_hash; |
| u64 rx_bytes, tx_bytes; |
| --- a/drivers/net/wireguard/queueing.c |
| +++ b/drivers/net/wireguard/queueing.c |
| @@ -9,8 +9,7 @@ struct multicore_worker __percpu * |
| wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) |
| { |
| int cpu; |
| - struct multicore_worker __percpu *worker = |
| - alloc_percpu(struct multicore_worker); |
| + struct multicore_worker __percpu *worker = alloc_percpu(struct multicore_worker); |
| |
| if (!worker) |
| return NULL; |
| @@ -23,7 +22,7 @@ wg_packet_percpu_multicore_worker_alloc( |
| } |
| |
| int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, |
| - bool multicore, unsigned int len) |
| + unsigned int len) |
| { |
| int ret; |
| |
| @@ -31,25 +30,78 @@ int wg_packet_queue_init(struct crypt_qu |
| ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); |
| if (ret) |
| return ret; |
| - if (function) { |
| - if (multicore) { |
| - queue->worker = wg_packet_percpu_multicore_worker_alloc( |
| - function, queue); |
| - if (!queue->worker) { |
| - ptr_ring_cleanup(&queue->ring, NULL); |
| - return -ENOMEM; |
| - } |
| - } else { |
| - INIT_WORK(&queue->work, function); |
| - } |
| + queue->worker = wg_packet_percpu_multicore_worker_alloc(function, queue); |
| + if (!queue->worker) { |
| + ptr_ring_cleanup(&queue->ring, NULL); |
| + return -ENOMEM; |
| } |
| return 0; |
| } |
| |
| -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) |
| +void wg_packet_queue_free(struct crypt_queue *queue) |
| { |
| - if (multicore) |
| - free_percpu(queue->worker); |
| + free_percpu(queue->worker); |
| WARN_ON(!__ptr_ring_empty(&queue->ring)); |
| ptr_ring_cleanup(&queue->ring, NULL); |
| } |
| + |
| +#define NEXT(skb) ((skb)->prev) |
| +#define STUB(queue) ((struct sk_buff *)&queue->empty) |
| + |
| +void wg_prev_queue_init(struct prev_queue *queue) |
| +{ |
| + NEXT(STUB(queue)) = NULL; |
| + queue->head = queue->tail = STUB(queue); |
| + queue->peeked = NULL; |
| + atomic_set(&queue->count, 0); |
| + BUILD_BUG_ON( |
| + offsetof(struct sk_buff, next) != offsetof(struct prev_queue, empty.next) - |
| + offsetof(struct prev_queue, empty) || |
| + offsetof(struct sk_buff, prev) != offsetof(struct prev_queue, empty.prev) - |
| + offsetof(struct prev_queue, empty)); |
| +} |
| + |
| +static void __wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb) |
| +{ |
| + WRITE_ONCE(NEXT(skb), NULL); |
| + WRITE_ONCE(NEXT(xchg_release(&queue->head, skb)), skb); |
| +} |
| + |
| +bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb) |
| +{ |
| + if (!atomic_add_unless(&queue->count, 1, MAX_QUEUED_PACKETS)) |
| + return false; |
| + __wg_prev_queue_enqueue(queue, skb); |
| + return true; |
| +} |
| + |
| +struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue) |
| +{ |
| + struct sk_buff *tail = queue->tail, *next = smp_load_acquire(&NEXT(tail)); |
| + |
| + if (tail == STUB(queue)) { |
| + if (!next) |
| + return NULL; |
| + queue->tail = next; |
| + tail = next; |
| + next = smp_load_acquire(&NEXT(next)); |
| + } |
| + if (next) { |
| + queue->tail = next; |
| + atomic_dec(&queue->count); |
| + return tail; |
| + } |
| + if (tail != READ_ONCE(queue->head)) |
| + return NULL; |
| + __wg_prev_queue_enqueue(queue, STUB(queue)); |
| + next = smp_load_acquire(&NEXT(tail)); |
| + if (next) { |
| + queue->tail = next; |
| + atomic_dec(&queue->count); |
| + return tail; |
| + } |
| + return NULL; |
| +} |
| + |
| +#undef NEXT |
| +#undef STUB |
| --- a/drivers/net/wireguard/queueing.h |
| +++ b/drivers/net/wireguard/queueing.h |
| @@ -17,12 +17,13 @@ struct wg_device; |
| struct wg_peer; |
| struct multicore_worker; |
| struct crypt_queue; |
| +struct prev_queue; |
| struct sk_buff; |
| |
| /* queueing.c APIs: */ |
| int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, |
| - bool multicore, unsigned int len); |
| -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); |
| + unsigned int len); |
| +void wg_packet_queue_free(struct crypt_queue *queue); |
| struct multicore_worker __percpu * |
| wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); |
| |
| @@ -135,8 +136,31 @@ static inline int wg_cpumask_next_online |
| return cpu; |
| } |
| |
| +void wg_prev_queue_init(struct prev_queue *queue); |
| + |
| +/* Multi producer */ |
| +bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb); |
| + |
| +/* Single consumer */ |
| +struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue); |
| + |
| +/* Single consumer */ |
| +static inline struct sk_buff *wg_prev_queue_peek(struct prev_queue *queue) |
| +{ |
| + if (queue->peeked) |
| + return queue->peeked; |
| + queue->peeked = wg_prev_queue_dequeue(queue); |
| + return queue->peeked; |
| +} |
| + |
| +/* Single consumer */ |
| +static inline void wg_prev_queue_drop_peeked(struct prev_queue *queue) |
| +{ |
| + queue->peeked = NULL; |
| +} |
| + |
| static inline int wg_queue_enqueue_per_device_and_peer( |
| - struct crypt_queue *device_queue, struct crypt_queue *peer_queue, |
| + struct crypt_queue *device_queue, struct prev_queue *peer_queue, |
| struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) |
| { |
| int cpu; |
| @@ -145,8 +169,9 @@ static inline int wg_queue_enqueue_per_d |
| /* We first queue this up for the peer ingestion, but the consumer |
| * will wait for the state to change to CRYPTED or DEAD before. |
| */ |
| - if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) |
| + if (unlikely(!wg_prev_queue_enqueue(peer_queue, skb))) |
| return -ENOSPC; |
| + |
| /* Then we queue it up in the device queue, which consumes the |
| * packet as soon as it can. |
| */ |
| @@ -157,9 +182,7 @@ static inline int wg_queue_enqueue_per_d |
| return 0; |
| } |
| |
| -static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, |
| - struct sk_buff *skb, |
| - enum packet_state state) |
| +static inline void wg_queue_enqueue_per_peer_tx(struct sk_buff *skb, enum packet_state state) |
| { |
| /* We take a reference, because as soon as we call atomic_set, the |
| * peer can be freed from below us. |
| @@ -167,14 +190,12 @@ static inline void wg_queue_enqueue_per_ |
| struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); |
| |
| atomic_set_release(&PACKET_CB(skb)->state, state); |
| - queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, |
| - peer->internal_id), |
| - peer->device->packet_crypt_wq, &queue->work); |
| + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, peer->internal_id), |
| + peer->device->packet_crypt_wq, &peer->transmit_packet_work); |
| wg_peer_put(peer); |
| } |
| |
| -static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, |
| - enum packet_state state) |
| +static inline void wg_queue_enqueue_per_peer_rx(struct sk_buff *skb, enum packet_state state) |
| { |
| /* We take a reference, because as soon as we call atomic_set, the |
| * peer can be freed from below us. |
| --- a/drivers/net/wireguard/receive.c |
| +++ b/drivers/net/wireguard/receive.c |
| @@ -444,7 +444,6 @@ packet_processed: |
| int wg_packet_rx_poll(struct napi_struct *napi, int budget) |
| { |
| struct wg_peer *peer = container_of(napi, struct wg_peer, napi); |
| - struct crypt_queue *queue = &peer->rx_queue; |
| struct noise_keypair *keypair; |
| struct endpoint endpoint; |
| enum packet_state state; |
| @@ -455,11 +454,10 @@ int wg_packet_rx_poll(struct napi_struct |
| if (unlikely(budget <= 0)) |
| return 0; |
| |
| - while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && |
| + while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL && |
| (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != |
| PACKET_STATE_UNCRYPTED) { |
| - __ptr_ring_discard_one(&queue->ring); |
| - peer = PACKET_PEER(skb); |
| + wg_prev_queue_drop_peeked(&peer->rx_queue); |
| keypair = PACKET_CB(skb)->keypair; |
| free = true; |
| |
| @@ -508,7 +506,7 @@ void wg_packet_decrypt_worker(struct wor |
| enum packet_state state = |
| likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ? |
| PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; |
| - wg_queue_enqueue_per_peer_napi(skb, state); |
| + wg_queue_enqueue_per_peer_rx(skb, state); |
| if (need_resched()) |
| cond_resched(); |
| } |
| @@ -531,12 +529,10 @@ static void wg_packet_consume_data(struc |
| if (unlikely(READ_ONCE(peer->is_dead))) |
| goto err; |
| |
| - ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, |
| - &peer->rx_queue, skb, |
| - wg->packet_crypt_wq, |
| - &wg->decrypt_queue.last_cpu); |
| + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb, |
| + wg->packet_crypt_wq, &wg->decrypt_queue.last_cpu); |
| if (unlikely(ret == -EPIPE)) |
| - wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); |
| + wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD); |
| if (likely(!ret || ret == -EPIPE)) { |
| rcu_read_unlock_bh(); |
| return; |
| --- a/drivers/net/wireguard/send.c |
| +++ b/drivers/net/wireguard/send.c |
| @@ -239,8 +239,7 @@ void wg_packet_send_keepalive(struct wg_ |
| wg_packet_send_staged_packets(peer); |
| } |
| |
| -static void wg_packet_create_data_done(struct sk_buff *first, |
| - struct wg_peer *peer) |
| +static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first) |
| { |
| struct sk_buff *skb, *next; |
| bool is_keepalive, data_sent = false; |
| @@ -262,22 +261,19 @@ static void wg_packet_create_data_done(s |
| |
| void wg_packet_tx_worker(struct work_struct *work) |
| { |
| - struct crypt_queue *queue = container_of(work, struct crypt_queue, |
| - work); |
| + struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work); |
| struct noise_keypair *keypair; |
| enum packet_state state; |
| struct sk_buff *first; |
| - struct wg_peer *peer; |
| |
| - while ((first = __ptr_ring_peek(&queue->ring)) != NULL && |
| + while ((first = wg_prev_queue_peek(&peer->tx_queue)) != NULL && |
| (state = atomic_read_acquire(&PACKET_CB(first)->state)) != |
| PACKET_STATE_UNCRYPTED) { |
| - __ptr_ring_discard_one(&queue->ring); |
| - peer = PACKET_PEER(first); |
| + wg_prev_queue_drop_peeked(&peer->tx_queue); |
| keypair = PACKET_CB(first)->keypair; |
| |
| if (likely(state == PACKET_STATE_CRYPTED)) |
| - wg_packet_create_data_done(first, peer); |
| + wg_packet_create_data_done(peer, first); |
| else |
| kfree_skb_list(first); |
| |
| @@ -306,16 +302,14 @@ void wg_packet_encrypt_worker(struct wor |
| break; |
| } |
| } |
| - wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, |
| - state); |
| + wg_queue_enqueue_per_peer_tx(first, state); |
| if (need_resched()) |
| cond_resched(); |
| } |
| } |
| |
| -static void wg_packet_create_data(struct sk_buff *first) |
| +static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first) |
| { |
| - struct wg_peer *peer = PACKET_PEER(first); |
| struct wg_device *wg = peer->device; |
| int ret = -EINVAL; |
| |
| @@ -323,13 +317,10 @@ static void wg_packet_create_data(struct |
| if (unlikely(READ_ONCE(peer->is_dead))) |
| goto err; |
| |
| - ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, |
| - &peer->tx_queue, first, |
| - wg->packet_crypt_wq, |
| - &wg->encrypt_queue.last_cpu); |
| + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first, |
| + wg->packet_crypt_wq, &wg->encrypt_queue.last_cpu); |
| if (unlikely(ret == -EPIPE)) |
| - wg_queue_enqueue_per_peer(&peer->tx_queue, first, |
| - PACKET_STATE_DEAD); |
| + wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD); |
| err: |
| rcu_read_unlock_bh(); |
| if (likely(!ret || ret == -EPIPE)) |
| @@ -393,7 +384,7 @@ void wg_packet_send_staged_packets(struc |
| packets.prev->next = NULL; |
| wg_peer_get(keypair->entry.peer); |
| PACKET_CB(packets.next)->keypair = keypair; |
| - wg_packet_create_data(packets.next); |
| + wg_packet_create_data(peer, packets.next); |
| return; |
| |
| out_invalid: |