xfrm: Add sub SAs
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index a52e12c..3ac0769 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -62,26 +62,27 @@ static void mlx5e_ipsec_handle_sw_limits(struct work_struct *_work) container_of(_work, struct mlx5e_ipsec_dwork, dwork.work); struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry; struct xfrm_state *x = sa_entry->x; + struct xfrm_sub_state *sx = x->sx; if (sa_entry->attrs.drop) return; - spin_lock_bh(&x->lock); + spin_lock_bh(&sx->lock); if (x->km.state == XFRM_STATE_EXPIRED) { sa_entry->attrs.drop = true; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&sx->lock); mlx5e_accel_ipsec_fs_modify(sa_entry); return; } if (x->km.state != XFRM_STATE_VALID) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&sx->lock); return; } xfrm_state_check_expire(x); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&sx->lock); queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, MLX5_IPSEC_RESCHED); @@ -90,18 +91,19 @@ static void mlx5e_ipsec_handle_sw_limits(struct work_struct *_work) static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_state *x = sa_entry->x; + struct xfrm_sub_state *sx = x->sx; u32 seq_bottom = 0; u32 esn, esn_msb; u8 overlap; switch (x->xso.dir) { case XFRM_DEV_OFFLOAD_IN: - esn = x->replay_esn->seq; - esn_msb = x->replay_esn->seq_hi; + esn = sx->replay_esn->seq; + esn_msb = sx->replay_esn->seq_hi; break; case XFRM_DEV_OFFLOAD_OUT: - esn = x->replay_esn->oseq; - esn_msb = x->replay_esn->oseq_hi; + esn = sx->replay_esn->oseq; + esn_msb = sx->replay_esn->oseq_hi; break; default: WARN_ON(true); @@ -110,11 +112,11 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) overlap = sa_entry->esn_state.overlap; - if (!x->replay_esn->replay_window) { + if (!sx->replay_esn->replay_window) { seq_bottom = esn; } else { - if (esn >= x->replay_esn->replay_window) - seq_bottom = esn - x->replay_esn->replay_window + 1; + if (esn >= sx->replay_esn->replay_window) + seq_bottom = esn - sx->replay_esn->replay_window + 1; if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO) esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom)); @@ -146,11 +148,12 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; + struct xfrm_sub_state *sx = x->sx; s64 start_value, n; - attrs->lft.hard_packet_limit = x->lft.hard_packet_limit; - attrs->lft.soft_packet_limit = x->lft.soft_packet_limit; - if (x->lft.soft_packet_limit == XFRM_INF) + attrs->lft.hard_packet_limit = sx->lft.hard_packet_limit; + attrs->lft.soft_packet_limit = sx->lft.soft_packet_limit; + if (sx->lft.soft_packet_limit == XFRM_INF) return; /* Compute hard limit initial value and number of rounds. @@ -221,9 +224,9 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, */ /* Start by estimating n and compute soft_value */ - n = (x->lft.soft_packet_limit - attrs->lft.hard_packet_limit) / BIT_ULL(31); + n = (sx->lft.soft_packet_limit - attrs->lft.hard_packet_limit) / BIT_ULL(31); start_value = attrs->lft.hard_packet_limit + n * BIT_ULL(31) - - x->lft.soft_packet_limit; + sx->lft.soft_packet_limit; /* Compare against constraints and adjust n */ if (n < 0) @@ -392,6 +395,7 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; + struct xfrm_sub_state *sx = x->sx; struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; struct aead_geniv_ctx *geniv_ctx; struct crypto_aead *aead; @@ -432,7 +436,7 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, x->xso.type != XFRM_DEV_OFFLOAD_PACKET) goto skip_replay_window; - switch (x->replay_esn->replay_window) { + switch (sx->replay_esn->replay_window) { case 32: attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT; @@ -488,6 +492,8 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, struct xfrm_state *x, struct netlink_ext_ack *extack) { + struct xfrm_sub_state *sx = x->sx; + if (x->props.aalgo != SADB_AALG_NONE) { NL_SET_ERR_MSG_MOD(extack, "Cannot offload authenticated xfrm states"); return -EINVAL; @@ -594,11 +600,11 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, return -EINVAL; } - if (x->replay_esn && x->xso.dir == XFRM_DEV_OFFLOAD_IN && - x->replay_esn->replay_window != 32 && - x->replay_esn->replay_window != 64 && - x->replay_esn->replay_window != 128 && - x->replay_esn->replay_window != 256) { + if (sx->replay_esn && x->xso.dir == XFRM_DEV_OFFLOAD_IN && + sx->replay_esn->replay_window != 32 && + sx->replay_esn->replay_window != 64 && + sx->replay_esn->replay_window != 128 && + sx->replay_esn->replay_window != 256) { NL_SET_ERR_MSG_MOD(extack, "Unsupported replay window size"); return -EINVAL; } @@ -608,26 +614,26 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, return -EINVAL; } - if (x->lft.soft_byte_limit >= x->lft.hard_byte_limit && - x->lft.hard_byte_limit != XFRM_INF) { + if (sx->lft.soft_byte_limit >= sx->lft.hard_byte_limit && + sx->lft.hard_byte_limit != XFRM_INF) { /* XFRM stack doesn't prevent such configuration :(. */ NL_SET_ERR_MSG_MOD(extack, "Hard byte limit must be greater than soft one"); return -EINVAL; } - if (!x->lft.soft_byte_limit || !x->lft.hard_byte_limit) { + if (!sx->lft.soft_byte_limit || !sx->lft.hard_byte_limit) { NL_SET_ERR_MSG_MOD(extack, "Soft/hard byte limits can't be 0"); return -EINVAL; } - if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit && - x->lft.hard_packet_limit != XFRM_INF) { + if (sx->lft.soft_packet_limit >= sx->lft.hard_packet_limit && + sx->lft.hard_packet_limit != XFRM_INF) { /* XFRM stack doesn't prevent such configuration :(. */ NL_SET_ERR_MSG_MOD(extack, "Hard packet limit must be greater than soft one"); return -EINVAL; } - if (!x->lft.soft_packet_limit || !x->lft.hard_packet_limit) { + if (!sx->lft.soft_packet_limit || !sx->lft.hard_packet_limit) { NL_SET_ERR_MSG_MOD(extack, "Soft/hard packet limits can't be 0"); return -EINVAL; } @@ -746,15 +752,16 @@ static int mlx5_ipsec_create_work(struct mlx5e_ipsec_sa_entry *sa_entry) static int mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_state *x = sa_entry->x; + struct xfrm_sub_state *sx = x->sx; struct mlx5e_ipsec_dwork *dwork; if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) return 0; - if (x->lft.soft_packet_limit == XFRM_INF && - x->lft.hard_packet_limit == XFRM_INF && - x->lft.soft_byte_limit == XFRM_INF && - x->lft.hard_byte_limit == XFRM_INF) + if (sx->lft.soft_packet_limit == XFRM_INF && + sx->lft.hard_packet_limit == XFRM_INF && + sx->lft.soft_byte_limit == XFRM_INF && + sx->lft.hard_byte_limit == XFRM_INF) return 0; dwork = kzalloc_obj(*dwork); @@ -1074,6 +1081,7 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) static void mlx5e_xfrm_update_stats(struct xfrm_state *x) { + struct xfrm_sub_state *sx = x->sx; struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct net *net = dev_net(x->xso.dev); @@ -1084,7 +1092,7 @@ static void mlx5e_xfrm_update_stats(struct xfrm_state *x) u64 packets, bytes, lastuse; size_t headers; - lockdep_assert(lockdep_is_held(&x->lock) || + lockdep_assert(lockdep_is_held(&sx->lock) || lockdep_is_held(&net->xfrm.xfrm_cfg_mutex) || lockdep_is_held(&net->xfrm.xfrm_state_lock)); @@ -1094,7 +1102,7 @@ static void mlx5e_xfrm_update_stats(struct xfrm_state *x) if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { mlx5_fc_query_cached(ipsec_rule->auth.fc, &auth_bytes, &auth_packets, &lastuse); - x->stats.integrity_failed += auth_packets; + sx->stats.integrity_failed += auth_packets; XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR, auth_packets); mlx5_fc_query_cached(ipsec_rule->trailer.fc, &trailer_bytes, @@ -1108,13 +1116,13 @@ static void mlx5e_xfrm_update_stats(struct xfrm_state *x) if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { mlx5_fc_query_cached(ipsec_rule->replay.fc, &replay_bytes, &replay_packets, &lastuse); - x->stats.replay += replay_packets; + sx->stats.replay += replay_packets; XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR, replay_packets); } mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse); success_packets = packets - auth_packets - trailer_packets - replay_packets; - x->curlft.packets += success_packets; + sx->curlft.packets += success_packets; /* NIC counts all bytes passed through flow steering and doesn't have * an ability to count payload data size which is needed for SA. * @@ -1128,7 +1136,7 @@ static void mlx5e_xfrm_update_stats(struct xfrm_state *x) headers += sizeof(struct ipv6hdr); success_bytes = bytes - auth_bytes - trailer_bytes - replay_bytes; - x->curlft.bytes += success_bytes - headers * success_packets; + sx->curlft.bytes += success_bytes - headers * success_packets; } static __be32 word_to_mask(int prefix)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 145677c..e7b2c24 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c
@@ -365,7 +365,7 @@ static void mlx5e_ipsec_handle_limits(struct mlx5e_ipsec_sa_entry *sa_entry) bool soft_arm, hard_arm; u64 hard_cnt; - lockdep_assert_held(&sa_entry->x->lock); + lockdep_assert_held(&sa_entry->x->sx->lock); soft_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, soft_lft_arm); hard_arm = !MLX5_GET(ipsec_aso, sa_entry->ctx, hard_lft_arm); @@ -450,7 +450,7 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) attrs = &sa_entry->attrs; - spin_lock_bh(&sa_entry->x->lock); + spin_lock_bh(&sa_entry->x->sx->lock); ret = mlx5e_ipsec_aso_query(sa_entry, NULL); if (ret) goto unlock; @@ -468,7 +468,7 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) } unlock: - spin_unlock_bh(&sa_entry->x->lock); + spin_unlock_bh(&sa_entry->x->sx->lock); if (need_modify) mlx5_accel_esp_modify_xfrm(sa_entry, &tmp); kfree(work); @@ -597,7 +597,7 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, u8 ds_cnt; int ret; - lockdep_assert_held(&sa_entry->x->lock); + lockdep_assert_held(&sa_entry->x->sx->lock); res = &mdev->mlx5e_res.hw_objs; spin_lock_bh(&aso->lock);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 6056106..eaa32236 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
@@ -153,7 +153,7 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb, void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x, struct xfrm_offload *xo) { - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; __u32 oseq = replay_esn->oseq; int iv_offset; __be64 seqno;
diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 3ab616b..0a31dc6 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h
@@ -185,7 +185,9 @@ enum xfrm_replay_mode { XFRM_REPLAY_MODE_ESN, }; -struct xfrm_sub_sa { +#define XFRM_MAX_SUB_STATES 1 + +struct xfrm_sub_state { /* Sub SA data */ spinlock_t ____cacheline_aligned lock; @@ -208,8 +210,6 @@ struct xfrm_sub_sa { struct xfrm_replay_state_esn *preplay_esn; /* --- cacheline 11 boundary (704 bytes) --- */ - struct hrtimer mtimer; - /* --- cacheline 12 boundary (768 bytes) --- */ /* Statistics */ struct xfrm_stats stats; @@ -217,9 +217,6 @@ struct xfrm_sub_sa { /* Replay detection notification settings */ u32 replay_maxage; u32 replay_maxdiff; - - /* Replay detection notification timer */ - struct timer_list rtimer; }; /* Full description of state of transformer. @@ -333,71 +330,39 @@ struct xfrm_state { const struct xfrm_mode_cbs *mode_cbs; void *mode_data; - /* 48 bytes hole */ + struct hrtimer mtimer; + + /* Replay detection notification timer */ + struct timer_list rtimer; + + /* Legacy and non IPsec stuff goes here. */ + + /* State for replay detection */ + struct xfrm_replay_state replay; + + /* Replay detection state at the time we sent the last notification */ + struct xfrm_replay_state preplay; + + union { + /* Data for care-of address */ + xfrm_address_t *coaddr; + /* compression algorithm for ipcomp */ + struct xfrm_algo *calg; + }; + + /* IPComp needs an IPIP tunnel for handling uncompressed packets */ + struct xfrm_state *tunnel; + + /* If a tunnel, number of users + 1 */ + atomic_t tunnel_users; + + u32 tfcpad; + + /* End of legacy suff. */ /* --- cacheline 8 boundary (512 bytes) --- */ /* Sub SA data */ - - union { - struct xfrm_sub_sa xs; - struct { - spinlock_t ____cacheline_aligned lock; - - /* --- cacheline 9 boundary (576 bytes) --- */ - struct xfrm_lifetime_cfg lft; - /* --- cacheline 10 boundary (640 bytes) --- */ - struct xfrm_lifetime_cur curlft; - - /* used to fix curlft->add_time when changing date */ - long saved_tmo; - - /* Last used time */ - time64_t lastused; - - /* State for replay detection */ - struct xfrm_replay_state_esn *replay_esn; - - /* Replay detection state at the time we sent the last notification */ - struct xfrm_replay_state_esn *preplay_esn; - - /* --- cacheline 11 boundary (704 bytes) --- */ - struct hrtimer mtimer; - - /* --- cacheline 12 boundary (768 bytes) --- */ - /* Statistics */ - struct xfrm_stats stats; - - /* Replay detection notification settings */ - u32 replay_maxage; - u32 replay_maxdiff; - - /* Replay detection notification timer */ - struct timer_list rtimer; - /* --- cacheline 13 boundary (832 bytes) was 40 bytes ago --- */ - - /* Legacy and non IPsec stuff goes here. */ - - /* State for replay detection */ - struct xfrm_replay_state replay; - - /* Replay detection state at the time we sent the last notification */ - struct xfrm_replay_state preplay; - - /* --- cacheline 14 boundary (896 bytes) --- */ - /* Data for care-of address */ - xfrm_address_t *coaddr; - - /* IPComp needs an IPIP tunnel for handling uncompressed packets */ - struct xfrm_state *tunnel; - - /* If a tunnel, number of users + 1 */ - atomic_t tunnel_users; - - u32 tfcpad; - - struct xfrm_algo *calg; - }; - }; + struct xfrm_sub_state *sx; }; @@ -2103,15 +2068,15 @@ static inline int xfrm_replay_clone(struct xfrm_state *x, struct xfrm_state *orig) { - x->replay_esn = kmemdup(orig->replay_esn, - xfrm_replay_state_esn_len(orig->replay_esn), + x->sx->replay_esn = kmemdup(orig->sx->replay_esn, + xfrm_replay_state_esn_len(orig->sx->replay_esn), GFP_KERNEL); - if (!x->replay_esn) + if (!x->sx->replay_esn) return -ENOMEM; - x->preplay_esn = kmemdup(orig->preplay_esn, - xfrm_replay_state_esn_len(orig->preplay_esn), + x->sx->preplay_esn = kmemdup(orig->sx->preplay_esn, + xfrm_replay_state_esn_len(orig->sx->preplay_esn), GFP_KERNEL); - if (!x->preplay_esn) + if (!x->sx->preplay_esn) return -ENOMEM; return 0;
diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 8e185b3..430b3ad8 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c
@@ -2699,11 +2699,13 @@ static u32 pktgen_dst_metrics[RTAX_MAX + 1] = { static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) { struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; + struct xfrm_sub_state *sx; int err = 0; struct net *net = dev_net(pkt_dev->odev); if (!x) return 0; + sx = x->sx; /* XXX: we dont support tunnel mode for now until * we resolve the dst issue */ @@ -2728,10 +2730,10 @@ static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR); goto error; } - spin_lock_bh(&x->lock); - x->curlft.bytes += skb->len; - x->curlft.packets++; - spin_unlock_bh(&x->lock); + spin_lock_bh(&sx->lock); + sx->curlft.bytes += skb->len; + sx->curlft.packets++; + spin_unlock_bh(&sx->lock); error: return err; }
diff --git a/net/ipv4/eesp4.c b/net/ipv4/eesp4.c index 0d26a27..867e904d 100644 --- a/net/ipv4/eesp4.c +++ b/net/ipv4/eesp4.c
@@ -14,10 +14,10 @@ static struct sock *eesp_find_tcp_sk(struct xfrm_state *x) __be16 sport, dport; struct sock *sk; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sk = inet_lookup_established(net, x->id.daddr.a4, dport, x->props.saddr.a4, sport, 0);
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 0498bbb..11448a1 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c
@@ -15,10 +15,10 @@ static struct sock *esp_find_tcp_sk(struct xfrm_state *x) __be16 sport, dport; struct sock *sk; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sk = inet_lookup_established(net, x->id.daddr.a4, dport, x->props.saddr.a4, sport, 0);
diff --git a/net/ipv4/ipcomp.c b/net/ipv4/ipcomp.c index 9a45aed..080ffc8 100644 --- a/net/ipv4/ipcomp.c +++ b/net/ipv4/ipcomp.c
@@ -63,7 +63,7 @@ static struct xfrm_state *ipcomp_tunnel_create(struct xfrm_state *x) t = xfrm_state_alloc(net); if (!t) goto out; - lockdep_set_class(&t->lock, &xfrm_state_lock_key); + lockdep_set_class(&t->sx->lock, &xfrm_state_lock_key); t->id.proto = IPPROTO_IPIP; t->id.spi = x->props.saddr.a4;
diff --git a/net/ipv6/eesp6.c b/net/ipv6/eesp6.c index 0a85475..599a62d 100644 --- a/net/ipv6/eesp6.c +++ b/net/ipv6/eesp6.c
@@ -27,10 +27,10 @@ static struct sock *eesp6_find_tcp_sk(struct xfrm_state *x) __be16 sport, dport; struct sock *sk; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sk = __inet6_lookup_established(net, &x->id.daddr.in6, dport, &x->props.saddr.in6, ntohs(sport), 0, 0);
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index b9fa72d..b29c85e 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c
@@ -29,10 +29,10 @@ static struct sock *esp6_find_tcp_sk(struct xfrm_state *x) __be16 sport, dport; struct sock *sk; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sk = __inet6_lookup_established(net, &x->id.daddr.in6, dport, &x->props.saddr.in6, ntohs(sport), 0, 0);
diff --git a/net/ipv6/ipcomp6.c b/net/ipv6/ipcomp6.c index 8607569..4f396e7 100644 --- a/net/ipv6/ipcomp6.c +++ b/net/ipv6/ipcomp6.c
@@ -80,7 +80,7 @@ static struct xfrm_state *ipcomp6_tunnel_create(struct xfrm_state *x) t = xfrm_state_alloc(net); if (!t) goto out; - lockdep_set_class(&t->lock, &xfrm_state_lock_key); + lockdep_set_class(&t->sx->lock, &xfrm_state_lock_key); t->id.proto = IPPROTO_IPV6; t->id.spi = xfrm6_tunnel_alloc_spi(net, (xfrm_address_t *)&x->props.saddr);
diff --git a/net/ipv6/mip6.c b/net/ipv6/mip6.c index 6a16a5b..f7b72f8 100644 --- a/net/ipv6/mip6.c +++ b/net/ipv6/mip6.c
@@ -122,11 +122,11 @@ static int mip6_destopt_input(struct xfrm_state *x, struct sk_buff *skb) struct ipv6_destopt_hdr *destopt = (struct ipv6_destopt_hdr *)skb->data; int err = destopt->nexthdr; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); if (!ipv6_addr_equal(&iph->saddr, (struct in6_addr *)x->coaddr) && !ipv6_addr_any((struct in6_addr *)x->coaddr)) err = -ENOENT; - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); return err; } @@ -162,9 +162,9 @@ static int mip6_destopt_output(struct xfrm_state *x, struct sk_buff *skb) len = ((char *)hao - (char *)dstopt) + sizeof(*hao); memcpy(&hao->addr, &iph->saddr, sizeof(hao->addr)); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); memcpy(&iph->saddr, x->coaddr, sizeof(iph->saddr)); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); WARN_ON(len != x->props.header_len); dstopt->hdrlen = (x->props.header_len >> 3) - 1; @@ -291,11 +291,11 @@ static int mip6_rthdr_input(struct xfrm_state *x, struct sk_buff *skb) struct rt2_hdr *rt2 = (struct rt2_hdr *)skb->data; int err = rt2->rt_hdr.nexthdr; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); if (!ipv6_addr_equal(&iph->daddr, (struct in6_addr *)x->coaddr) && !ipv6_addr_any((struct in6_addr *)x->coaddr)) err = -ENOENT; - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); return err; } @@ -325,9 +325,9 @@ static int mip6_rthdr_output(struct xfrm_state *x, struct sk_buff *skb) WARN_ON(rt2->rt_hdr.hdrlen != 2); memcpy(&rt2->addr, &iph->daddr, sizeof(rt2->addr)); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); memcpy(&iph->daddr, x->coaddr, sizeof(iph->daddr)); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); return 0; }
diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 699a001..01c5ff9 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c
@@ -285,18 +285,18 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, continue; } - spin_lock(&x->lock); + spin_lock(&x->sx->lock); if ((!i || (x->props.flags & XFRM_STATE_WILDRECV)) && likely(x->km.state == XFRM_STATE_VALID) && !xfrm_state_check_expire(x)) { - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); if (x->type->input(x, skb) > 0) { /* found a valid state */ break; } } else - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); xfrm_state_put(x); x = NULL; @@ -310,12 +310,12 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, sp->xvec[sp->len++] = x; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); - x->curlft.bytes += skb->len; - x->curlft.packets++; + x->sx->curlft.bytes += skb->len; + x->sx->curlft.packets++; - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); return 1;
diff --git a/net/key/af_key.c b/net/key/af_key.c index a166a88..68f3227 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c
@@ -899,10 +899,10 @@ static struct sk_buff *__pfkey_xfrm_state2msg(const struct xfrm_state *x, lifetime->sadb_lifetime_len = sizeof(struct sadb_lifetime)/sizeof(uint64_t); lifetime->sadb_lifetime_exttype = SADB_EXT_LIFETIME_HARD; - lifetime->sadb_lifetime_allocations = _X2KEY(x->lft.hard_packet_limit); - lifetime->sadb_lifetime_bytes = _X2KEY(x->lft.hard_byte_limit); - lifetime->sadb_lifetime_addtime = x->lft.hard_add_expires_seconds; - lifetime->sadb_lifetime_usetime = x->lft.hard_use_expires_seconds; + lifetime->sadb_lifetime_allocations = _X2KEY(x->sx->lft.hard_packet_limit); + lifetime->sadb_lifetime_bytes = _X2KEY(x->sx->lft.hard_byte_limit); + lifetime->sadb_lifetime_addtime = x->sx->lft.hard_add_expires_seconds; + lifetime->sadb_lifetime_usetime = x->sx->lft.hard_use_expires_seconds; } /* soft time */ if (hsc & 1) { @@ -910,20 +910,20 @@ static struct sk_buff *__pfkey_xfrm_state2msg(const struct xfrm_state *x, lifetime->sadb_lifetime_len = sizeof(struct sadb_lifetime)/sizeof(uint64_t); lifetime->sadb_lifetime_exttype = SADB_EXT_LIFETIME_SOFT; - lifetime->sadb_lifetime_allocations = _X2KEY(x->lft.soft_packet_limit); - lifetime->sadb_lifetime_bytes = _X2KEY(x->lft.soft_byte_limit); - lifetime->sadb_lifetime_addtime = x->lft.soft_add_expires_seconds; - lifetime->sadb_lifetime_usetime = x->lft.soft_use_expires_seconds; + lifetime->sadb_lifetime_allocations = _X2KEY(x->sx->lft.soft_packet_limit); + lifetime->sadb_lifetime_bytes = _X2KEY(x->sx->lft.soft_byte_limit); + lifetime->sadb_lifetime_addtime = x->sx->lft.soft_add_expires_seconds; + lifetime->sadb_lifetime_usetime = x->sx->lft.soft_use_expires_seconds; } /* current time */ lifetime = skb_put(skb, sizeof(struct sadb_lifetime)); lifetime->sadb_lifetime_len = sizeof(struct sadb_lifetime)/sizeof(uint64_t); lifetime->sadb_lifetime_exttype = SADB_EXT_LIFETIME_CURRENT; - lifetime->sadb_lifetime_allocations = x->curlft.packets; - lifetime->sadb_lifetime_bytes = x->curlft.bytes; - lifetime->sadb_lifetime_addtime = x->curlft.add_time; - lifetime->sadb_lifetime_usetime = x->curlft.use_time; + lifetime->sadb_lifetime_allocations = x->sx->curlft.packets; + lifetime->sadb_lifetime_bytes = x->sx->curlft.bytes; + lifetime->sadb_lifetime_addtime = x->sx->curlft.add_time; + lifetime->sadb_lifetime_usetime = x->sx->curlft.use_time; /* src address */ addr = skb_put(skb, sizeof(struct sadb_address) + sockaddr_size); addr->sadb_address_len = @@ -1152,17 +1152,17 @@ static struct xfrm_state * pfkey_msg2xfrm_state(struct net *net, lifetime = ext_hdrs[SADB_EXT_LIFETIME_HARD - 1]; if (lifetime != NULL) { - x->lft.hard_packet_limit = _KEY2X(lifetime->sadb_lifetime_allocations); - x->lft.hard_byte_limit = _KEY2X(lifetime->sadb_lifetime_bytes); - x->lft.hard_add_expires_seconds = lifetime->sadb_lifetime_addtime; - x->lft.hard_use_expires_seconds = lifetime->sadb_lifetime_usetime; + x->sx->lft.hard_packet_limit = _KEY2X(lifetime->sadb_lifetime_allocations); + x->sx->lft.hard_byte_limit = _KEY2X(lifetime->sadb_lifetime_bytes); + x->sx->lft.hard_add_expires_seconds = lifetime->sadb_lifetime_addtime; + x->sx->lft.hard_use_expires_seconds = lifetime->sadb_lifetime_usetime; } lifetime = ext_hdrs[SADB_EXT_LIFETIME_SOFT - 1]; if (lifetime != NULL) { - x->lft.soft_packet_limit = _KEY2X(lifetime->sadb_lifetime_allocations); - x->lft.soft_byte_limit = _KEY2X(lifetime->sadb_lifetime_bytes); - x->lft.soft_add_expires_seconds = lifetime->sadb_lifetime_addtime; - x->lft.soft_use_expires_seconds = lifetime->sadb_lifetime_usetime; + x->sx->lft.soft_packet_limit = _KEY2X(lifetime->sadb_lifetime_allocations); + x->sx->lft.soft_byte_limit = _KEY2X(lifetime->sadb_lifetime_bytes); + x->sx->lft.soft_add_expires_seconds = lifetime->sadb_lifetime_addtime; + x->sx->lft.soft_use_expires_seconds = lifetime->sadb_lifetime_usetime; } sec_ctx = ext_hdrs[SADB_X_EXT_SEC_CTX - 1]; @@ -1438,11 +1438,11 @@ static int pfkey_acquire(struct sock *sk, struct sk_buff *skb, const struct sadb if (x == NULL) return 0; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (x->km.state == XFRM_STATE_ACQ) x->km.state = XFRM_STATE_ERROR; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); xfrm_state_put(x); return 0; }
diff --git a/net/xfrm/xfrm_eesp.c b/net/xfrm/xfrm_eesp.c index 7859f2d..6ef3f54 100644 --- a/net/xfrm/xfrm_eesp.c +++ b/net/xfrm/xfrm_eesp.c
@@ -285,11 +285,11 @@ static int eesp_output_encap(struct xfrm_state *x, struct sk_buff *skb, eesph = ERR_PTR(-EOPNOTSUPP); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; encap_type = encap->encap_type; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); switch (encap_type) { default: @@ -346,10 +346,10 @@ int eesp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct eesp_info allocsize = ALIGN(tailen, L1_CACHE_BYTES); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); goto cow; } @@ -369,7 +369,7 @@ int eesp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct eesp_info pfrag->offset = pfrag->offset + allocsize; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); nfrags++; @@ -455,9 +455,9 @@ int eesp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct eesp_info allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); printk("exit2\n"); goto error_free; } @@ -469,7 +469,7 @@ int eesp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct eesp_info /* replace page frags in skb with new page */ __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len); pfrag->offset = pfrag->offset + allocsize; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1); err = skb_to_sgvec(skb, dsg,
diff --git a/net/xfrm/xfrm_esp.c b/net/xfrm/xfrm_esp.c index 8992899..4a0df7c 100644 --- a/net/xfrm/xfrm_esp.c +++ b/net/xfrm/xfrm_esp.c
@@ -355,11 +355,11 @@ static int esp_output_encap(struct xfrm_state *x, struct sk_buff *skb, esph = ERR_PTR(-EOPNOTSUPP); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); sport = encap->encap_sport; dport = encap->encap_dport; encap_type = encap->encap_type; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); switch (encap_type) { default: @@ -417,10 +417,10 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * allocsize = ALIGN(tailen, L1_CACHE_BYTES); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); goto cow; } @@ -439,7 +439,7 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * pfrag->offset = pfrag->offset + allocsize; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); nfrags++; @@ -527,9 +527,9 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); goto error_free; } @@ -540,7 +540,7 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * /* replace page frags in skb with new page */ __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len); pfrag->offset = pfrag->offset + allocsize; - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1); err = skb_to_sgvec(skb, dsg,
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index d0f0f5e..b990973 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c
@@ -514,7 +514,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (encap_type == -1) { async = 1; seq = XFRM_SKB_CB(skb)->seq.input.low; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); goto resume; } /* GRO call */ @@ -533,7 +533,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_audit_state_icvfail(x, skb, x->type->proto); - x->stats.integrity_failed++; + x->sx->stats.integrity_failed++; XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); goto drop; } @@ -629,7 +629,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) XFRM_SKB_CB(skb)->seq.input.low = seq; XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); if (unlikely(x->km.state != XFRM_STATE_VALID)) { if (x->km.state == XFRM_STATE_ACQ) @@ -664,7 +664,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) } if (!crypto_done) { - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); dev_hold(skb->dev); nexthdr = x->type->input(x, skb); @@ -675,14 +675,14 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) } dev_put(skb->dev); - spin_lock(&x->lock); + spin_lock(&x->sx->lock); } resume: if (nexthdr < 0) { if (nexthdr == -EBADMSG) { xfrm_audit_state_icvfail(x, skb, x->type->proto); - x->stats.integrity_failed++; + x->sx->stats.integrity_failed++; } XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); goto drop_unlock; @@ -698,11 +698,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_replay_advance(x, seq); - x->curlft.bytes += skb->len; - x->curlft.packets++; - x->lastused = ktime_get_real_seconds(); + x->sx->curlft.bytes += skb->len; + x->sx->curlft.packets++; + x->sx->lastused = ktime_get_real_seconds(); - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; @@ -780,7 +780,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) } drop_unlock: - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); drop: if (async) dev_put(skb->dev);
diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 97bc979e5..b886782 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c
@@ -1770,7 +1770,7 @@ static bool iptfs_enqueue(struct xfrm_iptfs_data *xtfs, struct sk_buff *skb) u64 newsz = xtfs->queue_size + skb->len; struct iphdr *iph; - assert_spin_locked(&xtfs->x->lock); + assert_spin_locked(&xtfs->x->sx->lock); if (newsz > xtfs->cfg.max_queue_size) return false; @@ -1878,7 +1878,7 @@ static int iptfs_output_collect(struct net *net, struct sock *sk, struct sk_buff /* We can be running on multiple cores and from the network softirq or * from user context depending on where the packet is coming from. */ - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); skb_list_walk_safe(segs, skb, nskb) { skb_mark_not_on_list(skb); @@ -1918,7 +1918,7 @@ static int iptfs_output_collect(struct net *net, struct sock *sk, struct sk_buff trace_iptfs_timer_start(xtfs, xtfs->init_delay_ns); } - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); return 0; } @@ -2331,12 +2331,12 @@ static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me) * ingress packets for us to process and transmit. */ - spin_lock(&x->lock); + spin_lock(&x->sx->lock); __skb_queue_head_init(&list); skb_queue_splice_init(&xtfs->queue, &list); xtfs->queue_size = 0; settime = xtfs->iptfs_settime; - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); /* After the above unlock, packets can begin queuing again, and the * timer can be set again, from another CPU either in softirq or user @@ -2708,11 +2708,11 @@ static void iptfs_destroy_state(struct xfrm_state *x) if (!xtfs) return; - spin_lock_bh(&xtfs->x->lock); + spin_lock_bh(&xtfs->x->sx->lock); hrtimer_cancel(&xtfs->iptfs_timer); __skb_queue_head_init(&list); skb_queue_splice_init(&xtfs->queue, &list); - spin_unlock_bh(&xtfs->x->lock); + spin_unlock_bh(&xtfs->x->sx->lock); while ((skb = __skb_dequeue(&list))) kfree_skb(skb);
diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 4589310..d93ce76 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c
@@ -170,9 +170,9 @@ static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) if (!interval) return 0; - spin_lock(&x->lock); + spin_lock(&x->sx->lock); - delta = (int)(ctx->now - x->lastused); + delta = (int)(ctx->now - x->sx->lastused); if (delta < interval) { x->nat_keepalive_expiration = ctx->now + interval - delta; next_run = x->nat_keepalive_expiration; @@ -184,7 +184,7 @@ static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) send_keepalive = true; } - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); if (send_keepalive) nat_keepalive_send(&ka);
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index a9652b4..ca94ec7 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c
@@ -492,6 +492,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) { struct dst_entry *dst = skb_dst(skb); struct xfrm_state *x = dst->xfrm; + struct xfrm_sub_state *sx = x->sx; struct net *net = xs_net(x); if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET) @@ -512,7 +513,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) goto error_nolock; } - spin_lock_bh(&x->lock); + spin_lock_bh(&sx->lock); if (unlikely(x->km.state != XFRM_STATE_VALID)) { XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID); @@ -532,11 +533,11 @@ static int xfrm_output_one(struct sk_buff *skb, int err) goto error; } - x->curlft.bytes += skb->len; - x->curlft.packets++; - x->lastused = ktime_get_real_seconds(); + sx->curlft.bytes += skb->len; + sx->curlft.packets++; + sx->lastused = ktime_get_real_seconds(); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&sx->lock); skb_dst_force(skb); if (!skb_dst(skb)) { @@ -575,7 +576,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) return 0; error: - spin_unlock_bh(&x->lock); + spin_unlock_bh(&sx->lock); error_nolock: kfree_skb(skb); out:
diff --git a/net/xfrm/xfrm_replay.c b/net/xfrm/xfrm_replay.c index dbdf8a3..dd1bdd4 100644 --- a/net/xfrm/xfrm_replay.c +++ b/net/xfrm/xfrm_replay.c
@@ -12,7 +12,7 @@ u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq) { u32 seq, seq_hi, bottom; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; if (!(x->props.flags & XFRM_STATE_ESN)) return 0; @@ -40,12 +40,13 @@ static void xfrm_replay_notify_esn(struct xfrm_state *x, int event); void xfrm_replay_notify(struct xfrm_state *x, int event) { + struct xfrm_sub_state *sx = x->sx; struct km_event c; /* we send notify messages in case * 1. we updated on of the sequence numbers, and the seqno difference - * is at least x->replay_maxdiff, in this case we also update the + * is at least sx->replay_maxdiff, in this case we also update the * timeout of our timer function - * 2. if x->replay_maxage has elapsed since last update, + * 2. if sx->replay_maxage has elapsed since last update, * and there were changes * * The state structure must be locked! @@ -64,9 +65,9 @@ void xfrm_replay_notify(struct xfrm_state *x, int event) switch (event) { case XFRM_REPLAY_UPDATE: - if (!x->replay_maxdiff || - ((x->replay.seq - x->preplay.seq < x->replay_maxdiff) && - (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff))) { + if (!sx->replay_maxdiff || + ((x->replay.seq - x->preplay.seq < sx->replay_maxdiff) && + (x->replay.oseq - x->preplay.oseq < sx->replay_maxdiff))) { if (x->xflags & XFRM_TIME_DEFER) event = XFRM_REPLAY_TIMEOUT; else @@ -90,8 +91,8 @@ void xfrm_replay_notify(struct xfrm_state *x, int event) c.data.aevent = event; km_state_notify(x, &c); - if (x->replay_maxage && - !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) + if (sx->replay_maxage && + !mod_timer(&x->rtimer, jiffies + sx->replay_maxage)) x->xflags &= ~XFRM_TIME_DEFER; } @@ -135,12 +136,12 @@ static int xfrm_replay_check_legacy(struct xfrm_state *x, diff = x->replay.seq - seq; if (diff >= x->props.replay_window) { - x->stats.replay_window++; + x->sx->stats.replay_window++; goto err; } if (x->replay.bitmap & (1U << diff)) { - x->stats.replay++; + x->sx->stats.replay++; goto err; } return 0; @@ -188,8 +189,9 @@ void xfrm_replay_advance(struct xfrm_state *x, __be32 net_seq) static int xfrm_replay_overflow_bmp(struct xfrm_state *x, struct sk_buff *skb) { + struct xfrm_sub_state *sx = x->sx; int err = 0; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = sx->replay_esn; struct net *net = xs_net(x); if (x->type->flags & XFRM_TYPE_REPLAY_PROT) { @@ -214,7 +216,7 @@ static int xfrm_replay_check_bmp(struct xfrm_state *x, struct sk_buff *skb, __be32 net_seq) { unsigned int bitnr, nr; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; u32 pos; u32 seq = ntohl(net_seq); u32 diff = replay_esn->seq - seq; @@ -229,7 +231,7 @@ static int xfrm_replay_check_bmp(struct xfrm_state *x, return 0; if (diff >= replay_esn->replay_window) { - x->stats.replay_window++; + x->sx->stats.replay_window++; goto err; } @@ -248,7 +250,7 @@ static int xfrm_replay_check_bmp(struct xfrm_state *x, return 0; err_replay: - x->stats.replay++; + x->sx->stats.replay++; err: xfrm_audit_state_replay(x, skb, net_seq); return -EINVAL; @@ -258,7 +260,7 @@ static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq) { unsigned int bitnr, nr, i; u32 diff; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; u32 seq = ntohl(net_seq); u32 pos; @@ -304,15 +306,16 @@ static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq) static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event) { + struct xfrm_sub_state *sx = x->sx; struct km_event c; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; - struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn; + struct xfrm_replay_state_esn *replay_esn = sx->replay_esn; + struct xfrm_replay_state_esn *preplay_esn = sx->preplay_esn; /* we send notify messages in case * 1. we updated on of the sequence numbers, and the seqno difference - * is at least x->replay_maxdiff, in this case we also update the + * is at least sx->replay_maxdiff, in this case we also update the * timeout of our timer function - * 2. if x->replay_maxage has elapsed since last update, + * 2. if sx->replay_maxage has elapsed since last update, * and there were changes * * The state structure must be locked! @@ -320,10 +323,10 @@ static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event) switch (event) { case XFRM_REPLAY_UPDATE: - if (!x->replay_maxdiff || - ((replay_esn->seq - preplay_esn->seq < x->replay_maxdiff) && + if (!sx->replay_maxdiff || + ((replay_esn->seq - preplay_esn->seq < sx->replay_maxdiff) && (replay_esn->oseq - preplay_esn->oseq - < x->replay_maxdiff))) { + < sx->replay_maxdiff))) { if (x->xflags & XFRM_TIME_DEFER) event = XFRM_REPLAY_TIMEOUT; else @@ -333,7 +336,7 @@ static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event) break; case XFRM_REPLAY_TIMEOUT: - if (memcmp(x->replay_esn, x->preplay_esn, + if (memcmp(sx->replay_esn, sx->preplay_esn, xfrm_replay_state_esn_len(replay_esn)) == 0) { x->xflags |= XFRM_TIME_DEFER; return; @@ -342,29 +345,30 @@ static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event) break; } - memcpy(x->preplay_esn, x->replay_esn, + memcpy(sx->preplay_esn, sx->replay_esn, xfrm_replay_state_esn_len(replay_esn)); c.event = XFRM_MSG_NEWAE; c.data.aevent = event; km_state_notify(x, &c); - if (x->replay_maxage && - !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) + if (sx->replay_maxage && + !mod_timer(&x->rtimer, jiffies + sx->replay_maxage)) x->xflags &= ~XFRM_TIME_DEFER; } static void xfrm_replay_notify_esn(struct xfrm_state *x, int event) { + struct xfrm_sub_state *sx = x->sx; u32 seq_diff, oseq_diff; struct km_event c; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; - struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn; + struct xfrm_replay_state_esn *replay_esn = sx->replay_esn; + struct xfrm_replay_state_esn *preplay_esn = sx->preplay_esn; /* we send notify messages in case * 1. we updated on of the sequence numbers, and the seqno difference - * is at least x->replay_maxdiff, in this case we also update the + * is at least sx->replay_maxdiff, in this case we also update the * timeout of our timer function - * 2. if x->replay_maxage has elapsed since last update, + * 2. if sx->replay_maxage has elapsed since last update, * and there were changes * * The state structure must be locked! @@ -372,7 +376,7 @@ static void xfrm_replay_notify_esn(struct xfrm_state *x, int event) switch (event) { case XFRM_REPLAY_UPDATE: - if (x->replay_maxdiff) { + if (sx->replay_maxdiff) { if (replay_esn->seq_hi == preplay_esn->seq_hi) seq_diff = replay_esn->seq - preplay_esn->seq; else @@ -386,8 +390,8 @@ static void xfrm_replay_notify_esn(struct xfrm_state *x, int event) oseq_diff = ~preplay_esn->oseq + replay_esn->oseq + 1; - if (seq_diff >= x->replay_maxdiff || - oseq_diff >= x->replay_maxdiff) + if (seq_diff >= sx->replay_maxdiff || + oseq_diff >= sx->replay_maxdiff) break; } @@ -399,7 +403,7 @@ static void xfrm_replay_notify_esn(struct xfrm_state *x, int event) break; case XFRM_REPLAY_TIMEOUT: - if (memcmp(x->replay_esn, x->preplay_esn, + if (memcmp(sx->replay_esn, sx->preplay_esn, xfrm_replay_state_esn_len(replay_esn)) == 0) { x->xflags |= XFRM_TIME_DEFER; return; @@ -408,21 +412,22 @@ static void xfrm_replay_notify_esn(struct xfrm_state *x, int event) break; } - memcpy(x->preplay_esn, x->replay_esn, + memcpy(sx->preplay_esn, sx->replay_esn, xfrm_replay_state_esn_len(replay_esn)); c.event = XFRM_MSG_NEWAE; c.data.aevent = event; km_state_notify(x, &c); - if (x->replay_maxage && - !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) + if (sx->replay_maxage && + !mod_timer(&x->rtimer, jiffies + sx->replay_maxage)) x->xflags &= ~XFRM_TIME_DEFER; } static int xfrm_replay_overflow_esn(struct xfrm_state *x, struct sk_buff *skb) { + struct xfrm_sub_state *sx = x->sx; int err = 0; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = sx->replay_esn; struct net *net = xs_net(x); if (x->type->flags & XFRM_TYPE_REPLAY_PROT) { @@ -453,7 +458,7 @@ static int xfrm_replay_check_esn(struct xfrm_state *x, { unsigned int bitnr, nr; u32 diff; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; u32 pos; u32 seq = ntohl(net_seq); u32 wsize = replay_esn->replay_window; @@ -482,7 +487,7 @@ static int xfrm_replay_check_esn(struct xfrm_state *x, } if (diff >= replay_esn->replay_window) { - x->stats.replay_window++; + x->sx->stats.replay_window++; goto err; } @@ -501,7 +506,7 @@ static int xfrm_replay_check_esn(struct xfrm_state *x, return 0; err_replay: - x->stats.replay++; + x->sx->stats.replay++; err: xfrm_audit_state_replay(x, skb, net_seq); return -EINVAL; @@ -527,7 +532,7 @@ static int xfrm_replay_recheck_esn(struct xfrm_state *x, { if (unlikely(XFRM_SKB_CB(skb)->seq.input.hi != htonl(xfrm_replay_seqhi(x, net_seq)))) { - x->stats.replay_window++; + x->sx->stats.replay_window++; return -EINVAL; } @@ -555,7 +560,7 @@ static void xfrm_replay_advance_esn(struct xfrm_state *x, __be32 net_seq) unsigned int bitnr, nr, i; int wrap; u32 diff, pos, seq, seq_hi; - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; if (!replay_esn->replay_window) return; @@ -652,7 +657,7 @@ static int xfrm_replay_overflow_offload_bmp(struct xfrm_state *x, struct sk_buff { int err = 0; struct xfrm_offload *xo = xfrm_offload(skb); - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; struct net *net = xs_net(x); __u32 oseq = replay_esn->oseq; @@ -692,7 +697,7 @@ static int xfrm_replay_overflow_offload_esn(struct xfrm_state *x, struct sk_buff { int err = 0; struct xfrm_offload *xo = xfrm_offload(skb); - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; struct net *net = xs_net(x); __u32 oseq = replay_esn->oseq; __u32 oseq_hi = replay_esn->oseq_hi; @@ -771,7 +776,7 @@ int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb) int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack) { - struct xfrm_replay_state_esn *replay_esn = x->replay_esn; + struct xfrm_replay_state_esn *replay_esn = x->sx->replay_esn; if (replay_esn) { if (replay_esn->replay_window >
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 2834a5c..1881739 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c
@@ -48,6 +48,7 @@ static void xfrm_state_gc_task(struct work_struct *work); static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024; static struct kmem_cache *xfrm_state_cache __ro_after_init; +static struct kmem_cache *xfrm_sub_state_cache __ro_after_init; static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); static HLIST_HEAD(xfrm_state_gc_list); @@ -604,6 +605,7 @@ EXPORT_SYMBOL(xfrm_state_free); static void xfrm_state_delete_tunnel(struct xfrm_state *x); static void xfrm_state_gc_destroy(struct xfrm_state *x) { + struct xfrm_sub_state *sx = x->sx; if (x->mode_cbs && x->mode_cbs->destroy_state) x->mode_cbs->destroy_state(x); hrtimer_cancel(&x->mtimer); @@ -614,8 +616,8 @@ static void xfrm_state_gc_destroy(struct xfrm_state *x) kfree(x->calg); kfree(x->encap); kfree(x->coaddr); - kfree(x->replay_esn); - kfree(x->preplay_esn); + kfree(sx->replay_esn); + kfree(sx->preplay_esn); xfrm_unset_type_offload(x); xfrm_state_delete_tunnel(x); if (x->type) { @@ -648,39 +650,40 @@ static void xfrm_state_gc_task(struct work_struct *work) static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) { struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer); + struct xfrm_sub_state *sx = x->sx; enum hrtimer_restart ret = HRTIMER_NORESTART; time64_t now = ktime_get_real_seconds(); time64_t next = TIME64_MAX; int warn = 0; int err = 0; - spin_lock(&x->lock); + spin_lock(&sx->lock); xfrm_dev_state_update_stats(x); if (x->km.state == XFRM_STATE_DEAD) goto out; if (x->km.state == XFRM_STATE_EXPIRED) goto expired; - if (x->lft.hard_add_expires_seconds) { - time64_t tmo = x->lft.hard_add_expires_seconds + - x->curlft.add_time - now; + if (sx->lft.hard_add_expires_seconds) { + time64_t tmo = sx->lft.hard_add_expires_seconds + + sx->curlft.add_time - now; if (tmo <= 0) { if (x->xflags & XFRM_SOFT_EXPIRE) { /* enter hard expire without soft expire first?! * setting a new date could trigger this. * workaround: fix x->curflt.add_time by below: */ - x->curlft.add_time = now - x->saved_tmo - 1; - tmo = x->lft.hard_add_expires_seconds - x->saved_tmo; + sx->curlft.add_time = now - sx->saved_tmo - 1; + tmo = sx->lft.hard_add_expires_seconds - sx->saved_tmo; } else goto expired; } if (tmo < next) next = tmo; } - if (x->lft.hard_use_expires_seconds) { - time64_t tmo = x->lft.hard_use_expires_seconds + - (READ_ONCE(x->curlft.use_time) ? : now) - now; + if (sx->lft.hard_use_expires_seconds) { + time64_t tmo = sx->lft.hard_use_expires_seconds + + (READ_ONCE(sx->curlft.use_time) ? : now) - now; if (tmo <= 0) goto expired; if (tmo < next) @@ -688,21 +691,21 @@ static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) } if (x->km.dying) goto resched; - if (x->lft.soft_add_expires_seconds) { - time64_t tmo = x->lft.soft_add_expires_seconds + - x->curlft.add_time - now; + if (sx->lft.soft_add_expires_seconds) { + time64_t tmo = sx->lft.soft_add_expires_seconds + + sx->curlft.add_time - now; if (tmo <= 0) { warn = 1; x->xflags &= ~XFRM_SOFT_EXPIRE; } else if (tmo < next) { next = tmo; x->xflags |= XFRM_SOFT_EXPIRE; - x->saved_tmo = tmo; + sx->saved_tmo = tmo; } } - if (x->lft.soft_use_expires_seconds) { - time64_t tmo = x->lft.soft_use_expires_seconds + - (READ_ONCE(x->curlft.use_time) ? : now) - now; + if (sx->lft.soft_use_expires_seconds) { + time64_t tmo = sx->lft.soft_use_expires_seconds + + (READ_ONCE(sx->curlft.use_time) ? : now) - now; if (tmo <= 0) warn = 1; else if (tmo < next) @@ -731,15 +734,36 @@ static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me) xfrm_audit_state_delete(x, err ? 0 : 1, true); out: - spin_unlock(&x->lock); + spin_unlock(&sx->lock); return ret; } static void xfrm_replay_timer_handler(struct timer_list *t); +static struct xfrm_sub_state *xfrm_sub_state_alloc(struct xfrm_state *x) +{ + struct xfrm_sub_state *sx; + + sx = kmem_cache_zalloc(xfrm_sub_state_cache, GFP_ATOMIC); + if (!sx) + return NULL; + + sx->curlft.add_time = ktime_get_real_seconds(); + sx->lft.soft_byte_limit = XFRM_INF; + sx->lft.soft_packet_limit = XFRM_INF; + sx->lft.hard_byte_limit = XFRM_INF; + sx->lft.hard_packet_limit = XFRM_INF; + sx->replay_maxage = 0; + sx->replay_maxdiff = 0; + spin_lock_init(&sx->lock); + + return sx; +} + struct xfrm_state *xfrm_state_alloc(struct net *net) { struct xfrm_state *x; + struct xfrm_sub_state *sx; x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC); @@ -753,20 +777,22 @@ struct xfrm_state *xfrm_state_alloc(struct net *net) INIT_HLIST_NODE(&x->bysrc); INIT_HLIST_NODE(&x->byspi); INIT_HLIST_NODE(&x->byseq); + x->pcpu_num = UINT_MAX; + x->mode_data = NULL; + hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT); timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0); - x->curlft.add_time = ktime_get_real_seconds(); - x->lft.soft_byte_limit = XFRM_INF; - x->lft.soft_packet_limit = XFRM_INF; - x->lft.hard_byte_limit = XFRM_INF; - x->lft.hard_packet_limit = XFRM_INF; - x->replay_maxage = 0; - x->replay_maxdiff = 0; - x->pcpu_num = UINT_MAX; - spin_lock_init(&x->lock); - x->mode_data = NULL; } + + sx = xfrm_sub_state_alloc(x); + if (!sx) { + kmem_cache_free(xfrm_state_cache, x); + return NULL; + } + + x->sx = sx; + return x; } EXPORT_SYMBOL(xfrm_state_alloc); @@ -862,9 +888,9 @@ int xfrm_state_delete(struct xfrm_state *x) { int err; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); err = __xfrm_state_delete(x); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); return err; } @@ -1338,7 +1364,7 @@ static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x, ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) || (*best)->km.dying > x->km.dying || ((*best)->km.dying == x->km.dying && - (*best)->curlft.add_time < x->curlft.add_time)) + (*best)->sx->curlft.add_time < x->sx->curlft.add_time)) *best = x; } else if (x->km.state == XFRM_STATE_ACQ) { if (!*best || x->pcpu_num == pcpu_id) @@ -1591,7 +1617,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, x->xso.type); } - x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; + x->sx->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL_SOFT); @@ -1723,6 +1749,7 @@ static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 sp static void __xfrm_state_insert(struct xfrm_state *x) { + struct xfrm_sub_state *sx = x->sx; struct net *net = xs_net(x); unsigned int h; @@ -1760,8 +1787,8 @@ static void __xfrm_state_insert(struct xfrm_state *x) } hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); - if (x->replay_maxage) - mod_timer(&x->rtimer, jiffies + x->replay_maxage); + if (sx->replay_maxage) + mod_timer(&x->rtimer, jiffies + sx->replay_maxage); net->xfrm.state_num++; @@ -1869,7 +1896,7 @@ static struct xfrm_state *__find_acq_core(struct net *net, x->if_id = if_id; x->mark.v = m->v; x->mark.m = m->m; - x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; + x->sx->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; xfrm_state_hold(x); hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), @@ -1978,14 +2005,18 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, struct xfrm_encap_tmpl *encap, struct xfrm_migrate *m) { + struct xfrm_sub_state *orig_sx = orig->sx; struct net *net = xs_net(orig); struct xfrm_state *x = xfrm_state_alloc(net); + struct xfrm_sub_state *sx; + if (!x) goto out; + sx = x->sx; memcpy(&x->id, &orig->id, sizeof(x->id)); memcpy(&x->sel, &orig->sel, sizeof(x->sel)); - memcpy(&x->lft, &orig->lft, sizeof(x->lft)); + memcpy(&sx->lft, &orig_sx->lft, sizeof(sx->lft)); x->props.mode = orig->props.mode; x->props.replay_window = orig->props.replay_window; x->props.reqid = orig->props.reqid; @@ -2042,7 +2073,7 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, goto error; } - if (orig->replay_esn) { + if (orig_sx->replay_esn) { if (xfrm_replay_clone(x, orig)) goto error; } @@ -2056,15 +2087,15 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, x->pcpu_num = orig->pcpu_num; x->if_id = orig->if_id; x->tfcpad = orig->tfcpad; - x->replay_maxdiff = orig->replay_maxdiff; - x->replay_maxage = orig->replay_maxage; - memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft)); + sx->replay_maxdiff = orig_sx->replay_maxdiff; + sx->replay_maxage = orig_sx->replay_maxage; + memcpy(&sx->curlft, &orig_sx->curlft, sizeof(sx->curlft)); x->km.state = orig->km.state; x->km.seq = orig->km.seq; x->replay = orig->replay; x->preplay = orig->preplay; x->mapping_maxage = orig->mapping_maxage; - x->lastused = orig->lastused; + sx->lastused = orig_sx->lastused; x->new_mapping = 0; x->new_mapping_sport = 0; x->dir = orig->dir; @@ -2185,6 +2216,7 @@ EXPORT_SYMBOL(xfrm_state_migrate); int xfrm_state_update(struct xfrm_state *x) { + struct xfrm_sub_state *sx = x->sx; struct xfrm_state *x1, *to_put; int err; int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY); @@ -2237,7 +2269,7 @@ int xfrm_state_update(struct xfrm_state *x) } err = -EINVAL; - spin_lock_bh(&x1->lock); + spin_lock_bh(&x1->sx->lock); if (likely(x1->km.state == XFRM_STATE_VALID)) { if (x->encap && x1->encap && x->encap->encap_type == x1->encap->encap_type) @@ -2250,12 +2282,12 @@ int xfrm_state_update(struct xfrm_state *x) } if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel))) memcpy(&x1->sel, &x->sel, sizeof(x1->sel)); - memcpy(&x1->lft, &x->lft, sizeof(x1->lft)); + memcpy(&x1->sx->lft, &sx->lft, sizeof(x1->sx->lft)); x1->km.dying = 0; hrtimer_start(&x1->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); - if (READ_ONCE(x1->curlft.use_time)) + if (READ_ONCE(x1->sx->curlft.use_time)) xfrm_state_check_expire(x1); if (x->props.smark.m || x->props.smark.v || x->if_id) { @@ -2278,7 +2310,7 @@ int xfrm_state_update(struct xfrm_state *x) } fail: - spin_unlock_bh(&x1->lock); + spin_unlock_bh(&x1->sx->lock); xfrm_state_put(x1); @@ -2288,6 +2320,7 @@ EXPORT_SYMBOL(xfrm_state_update); int xfrm_state_check_expire(struct xfrm_state *x) { + struct xfrm_sub_state *sx = x->sx; /* All counters which are needed to decide if state is expired * are handled by SW for non-packet offload modes. Simply skip * the following update and save extra boilerplate in drivers. @@ -2295,19 +2328,19 @@ int xfrm_state_check_expire(struct xfrm_state *x) if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) xfrm_dev_state_update_stats(x); - if (!READ_ONCE(x->curlft.use_time)) - WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds()); + if (!READ_ONCE(sx->curlft.use_time)) + WRITE_ONCE(sx->curlft.use_time, ktime_get_real_seconds()); - if (x->curlft.bytes >= x->lft.hard_byte_limit || - x->curlft.packets >= x->lft.hard_packet_limit) { + if (sx->curlft.bytes >= sx->lft.hard_byte_limit || + sx->curlft.packets >= sx->lft.hard_packet_limit) { x->km.state = XFRM_STATE_EXPIRED; hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT); return -EINVAL; } if (!x->km.dying && - (x->curlft.bytes >= x->lft.soft_byte_limit || - x->curlft.packets >= x->lft.soft_packet_limit)) { + (sx->curlft.bytes >= sx->lft.soft_byte_limit || + sx->curlft.packets >= sx->lft.soft_packet_limit)) { x->km.dying = 1; km_state_expired(x, 0, 0); } @@ -2590,7 +2623,7 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, u32 range = high - low + 1; __be32 newspi = 0; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); if (x->km.state == XFRM_STATE_DEAD) { NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state"); goto unlock; @@ -2636,7 +2669,7 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, NL_SET_ERR_MSG(extack, "No SPI available in the requested range"); unlock: - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); return err; } @@ -2727,9 +2760,9 @@ EXPORT_SYMBOL(xfrm_state_walk_done); static void xfrm_replay_timer_handler(struct timer_list *t) { - struct xfrm_state *x = timer_container_of(x, t, rtimer); + struct xfrm_state *x = container_of(t, struct xfrm_state, rtimer); - spin_lock(&x->lock); + spin_lock(&x->sx->lock); if (x->km.state == XFRM_STATE_VALID) { if (xfrm_aevent_is_on(xs_net(x))) @@ -2738,7 +2771,7 @@ static void xfrm_replay_timer_handler(struct timer_list *t) x->xflags |= XFRM_TIME_DEFER; } - spin_unlock(&x->lock); + spin_unlock(&x->sx->lock); } static LIST_HEAD(xfrm_km_list); @@ -3273,9 +3306,12 @@ int __net_init xfrm_state_init(struct net *net) struct hlist_head *ndst, *nsrc, *nspi, *nseq; unsigned int sz; - if (net_eq(net, &init_net)) + if (net_eq(net, &init_net)) { xfrm_state_cache = KMEM_CACHE(xfrm_state, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + xfrm_sub_state_cache = KMEM_CACHE(xfrm_sub_state, + SLAB_HWCACHE_ALIGN | SLAB_PANIC); + } INIT_LIST_HEAD(&net->xfrm.state_all);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index f380902..a713849 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c
@@ -814,9 +814,10 @@ static inline unsigned int xfrm_user_sec_ctx_size(struct xfrm_sec_ctx *xfrm_ctx) static void copy_from_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p) { + struct xfrm_sub_state *sx = x->sx; memcpy(&x->id, &p->id, sizeof(x->id)); memcpy(&x->sel, &p->sel, sizeof(x->sel)); - memcpy(&x->lft, &p->lft, sizeof(x->lft)); + memcpy(&sx->lft, &p->lft, sizeof(sx->lft)); x->props.mode = p->mode; x->props.replay_window = min_t(unsigned int, p->replay_window, sizeof(x->replay.bitmap) * 8); @@ -844,12 +845,12 @@ static void xfrm_update_ae_params(struct xfrm_state *x, struct nlattr **attrs, struct nlattr *rt = attrs[XFRMA_REPLAY_THRESH]; struct nlattr *mt = attrs[XFRMA_MTIMER_THRESH]; - if (re && x->replay_esn && x->preplay_esn) { + if (re && x->sx->replay_esn && x->sx->preplay_esn) { struct xfrm_replay_state_esn *replay_esn; replay_esn = nla_data(re); - memcpy(x->replay_esn, replay_esn, + memcpy(x->sx->replay_esn, replay_esn, xfrm_replay_state_esn_len(replay_esn)); - memcpy(x->preplay_esn, replay_esn, + memcpy(x->sx->preplay_esn, replay_esn, xfrm_replay_state_esn_len(replay_esn)); } @@ -863,17 +864,17 @@ static void xfrm_update_ae_params(struct xfrm_state *x, struct nlattr **attrs, if (lt) { struct xfrm_lifetime_cur *ltime; ltime = nla_data(lt); - x->curlft.bytes = ltime->bytes; - x->curlft.packets = ltime->packets; - x->curlft.add_time = ltime->add_time; - x->curlft.use_time = ltime->use_time; + x->sx->curlft.bytes = ltime->bytes; + x->sx->curlft.packets = ltime->packets; + x->sx->curlft.add_time = ltime->add_time; + x->sx->curlft.use_time = ltime->use_time; } if (et) - x->replay_maxage = nla_get_u32(et); + x->sx->replay_maxage = nla_get_u32(et); if (rt) - x->replay_maxdiff = nla_get_u32(rt); + x->sx->replay_maxdiff = nla_get_u32(rt); if (mt) x->mapping_maxage = nla_get_u32(mt); @@ -975,14 +976,14 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, goto error; } - if ((err = xfrm_alloc_replay_state_esn(&x->replay_esn, &x->preplay_esn, + if ((err = xfrm_alloc_replay_state_esn(&x->sx->replay_esn, &x->sx->preplay_esn, attrs[XFRMA_REPLAY_ESN_VAL]))) goto error; x->km.seq = p->seq; - x->replay_maxdiff = net->xfrm.sysctl_aevent_rseqth; + x->sx->replay_maxdiff = net->xfrm.sysctl_aevent_rseqth; /* sysctl_xfrm_aevent_etime is in 100ms units */ - x->replay_maxage = (net->xfrm.sysctl_aevent_etime*HZ)/XFRM_AE_ETH_M; + x->sx->replay_maxage = (net->xfrm.sysctl_aevent_etime*HZ)/XFRM_AE_ETH_M; if ((err = xfrm_init_replay(x, extack))) goto error; @@ -1134,16 +1135,17 @@ static int xfrm_del_sa(struct sk_buff *skb, struct nlmsghdr *nlh, static void copy_to_user_state(struct xfrm_state *x, struct xfrm_usersa_info *p) { + struct xfrm_sub_state *sx = x->sx; memset(p, 0, sizeof(*p)); memcpy(&p->id, &x->id, sizeof(p->id)); memcpy(&p->sel, &x->sel, sizeof(p->sel)); - memcpy(&p->lft, &x->lft, sizeof(p->lft)); + memcpy(&p->lft, &sx->lft, sizeof(p->lft)); if (x->xso.dev) xfrm_dev_state_update_stats(x); - memcpy(&p->curlft, &x->curlft, sizeof(p->curlft)); - put_unaligned(x->stats.replay_window, &p->stats.replay_window); - put_unaligned(x->stats.replay, &p->stats.replay); - put_unaligned(x->stats.integrity_failed, &p->stats.integrity_failed); + memcpy(&p->curlft, &sx->curlft, sizeof(p->curlft)); + put_unaligned(sx->stats.replay_window, &p->stats.replay_window); + put_unaligned(sx->stats.replay, &p->stats.replay); + put_unaligned(sx->stats.integrity_failed, &p->stats.integrity_failed); memcpy(&p->saddr, &x->props.saddr, sizeof(p->saddr)); p->mode = x->props.mode; p->replay_window = x->props.replay_window; @@ -1354,8 +1356,8 @@ static int copy_to_user_state_extra(struct xfrm_state *x, if (ret) goto out; } - if (x->lastused) { - ret = nla_put_u64_64bit(skb, XFRMA_LASTUSED, x->lastused, + if (x->sx->lastused) { + ret = nla_put_u64_64bit(skb, XFRMA_LASTUSED, x->sx->lastused, XFRMA_PAD); if (ret) goto out; @@ -1398,10 +1400,10 @@ static int copy_to_user_state_extra(struct xfrm_state *x, if (ret) goto out; - if (x->replay_esn) + if (x->sx->replay_esn) ret = nla_put(skb, XFRMA_REPLAY_ESN_VAL, - xfrm_replay_state_esn_len(x->replay_esn), - x->replay_esn); + xfrm_replay_state_esn_len(x->sx->replay_esn), + x->sx->replay_esn); else ret = nla_put(skb, XFRMA_REPLAY_VAL, sizeof(x->replay), &x->replay); @@ -2667,8 +2669,8 @@ static int xfrm_flush_sa(struct sk_buff *skb, struct nlmsghdr *nlh, static inline unsigned int xfrm_aevent_msgsize(struct xfrm_state *x) { - unsigned int replay_size = x->replay_esn ? - xfrm_replay_state_esn_len(x->replay_esn) : + unsigned int replay_size = x->sx->replay_esn ? + xfrm_replay_state_esn_len(x->sx->replay_esn) : sizeof(struct xfrm_replay_state); return NLMSG_ALIGN(sizeof(struct xfrm_aevent_id)) @@ -2684,6 +2686,7 @@ static inline unsigned int xfrm_aevent_msgsize(struct xfrm_state *x) static int build_aevent(struct sk_buff *skb, struct xfrm_state *x, const struct km_event *c) { + struct xfrm_sub_state *sx = x->sx; struct xfrm_aevent_id *id; struct nlmsghdr *nlh; int err; @@ -2702,29 +2705,29 @@ static int build_aevent(struct sk_buff *skb, struct xfrm_state *x, const struct id->reqid = x->props.reqid; id->flags = c->data.aevent; - if (x->replay_esn) { + if (sx->replay_esn) { err = nla_put(skb, XFRMA_REPLAY_ESN_VAL, - xfrm_replay_state_esn_len(x->replay_esn), - x->replay_esn); + xfrm_replay_state_esn_len(sx->replay_esn), + sx->replay_esn); } else { err = nla_put(skb, XFRMA_REPLAY_VAL, sizeof(x->replay), &x->replay); } if (err) goto out_cancel; - err = nla_put_64bit(skb, XFRMA_LTIME_VAL, sizeof(x->curlft), &x->curlft, + err = nla_put_64bit(skb, XFRMA_LTIME_VAL, sizeof(sx->curlft), &sx->curlft, XFRMA_PAD); if (err) goto out_cancel; if (id->flags & XFRM_AE_RTHR) { - err = nla_put_u32(skb, XFRMA_REPLAY_THRESH, x->replay_maxdiff); + err = nla_put_u32(skb, XFRMA_REPLAY_THRESH, sx->replay_maxdiff); if (err) goto out_cancel; } if (id->flags & XFRM_AE_ETHR) { err = nla_put_u32(skb, XFRMA_ETIMER_THRESH, - x->replay_maxage * 10 / HZ); + sx->replay_maxage * 10 / HZ); if (err) goto out_cancel; } @@ -2785,21 +2788,21 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, * gets lock (the concern is things getting updated * while we are still reading) - jhs */ - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); c.data.aevent = p->flags; c.seq = nlh->nlmsg_seq; c.portid = nlh->nlmsg_pid; err = build_aevent(r_skb, x, &c); if (err < 0) { - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); xfrm_state_put(x); kfree_skb(r_skb); return err; } err = nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, NETLINK_CB(skb).portid); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); xfrm_state_put(x); return err; } @@ -2842,13 +2845,13 @@ static int xfrm_new_ae(struct sk_buff *skb, struct nlmsghdr *nlh, goto out; } - err = xfrm_replay_verify_len(x->replay_esn, re, extack); + err = xfrm_replay_verify_len(x->sx->replay_esn, re, extack); if (err) goto out; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); xfrm_update_ae_params(x, attrs, 1); - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); c.event = nlh->nlmsg_type; c.seq = nlh->nlmsg_seq; @@ -2975,7 +2978,7 @@ static int xfrm_add_sa_expire(struct sk_buff *skb, struct nlmsghdr *nlh, if (x == NULL) return err; - spin_lock_bh(&x->lock); + spin_lock_bh(&x->sx->lock); err = -EINVAL; if (x->km.state != XFRM_STATE_VALID) { NL_SET_ERR_MSG(extack, "SA must be in VALID state"); @@ -2990,7 +2993,7 @@ static int xfrm_add_sa_expire(struct sk_buff *skb, struct nlmsghdr *nlh, } err = 0; out: - spin_unlock_bh(&x->lock); + spin_unlock_bh(&x->sx->lock); xfrm_state_put(x); return err; } @@ -3673,8 +3676,8 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) l += nla_total_size(sizeof(*x->encap)); if (x->tfcpad) l += nla_total_size(sizeof(x->tfcpad)); - if (x->replay_esn) - l += nla_total_size(xfrm_replay_state_esn_len(x->replay_esn)); + if (x->sx->replay_esn) + l += nla_total_size(xfrm_replay_state_esn_len(x->sx->replay_esn)); else l += nla_total_size(sizeof(struct xfrm_replay_state)); if (x->security) @@ -3695,7 +3698,7 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) if (x->pcpu_num != UINT_MAX) l += nla_total_size(sizeof(x->pcpu_num)); - /* Must count x->lastused as it may become non-zero behind our back. */ + /* Must count x->sx->lastused as it may become non-zero behind our back. */ l += nla_total_size_64bit(sizeof(u64)); if (x->mapping_maxage)