| From c967674f8a3b5775a86005f823c612e93a242b53 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Wed, 23 Jun 2021 14:01:28 +0200 |
| Subject: x86/fpu: Fix copy_xstate_to_kernel() gap handling |
| |
| From: Thomas Gleixner <tglx@linutronix.de> |
| |
| [ Upstream commit 9625895011d130033d1bc7aac0d77a9bf68ff8a6 ] |
| |
| The gap handling in copy_xstate_to_kernel() is wrong when XSAVES is in |
| use. |
| |
| Using init_fpstate for copying the init state of features which are |
| not set in the xstate header is only correct for the legacy area, but |
| not for the extended features area because when XSAVES is in use then |
| init_fpstate is in compacted form which means the xstate offsets which |
| are used to copy from init_fpstate are not valid. |
| |
| Fortunately, this is not a real problem today because all extended |
| features in use have an all-zeros init state, but it is wrong |
| nevertheless and with a potentially dynamically sized init_fpstate this |
| would result in an access outside of the init_fpstate. |
| |
| Fix this by keeping track of the last copied state in the target buffer and |
| explicitly zero it when there is a feature or alignment gap. |
| |
| Use the compacted offset when accessing the extended feature space in |
| init_fpstate. |
| |
| As this is not a functional issue on older kernels this is intentionally |
| not tagged for stable. |
| |
| Fixes: b8be15d58806 ("x86/fpu/xstate: Re-enable XSAVES") |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Signed-off-by: Borislav Petkov <bp@suse.de> |
| Reviewed-by: Borislav Petkov <bp@suse.de> |
| Link: https://lkml.kernel.org/r/20210623121451.294282032@linutronix.de |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| arch/x86/kernel/fpu/xstate.c | 105 ++++++++++++++++++++--------------- |
| 1 file changed, 61 insertions(+), 44 deletions(-) |
| |
| diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c |
| index 451435d7ff41..038a119114c4 100644 |
| --- a/arch/x86/kernel/fpu/xstate.c |
| +++ b/arch/x86/kernel/fpu/xstate.c |
| @@ -1084,20 +1084,10 @@ static inline bool xfeatures_mxcsr_quirk(u64 xfeatures) |
| return true; |
| } |
| |
| -static void fill_gap(struct membuf *to, unsigned *last, unsigned offset) |
| +static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, |
| + void *init_xstate, unsigned int size) |
| { |
| - if (*last >= offset) |
| - return; |
| - membuf_write(to, (void *)&init_fpstate.xsave + *last, offset - *last); |
| - *last = offset; |
| -} |
| - |
| -static void copy_part(struct membuf *to, unsigned *last, unsigned offset, |
| - unsigned size, void *from) |
| -{ |
| - fill_gap(to, last, offset); |
| - membuf_write(to, from, size); |
| - *last = offset + size; |
| + membuf_write(to, from_xstate ? xstate : init_xstate, size); |
| } |
| |
| /* |
| @@ -1109,10 +1099,10 @@ static void copy_part(struct membuf *to, unsigned *last, unsigned offset, |
| */ |
| void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave) |
| { |
| + const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); |
| + struct xregs_state *xinit = &init_fpstate.xsave; |
| struct xstate_header header; |
| - const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr); |
| - unsigned size = to.left; |
| - unsigned last = 0; |
| + unsigned int zerofrom; |
| int i; |
| |
| /* |
| @@ -1122,41 +1112,68 @@ void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave) |
| header.xfeatures = xsave->header.xfeatures; |
| header.xfeatures &= xfeatures_mask_user(); |
| |
| - if (header.xfeatures & XFEATURE_MASK_FP) |
| - copy_part(&to, &last, 0, off_mxcsr, &xsave->i387); |
| - if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM)) |
| - copy_part(&to, &last, off_mxcsr, |
| - MXCSR_AND_FLAGS_SIZE, &xsave->i387.mxcsr); |
| - if (header.xfeatures & XFEATURE_MASK_FP) |
| - copy_part(&to, &last, offsetof(struct fxregs_state, st_space), |
| - 128, &xsave->i387.st_space); |
| - if (header.xfeatures & XFEATURE_MASK_SSE) |
| - copy_part(&to, &last, xstate_offsets[XFEATURE_SSE], |
| - 256, &xsave->i387.xmm_space); |
| - /* |
| - * Fill xsave->i387.sw_reserved value for ptrace frame: |
| - */ |
| - copy_part(&to, &last, offsetof(struct fxregs_state, sw_reserved), |
| - 48, xstate_fx_sw_bytes); |
| - /* |
| - * Copy xregs_state->header: |
| - */ |
| - copy_part(&to, &last, offsetof(struct xregs_state, header), |
| - sizeof(header), &header); |
| + /* Copy FP state up to MXCSR */ |
| + copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, |
| + &xinit->i387, off_mxcsr); |
| + |
| + /* Copy MXCSR when SSE or YMM are set in the feature mask */ |
| + copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), |
| + &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, |
| + MXCSR_AND_FLAGS_SIZE); |
| + |
| + /* Copy the remaining FP state */ |
| + copy_feature(header.xfeatures & XFEATURE_MASK_FP, |
| + &to, &xsave->i387.st_space, &xinit->i387.st_space, |
| + sizeof(xsave->i387.st_space)); |
| + |
| + /* Copy the SSE state - shared with YMM, but independently managed */ |
| + copy_feature(header.xfeatures & XFEATURE_MASK_SSE, |
| + &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, |
| + sizeof(xsave->i387.xmm_space)); |
| + |
| + /* Zero the padding area */ |
| + membuf_zero(&to, sizeof(xsave->i387.padding)); |
| + |
| + /* Copy xsave->i387.sw_reserved */ |
| + membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); |
| + |
| + /* Copy the user space relevant state of @xsave->header */ |
| + membuf_write(&to, &header, sizeof(header)); |
| + |
| + zerofrom = offsetof(struct xregs_state, extended_state_area); |
| |
| for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { |
| /* |
| - * Copy only in-use xstates: |
| + * The ptrace buffer is in non-compacted XSAVE format. |
| + * In non-compacted format disabled features still occupy |
| + * state space, but there is no state to copy from in the |
| + * compacted init_fpstate. The gap tracking will zero this |
| + * later. |
| */ |
| - if ((header.xfeatures >> i) & 1) { |
| - void *src = __raw_xsave_addr(xsave, i); |
| + if (!(xfeatures_mask_user() & BIT_ULL(i))) |
| + continue; |
| |
| - copy_part(&to, &last, xstate_offsets[i], |
| - xstate_sizes[i], src); |
| - } |
| + /* |
| + * If there was a feature or alignment gap, zero the space |
| + * in the destination buffer. |
| + */ |
| + if (zerofrom < xstate_offsets[i]) |
| + membuf_zero(&to, xstate_offsets[i] - zerofrom); |
| + |
| + copy_feature(header.xfeatures & BIT_ULL(i), &to, |
| + __raw_xsave_addr(xsave, i), |
| + __raw_xsave_addr(xinit, i), |
| + xstate_sizes[i]); |
| |
| + /* |
| + * Keep track of the last copied state in the non-compacted |
| + * target buffer for gap zeroing. |
| + */ |
| + zerofrom = xstate_offsets[i] + xstate_sizes[i]; |
| } |
| - fill_gap(&to, &last, size); |
| + |
| + if (to.left) |
| + membuf_zero(&to, to.left); |
| } |
| |
| /* |
| -- |
| 2.30.2 |
| |