Merge branch into tip/master: 'x86/tdx'
# New commits in x86/tdx:
6712564c884d ("x86/virt/tdx: Enable TDX module runtime updates")
73be1bb72f4c ("x86/virt/tdx: Refresh TDX module version after update")
bd0ba697612a ("coco/tdx-host: Lock out module updates when reading version")
eb71a4c94061 ("x86/virt/seamldr: Add module update locking")
069be08012cf ("x86/virt/tdx: Restore TDX module state")
f74245e39c21 ("x86/virt/seamldr: Initialize the newly-installed TDX module")
d909333bf655 ("x86/virt/seamldr: Install a new TDX module")
522bacc2fbac ("x86/virt/tdx: Reset software states during TDX module shutdown")
146ac22b2b96 ("x86/virt/seamldr: Shut down the current TDX module")
c507e80de947 ("x86/virt/seamldr: Abort updates after a failed step")
e16ce07a9053 ("x86/virt/seamldr: Introduce skeleton for TDX module updates")
35621312a061 ("x86/virt/seamldr: Allocate and populate a module update request")
000c293c24bc ("coco/tdx-host: Implement firmware upload sysfs ABI for TDX module updates")
56b46fe202f8 ("coco/tdx-host: Don't expose P-SEAMLDR information on CPUs with erratum")
b094b1684fef ("coco/tdx-host: Expose P-SEAMLDR information via sysfs")
fcbc30f0d66f ("x86/virt/seamldr: Add a helper to retrieve P-SEAMLDR information")
b434b916fed3 ("x86/virt/seamldr: Introduce a wrapper for P-SEAMLDR SEAMCALLs")
e4afd39aefd8 ("coco/tdx-host: Expose TDX module version")
c6a2ea2cfa6a ("coco/tdx-host: Introduce a "tdx_host" device")
0a7808c1b5ff ("x86/virt/tdx: Move low level SEAMCALL helpers out of <asm/tdx.h>")
2818e8c8a46d ("x86/virt/tdx: Move TDX_FEATURES0 bits to asm/tdx.h")
332d5758bbad ("x86/virt/tdx: Consolidate TDX global initialization states")
2f410fa074fb ("x86/virt/tdx: Move TDX global initialization states to file scope")
394d7f52d844 ("x86/virt/tdx: Clarify try_init_module_global() result caching")
5209e5bfe5ca ("x86/virt/tdx: Remove kexec docs")
5b25f249be32 ("x86/tdx: Disable the TDX module during kexec and kdump")
b7d2173946ef ("x86/virt/tdx: Add SEAMCALL wrapper for TDH.SYS.DISABLE")
597bdf6e068e ("x86/virt/tdx: Pull kexec cache flush logic into arch/x86")
53642715861e ("x86/tdx: Move TDX architectural error codes into <asm/shared/tdx_errno.h>")
Signed-off-by: Ingo Molnar <mingo@kernel.org>
diff --git a/.mailmap b/.mailmap
index 34acd34..a009f73d 100644
--- a/.mailmap
+++ b/.mailmap
@@ -19,6 +19,7 @@
Ahmad Masri <quic_amasri@quicinc.com> <amasri@codeaurora.org>
Adam Oldham <oldhamca@gmail.com>
Adam Radford <aradford@gmail.com>
+Aditya Garg <gargaditya08@proton.me> <gargaditya08@live.com>
Adriana Reus <adi.reus@gmail.com> <adriana.reus@intel.com>
Adrian Bunk <bunk@stusta.de>
Ajay Kaher <ajay.kaher@broadcom.com> <akaher@vmware.com>
@@ -207,6 +208,7 @@
Colin Ian King <colin.i.king@gmail.com> <colin.king@canonical.com>
Corey Minyard <minyard@acm.org>
Damian Hobson-Garcia <dhobsong@igel.co.jp>
+Dan Carpenter <error27@gmail.com> <dan.carpenter@linaro.org>
Dan Carpenter <error27@gmail.com> <dan.carpenter@oracle.com>
Dan Williams <djbw@kernel.org> <dan.j.williams@intel.com>
Daniel Borkmann <daniel@iogearbox.net> <danborkmann@googlemail.com>
@@ -261,8 +263,9 @@
Enric Balletbo i Serra <eballetbo@kernel.org> <eballetbo@iseebcn.com>
Erik Kaneda <erik.kaneda@intel.com> <erik.schmauss@intel.com>
Ethan Carter Edwards <ethan@ethancedwards.com> Ethan Edwards <ethancarteredwards@gmail.com>
-Eugen Hristev <eugen.hristev@linaro.org> <eugen.hristev@microchip.com>
-Eugen Hristev <eugen.hristev@linaro.org> <eugen.hristev@collabora.com>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@microchip.com>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@linaro.org>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@collabora.com>
Evgeniy Polyakov <johnpol@2ka.mipt.ru>
Ezequiel Garcia <ezequiel@vanguardiasur.com.ar> <ezequiel@collabora.com>
Faith Ekstrand <faith.ekstrand@collabora.com> <jason@jlekstrand.net>
@@ -337,6 +340,7 @@
Herbert Xu <herbert@gondor.apana.org.au>
Huacai Chen <chenhuacai@kernel.org> <chenhc@lemote.com>
Huacai Chen <chenhuacai@kernel.org> <chenhuacai@loongson.cn>
+Ian Ray <ian.ray@gehealthcare.com> <ian.ray@ge.com>
Ignat Korchagin <ignat@linux.win> <ignat@cloudflare.com>
Igor Korotin <igor.korotin@linux.dev> <igor.korotin.linux@gmail.com>
Ike Panhc <ikepanhc@gmail.com> <ike.pan@canonical.com>
@@ -495,6 +499,7 @@
Leon Romanovsky <leon@kernel.org> <leonro@mellanox.com>
Leon Romanovsky <leon@kernel.org> <leonro@nvidia.com>
Leo Yan <leo.yan@linux.dev> <leo.yan@linaro.org>
+Liam R. Howlett <liam@infradead.org> <Liam.Howlett@oracle.com>
Liam Mark <quic_lmark@quicinc.com> <lmark@codeaurora.org>
Linas Vepstas <linas@austin.ibm.com>
Linus Lüssing <linus.luessing@c0d3.blue> <linus.luessing@ascom.ch>
@@ -505,6 +510,8 @@
Linus Walleij <linusw@kernel.org> <linus.walleij@linaro.org>
Linus Walleij <linusw@kernel.org> <triad@df.lth.se>
<linux-hardening@vger.kernel.org> <kernel-hardening@lists.openwall.com>
+Li Wang <li.wang@linux.dev> <liwang@redhat.com>
+Li Wang <li.wang@linux.dev> <wangli.ahau@gmail.com>
Li Yang <leoyang.li@nxp.com> <leoli@freescale.com>
Li Yang <leoyang.li@nxp.com> <leo@zh-kernel.org>
Lior David <quic_liord@quicinc.com> <liord@codeaurora.org>
@@ -579,6 +586,8 @@
Md Sadre Alam <quic_mdalam@quicinc.com> <mdalam@codeaurora.org>
Miaoqing Pan <quic_miaoqing@quicinc.com> <miaoqing@codeaurora.org>
Michael Buesch <m@bues.ch>
+Michal Grzeschik <mgr@kernel.org> <m.grzeschik@pengutronix.de>
+Michal Grzeschik <mgr@kernel.org> <mgr@pengutronix.de>
Michael Riesch <michael.riesch@collabora.com> <michael.riesch@wolfvision.net>
Michal Simek <michal.simek@amd.com> <michal.simek@xilinx.com>
Michel Dänzer <michel@tungstengraphics.com>
@@ -677,6 +686,7 @@
Peter Hilber <peter.hilber@oss.qualcomm.com> <quic_philber@quicinc.com>
Peter Oruba <peter.oruba@amd.com>
Peter Oruba <peter@oruba.de>
+Peter Rosin <peda@lysator.liu.se> <peda@axentia.se>
Pierre-Louis Bossart <pierre-louis.bossart@linux.dev> <pierre-louis.bossart@linux.intel.com>
Pratyush Anand <pratyush.anand@gmail.com> <pratyush.anand@st.com>
Pratyush Yadav <pratyush@kernel.org> <ptyadav@amazon.de>
@@ -687,6 +697,7 @@
Puranjay Mohan <puranjay@kernel.org> <puranjay12@gmail.com>
Qais Yousef <qyousef@layalina.io> <qais.yousef@imgtec.com>
Qais Yousef <qyousef@layalina.io> <qais.yousef@arm.com>
+Qi Zheng <qi.zheng@linux.dev> <zhengqi.arch@bytedance.com>
Quentin Monnet <qmo@kernel.org> <quentin.monnet@netronome.com>
Quentin Monnet <qmo@kernel.org> <quentin@isovalent.com>
Quentin Perret <qperret@qperret.net> <quentin.perret@arm.com>
@@ -850,6 +861,7 @@
Tobias Klauser <tklauser@distanz.ch> <tklauser@nuerscht.ch>
Tobias Klauser <tklauser@distanz.ch> <tklauser@xenon.tklauser.home>
Todor Tomov <todor.too@gmail.com> <todor.tomov@linaro.org>
+Tomasz Jeznach <tomasz.jeznach@linux.dev> <tjeznach@rivosinc.com>
Tony Luck <tony.luck@intel.com>
Trilok Soni <quic_tsoni@quicinc.com> <tsoni@codeaurora.org>
TripleX Chung <xxx.phy@gmail.com> <triplex@zh-kernel.org>
diff --git a/CREDITS b/CREDITS
index 17962bd..42a7908 100644
--- a/CREDITS
+++ b/CREDITS
@@ -2804,6 +2804,7 @@
N: William (Bill) Metzenthen
E: billm@suburbia.net
+E: billm@melbpc.org.au
D: Author of the FPU emulator.
D: Minor kernel hacker for other lost causes (Hercules mono, etc).
S: 22 Parker Street
diff --git a/Documentation/.renames.txt b/Documentation/.renames.txt
index 43d4475..aa7e5aa 100644
--- a/Documentation/.renames.txt
+++ b/Documentation/.renames.txt
@@ -786,6 +786,7 @@
networking/bpf_flow_dissector bpf/prog_flow_dissector
networking/cxacru networking/device_drivers/atm/cxacru
networking/defza networking/device_drivers/fddi/defza
+networking/device_drivers/3com/3c509 networking/device_drivers/ethernet/3com/3c509
networking/device_drivers/3com/vortex networking/device_drivers/ethernet/3com/vortex
networking/device_drivers/amazon/ena networking/device_drivers/ethernet/amazon/ena
networking/device_drivers/aquantia/atlantic networking/device_drivers/ethernet/aquantia/atlantic
diff --git a/Documentation/ABI/obsolete/sysfs-selinux-user b/Documentation/ABI/removed/sysfs-selinux-user
similarity index 100%
rename from Documentation/ABI/obsolete/sysfs-selinux-user
rename to Documentation/ABI/removed/sysfs-selinux-user
diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst
index 9f8e273..7c7cd45 100644
--- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst
+++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst
@@ -47,21 +47,19 @@
Called when swp_entry's refcnt goes down to 0. A charge against swap
disappears.
-3. charge-commit-cancel
+3. charge-commit
=======================
Memcg pages are charged in two steps:
- mem_cgroup_try_charge()
- - mem_cgroup_commit_charge() or mem_cgroup_cancel_charge()
+ - commit_charge()
At try_charge(), there are no flags to say "this page is charged".
at this point, usage += PAGE_SIZE.
At commit(), the page is associated with the memcg.
- At cancel(), simply usage -= PAGE_SIZE.
-
Under below explanation, we assume CONFIG_SWAP=y.
4. Anonymous
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 8ad0b27..6efd009 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -220,7 +220,7 @@
memory_hugetlb_accounting
Count HugeTLB memory usage towards the cgroup's overall
memory usage for the memory controller (for the purpose of
- statistics reporting and memory protetion). This is a new
+ statistics reporting and memory protection). This is a new
behavior that could regress existing setups, so it must be
explicitly opted in with this mount option.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 4d0f545..91a18d1 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -789,24 +789,6 @@
cio_ignore= [S390]
See Documentation/arch/s390/common_io.rst for details.
- clearcpuid=X[,X...] [X86]
- Disable CPUID feature X for the kernel. See
- arch/x86/include/asm/cpufeatures.h for the valid bit
- numbers X. Note the Linux-specific bits are not necessarily
- stable over kernel options, but the vendor-specific
- ones should be.
- X can also be a string as appearing in the flags: line
- in /proc/cpuinfo which does not have the above
- instability issue. However, not all features have names
- in /proc/cpuinfo.
- Note that using this option will taint your kernel.
- Also note that user programs calling CPUID directly
- or using the feature without checking anything
- will still see it. This just prevents it from
- being used by the kernel or shown in /proc/cpuinfo.
- Also note the kernel might malfunction if you disable
- some critical bits.
-
clk_ignore_unused
[CLK]
Prevents the clock framework from automatically gating
@@ -4437,10 +4419,6 @@
These settings can be accessed at runtime via
the nmi_watchdog and hardlockup_panic sysctls.
- no387 [BUGS=X86-32] Tells the kernel to use the 387 maths
- emulation library even if a 387 maths coprocessor
- is present.
-
no4lvl [RISCV,EARLY] Disable 4-level and 5-level paging modes.
Forces kernel to use 3-level paging instead.
@@ -7236,6 +7214,18 @@
Not specifying this option is equivalent to
spec_store_bypass_disable=auto.
+ split_llc=
+ [X86,EARLY] Split the LLC N-ways
+
+ When set, the LLC is split this many ways by matching
+ 'core_id % n'. This is setup before SMP bringup and
+ used during SMP bringup before it knows the full
+ topology. If your core count doesn't nicely divide by
+ the number given, you get to keep the pieces.
+
+ This is mostly a debug feature to emulate multiple LLCs
+ on hardware that only have a single LLC.
+
split_lock_detect=
[X86] Enable split lock detection or bus lock detection
diff --git a/Documentation/admin-guide/laptops/uniwill-laptop.rst b/Documentation/admin-guide/laptops/uniwill-laptop.rst
index 5613348..1f3ca84 100644
--- a/Documentation/admin-guide/laptops/uniwill-laptop.rst
+++ b/Documentation/admin-guide/laptops/uniwill-laptop.rst
@@ -43,6 +43,11 @@
Battery Charging Control
------------------------
+.. warning:: Some devices do not properly implement the charging threshold interface. Forcing
+ the driver to enable access to said interface on such devices might damage the
+ battery [1]_. Because of this the driver will not enable said feature even when
+ using the ``force`` module parameter.
+
The ``uniwill-laptop`` driver supports controlling the battery charge limit. This happens over
the standard ``charge_control_end_threshold`` power supply sysfs attribute. All values
between 1 and 100 percent are supported.
@@ -70,3 +75,8 @@
allow it.
See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details.
+
+References
+==========
+
+.. [1] https://www.reddit.com/r/XMG_gg/comments/ld9yyf/battery_limit_hidden_function_discovered_on/
diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst
index f8e7050..a95e2eb 100644
--- a/Documentation/admin-guide/pm/amd-pstate.rst
+++ b/Documentation/admin-guide/pm/amd-pstate.rst
@@ -358,9 +358,9 @@
The amd-pstate driver supports dynamically selecting the energy performance
profile based on whether the machine is running on AC or DC power.
-Whether this behavior is enabled by default depends on the kernel
-config option `CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`. This behavior can also be overridden
-at runtime by the sysfs file ``/sys/devices/system/cpu/cpufreq/policyX/dynamic_epp``.
+Whether this behavior is enabled by default depends on the kernel command line option
+``amd_dynamic_epp`` is set. This behavior can also be overridden
+at runtime by the sysfs file ``/sys/devices/system/cpu/amd_pstate/dynamic_epp``.
When set to enabled, the driver will select a different energy performance
profile when the machine is running on battery or AC power. The driver will
@@ -485,9 +485,8 @@
``amd_dynamic_epp``
When AMD pstate is in auto mode, dynamic EPP will control whether the kernel
-autonomously changes the EPP mode. The default is configured by
-``CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`` but can be explicitly enabled with
-``amd_dynamic_epp=enable`` or disabled with ``amd_dynamic_epp=disable``.
+autonomously changes the EPP mode. The default is disabled. It can be enabled
+with the kernel parameter ``amd_dynamic_epp=enable``.
User Space Interface in ``sysfs`` - General
===========================================
diff --git a/Documentation/admin-guide/pm/intel_pstate.rst b/Documentation/admin-guide/pm/intel_pstate.rst
index fde967b..25fe5d8 100644
--- a/Documentation/admin-guide/pm/intel_pstate.rst
+++ b/Documentation/admin-guide/pm/intel_pstate.rst
@@ -355,11 +355,12 @@
one core, ``intel_pstate`` assigns performance-based priorities to CPUs. Namely,
the priority of a given CPU reflects its highest HWP performance level which
causes the CPU scheduler to generally prefer more performant CPUs, so the less
-performant CPUs are used when the other ones are fully loaded. However, SMT
-siblings (that is, logical CPUs sharing one physical core) are treated in a
-special way such that if one of them is in use, the effective priority of the
-other ones is lowered below the priorities of the CPUs located in the other
-physical cores.
+performant CPUs are used when the other ones are fully loaded. SMT siblings
+(that is, logical CPUs sharing one physical core) are given the same priority.
+The scheduler can pull tasks from lower-priority cores and place them on any
+sibling. Since the scheduler spreads tasks among physical cores, tasks will be
+placed on the SMT siblings of physical cores only after all physical cores are
+busy.
This approach maximizes performance in the majority of cases, but unfortunately
it also leads to excessive energy usage in some important scenarios, like video
diff --git a/Documentation/arch/loongarch/irq-chip-model.rst b/Documentation/arch/loongarch/irq-chip-model.rst
index 8f5c334..774d40d 100644
--- a/Documentation/arch/loongarch/irq-chip-model.rst
+++ b/Documentation/arch/loongarch/irq-chip-model.rst
@@ -181,6 +181,41 @@
| Devices |
+---------+
+Advanced Extended IRQ model (with redirection)
+==============================================
+
+In this model, IPI (Inter-Processor Interrupt) and CPU Local Timer interrupt go
+to CPUINTC directly, CPU UARTS interrupts go to LIOINTC, PCH-MSI interrupts go
+to REDIRECT for remapping it to AVECINTC, and then go to CPUINTC directly, while
+all other devices interrupts go to PCH-PIC/PCH-LPC and gathered by EIOINTC, and
+then go to CPUINTC directly::
+
+ +-----+ +-----------------------+ +-------+
+ | IPI | --> | CPUINTC | <-- | Timer |
+ +-----+ +-----------------------+ +-------+
+ ^ ^ ^
+ | | |
+ | +----------+ |
+ +---------+ | AVECINTC | +---------+ +-------+
+ | EIOINTC | +----------+ | LIOINTC | <-- | UARTs |
+ +---------+ | REDIRECT | +---------+ +-------+
+ ^ +----------+
+ | ^
+ | |
+ +---------+ +---------+
+ | PCH-PIC | | PCH-MSI |
+ +---------+ +---------+
+ ^ ^ ^
+ | | |
+ +---------+ +---------+ +---------+
+ | Devices | | PCH-LPC | | Devices |
+ +---------+ +---------+ +---------+
+ ^
+ |
+ +---------+
+ | Devices |
+ +---------+
+
ACPI-related definitions
========================
diff --git a/Documentation/arch/riscv/cmodx.rst b/Documentation/arch/riscv/cmodx.rst
index 40ba53b..cbfa812 100644
--- a/Documentation/arch/riscv/cmodx.rst
+++ b/Documentation/arch/riscv/cmodx.rst
@@ -21,13 +21,13 @@
enable or disable the redirection. In the case of RISC-V, 2 instructions,
AUIPC + JALR, are required to compose a function call. However, it is impossible
to patch 2 instructions and expect that a concurrent read-side executes them
-without a race condition. This series makes atmoic code patching possible in
+without a race condition. This series makes atomic code patching possible in
RISC-V ftrace. Kernel preemption makes things even worse as it allows the old
state to persist across the patching process with stop_machine().
In order to get rid of stop_machine() and run dynamic ftrace with full kernel
preemption, we partially initialize each patchable function entry at boot-time,
-setting the first instruction to AUIPC, and the second to NOP. Now, atmoic
+setting the first instruction to AUIPC, and the second to NOP. Now, atomic
patching is possible because the kernel only has to update one instruction.
According to Ziccif, as long as an instruction is naturally aligned, the ISA
guarantee an atomic update.
@@ -36,8 +36,8 @@
is limited to +-2K from the predetermined target, ftrace_caller, due to the lack
of immediate encoding space in RISC-V. To address the issue, we introduce
CALL_OPS, where an 8B naturally align metadata is added in front of each
-pacthable function. The metadata is resolved at the first trampoline, then the
-execution can be derect to another custom trampoline.
+patchable function. The metadata is resolved at the first trampoline, then the
+execution can be directed to another custom trampoline.
CMODX in the User Space
-----------------------
diff --git a/Documentation/arch/riscv/zicfilp.rst b/Documentation/arch/riscv/zicfilp.rst
index ab7d8e6..12b35969 100644
--- a/Documentation/arch/riscv/zicfilp.rst
+++ b/Documentation/arch/riscv/zicfilp.rst
@@ -78,7 +78,7 @@
Per-task indirect branch tracking state can be monitored and
controlled via the :c:macro:`PR_GET_CFI` and :c:macro:`PR_SET_CFI`
-``prctl()` arguments (respectively), by supplying
+``prctl()`` arguments (respectively), by supplying
:c:macro:`PR_CFI_BRANCH_LANDING_PADS` as the second argument. These
are architecture-agnostic, and will return -EINVAL if the underlying
functionality is not supported.
diff --git a/Documentation/arch/x86/cpuinfo.rst b/Documentation/arch/x86/cpuinfo.rst
index 9f2e47c..17fce95 100644
--- a/Documentation/arch/x86/cpuinfo.rst
+++ b/Documentation/arch/x86/cpuinfo.rst
@@ -187,6 +187,10 @@
Protection can be disabled using clearcpuid=514. The number 514 is calculated
from #define X86_FEATURE_UMIP (16*32 + 2).
+DO NOT USE this cmdline option in production - it is meant to be used only as
+a quick'n'dirty debugging aid to rule out a feature-enabling code is the
+culprit. If you use it, it'll taint the kernel.
+
In addition, there exists a variety of custom command-line parameters that
disable specific features. The list of parameters includes, but is not limited
to, nofsgsbase, nosgx, noxsave, etc. 5-level paging can also be disabled using
diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst
index 9b4afca..6de26d1 100644
--- a/Documentation/core-api/cpu_hotplug.rst
+++ b/Documentation/core-api/cpu_hotplug.rst
@@ -45,11 +45,6 @@
This option is limited to the X86 and S390 architecture.
-``cpu0_hotplug``
- Allow to shutdown CPU0.
-
- This option is limited to the X86 architecture.
-
CPU maps
========
diff --git a/Documentation/crypto/krb5.rst b/Documentation/crypto/krb5.rst
index beffa01..f62e07a 100644
--- a/Documentation/crypto/krb5.rst
+++ b/Documentation/crypto/krb5.rst
@@ -158,13 +158,22 @@
When a message has been received, the location and size of the data with the
message can be determined by calling::
- void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
- enum krb5_crypto_mode mode,
- size_t *_offset, size_t *_len);
+ int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t *_offset, size_t *_len);
The caller provides the offset and length of the message to the function, which
then alters those values to indicate the region containing the data (plus any
-padding). It is up to the caller to determine how much padding there is.
+padding). It is up to the caller to determine how much padding there is. The
+function returns an error if the length is too small or if the mode is
+unsupported. An additional function::
+
+ int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t len, size_t min_content);
+
+is provided to just do a basic check that the decrypted/verified message would
+have a sufficient minimum payload.
Preparation Functions
---------------------
diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst
index 54d9ee2..8e71e1e 100644
--- a/Documentation/dev-tools/context-analysis.rst
+++ b/Documentation/dev-tools/context-analysis.rst
@@ -17,7 +17,7 @@
CONFIG_WARN_CONTEXT_ANALYSIS=y
-The feature requires Clang 22 or later.
+The feature requires Clang 23 or later.
The analysis is *opt-in by default*, and requires declaring which modules and
subsystems should be analyzed in the respective `Makefile`::
diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
index 876bf90..ccb6b8d 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
@@ -30,6 +30,16 @@
maxItems: 1
clocks:
+ minItems: 1
+ maxItems: 2
+
+ clock-names:
+ minItems: 1
+ items:
+ - const: core
+ - const: iface
+
+ power-domains:
maxItems: 1
operating-points-v2: true
@@ -44,6 +54,25 @@
additionalProperties: false
+allOf:
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,eliza-inline-crypto-engine
+ - qcom,milos-inline-crypto-engine
+
+ then:
+ required:
+ - power-domains
+ - clock-names
+ properties:
+ clocks:
+ minItems: 2
+ clock-names:
+ minItems: 2
+
examples:
- |
#include <dt-bindings/clock/qcom,sm8550-gcc.h>
@@ -52,7 +81,11 @@
compatible = "qcom,sm8550-inline-crypto-engine",
"qcom,inline-crypto-engine";
reg = <0x01d88000 0x8000>;
- clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+ clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
+ <&gcc GCC_UFS_PHY_AHB_CLK>;
+ clock-names = "core",
+ "iface";
+ power-domains = <&gcc UFS_PHY_GDSC>;
operating-points-v2 = <&ice_opp_table>;
diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
index 8239adb..094a638 100644
--- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
@@ -219,6 +219,7 @@
- required:
- "#sound-dai-cells"
else:
+ $ref: /schemas/sound/dai-common.yaml#
properties:
aux-bus: false
required:
@@ -243,7 +244,7 @@
clocks:
minItems: 5
maxItems: 5
- clocks-names:
+ clock-names:
minItems: 5
maxItems: 5
@@ -264,7 +265,7 @@
clocks:
minItems: 5
maxItems: 6
- clocks-names:
+ clock-names:
minItems: 5
maxItems: 6
@@ -277,7 +278,6 @@
- qcom,sc8180x-dp
- qcom,sdm845-dp
- qcom,sm8350-dp
- - qcom,sm8650-dp
then:
properties:
reg:
@@ -286,6 +286,24 @@
clocks:
minItems: 6
maxItems: 6
+ clock-names:
+ minItems: 6
+ maxItems: 6
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,sm8650-dp
+ then:
+ properties:
+ reg:
+ minItems: 5
+ maxItems: 9
+ clocks:
+ minItems: 6
+ maxItems: 6
clocks-names:
minItems: 6
maxItems: 6
@@ -306,7 +324,7 @@
clocks:
minItems: 6
maxItems: 8
- clocks-names:
+ clock-names:
minItems: 6
maxItems: 8
@@ -326,7 +344,7 @@
clocks:
minItems: 5
maxItems: 6
- clocks-names:
+ clock-names:
minItems: 5
maxItems: 6
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
index 47938d1..bd4ba91 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
@@ -119,7 +119,7 @@
mdss_mdp: display-controller@ae01000 {
compatible = "qcom,eliza-dpu";
reg = <0x0ae01000 0x93000>,
- <0x0aeb0000 0x2008>;
+ <0x0aeb0000 0x3000>;
reg-names = "mdp",
"vbif";
@@ -304,7 +304,7 @@
mdss_dsi0_phy: phy@ae95000 {
compatible = "qcom,eliza-dsi-phy-4nm", "qcom,sm8650-dsi-phy-4nm";
reg = <0x0ae95000 0x200>,
- <0x0ae95200 0x280>,
+ <0x0ae95200 0x300>,
<0x0ae95500 0x400>;
reg-names = "dsi_phy",
"dsi_phy_lane",
@@ -388,7 +388,7 @@
mdss_dsi1_phy: phy@ae97000 {
compatible = "qcom,eliza-dsi-phy-4nm", "qcom,sm8650-dsi-phy-4nm";
reg = <0x0ae97000 0x200>,
- <0x0ae97200 0x280>,
+ <0x0ae97200 0x300>,
<0x0ae97500 0x400>;
reg-names = "dsi_phy",
"dsi_phy_lane",
@@ -407,11 +407,15 @@
displayport-controller@af54000 {
compatible = "qcom,eliza-dp", "qcom,sm8650-dp";
- reg = <0xaf54000 0x104>,
- <0xaf54200 0xc0>,
- <0xaf55000 0x770>,
- <0xaf56000 0x9c>,
- <0xaf57000 0x9c>;
+ reg = <0x0af54000 0x200>,
+ <0x0af54200 0x200>,
+ <0x0af55000 0xc00>,
+ <0x0af56000 0x400>,
+ <0x0af57000 0x400>,
+ <0x0af58000 0x400>,
+ <0x0af59000 0x400>,
+ <0x0af5a000 0x600>,
+ <0x0af5b000 0x600>;
interrupts-extended = <&mdss 12>;
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
index dccac52..134321b 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
@@ -70,7 +70,7 @@
display-controller@ae01000 {
compatible = "qcom,sm8650-dpu";
reg = <0x0ae01000 0x8f000>,
- <0x0aeb0000 0x2008>;
+ <0x0aeb0000 0x3000>;
reg-names = "mdp", "vbif";
clocks = <&gcc_axi_clk>,
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
index a1c53e1..0f7f795 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
@@ -112,7 +112,7 @@
display-controller@ae01000 {
compatible = "qcom,sm8650-dpu";
reg = <0x0ae01000 0x8f000>,
- <0x0aeb0000 0x2008>;
+ <0x0aeb0000 0x3000>;
reg-names = "mdp", "vbif";
clocks = <&gcc_axi_clk>,
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
index a38c226..46dc0d2 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
@@ -117,7 +117,7 @@
display-controller@ae01000 {
compatible = "qcom,sm8750-dpu";
reg = <0x0ae01000 0x93000>,
- <0x0aeb0000 0x2008>;
+ <0x0aeb0000 0x3000>;
reg-names = "mdp",
"vbif";
@@ -389,11 +389,15 @@
displayport-controller@af54000 {
compatible = "qcom,sm8750-dp", "qcom,sm8650-dp";
- reg = <0xaf54000 0x104>,
- <0xaf54200 0xc0>,
- <0xaf55000 0x770>,
- <0xaf56000 0x9c>,
- <0xaf57000 0x9c>;
+ reg = <0x0af54000 0x200>,
+ <0x0af54200 0x200>,
+ <0x0af55000 0xc00>,
+ <0x0af56000 0x400>,
+ <0x0af57000 0x400>,
+ <0x0af58000 0x400>,
+ <0x0af59000 0x400>,
+ <0x0af5a000 0x600>,
+ <0x0af5b000 0x600>;
interrupts-extended = <&mdss 12>;
diff --git a/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml b/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml
index c4cc8af..7b59b60 100644
--- a/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml
@@ -16,10 +16,15 @@
properties:
compatible:
- enum:
- - amlogic,meson6-i2c # Meson6, Meson8 and compatible SoCs
- - amlogic,meson-gxbb-i2c # GXBB and compatible SoCs
- - amlogic,meson-axg-i2c # AXG and compatible SoCs
+ oneOf:
+ - items:
+ - enum:
+ - amlogic,t7-i2c
+ - const: amlogic,meson-axg-i2c
+ - enum:
+ - amlogic,meson6-i2c # Meson6, Meson8 and compatible SoCs
+ - amlogic,meson-gxbb-i2c # GXBB and compatible SoCs
+ - amlogic,meson-axg-i2c # AXG and compatible SoCs
reg:
maxItems: 1
diff --git a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
index 500a965..9e59200 100644
--- a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
@@ -22,7 +22,9 @@
compatible:
oneOf:
- items:
- - const: apple,t6020-i2c
+ - enum:
+ - apple,t6020-i2c
+ - apple,t8122-i2c
- const: apple,t8103-i2c
- items:
- enum:
diff --git a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
index d0fad93..d266719 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
@@ -38,6 +38,8 @@
- amlogic,a4-gpio-intc
- amlogic,a4-gpio-ao-intc
- amlogic,a5-gpio-intc
+ - amlogic,a9-gpio-intc
+ - amlogic,a9-gpio-ao-intc
- amlogic,c3-gpio-intc
- amlogic,s6-gpio-intc
- amlogic,s7-gpio-intc
@@ -56,7 +58,7 @@
amlogic,channel-interrupts:
description: Array with the upstream hwirq numbers
minItems: 2
- maxItems: 12
+ maxItems: 20
$ref: /schemas/types.yaml#/definitions/uint32-array
required:
@@ -76,9 +78,20 @@
amlogic,channel-interrupts:
maxItems: 2
else:
- properties:
- amlogic,channel-interrupts:
- minItems: 8
+ if:
+ properties:
+ compatible:
+ contains:
+ const: amlogic,a9-gpio-ao-intc
+ then:
+ properties:
+ amlogic,channel-interrupts:
+ minItems: 20
+ else:
+ properties:
+ amlogic,channel-interrupts:
+ minItems: 8
+ maxItems: 12
additionalProperties: false
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
deleted file mode 100644
index 258d21f..0000000
--- a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/interrupt-controller/aspeed,ast2700-intc.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Aspeed AST2700 Interrupt Controller
-
-description:
- This interrupt controller hardware is second level interrupt controller that
- is hooked to a parent interrupt controller. It's useful to combine multiple
- interrupt sources into 1 interrupt to parent interrupt controller.
-
-maintainers:
- - Kevin Chen <kevin_chen@aspeedtech.com>
-
-properties:
- compatible:
- enum:
- - aspeed,ast2700-intc-ic
-
- reg:
- maxItems: 1
-
- interrupt-controller: true
-
- '#interrupt-cells':
- const: 1
- description:
- The first cell is the IRQ number, the second cell is the trigger
- type as defined in interrupt.txt in this directory.
-
- interrupts:
- minItems: 1
- maxItems: 10
- description: |
- Depend to which INTC0 or INTC1 used.
- INTC0 and INTC1 are two kinds of interrupt controller with enable and raw
- status registers for use.
- INTC0 is used to assert GIC if interrupt in INTC1 asserted.
- INTC1 is used to assert INTC0 if interrupt of modules asserted.
- +-----+ +-------+ +---------+---module0
- | GIC |---| INTC0 |--+--| INTC1_0 |---module2
- | | | | | | |---...
- +-----+ +-------+ | +---------+---module31
- |
- | +---------+---module0
- +---| INTC1_1 |---module2
- | | |---...
- | +---------+---module31
- ...
- | +---------+---module0
- +---| INTC1_5 |---module2
- | |---...
- +---------+---module31
-
-required:
- - compatible
- - reg
- - interrupt-controller
- - '#interrupt-cells'
- - interrupts
-
-additionalProperties: false
-
-examples:
- - |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
-
- bus {
- #address-cells = <2>;
- #size-cells = <2>;
-
- interrupt-controller@12101b00 {
- compatible = "aspeed,ast2700-intc-ic";
- reg = <0 0x12101b00 0 0x10>;
- #interrupt-cells = <1>;
- interrupt-controller;
- interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
- };
- };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml
new file mode 100644
index 0000000..a62f0fd
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/aspeed,ast2700-interrupt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ASPEED AST2700 Interrupt Controllers (INTC0/INTC1)
+
+description: |
+ The ASPEED AST2700 SoC integrates two interrupt controller designs:
+
+ - INTC0: Primary controller that routes interrupt sources to upstream,
+ processor-specific interrupt controllers
+
+ - INTC1: Secondary controller whose interrupt outputs feed into INTC0
+
+ The SoC contains four processors to which interrupts can be routed:
+
+ - PSP: Primary Service Processor (Cortex-A35)
+ - SSP: Secondary Service Processor (Cortex-M4)
+ - TSP: Tertiary Service Processor (Cortex-M4)
+ - BMCU: Boot MCU (a RISC-V microcontroller)
+
+ The following diagram illustrates the overall architecture of the
+ ASPEED AST2700 interrupt controllers:
+
+ +-----------+ +-----------+
+ | INTC0 | | INTC1(0) |
+ +-----------+ +-----------+
+ | Router | +-----------+ | Router |
+ | out int | +Peripheral + | out int |
+ +-----------+ | 0 0 <-+Controllers+ | INTM | +-----------+
+ |PSP GIC <-|---+ . . | +-----------+ | . . <-+Peripheral +
+ +-----------+ | . . | | . . | +Controllers+
+ +-----------+ | . . | | . . | +-----------+
+ |SSP NVIC <-|---+ . . <----------------+ . . |
+ +-----------+ | . . | | . . |
+ +-----------+ | . . <-------- | . . |
+ |TSP NVIC <-|---+ . . | | ----+ . . |
+ +-----------+ | . . | | | | O P |
+ | . . | | | +-----------+
+ | . . <---- | --------------------
+ | . . | | | +-----------+ |
+ | M N | | ---------+ INTC1(1) | |
+ +-----------+ | +-----------+ |
+ | . |
+ | +-----------+ |
+ -------------+ INTC1(N) | |
+ +-----------+ |
+ +--------------+ |
+ + BMCU APLIC <-+---------------------------------------------
+ +--------------+
+
+ INTC0 supports:
+ - 128 local peripheral interrupt inputs
+ - Fan-in from up to three INTC1 instances via banked interrupt lines (INTM)
+ - Local peripheral interrupt outputs
+ - Merged interrupt outputs
+ - Software interrupt outputs (SWINT)
+ - Configurable interrupt routes targeting the PSP, SSP, and TSP
+
+ INTC1 supports:
+ - 192 local peripheral interrupt inputs
+ - Banked interrupt outputs (INTM, 5 x 6 banks x 32 interrupts per bank)
+ - Configurable interrupt routes targeting the PSP, SSP, TSP, and BMCU
+
+ One INTC1 instance is always present, on the SoC's IO die. A further two
+ instances may be attached to the SoC's one INTC0 instance via LTPI (LVDS
+ Tunneling Protocol & Interface).
+
+ Interrupt numbering model
+ -------------------------
+ The binding uses a controller-local numbering model. Peripheral device
+ nodes use the INTCx local interrupt number (hwirq) in their 'interrupts' or
+ 'interrupts-extended' properties.
+
+ For AST2700, INTC0 exposes the following (inclusive) input ranges:
+
+ - 000..479: Independent interrupts
+ - 480..489: INTM0-INTM9
+ - 490..499: INTM10-INTM19
+ - 500..509: INTM20-INTM29
+ - 510..519: INTM30-INTM39
+ - 520..529: INTM40-INTM49
+
+ INTC0's (inclusive) output ranges are as follows:
+
+ - 000..127: 1:1 local peripheral interrupt output to PSP
+ - 144..151: Software interrupts from the SSP output to PSP
+ - 152..159: Software interrupts from the TSP output to PSP
+ - 192..201: INTM0-INTM9 banked outputs to PSP
+ - 208..217: INTM30-INTM39 banked outputs to PSP
+ - 224..233: INTM40-INTM49 banked outputs to PSP
+ - 256..383: 1:1 local peripheral interrupt output to SSP
+ - 384..393: INTM10-INTM19 banked outputs to SSP
+ - 400..407: Software interrupts from the PSP output to SSP
+ - 408..415: Software interrupts from the TSP output to SSP
+ - 426..553: 1:1 local peripheral interrupt output to TSP
+ - 554..563: INTM20-INTM29 banked outputs to TSP
+ - 570..577: Software interrupts from the PSP output to TSP
+ - 578..585: Software interrupts from the SSP output to TSP
+
+ Inputs and outputs for INTC1 instances are context-dependent. However, for the
+ first instance of INTC1, the (inclusive) output ranges are:
+
+ - 00..05: INTM0-INTM5
+ - 10..15: INTM10-INTM15
+ - 20..25: INTM20-INTM25
+ - 30..35: INTM30-INTM35
+ - 40..45: INTM40-INTM45
+ - 50..50: BootMCU
+
+maintainers:
+ - Ryan Chen <ryan_chen@aspeedtech.com>
+ - Andrew Jeffery <andrew@codeconstruct.com.au>
+
+properties:
+ compatible:
+ enum:
+ - aspeed,ast2700-intc0
+ - aspeed,ast2700-intc1
+
+ reg:
+ maxItems: 1
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 1
+ description: Single cell encoding the INTC local interrupt number (hwirq).
+
+ aspeed,interrupt-ranges:
+ description: |
+ Describes how ranges of controller output pins are routed to a parent
+ interrupt controller.
+
+ Each range entry is encoded as:
+
+ <out count phandle parent-specifier...>
+
+ where:
+ - out: First controller interrupt output index in the range.
+ - count: Number of consecutive controller interrupt outputs and parent
+ interrupt inputs in this range.
+ - phandle: Phandle to the parent interrupt controller node.
+ - parent-specifier: Interrupt specifier, as defined by the parent
+ interrupt controller binding.
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 3
+ items:
+ description: Range descriptors with a parent interrupt specifier.
+
+required:
+ - compatible
+ - reg
+ - interrupt-controller
+ - '#interrupt-cells'
+ - aspeed,interrupt-ranges
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ interrupt-controller@12100000 {
+ compatible = "aspeed,ast2700-intc0";
+ reg = <0x12100000 0x3b00>;
+ interrupt-parent = <&gic>;
+ interrupt-controller;
+ #interrupt-cells = <1>;
+
+ aspeed,interrupt-ranges =
+ <0 128 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+ <144 8 &gic GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+ <152 8 &gic GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
+ <192 10 &gic GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+ <208 10 &gic GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+ <224 10 &gic GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+ <256 128 &ssp_nvic 0 0>,
+ <384 10 &ssp_nvic 160 0>,
+ <400 8 &ssp_nvic 144 0>,
+ <408 8 &ssp_nvic 152 0>,
+ <426 128 &tsp_nvic 0 0>,
+ <554 10 &tsp_nvic 160 0>,
+ <570 8 &tsp_nvic 144 0>,
+ <578 8 &tsp_nvic 152 0>;
+ };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
index 5536319..44c0978 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
@@ -52,6 +52,25 @@
- description: primary per-CPU IRQ
- description: shadow IRQ number
+ econet,cpu-interrupt-map:
+ $ref: /schemas/types.yaml#/definitions/uint32-matrix
+ description:
+ When running in VEIC mode, the hardware re-routes interrupts from the
+ CPU interrupt controller core to the "external" interrupt controller
+ (this device). It then prioritizes them and sends them back to the CPU
+ along with its own interrupts. The CPU hardware handles interrupts using
+ a special dispatch table (the normal interrupt handler is not invoked).
+ In this interrupt controller, the CPU interrupts are renumbered as they
+ are merged with this controller's own hardware interrupts.
+
+ This is the inverse of an interrupt-map, mapping which interrupts from
+ this controller must be routed back to the CPU interrupt domain for
+ correct handling there.
+ items:
+ items:
+ - description: The interrupt number as received in this controller
+ - description: The interrupt number to be dispatched on the CPU intc
+
required:
- compatible
- reg
@@ -74,5 +93,6 @@
interrupts = <2>;
econet,shadow-interrupts = <7 2>, <8 3>, <13 12>, <30 29>;
+ econet,cpu-interrupt-map = <7 0>, <8 1>;
};
...
diff --git a/Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
similarity index 68%
rename from Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
rename to Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
index ada5788..d8a0a38 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
@@ -1,13 +1,13 @@
# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
%YAML 1.2
---
-$id: http://devicetree.org/schemas/interrupt-controller/starfive,jh8100-intc.yaml#
+$id: http://devicetree.org/schemas/interrupt-controller/starfive,jhb100-intc.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: StarFive External Interrupt Controller
description:
- StarFive SoC JH8100 contain a external interrupt controller. It can be used
+ StarFive SoC JHB100 contain a external interrupt controller. It can be used
to handle high-level input interrupt signals. It also send the output
interrupt signal to RISC-V PLIC.
@@ -16,19 +16,11 @@
properties:
compatible:
- const: starfive,jh8100-intc
+ const: starfive,jhb100-intc
reg:
maxItems: 1
- clocks:
- description: APB clock for the interrupt controller
- maxItems: 1
-
- resets:
- description: APB reset for the interrupt controller
- maxItems: 1
-
interrupts:
maxItems: 1
@@ -40,8 +32,6 @@
required:
- compatible
- reg
- - clocks
- - resets
- interrupts
- interrupt-controller
- "#interrupt-cells"
@@ -51,10 +41,8 @@
examples:
- |
interrupt-controller@12260000 {
- compatible = "starfive,jh8100-intc";
+ compatible = "starfive,jhb100-intc";
reg = <0x12260000 0x10000>;
- clocks = <&syscrg_ne 76>;
- resets = <&syscrg_ne 13>;
interrupts = <45>;
interrupt-controller;
#interrupt-cells = <1>;
diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
index 7c4d617..f5c584c 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
@@ -24,6 +24,7 @@
compatible:
items:
- enum:
+ - qcom,eliza-ipcc
- qcom,glymur-ipcc
- qcom,kaanapali-ipcc
- qcom,milos-ipcc
diff --git a/Documentation/devicetree/bindings/net/eswin,eic7700-eth.yaml b/Documentation/devicetree/bindings/net/eswin,eic7700-eth.yaml
index 91e8cd1..65882ff 100644
--- a/Documentation/devicetree/bindings/net/eswin,eic7700-eth.yaml
+++ b/Documentation/devicetree/bindings/net/eswin,eic7700-eth.yaml
@@ -73,15 +73,27 @@
HSP CSR is to control and get status of different high-speed peripherals
(such as Ethernet, USB, SATA, etc.) via register, which can tune
board-level's parameters of PHY, etc.
+
+ Additional background information about the High-Speed Subsystem
+ and the HSP CSR block is available in Chapter 10 ("High-Speed Interface")
+ of the EIC7700X SoC Technical Reference Manual, Part 4
+ (EIC7700X_SoC_Technical_Reference_Manual_Part4.pdf). The manual is
+ publicly available at
+ https://github.com/eswincomputing/EIC7700X-SoC-Technical-Reference-Manual/releases
+
+ This reference is provided for background information only.
$ref: /schemas/types.yaml#/definitions/phandle-array
items:
- - items:
+ - minItems: 4
+ items:
- description: Phandle to HSP(High-Speed Peripheral) device
- description: Offset of phy control register for internal
or external clock selection
- description: Offset of AXI clock controller Low-Power request
register
- description: Offset of register controlling TX/RX clock delay
+ - description: Optional offset of register controlling TXD delay
+ - description: Optional offset of register controlling RXD delay
required:
- compatible
@@ -116,7 +128,7 @@
reset-names = "stmmaceth";
rx-internal-delay-ps = <200>;
tx-internal-delay-ps = <200>;
- eswin,hsp-sp-csr = <&hsp_sp_csr 0x100 0x108 0x118>;
+ eswin,hsp-sp-csr = <&hsp_sp_csr 0x100 0x108 0x118 0x114 0x11c>;
snps,axi-config = <&stmmac_axi_setup>;
snps,aal;
snps,fixed-burst;
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
index ed698c9..becc7a1 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
@@ -18,7 +18,9 @@
description: Phandles of rt5650 and rt5514 codecs
items:
- description: phandle of rt5650 codec
+ maxItems: 1
- description: phandle of rt5514 codec
+ maxItems: 1
mediatek,platform:
$ref: /schemas/types.yaml#/definitions/phandle
diff --git a/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml b/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
index 1d10cfb..504df31 100644
--- a/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
+++ b/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
@@ -21,6 +21,9 @@
- fsl,ls2080a-qspi
- spacemit,k1-qspi
- items:
+ - const: spacemit,k3-qspi
+ - const: spacemit,k1-qspi
+ - items:
- enum:
- fsl,ls1043a-qspi
- const: fsl,ls1021a-qspi
diff --git a/Documentation/devicetree/bindings/spi/qcom,spi-qpic-snand.yaml b/Documentation/devicetree/bindings/spi/qcom,spi-qpic-snand.yaml
index 7d0571f..829da22 100644
--- a/Documentation/devicetree/bindings/spi/qcom,spi-qpic-snand.yaml
+++ b/Documentation/devicetree/bindings/spi/qcom,spi-qpic-snand.yaml
@@ -25,6 +25,7 @@
- items:
- enum:
- qcom,ipq5018-snand
+ - qcom,ipq5210-snand
- qcom,ipq5332-snand
- qcom,ipq5424-snand
- const: qcom,ipq9574-snand
diff --git a/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml b/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
index 9898dc7..6d41fb1 100644
--- a/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
+++ b/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
@@ -14,6 +14,7 @@
oneOf:
- const: fsl,imx1-gpt
- const: fsl,imx21-gpt
+ - const: fsl,imx25-epit
- items:
- const: fsl,imx27-gpt
- const: fsl,imx21-gpt
diff --git a/Documentation/devicetree/bindings/usb/eswin,eic7700-usb.yaml b/Documentation/devicetree/bindings/usb/eswin,eic7700-usb.yaml
index 41c3b1b..6582606 100644
--- a/Documentation/devicetree/bindings/usb/eswin,eic7700-usb.yaml
+++ b/Documentation/devicetree/bindings/usb/eswin,eic7700-usb.yaml
@@ -41,12 +41,13 @@
- const: usb_en
resets:
- maxItems: 2
+ maxItems: 3
reset-names:
items:
- const: vaux
- const: usb_rst
+ - const: usb_phy
eswin,hsp-sp-csr:
description:
@@ -85,8 +86,8 @@
interrupt-parent = <&plic>;
interrupts = <85>;
interrupt-names = "peripheral";
- resets = <&reset 84>, <&hspcrg 2>;
- reset-names = "vaux", "usb_rst";
+ resets = <&reset 84>, <&hspcrg 2>, <&hspcrg 4>;
+ reset-names = "vaux", "usb_rst", "usb_phy";
dr_mode = "peripheral";
maximum-speed = "high-speed";
phy_type = "utmi";
diff --git a/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml b/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
index a3d15f2..e1887e4 100644
--- a/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
+++ b/Documentation/devicetree/bindings/usb/ti,omap4-musb.yaml
@@ -81,9 +81,7 @@
const: usb2-phy
usb-phy:
- $ref: /schemas/types.yaml#/definitions/phandle-array
- description: Phandle for the PHY device.
- deprecated: true
+ maxItems: 1
ctrl-module:
$ref: /schemas/types.yaml#/definitions/phandle
@@ -96,6 +94,9 @@
- interrupts
- interrupt-names
+allOf:
+ - $ref: usb-hcd.yaml#
+
unevaluatedProperties: false
examples:
diff --git a/Documentation/filesystems/isofs.rst b/Documentation/filesystems/isofs.rst
index 08fd469..2a30999b 100644
--- a/Documentation/filesystems/isofs.rst
+++ b/Documentation/filesystems/isofs.rst
@@ -57,7 +57,7 @@
Recommended documents about ISO 9660 standard are located at:
- http://www.y-adagio.com/
-- ftp://ftp.ecma.ch/ecma-st/Ecma-119.pdf
+- https://ecma-international.org/wp-content/uploads/ECMA-119_2nd_edition_december_1987.pdf
Quoting from the PDF "This 2nd Edition of Standard ECMA-119 is technically
identical with ISO 9660.", so it is a valid and gratis substitute of the
diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst
index b003bed..e4b66af 100644
--- a/Documentation/filesystems/resctrl.rst
+++ b/Documentation/filesystems/resctrl.rst
@@ -427,9 +427,9 @@
Two MBM events are supported by default: mbm_local_bytes and mbm_total_bytes.
Each MBM event's sub-directory contains a file named "event_filter" that is
- used to view and modify which memory transactions the MBM event is configured
- with. The file is accessible only when "mbm_event" counter assignment mode is
- enabled.
+ used to view and (if writable) modify which memory transactions the MBM event
+ is configured with. The file is accessible only when "mbm_event" counter
+ assignment mode is enabled.
List of memory transaction types supported:
@@ -454,9 +454,8 @@
# cat /sys/fs/resctrl/info/L3_MON/event_configs/mbm_local_bytes/event_filter
local_reads,local_non_temporal_writes,local_reads_slow_memory
- Modify the event configuration by writing to the "event_filter" file within
- the "event_configs" directory. The read/write "event_filter" file contains the
- configuration of the event that reflects which memory transactions are counted by it.
+ The memory transactions the MBM event is configured with can be changed
+ if "event_filter" is writable.
For example::
@@ -480,6 +479,12 @@
"1":
Auto assignment is enabled.
+ Automatic counter assignment is done with best effort. If auto
+ assignment is enabled but there are not enough available counters then
+ monitor group creation could succeed while one or more events belonging
+ to the group may not have a counter assigned in all domains. Consult
+ mbm_L3_assignments for counter assignment states of the new groups.
+
Example::
# echo 0 > /sys/fs/resctrl/info/L3_MON/mbm_assign_on_mkdir
@@ -570,6 +575,11 @@
then the task must already belong to the CTRL_MON parent of this
group. The task is removed from any previous MON group.
+ When writing to this file, a task id of 0 is interpreted as the
+ task id of the currently running task. On reading the file, a task
+ id of 0 will never be shown and there is no representation of the
+ idle tasks. Instead, a CPU's idle task is always considered as a
+ member of the group owning the CPU.
"cpus":
Reading this file shows a bitmask of the logical CPUs owned by
diff --git a/Documentation/hwmon/sy7636a-hwmon.rst b/Documentation/hwmon/sy7636a-hwmon.rst
index 0143ce0e..03d866a 100644
--- a/Documentation/hwmon/sy7636a-hwmon.rst
+++ b/Documentation/hwmon/sy7636a-hwmon.rst
@@ -22,5 +22,5 @@
sysfs-Interface
---------------
-temp0_input
+temp1_input
- Temperature of external NTC (milli-degree C)
diff --git a/Documentation/hwmon/yogafan.rst b/Documentation/hwmon/yogafan.rst
index c553a38..6876194 100644
--- a/Documentation/hwmon/yogafan.rst
+++ b/Documentation/hwmon/yogafan.rst
@@ -135,4 +135,4 @@
4. **Lenovo IdeaPad Laptop Driver:** Reference for DMI-based hardware
feature gating in Lenovo laptops.
- https://github.com/torvalds/linux/blob/master/drivers/platform/x86/ideapad-laptop.c
+ https://github.com/torvalds/linux/blob/master/drivers/platform/x86/lenovo/ideapad-laptop.c
diff --git a/Documentation/locking/robust-futex-ABI.rst b/Documentation/locking/robust-futex-ABI.rst
index f24904f..5e6a066 100644
--- a/Documentation/locking/robust-futex-ABI.rst
+++ b/Documentation/locking/robust-futex-ABI.rst
@@ -153,6 +153,9 @@
3) release the futex lock, and
4) clear the 'lock_op_pending' word.
+Please note that the removal of a robust futex purely in userspace is
+racy. Refer to the next chapter to learn more and how to avoid this.
+
On exit, the kernel will consider the address stored in
'list_op_pending' and the address of each 'lock word' found by walking
the list starting at 'head'. For each such address, if the bottom 30
@@ -182,3 +185,44 @@
When the kernel sees a list entry whose 'lock word' doesn't have the
current threads TID in the lower 30 bits, it does nothing with that
entry, and goes on to the next entry.
+
+Robust release is racy
+----------------------
+
+The removal of a robust futex from the list is racy when doing it solely in
+userspace. Quoting Thomas Gleixner for the explanation:
+
+ The robust futex unlock mechanism is racy in respect to the clearing of the
+ robust_list_head::list_op_pending pointer because unlock and clearing the
+ pointer are not atomic. The race window is between the unlock and clearing
+ the pending op pointer. If the task is forced to exit in this window, exit
+ will access a potentially invalid pending op pointer when cleaning up the
+ robust list. That happens if another task manages to unmap the object
+ containing the lock before the cleanup, which results in an UAF. In the
+ worst case this UAF can lead to memory corruption when unrelated content
+ has been mapped to the same address by the time the access happens.
+
+A full in-depth analysis can be read at
+https://lore.kernel.org/lkml/20260316162316.356674433@kernel.org/
+
+To overcome that, the kernel needs to participate in the lock release operation.
+This ensures that the release happens "atomically" with regard to releasing
+the lock and removing the address from ``list_op_pending``. If the release is
+interrupted by a signal, the kernel will also verify if it interrupted the
+release operation.
+
+For the contended unlock case, where other threads are waiting for the lock
+release, there's the ``FUTEX_ROBUST_UNLOCK`` operation feature flag for the
+``futex()`` system call, which must be used with one of the following
+operations: ``FUTEX_WAKE``, ``FUTEX_WAKE_BITSET`` or ``FUTEX_UNLOCK_PI``.
+The kernel will release the lock (set the futex word to zero), clean the
+``list_op_pending`` field. Then, it will proceed with the normal wake path.
+
+For the non-contended path, there's still a race between checking the futex word
+and clearing the ``list_op_pending`` field. To solve this without the need of a
+complete system call, userspace should call the virtual syscall
+``__vdso_futex_robust_listXX_try_unlock()`` (where XX is either 32 or 64,
+depending on the size of the pointer). If the vDSO call succeeds, it means that
+it released the lock and cleared ``list_op_pending``. If it fails, that means
+that there are waiters for this lock and a call to ``futex()`` syscall with
+``FUTEX_ROBUST_UNLOCK`` is needed.
diff --git a/Documentation/locking/robust-futexes.rst b/Documentation/locking/robust-futexes.rst
index 6361fb0..1423f53 100644
--- a/Documentation/locking/robust-futexes.rst
+++ b/Documentation/locking/robust-futexes.rst
@@ -94,7 +94,7 @@
locks to be cleaned up?
In the common case, at do_exit() time, there is no list registered, so
-the cost of robust futexes is just a simple current->robust_list != NULL
+the cost of robust futexes is just a current->futex.robust_list != NULL
comparison. If the thread has registered a list, then normally the list
is empty. If the thread/process crashed or terminated in some incorrect
way then the list might be non-empty: in this case the kernel carefully
@@ -178,9 +178,9 @@
size_t __user *len_ptr);
List registration is very fast: the pointer is simply stored in
-current->robust_list. [Note that in the future, if robust futexes become
-widespread, we could extend sys_clone() to register a robust-list head
-for new threads, without the need of another syscall.]
+current->futex.robust_list. [Note that in the future, if robust futexes
+become widespread, we could extend sys_clone() to register a robust-list
+head for new threads, without the need of another syscall.]
So there is virtually zero overhead for tasks not using robust futexes,
and even for robust futex users, there is only one extra syscall per
diff --git a/Documentation/netlink/genetlink-c.yaml b/Documentation/netlink/genetlink-c.yaml
index 57f59fe..4ea31e8 100644
--- a/Documentation/netlink/genetlink-c.yaml
+++ b/Documentation/netlink/genetlink-c.yaml
@@ -69,6 +69,15 @@
header:
description: For C-compatible languages, header which already defines this value.
type: string
+ scope:
+ description: |
+ Visibility of this definition. "uapi" (default) renders into
+ the uAPI header, "kernel" renders into the kernel-side
+ generated header, "user" renders into the user-side
+ generated header. When combined with `header:`, the
+ definition is not rendered, and the named header is
+ included only by code matching the scope.
+ enum: [ uapi, kernel, user ]
type:
enum: [ const, enum, flags ]
doc:
diff --git a/Documentation/netlink/genetlink-legacy.yaml b/Documentation/netlink/genetlink-legacy.yaml
index 66fb865..f9c4474 100644
--- a/Documentation/netlink/genetlink-legacy.yaml
+++ b/Documentation/netlink/genetlink-legacy.yaml
@@ -83,6 +83,15 @@
header:
description: For C-compatible languages, header which already defines this value.
type: string
+ scope:
+ description: |
+ Visibility of this definition. "uapi" (default) renders into
+ the uAPI header, "kernel" renders into the kernel-side
+ generated header, "user" renders into the user-side
+ generated header. When combined with `header:`, the
+ definition is not rendered, and the named header is
+ included only by code matching the scope.
+ enum: [ uapi, kernel, user ]
type:
enum: [ const, enum, flags, struct ] # Trim
doc:
diff --git a/Documentation/netlink/genetlink.yaml b/Documentation/netlink/genetlink.yaml
index a1194d5..d3f3f33 100644
--- a/Documentation/netlink/genetlink.yaml
+++ b/Documentation/netlink/genetlink.yaml
@@ -55,6 +55,15 @@
header:
description: For C-compatible languages, header which already defines this value.
type: string
+ scope:
+ description: |
+ Visibility of this definition. "uapi" (default) renders into
+ the uAPI header, "kernel" renders into the kernel-side
+ generated header, "user" renders into the user-side
+ generated header. When combined with `header:`, the
+ definition is not rendered, and the named header is
+ included only by code matching the scope.
+ enum: [ uapi, kernel, user ]
type:
enum: [ const, enum, flags ]
doc:
diff --git a/Documentation/netlink/netlink-raw.yaml b/Documentation/netlink/netlink-raw.yaml
index dd98dda..4c436b5 100644
--- a/Documentation/netlink/netlink-raw.yaml
+++ b/Documentation/netlink/netlink-raw.yaml
@@ -87,6 +87,15 @@
header:
description: For C-compatible languages, header which already defines this value.
type: string
+ scope:
+ description: |
+ Visibility of this definition. "uapi" (default) renders into
+ the uAPI header, "kernel" renders into the kernel-side
+ generated header, "user" renders into the user-side
+ generated header. When combined with `header:`, the
+ definition is not rendered, and the named header is
+ included only by code matching the scope.
+ enum: [ uapi, kernel, user ]
type:
enum: [ const, enum, flags, struct ] # Trim
doc:
diff --git a/Documentation/netlink/specs/handshake.yaml b/Documentation/netlink/specs/handshake.yaml
index 95c3fad..1024297 100644
--- a/Documentation/netlink/specs/handshake.yaml
+++ b/Documentation/netlink/specs/handshake.yaml
@@ -13,6 +13,12 @@
definitions:
-
+ type: const
+ name: max-errno
+ value: 4095
+ header: linux/err.h
+ scope: kernel
+ -
type: enum
name: handler-class
value-start: 0
@@ -80,6 +86,8 @@
-
name: status
type: u32
+ checks:
+ max: max-errno
-
name: sockfd
type: s32
diff --git a/Documentation/netlink/specs/net_shaper.yaml b/Documentation/netlink/specs/net_shaper.yaml
index 3f2ad77..de01f92 100644
--- a/Documentation/netlink/specs/net_shaper.yaml
+++ b/Documentation/netlink/specs/net_shaper.yaml
@@ -34,6 +34,11 @@
definitions:
-
+ type: const
+ name: max-handle-id
+ value: 0x3fffffe
+ scope: kernel
+ -
type: enum
name: scope
doc: Defines the shaper @id interpretation.
@@ -140,6 +145,8 @@
-
name: id
type: u32
+ checks:
+ max: max-handle-id
doc: |
Numeric identifier of a shaper. The id semantic depends on
the scope. For @queue scope it's the queue id and for @node
diff --git a/Documentation/netlink/specs/psp.yaml b/Documentation/netlink/specs/psp.yaml
index 100c36cd..bfcd6e4 100644
--- a/Documentation/netlink/specs/psp.yaml
+++ b/Documentation/netlink/specs/psp.yaml
@@ -188,6 +188,7 @@
name: dev-set
doc: Set the configuration of a PSP device.
attribute-set: dev
+ flags: [admin-perm]
do:
request:
attributes:
@@ -207,6 +208,7 @@
name: key-rotate
doc: Rotate the device key.
attribute-set: dev
+ flags: [admin-perm]
do:
request:
attributes:
diff --git a/Documentation/networking/device_drivers/ethernet/3com/3c509.rst b/Documentation/networking/device_drivers/ethernet/3com/3c509.rst
new file mode 100644
index 0000000..a8c5e5e
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/3com/3c509.rst
@@ -0,0 +1,249 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================================================================
+Linux and the 3Com EtherLink III Series Ethercards (driver v1.18c and higher)
+=============================================================================
+
+This file contains the instructions and caveats for v1.18c and higher versions
+of the 3c509 driver. You should not use the driver without reading this file.
+
+release 1.0
+
+28 February 2002
+
+Current maintainer (corrections to):
+ Maciej W. Rozycki <macro@orcam.me.uk>
+
+Introduction
+============
+
+The following are notes and information on using the 3Com EtherLink III series
+ethercards in Linux. These cards are commonly known by the most widely-used
+card's 3Com model number, 3c509. They are all 10mb/s ISA-bus cards and shouldn't
+be (but sometimes are) confused with the similarly-numbered PCI-bus "3c905"
+(aka "Vortex" or "Boomerang") series. Kernel support for the 3c509 family is
+provided by the module 3c509.c, which has code to support all of the following
+models:
+
+ - 3c509 (original ISA card)
+ - 3c509B (later revision of the ISA card; supports full-duplex)
+ - 3c589 (PCMCIA)
+ - 3c589B (later revision of the 3c589; supports full-duplex)
+ - 3c579 (EISA)
+
+Large portions of this documentation were heavily borrowed from the guide
+written the original author of the 3c509 driver, Donald Becker. The master
+copy of that document, which contains notes on older versions of the driver,
+currently resides on Scyld web server: http://www.scyld.com/.
+
+
+Special Driver Features
+=======================
+
+Overriding card settings
+
+The driver allows boot- or load-time overriding of the card's detected IOADDR,
+IRQ, and transceiver settings, although this capability shouldn't generally be
+needed except to enable full-duplex mode (see below). An example of the syntax
+for LILO parameters for doing this::
+
+ ether=10,0x310,3,0x3c509,eth0
+
+This configures the first found 3c509 card for IRQ 10, base I/O 0x310, and
+transceiver type 3 (10base2). The flag "0x3c509" must be set to avoid conflicts
+with other card types when overriding the I/O address. When the driver is
+loaded as a module, only the IRQ may be overridden. For example,
+setting two cards to IRQ10 and IRQ11 is done by using the irq module
+option::
+
+ options 3c509 irq=10,11
+
+
+Full-duplex mode
+================
+
+The v1.18c driver added support for the 3c509B's full-duplex capabilities.
+In order to enable and successfully use full-duplex mode, three conditions
+must be met:
+
+(a) You must have a Etherlink III card model whose hardware supports full-
+duplex operations. Currently, the only members of the 3c509 family that are
+positively known to support full-duplex are the 3c509B (ISA bus) and 3c589B
+(PCMCIA) cards. Cards without the "B" model designation do *not* support
+full-duplex mode; these include the original 3c509 (no "B"), the original
+3c589, the 3c529 (MCA bus), and the 3c579 (EISA bus).
+
+(b) You must be using your card's 10baseT transceiver (i.e., the RJ-45
+connector), not its AUI (thick-net) or 10base2 (thin-net/coax) interfaces.
+AUI and 10base2 network cabling is physically incapable of full-duplex
+operation.
+
+(c) Most importantly, your 3c509B must be connected to a link partner that is
+itself full-duplex capable. This is almost certainly one of two things: a full-
+duplex-capable Ethernet switch (*not* a hub), or a full-duplex-capable NIC on
+another system that's connected directly to the 3c509B via a crossover cable.
+
+Full-duplex mode can be enabled using 'ethtool'.
+
+.. warning::
+
+ Extremely important caution concerning full-duplex mode
+
+ Understand that the 3c509B's hardware's full-duplex support is much more
+ limited than that provide by more modern network interface cards. Although
+ at the physical layer of the network it fully supports full-duplex operation,
+ the card was designed before the current Ethernet auto-negotiation (N-way)
+ spec was written. This means that the 3c509B family ***cannot and will not
+ auto-negotiate a full-duplex connection with its link partner under any
+ circumstances, no matter how it is initialized***. If the full-duplex mode
+ of the 3c509B is enabled, its link partner will very likely need to be
+ independently _forced_ into full-duplex mode as well; otherwise various nasty
+ failures will occur - at the very least, you'll see massive numbers of packet
+ collisions. This is one of very rare circumstances where disabling auto-
+ negotiation and forcing the duplex mode of a network interface card or switch
+ would ever be necessary or desirable.
+
+
+Available Transceiver Types
+===========================
+
+For versions of the driver v1.18c and above, the available transceiver types are:
+
+== =========================================================================
+0 transceiver type from EEPROM config (normally 10baseT); force half-duplex
+1 AUI (thick-net / DB15 connector)
+2 (undefined)
+3 10base2 (thin-net == coax / BNC connector)
+4 10baseT (RJ-45 connector); force half-duplex mode
+8 transceiver type and duplex mode taken from card's EEPROM config settings
+12 10baseT (RJ-45 connector); force full-duplex mode
+== =========================================================================
+
+Prior to driver version 1.18c, only transceiver codes 0-4 were supported. Note
+that the new transceiver codes 8 and 12 are the *only* ones that will enable
+full-duplex mode, no matter what the card's detected EEPROM settings might be.
+This insured that merely upgrading the driver from an earlier version would
+never automatically enable full-duplex mode in an existing installation;
+it must always be explicitly enabled via one of these code in order to be
+activated.
+
+The transceiver type can be changed using 'ethtool'.
+
+
+Interpretation of error messages and common problems
+----------------------------------------------------
+
+Error Messages
+^^^^^^^^^^^^^^
+
+eth0: Infinite loop in interrupt, status 2011.
+These are "mostly harmless" message indicating that the driver had too much
+work during that interrupt cycle. With a status of 0x2011 you are receiving
+packets faster than they can be removed from the card. This should be rare
+or impossible in normal operation. Possible causes of this error report are:
+
+ - a "green" mode enabled that slows the processor down when there is no
+ keyboard activity.
+
+ - some other device or device driver hogging the bus or disabling interrupts.
+ Check /proc/interrupts for excessive interrupt counts. The timer tick
+ interrupt should always be incrementing faster than the others.
+
+No received packets
+^^^^^^^^^^^^^^^^^^^
+
+If a 3c509, 3c562 or 3c589 can successfully transmit packets, but never
+receives packets (as reported by /proc/net/dev or 'ifconfig') you likely
+have an interrupt line problem. Check /proc/interrupts to verify that the
+card is actually generating interrupts. If the interrupt count is not
+increasing you likely have a physical conflict with two devices trying to
+use the same ISA IRQ line. The common conflict is with a sound card on IRQ10
+or IRQ5, and the easiest solution is to move the 3c509 to a different
+interrupt line. If the device is receiving packets but 'ping' doesn't work,
+you have a routing problem.
+
+Tx Carrier Errors Reported in /proc/net/dev
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+If an EtherLink III appears to transmit packets, but the "Tx carrier errors"
+field in /proc/net/dev increments as quickly as the Tx packet count, you
+likely have an unterminated network or the incorrect media transceiver selected.
+
+3c509B card is not detected on machines with an ISA PnP BIOS.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the updated driver works with most PnP BIOS programs, it does not work
+with all. This can be fixed by disabling PnP support using the 3Com-supplied
+setup program.
+
+3c509 card is not detected on overclocked machines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Increase the delay time in id_read_eeprom() from the current value, 500,
+to an absurdly high value, such as 5000.
+
+
+Decoding Status and Error Messages
+----------------------------------
+
+
+The bits in the main status register are:
+
+===== ======================================
+value description
+===== ======================================
+0x01 Interrupt latch
+0x02 Tx overrun, or Rx underrun
+0x04 Tx complete
+0x08 Tx FIFO room available
+0x10 A complete Rx packet has arrived
+0x20 A Rx packet has started to arrive
+0x40 The driver has requested an interrupt
+0x80 Statistics counter nearly full
+===== ======================================
+
+The bits in the transmit (Tx) status word are:
+
+===== ============================================
+value description
+===== ============================================
+0x02 Out-of-window collision.
+0x04 Status stack overflow (normally impossible).
+0x08 16 collisions.
+0x10 Tx underrun (not enough PCI bus bandwidth).
+0x20 Tx jabber.
+0x40 Tx interrupt requested.
+0x80 Status is valid (this should always be set).
+===== ============================================
+
+
+When a transmit error occurs the driver produces a status message such as::
+
+ eth0: Transmit error, Tx status register 82
+
+The two values typically seen here are:
+
+0x82
+^^^^
+
+Out of window collision. This typically occurs when some other Ethernet
+host is incorrectly set to full duplex on a half duplex network.
+
+0x88
+^^^^
+
+16 collisions. This typically occurs when the network is exceptionally busy
+or when another host doesn't correctly back off after a collision. If this
+error is mixed with 0x82 errors it is the result of a host incorrectly set
+to full duplex (see above).
+
+Both of these errors are the result of network problems that should be
+corrected. They do not represent driver malfunction.
+
+
+Revision history (this file)
+============================
+
+28Feb02 v1.0 DR New; major portions based on Becker original 3c509 docs
+
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst
index 64621c2..1d25be4 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -10,6 +10,7 @@
.. toctree::
:maxdepth: 2
+ 3com/3c509
3com/vortex
amazon/ena
altera/altera_tse
diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst
index dbd6ea1..aa7c959 100644
--- a/Documentation/process/index.rst
+++ b/Documentation/process/index.rst
@@ -86,6 +86,7 @@
debugging/index
handling-regressions
security-bugs
+ threat-model
cve
embargoed-hardware-issues
diff --git a/Documentation/process/security-bugs.rst b/Documentation/process/security-bugs.rst
index 27b028e..3c51ddd 100644
--- a/Documentation/process/security-bugs.rst
+++ b/Documentation/process/security-bugs.rst
@@ -66,6 +66,42 @@
the issue appear. It is useful to share them, as they can be helpful to
keep end users protected during the time it takes them to apply the fix.
+What qualifies as a security bug
+--------------------------------
+
+It is important that most bugs are handled publicly so as to involve the widest
+possible audience and find the best solution. By nature, bugs that are handled
+in closed discussions between a small set of participants are less likely to
+produce the best possible fix (e.g., risk of missing valid use cases, limited
+testing abilities).
+
+It turns out that the majority of the bugs reported via the security team are
+just regular bugs that have been improperly qualified as security bugs due to
+a lack of awareness of the Linux kernel's threat model, as described in
+Documentation/process/threat-model.rst, and ought to have been sent through
+the normal channels described in Documentation/admin-guide/reporting-issues.rst
+instead.
+
+The security list exists for urgent bugs that grant an attacker a capability
+they are not supposed to have on a correctly configured production system, and
+can be easily exploited, representing an imminent threat to many users. Before
+reporting, consider whether the issue actually crosses a trust boundary on such
+a system.
+
+**If you resorted to AI assistance to identify a bug, you must treat it as
+public**. While you may have valid reasons to believe it is not, the security
+team's experience shows that bugs discovered this way systematically surface
+simultaneously across multiple researchers, often on the same day. In this
+case, do not publicly share a reproducer, as this could cause unintended harm;
+just mention that one is available and maintainers might ask for it privately
+if they need it.
+
+If you are unsure whether an issue qualifies, err on the side of reporting
+privately: the security team would rather triage a borderline report than miss
+a real vulnerability. Reporting ordinary bugs to the security list, however,
+does not make them move faster and consumes triage capacity that other reports
+need.
+
Identifying contacts
--------------------
@@ -74,7 +110,7 @@
not send it to a public list at this stage, unless you have good reasons to
consider the issue as being public or trivial to discover (e.g. result of a
widely available automated vulnerability scanning tool that can be repeated by
-anyone).
+anyone, or use of AI-based tools).
If you're sending a report for issues affecting multiple parts in the kernel,
even if they're fairly similar issues, please send individual messages (think
@@ -131,6 +167,64 @@
will receive instructions about whom to contact, if needed. Your message may
equally be forwarded as-is to the relevant maintainers.
+Responsible use of AI to find bugs
+----------------------------------
+
+A significant fraction of bug reports submitted to the security team are
+actually the result of code reviews assisted by AI tools. While this can be an
+efficient means to find bugs in rarely explored areas, it causes an overload on
+maintainers, who are sometimes forced to ignore such reports due to their poor
+quality or accuracy. As such, reporters must be particularly cautious about a
+number of points which tend to make these reports needlessly difficult to
+handle:
+
+ * **Length**: AI-generated reports tend to be excessively long, containing
+ multiple sections and excessive detail. This makes it difficult to spot
+ important information such as affected files, versions, and impact. Please
+ ensure that a clear summary of the problem and all critical details are
+ presented first. Do not require triage engineers to scan multiple pages of
+ text. Configure your tools to produce concise, human-style reports.
+
+ * **Formatting**: Most AI-generated reports are littered with Markdown tags.
+ These decorations complicate the search for important information and do
+ not survive the quoting processes involved in forwarding or replying.
+ Please **always convert your report to plain text** without any formatting
+ decorations before sending it.
+
+ * **Impact Evaluation**: Many AI-generated reports lack an understanding
+ of the kernel's threat model (see Documentation/process/threat-model.rst)
+ and go to great lengths inventing theoretical consequences. This adds
+ noise and complicates triage. Please stick to verifiable facts (e.g.,
+ "this bug permits any user to gain CAP_NET_ADMIN") without enumerating
+ speculative implications. Have your tool read this documentation as
+ part of the evaluation process.
+
+ * **Reproducer**: AI-based tools are often capable of generating reproducers.
+ Please always ensure your tool provides one and **test it thoroughly**. If
+ the reproducer does not work, or if the tool cannot produce one, the
+ validity of the report should be seriously questioned. Note that since the
+ report will be posted to a public list, the reproducer should only be
+ shared upon maintainers' request.
+
+ * **Propose a Fix**: Many AI tools are actually better at writing code than
+ evaluating it. Please ask your tool to propose a fix and **test it** before
+ reporting the problem. If the fix cannot be tested because it relies on
+ rare hardware or almost extinct network protocols, the issue is likely not
+ a security bug. In any case, if a fix is proposed, it must adhere to
+ Documentation/process/submitting-patches.rst and include a 'Fixes:' tag
+ designating the commit that introduced the bug.
+
+Failure to consider these points exposes your report to the risk of being
+ignored.
+
+Use common sense when evaluating the report. If the affected file has not been
+touched for more than one year and is maintained by a single individual, it is
+likely that usage has declined and exposed users are virtually non-existent
+(e.g., drivers for very old hardware, obsolete filesystems). In such cases,
+there is no need to consume a maintainer's time with an unimportant report. If
+the issue is clearly trivial and publicly discoverable, you should report it
+directly to the public mailing lists.
+
Sending the report
------------------
@@ -148,7 +242,15 @@
or cannot effectively discuss their findings may be abandoned if the
communication does not quickly improve.
-The report must be sent to maintainers, with the security team in ``Cc:``.
+The report must be sent to maintainers. If there are two or fewer
+recipients in your message, you must also always Cc: the Linux kernel
+security team who will ensure the message is delivered to the proper
+people, and will be able to assist small maintainer teams with processes
+they may not be familiar with. For larger teams, Cc: the Linux kernel
+security team for your first few reports or when seeking specific help,
+such as when resending a message which got no response within a week.
+Once you have become comfortable with the process for a few reports, it is
+no longer necessary to Cc: the security list when sending to large teams.
The Linux kernel security team can be contacted by email at
<security@kernel.org>. This is a private list of security officers
who will help verify the bug report and assist developers working on a fix.
diff --git a/Documentation/process/threat-model.rst b/Documentation/process/threat-model.rst
new file mode 100644
index 0000000..f177b8d
--- /dev/null
+++ b/Documentation/process/threat-model.rst
@@ -0,0 +1,235 @@
+The Linux Kernel threat model
+=============================
+
+There are a lot of assumptions regarding what the kernel does and does not
+protect against. These assumptions tend to cause confusion for bug reports
+(:doc:`security-related ones <security-bugs>` vs :doc:`non-security ones
+<../admin-guide/reporting-issues>`), and can complicate security enforcement
+when the responsibilities for some boundaries is not clear between the kernel,
+distros, administrators and users.
+
+This document tries to clarify the responsibilities of the kernel in this
+domain.
+
+The kernel's responsibilities
+-----------------------------
+
+The kernel abstracts access to local hardware resources and to remote systems
+in a way that allows multiple local users to get a fair share of the available
+resources granted to them, and, when the underlying hardware permits, to assign
+a level of confidentiality to their communications and to the data they are
+processing or storing.
+
+The kernel assumes that the underlying hardware behaves according to its
+specifications. This includes the integrity of the CPU's instruction set, the
+transparency of the branch prediction unit and the cache units, the consistency
+of the Memory Management Unit (MMU), the isolation of DMA-capable peripherals
+(e.g., via IOMMU), state transitions in controllers, ranges of values read from
+registers, the respect of documented hardware limitations, etc.
+
+When hardware fails to maintain its specified isolation (e.g., CPU bugs,
+side-channels, hardware response to unexpected inputs), the kernel will usually
+attempt to implement reasonable mitigations. These are best-effort measures
+intended to reduce the attack surface or elevate the cost of an attack within
+the limits of the hardware's facilities; they do not constitute a
+kernel-provided safety guarantee.
+
+Users always perform their activities under the authority of an administrator
+who is able to grant or deny various types of permissions that may affect how
+users benefit from available resources, or the level of confidentiality of
+their activities. Administrators may also delegate all or part of their own
+permissions to some users, particularly via capabilities but not only. All this
+is performed via configuration (sysctl, file-system permissions etc).
+
+The Linux Kernel applies a certain collection of default settings that match
+its threat model. Distros have their own threat model and will come with their
+own configuration presets, that the administrator may have to adjust to better
+suit their expectations (relax or restrict).
+
+By default, the Linux Kernel guarantees the following protections when running
+on common processors featuring privilege levels and memory management units:
+
+* **User-based isolation**: an unprivileged user may restrict access to their
+ own data from other unprivileged users running on the same system. This
+ includes:
+
+ * stored data, via file system permissions
+ * in-memory data (pages are not accessible by default to other users)
+ * process activity (ptrace is not permitted to other users)
+ * inter-process communication (other users may not observe data exchanged via
+ UNIX domain sockets or other IPC mechanisms).
+ * network communications within the same or with other systems
+
+* **Capability-based protection**:
+
+ * users not having elevated capabilities (including but not limited to
+ CAP_SYS_ADMIN) may not alter the
+ kernel's configuration, memory nor state, change other users' view of the
+ file system layout, grant any user capabilities they do not have, nor
+ affect the system's availability (shutdown, reboot, panic, hang, or making
+ the system unresponsive via unbounded resource exhaustion).
+ * users not having the ``CAP_NET_ADMIN`` capability may not alter the network
+ configuration, intercept nor spoof network communications from other users
+ nor systems.
+ * users not having ``CAP_SYS_PTRACE`` may not observe other users' processes
+ activities.
+
+When ``CONFIG_USER_NS`` is set, the kernel also permits unprivileged users to
+create their own user namespace in which they have all capabilities, but with a
+number of restrictions (they may not perform actions that have impacts on the
+initial user namespace, such as changing time, loading modules or mounting
+block devices). Please refer to ``user_namespaces(7)`` for more details, the
+possibilities of user namespaces are not covered in this document.
+
+The kernel also offers a lot of troubleshooting and debugging facilities, which
+can constitute attack vectors when placed in wrong hands. While some of them
+are designed to be accessible to regular local users with a low risk (e.g.
+kernel logs via ``/proc/kmsg``), some would expose enough information to
+represent a risk in most places and the decision to expose them is under the
+administrator's responsibility (perf events, traces), and others are not
+designed to be accessed by non-privileged users (e.g. debugfs). Access to these
+facilities by a user who has been explicitly granted permission by an
+administrator does not constitute a security breach.
+
+Bugs that permit to violate the principles above constitute security breaches.
+However, bugs that permit one violation only once another one was already
+achieved are only weaknesses. The kernel applies a number of self-protection
+measures whose purpose is to avoid crossing a security boundary when certain
+classes of bugs are found, but a failure of these extra protections do not
+constitute a vulnerability alone.
+
+What does not constitute a security bug
+---------------------------------------
+
+In the Linux kernel's threat model, the following classes of problems are
+**NOT** considered as Linux Kernel security bugs. However, when it is believed
+that the kernel could do better, they should be reported, so that they can be
+reviewed and fixed where reasonably possible, but they will be handled as any
+regular bug:
+
+* **Configuration**:
+
+ * outdated kernels and particularly end-of-life branches are out of the scope
+ of the kernel's threat model: administrators are responsible for keeping
+ their system up to date. For a bug to qualify as a security bug, it must be
+ demonstrated that it affects actively maintained versions.
+
+ * build-level: changes to the kernel configuration that are explicitly
+ documented as lowering the security level (e.g. ``CONFIG_NOMMU``), or
+ targeted at developers only.
+
+ * OS-level: changes to command line parameters, sysctls, filesystem
+ permissions, user capabilities, exposure of privileged interfaces, that
+ explicitly increase exposure by either offering non-default access to
+ unprivileged users, or reduce the kernel's ability to enforce some
+ protections or mitigations. Example: write access to procfs or debugfs.
+
+ * issues triggered only when using features intended for development or
+ debugging (e.g., LOCKDEP, KASAN, FAULT_INJECTION): these features are known
+ to introduce overhead and potential instability and are not intended for
+ production use.
+
+ * issues affecting drivers exposed under CONFIG_STAGING, as well as features
+ marked EXPERIMENTAL in the configuration.
+
+ * loading of explicitly insecure/broken/staging modules, and generally any
+ using any subsystem marked as experimental or not intended for production
+ use.
+
+ * running out-of-tree modules or unofficial kernel forks; these should be
+ reported to the relevant vendor.
+
+* **Excess of initial privileges**:
+
+ * actions performed by a user already possessing the privileges required to
+ perform that action or modify that state (e.g. ``CAP_SYS_ADMIN``,
+ ``CAP_NET_ADMIN``, ``CAP_SYS_RAWIO``, ``CAP_SYS_MODULE`` with no further
+ boundary being crossed).
+
+ * actions performed in user namespace that do not bypass the restrictions
+ imposed to the initial user (e.g. ptrace usage, signal delivery, resource
+ usage, access to FS/device/sysctl/memory, network binding, system/network
+ configuration etc).
+
+ * anything performed by the root user in the initial namespace (e.g. kernel
+ oops when writing to a privileged device).
+
+* **Out of production use**:
+
+ This covers theoretical/probabilistic attacks that rely on laboratory
+ conditions with zero system noise, or those requiring an unrealistic number
+ of attempts (e.g., billions of trials) that would be detected by standard
+ system monitoring long before success, such as:
+
+ * prediction of random numbers that only works in a totally silent
+ environment (such as IP ID, TCP ports or sequence numbers that can only be
+ guessed in a lab).
+
+ * activity observation and information leaks based on probabilistic
+ approaches that are prone to measurement noise and not realistically
+ reproducible on a production system.
+
+ * issues that can only be triggered by heavy attacks (e.g. brute force) whose
+ impact on the system makes it unlikely or impossible to remain undetected
+ before they succeed (e.g. consuming all memory before succeeding).
+
+ * problems seen only under development simulators, emulators, or combinations
+ that do not exist on real systems at the time of reporting (issues
+ involving tens of millions of threads, tens of thousands of CPUs,
+ unrealistic CPU frequencies, RAM sizes or disk capacities, network speeds.
+
+ * issues whose reproduction requires hardware modification or emulation,
+ including fake USB devices that pretend to be another one.
+
+ * as well as issues that can be triggered at a cost that is orders of
+ magnitude higher than the expected benefits (e.g. fully functional keyboard
+ emulator only to retrieve 7 uninitialized bytes in a structure, or
+ brute-force method involving millions of connection attempts to guess a
+ port number).
+
+* **Hardening failures**:
+
+ * ability to bypass some of the kernel's hardening measures with no
+ demonstrable exploit path (e.g. ASLR bypass, events timing or probing with
+ no demonstrable consequence). These are just weaknesses, not
+ vulnerabilities.
+
+ * missing argument checks and failure to report certain errors with no
+ immediate consequence.
+
+* **Random information leaks**:
+
+ This concerns information leaks of small data parts that happen to be there
+ and that cannot be chosen by the attacker, or face access restrictions:
+
+ * structure padding reported by syscalls or other interfaces.
+
+ * identifiers, partial data, non-terminated strings reported in error
+ messages.
+
+ * Leaks of kernel memory addresses/pointers do not constitute an immediately
+ exploitable vector and are not security bugs, though they must be reported
+ and fixed.
+
+* **Crafted file system images**:
+
+ * bugs triggered by mounting a corrupted or maliciously crafted file system
+ image are generally not security bugs, as the kernel assumes the underlying
+ storage media is under the administrator's control, unless the filesystem
+ driver is specifically documented as being hardened against untrusted media.
+
+ * issues that are resolved, mitigated, or detected by running a filesystem
+ consistency check (fsck) on the image prior to mounting.
+
+* **Physical access**:
+
+ Issues that require physical access to the machine, hardware modification, or
+ the use of specialized hardware (e.g., logic analyzers, DMA-attack tools over
+ PCI-E/Thunderbolt) are out of scope unless the system is explicitly
+ configured with technologies meant to defend against such attacks
+ (e.g. IOMMU).
+
+* **Functional and performance regressions**:
+
+ Any issue that can be mitigated by setting proper permissions and limits
+ doesn't qualify as a security bug.
diff --git a/Documentation/sound/codecs/cs35l56.rst b/Documentation/sound/codecs/cs35l56.rst
index d5363b0..b3f8c1c 100644
--- a/Documentation/sound/codecs/cs35l56.rst
+++ b/Documentation/sound/codecs/cs35l56.rst
@@ -40,7 +40,7 @@
*For systems using SoundWire*: sound/soc/codecs/cs35l56.c and associated files
-*For systems using HDA*: sound/pci/hda/cs35l56_hda.c
+*For systems using HDA*: sound/hda/codecs/side-codecs/cs35l56_hda.c
Firmware
========
diff --git a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst
index d4ff80d..87b58ae 100644
--- a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst
+++ b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst
@@ -174,6 +174,40 @@
| Devices |
+---------+
+高级扩展IRQ模型 (带重定向)
+==========================
+
+在这种模型里面，IPI（Inter-Processor Interrupt）和CPU本地时钟中断直接发送到CPUINTC，
+CPU串口（UARTs）中断发送到LIOINTC，PCH-MSI中断首先发送到REDIRECT模块,完成重定向后发
+送到AVECINTC，而后通过AVECINTC直接送达CPUINTC，而其他所有设备的中断则分别发送到所连
+接的PCH-PIC/PCH-LPC，然后由EIOINTC统一收集，再直接到达CPUINTC::
+
+ +-----+ +-----------------------+ +-------+
+ | IPI | --> | CPUINTC | <-- | Timer |
+ +-----+ +-----------------------+ +-------+
+ ^ ^ ^
+ | | |
+ | +----------+ |
+ +---------+ | AVECINTC | +---------+ +-------+
+ | EIOINTC | +----------+ | LIOINTC | <-- | UARTs |
+ +---------+ | REDIRECT | +---------+ +-------+
+ ^ +----------+
+ | ^
+ | |
+ +---------+ +---------+
+ | PCH-PIC | | PCH-MSI |
+ +---------+ +---------+
+ ^ ^ ^
+ | | |
+ +---------+ +---------+ +---------+
+ | Devices | | PCH-LPC | | Devices |
+ +---------+ +---------+ +---------+
+ ^
+ |
+ +---------+
+ | Devices |
+ +---------+
+
ACPI相关的定义
==============
diff --git a/Documentation/userspace-api/rseq.rst b/Documentation/userspace-api/rseq.rst
index 3cd27a3..8549a6c 100644
--- a/Documentation/userspace-api/rseq.rst
+++ b/Documentation/userspace-api/rseq.rst
@@ -24,6 +24,97 @@
Allows to implement per CPU data efficiently. Documentation is in code and
selftests. :(
+Optimized RSEQ V2
+-----------------
+
+On architectures which utilize the generic entry code and generic TIF bits
+the kernel supports runtime optimizations for RSEQ, which also enable
+enhanced features like scheduler time slice extensions.
+
+To enable them a task has to register the RSEQ region with at least the
+length advertised by getauxval(AT_RSEQ_FEATURE_SIZE).
+
+If existing binaries register with RSEQ_ORIG_SIZE (32 bytes), the kernel
+keeps the legacy low performance mode enabled to fulfil the expectations
+of existing users regarding the original RSEQ implementation behaviour.
+
+The following table documents the ABI and behavioral guarantees of the
+legacy and the optimized V2 mode.
+
+.. list-table:: RSEQ modes
+ :header-rows: 1
+
+ * - Nr
+ - What
+
+ - Legacy
+ - Optimized V2
+
+ * - 1
+ - The cpu_id_start, cpu_id, node_id and mm_cid fields (User mode read
+ only)
+ .. Legacy
+ - Updated by the kernel unconditionally after each context switch and
+ before signal delivery
+ .. Optimized V2
+ - Updated by the kernel if and only if they change, i.e. if the task
+ is migrated or mm_cid changes
+
+ * - 2
+ - The rseq_cs critical section field
+ .. Legacy
+ - Evaluated and handled unconditionally after each context switch and
+ before signal delivery
+ .. Optimized V2
+ - Evaluated and handled conditionally only when user space was
+ interrupted and was scheduled out or before delivering a signal in
+ the interrupted context.
+
+ * - 3
+ - Read only fields
+ .. Legacy
+ - No strict enforcement except in debug mode
+ .. Optimized V2
+ - Strict enforcement
+
+ * - 4
+ - membarrier(...RSEQ)
+ .. Legacy
+ - All running threads of the process are interrupted and the ID fields
+ are rewritten and eventually active critical sections are aborted
+ before they return to user space. All threads which are scheduled
+ out whether voluntary or not are covered by #1/#2 above.
+ .. Optimized V2
+ - All running threads of the process are interrupted and eventually
+ active critical sections are aborted before these threads return to
+ user space. The ID fields are only updated if changed as a
+ consequence of the interrupt. All threads which are scheduled out
+ whether voluntary or not are covered by #1/#2 above.
+
+ * - 5
+ - Time slice extensions
+ .. Legacy
+ - Not supported
+ .. Optimized V2
+ - Supported
+
+The legacy mode is obviously less performant as it does unconditional
+updates and critical section checks even if not strictly required by the
+ABI contract. That can't be changed anymore as some users depend on that
+observed behavior, which in turn enables them to violate the ABI and
+overwrite the cpu_id_start field for their own purposes. This is obviously
+discouraged as it renders RSEQ incompatible with the intended usage and
+breaks the expectation of other libraries in the same application.
+
+The ABI compliant optimized v2 mode, which respects the read only fields,
+does not require unconditional updates and therefore is way more
+performant. The kernel validates the read only fields for compliance. If
+user space modifies them, the process is killed. Compliant usage allows
+multiple libraries in the same application to benefit from the RSEQ
+functionality without disturbing each other. The ABI compliant optimized v2
+mode also enables extended RSEQ features like time slice extensions.
+
+
Scheduler time slice extensions
-------------------------------
@@ -37,7 +128,8 @@
* Enabled at boot time (default is enabled)
- * A rseq userspace pointer has been registered for the thread
+ * A rseq userspace pointer has been registered for the thread in
+ optimized V2 mode
The thread has to enable the functionality via prctl(2)::
diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst
index b2395dd..bd04a90 100644
--- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst
+++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst
@@ -656,8 +656,8 @@
See [white-paper]_, [api-spec]_, [amd-apm]_, [kvm-forum]_, and [snp-fw-abi]_
for more info.
-.. [white-paper] https://developer.amd.com/wordpress/media/2013/12/AMD_Memory_Encryption_Whitepaper_v7-Public.pdf
-.. [api-spec] https://support.amd.com/TechDocs/55766_SEV-KM_API_Specification.pdf
-.. [amd-apm] https://support.amd.com/TechDocs/24593.pdf (section 15.34)
+.. [white-paper] https://docs.amd.com/v/u/en-US/memory-encryption-white-paper
+.. [api-spec] https://docs.amd.com/v/u/en-US/55766_PUB_3.24_SEV_API
+.. [amd-apm] https://docs.amd.com/v/u/en-US/24593_3.44_APM_Vol2 (section 15.34)
.. [kvm-forum] https://www.linux-kvm.org/images/7/74/02x08A-Thomas_Lendacky-AMDs_Virtualizatoin_Memory_Encryption_Technology.pdf
-.. [snp-fw-abi] https://www.amd.com/system/files/TechDocs/56860.pdf
+.. [snp-fw-abi] https://www.amd.com/content/dam/amd/en/documents/developer/56860.pdf
diff --git a/MAINTAINERS b/MAINTAINERS
index 2fb1c75..f11a099 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -68,6 +68,12 @@
first. When adding to this list, please keep the entries in
alphabetical order.
+3C509 NETWORK DRIVER
+M: "Maciej W. Rozycki" <macro@orcam.me.uk>
+L: netdev@vger.kernel.org
+S: Maintained
+F: drivers/net/ethernet/3com/3c509.c
+
3C59X NETWORK DRIVER
M: Steffen Klassert <klassert@kernel.org>
L: netdev@vger.kernel.org
@@ -1115,10 +1121,9 @@
F: drivers/usb/gadget/udc/amd5536udc.*
AMD GEODE PROCESSOR/CHIPSET SUPPORT
-M: Andres Salomon <dilinger@queued.net>
L: linux-geode@lists.infradead.org (moderated for non-subscribers)
-S: Supported
-W: http://www.amd.com/us-en/ConnectivitySolutions/TechnicalResources/0,,50_2334_2452_11363,00.html
+S: Orphaned
+F: arch/x86/platform/geode/
F: arch/x86/include/asm/geode.h
F: drivers/char/hw_random/geode-rng.c
F: drivers/crypto/geode*
@@ -1144,8 +1149,9 @@
F: drivers/platform/x86/amd/hfi/
AMD IOMMU (AMD-VI)
-M: Joerg Roedel <joro@8bytes.org>
+M: Joerg Roedel (AMD) <joro@8bytes.org>
R: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
+R: Vasant Hegde <vasant.hegde@amd.com>
L: iommu@lists.linux.dev
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
@@ -2015,7 +2021,7 @@
F: drivers/hwmon/aquacomputer_d5next.c
AQUANTIA ETHERNET DRIVER (atlantic)
-M: Igor Russkikh <irusskikh@marvell.com>
+M: Sukhdeep Singh <sukhdeeps@marvell.com>
L: netdev@vger.kernel.org
S: Maintained
W: https://www.marvell.com/
@@ -2024,7 +2030,7 @@
F: drivers/net/ethernet/aquantia/atlantic/
AQUANTIA ETHERNET DRIVER PTP SUBSYSTEM
-M: Egor Pomozov <epomozov@marvell.com>
+M: Sukhdeep Singh <sukhdeeps@marvell.com>
L: netdev@vger.kernel.org
S: Maintained
W: http://www.aquantia.com
@@ -2058,7 +2064,7 @@
F: drivers/gpu/drm/tiny/arcpgu.c
ARCNET NETWORK LAYER
-M: Michael Grzeschik <m.grzeschik@pengutronix.de>
+M: Michael Grzeschik <mgr@kernel.org>
L: netdev@vger.kernel.org
S: Maintained
F: drivers/net/arcnet/
@@ -3361,7 +3367,9 @@
F: drivers/tty/serial/rda-uart.c
ARM/REALTEK ARCHITECTURE
-M: Andreas Färber <afaerber@suse.de>
+M: James Tai <james.tai@realtek.com>
+M: Yu-Chun Lin <eleanor.lin@realtek.com>
+R: Andreas Färber <afaerber@suse.com>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
L: linux-realtek-soc@lists.infradead.org (moderated for non-subscribers)
S: Maintained
@@ -3369,6 +3377,7 @@
F: arch/arm/boot/dts/realtek/
F: arch/arm/mach-realtek/
F: arch/arm64/boot/dts/realtek/
+F: drivers/pinctrl/realtek/
ARM/RISC-V/RENESAS ARCHITECTURE
M: Geert Uytterhoeven <geert+renesas@glider.be>
@@ -4181,8 +4190,8 @@
F: net/atm/
ATMEL MACB ETHERNET DRIVER
-M: Nicolas Ferre <nicolas.ferre@microchip.com>
-M: Claudiu Beznea <claudiu.beznea@tuxon.dev>
+M: Théo Lebrun <theo.lebrun@bootlin.com>
+R: Conor Dooley <conor.dooley@microchip.com>
S: Maintained
F: drivers/net/ethernet/cadence/
@@ -4299,18 +4308,16 @@
F: drivers/video/backlight/aw99706.c
AXENTIA ARM DEVICES
-M: Peter Rosin <peda@axentia.se>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
-S: Maintained
+S: Orphan
F: arch/arm/boot/dts/microchip/at91-linea.dtsi
F: arch/arm/boot/dts/microchip/at91-natte.dtsi
F: arch/arm/boot/dts/microchip/at91-nattis-2-natte-2.dts
F: arch/arm/boot/dts/microchip/at91-tse850-3.dts
AXENTIA ASOC DRIVERS
-M: Peter Rosin <peda@axentia.se>
L: linux-sound@vger.kernel.org
-S: Maintained
+S: Orphan
F: Documentation/devicetree/bindings/sound/axentia,*
F: sound/soc/atmel/tse850-pcm5142.c
@@ -6358,6 +6365,7 @@
COMMON CLK FRAMEWORK
M: Michael Turquette <mturquette@baylibre.com>
M: Stephen Boyd <sboyd@kernel.org>
+R: Brian Masney <bmasney@redhat.com>
L: linux-clk@vger.kernel.org
S: Maintained
Q: http://patchwork.kernel.org/project/linux-clk/list/
@@ -6526,7 +6534,7 @@
CONTROL GROUP - CPUSET
M: Waiman Long <longman@redhat.com>
-R: Chen Ridong <chenridong@huaweicloud.com>
+R: Ridong Chen <ridong.chen@linux.dev>
L: cgroups@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git
@@ -6667,6 +6675,7 @@
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git smp/core
F: include/linux/cpu.h
F: include/linux/cpuhotplug.h
+F: include/linux/cpuhplock.h
F: include/linux/smpboot.h
F: kernel/cpu.c
F: kernel/smpboot.*
@@ -7077,6 +7086,12 @@
F: include/linux/debugobjects.h
F: lib/debugobjects.c
+DEC LANCE NETWORK DRIVER
+M: "Maciej W. Rozycki" <macro@orcam.me.uk>
+L: netdev@vger.kernel.org
+S: Maintained
+F: drivers/net/ethernet/amd/declance.c
+
DECSTATION PLATFORM SUPPORT
M: "Maciej W. Rozycki" <macro@orcam.me.uk>
L: linux-mips@vger.kernel.org
@@ -7873,7 +7888,7 @@
DRM DRIVER FOR APPLE TOUCH BARS
M: Aun-Ali Zaidi <admin@kodeit.net>
-M: Aditya Garg <gargaditya08@live.com>
+M: Aditya Garg <gargaditya08@proton.me>
L: dri-devel@lists.freedesktop.org
S: Maintained
T: git https://gitlab.freedesktop.org/drm/misc/kernel.git
@@ -8193,10 +8208,9 @@
CORE DRIVER FOR NVIDIA GPUS [RUST]
M: Danilo Krummrich <dakr@kernel.org>
M: Alexandre Courbot <acourbot@nvidia.com>
-L: nouveau@lists.freedesktop.org
+L: nova-gpu@lists.linux.dev
S: Supported
W: https://rust-for-linux.com/nova-gpu-driver
-Q: https://patchwork.freedesktop.org/project/nouveau/
B: https://gitlab.freedesktop.org/drm/nova/-/issues
C: irc://irc.oftc.net/nouveau
T: git https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-next
@@ -8205,10 +8219,9 @@
DRM DRIVER FOR NVIDIA GPUS [RUST]
M: Danilo Krummrich <dakr@kernel.org>
-L: nouveau@lists.freedesktop.org
+L: nova-gpu@lists.linux.dev
S: Supported
W: https://rust-for-linux.com/nova-gpu-driver
-Q: https://patchwork.freedesktop.org/project/nouveau/
B: https://gitlab.freedesktop.org/drm/nova/-/issues
C: irc://irc.oftc.net/nouveau
T: git https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-next
@@ -10102,12 +10115,6 @@
F: drivers/fpga/
F: include/linux/fpga/
-FPU EMULATOR
-M: Bill Metzenthen <billm@melbpc.org.au>
-S: Maintained
-W: https://floatingpoint.billm.au/
-F: arch/x86/math-emu/
-
FRAMEBUFFER CONSOLE
M: Helge Deller <deller@gmx.de>
M: Thomas Zimmermann <tzimmermann@suse.de>
@@ -10823,7 +10830,7 @@
F: lib/generic-radix-tree.c
GENERIC RESISTIVE TOUCHSCREEN ADC DRIVER
-M: Eugen Hristev <eugen.hristev@microchip.com>
+M: Eugen Hristev <ehristev@kernel.org>
L: linux-input@vger.kernel.org
S: Maintained
F: drivers/input/touchscreen/resistive-adc-touch.c
@@ -10857,6 +10864,7 @@
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/vdso
F: include/asm-generic/vdso/vsyscall.h
+F: include/linux/vdso_datastore.h
F: include/vdso/
F: kernel/time/namespace_vdso.c
F: kernel/time/vsyscall.c
@@ -12046,7 +12054,7 @@
F: drivers/i2c/busses/i2c-nvidia-gpu.c
I2C MUXES
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-i2c@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/i2c/i2c-arb*
@@ -12447,7 +12455,7 @@
F: include/linux/iio/backend.h
IIO DIGITAL POTENTIOMETER DAC
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-iio@vger.kernel.org
S: Maintained
F: Documentation/ABI/testing/sysfs-bus-iio-dac-dpot-dac
@@ -12455,7 +12463,7 @@
F: drivers/iio/dac/dpot-dac.c
IIO ENVELOPE DETECTOR
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-iio@vger.kernel.org
S: Maintained
F: Documentation/ABI/testing/sysfs-bus-iio-adc-envelope-detector
@@ -12471,7 +12479,7 @@
F: drivers/iio/test/iio-test-gts.c
IIO MULTIPLEXER
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-iio@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/iio/multiplexer/io-channel-mux.yaml
@@ -12502,7 +12510,7 @@
F: tools/iio/
IIO UNIT CONVERTER
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-iio@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/iio/afe/current-sense-amplifier.yaml
@@ -12779,7 +12787,6 @@
M: Liam Girdwood <liam.r.girdwood@linux.intel.com>
M: Peter Ujfalusi <peter.ujfalusi@linux.intel.com>
M: Bard Liao <yung-chuan.liao@linux.intel.com>
-M: Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
M: Kai Vehmanen <kai.vehmanen@linux.intel.com>
R: Pierre-Louis Bossart <pierre-louis.bossart@linux.dev>
L: linux-sound@vger.kernel.org
@@ -13472,7 +13479,7 @@
F: include/linux/iova.h
IOMMU SUBSYSTEM
-M: Joerg Roedel <joro@8bytes.org>
+M: Joerg Roedel (AMD) <joro@8bytes.org>
M: Will Deacon <will@kernel.org>
R: Robin Murphy <robin.murphy@arm.com>
L: iommu@lists.linux.dev
@@ -13860,7 +13867,7 @@
R: Dave Young <ruirui.yang@linux.dev>
L: kexec@lists.infradead.org
S: Maintained
-W: http://lse.sourceforge.net/kdump/
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
F: Documentation/admin-guide/kdump/
F: fs/proc/vmcore.c
F: include/linux/crash_core.h
@@ -14053,6 +14060,7 @@
M: Marc Zyngier <maz@kernel.org>
M: Oliver Upton <oupton@kernel.org>
R: Joey Gouly <joey.gouly@arm.com>
+R: Steffen Eiden <seiden@linux.ibm.com>
R: Suzuki K Poulose <suzuki.poulose@arm.com>
R: Zenghui Yu <yuzenghui@huawei.com>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -14177,6 +14185,7 @@
M: Pratyush Yadav <pratyush@kernel.org>
L: kexec@lists.infradead.org
W: http://kernel.org/pub/linux/utils/kernel/kexec/
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
F: include/linux/kexec.h
F: include/uapi/linux/kexec.h
F: kernel/kexec*
@@ -14893,6 +14902,7 @@
M: Pasha Tatashin <pasha.tatashin@soleen.com>
M: Mike Rapoport <rppt@kernel.org>
M: Pratyush Yadav <pratyush@kernel.org>
+L: kexec@lists.infradead.org
L: linux-kernel@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
@@ -15252,7 +15262,7 @@
M: Cyril Hrubis <chrubis@suse.cz>
M: Jan Stancek <jstancek@redhat.com>
M: Petr Vorel <pvorel@suse.cz>
-M: Li Wang <liwang@redhat.com>
+M: Li Wang <li.wang@linux.dev>
M: Yang Xu <xuyang2018.jy@fujitsu.com>
M: Xiao Yang <yangx.jy@fujitsu.com>
L: ltp@lists.linux.it (subscribers-only)
@@ -15399,7 +15409,7 @@
F: net/mctp/
MAPLE TREE
-M: Liam R. Howlett <Liam.Howlett@oracle.com>
+M: Liam R. Howlett <liam@infradead.org>
R: Alice Ryhl <aliceryhl@google.com>
R: Andrew Ballance <andrewjballance@gmail.com>
L: maple-tree@lists.infradead.org
@@ -15719,7 +15729,7 @@
F: drivers/media/i2c/max96717.c
MAX9860 MONO AUDIO VOICE CODEC DRIVER
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-sound@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/sound/max9860.txt
@@ -15934,7 +15944,7 @@
F: drivers/net/can/spi/mcp251xfd/
MCP4018 AND MCP4531 MICROCHIP DIGITAL POTENTIOMETER DRIVERS
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-iio@vger.kernel.org
S: Maintained
F: Documentation/ABI/testing/sysfs-bus-iio-potentiometer-mcp4531
@@ -16492,7 +16502,7 @@
MEGACHIPS STDPXXXX-GE-B850V3-FW LVDS/DP++ BRIDGES
M: Peter Senna Tschudin <peter.senna@gmail.com>
-M: Ian Ray <ian.ray@ge.com>
+M: Ian Ray <ian.ray@gehealthcare.com>
M: Martyn Welch <martyn.welch@collabora.co.uk>
S: Maintained
F: Documentation/devicetree/bindings/display/bridge/megachips-stdpxxxx-ge-b850v3-fw.txt
@@ -16759,7 +16769,7 @@
M: Andrew Morton <akpm@linux-foundation.org>
M: David Hildenbrand <david@kernel.org>
R: Lorenzo Stoakes <ljs@kernel.org>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
R: Vlastimil Babka <vbabka@kernel.org>
R: Mike Rapoport <rppt@kernel.org>
R: Suren Baghdasaryan <surenb@google.com>
@@ -16805,7 +16815,7 @@
F: mm/util.c
F: mm/vmpressure.c
F: mm/vmstat.c
-N: include/linux/page[-_]*
+N: include\/linux\/page[-_][a-zA-Z]*
MEMORY MANAGEMENT - EXECMEM
M: Andrew Morton <akpm@linux-foundation.org>
@@ -16895,7 +16905,7 @@
M: Andrew Morton <akpm@linux-foundation.org>
M: David Hildenbrand <david@kernel.org>
R: Lorenzo Stoakes <ljs@kernel.org>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
R: Vlastimil Babka <vbabka@kernel.org>
R: Mike Rapoport <rppt@kernel.org>
R: Suren Baghdasaryan <surenb@google.com>
@@ -16962,6 +16972,7 @@
F: include/linux/compaction.h
F: include/linux/gfp.h
F: include/linux/page-isolation.h
+F: include/linux/pageblock-flags.h
F: mm/compaction.c
F: mm/debug_page_alloc.c
F: mm/debug_page_ref.c
@@ -16983,7 +16994,7 @@
M: Johannes Weiner <hannes@cmpxchg.org>
R: David Hildenbrand <david@kernel.org>
R: Michal Hocko <mhocko@kernel.org>
-R: Qi Zheng <zhengqi.arch@bytedance.com>
+R: Qi Zheng <qi.zheng@linux.dev>
R: Shakeel Butt <shakeel.butt@linux.dev>
R: Lorenzo Stoakes <ljs@kernel.org>
L: linux-mm@kvack.org
@@ -16996,7 +17007,7 @@
M: David Hildenbrand <david@kernel.org>
M: Lorenzo Stoakes <ljs@kernel.org>
R: Rik van Riel <riel@surriel.com>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
R: Vlastimil Babka <vbabka@kernel.org>
R: Harry Yoo <harry@kernel.org>
R: Jann Horn <jannh@google.com>
@@ -17043,7 +17054,7 @@
M: Lorenzo Stoakes <ljs@kernel.org>
R: Zi Yan <ziy@nvidia.com>
R: Baolin Wang <baolin.wang@linux.alibaba.com>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
R: Nico Pache <npache@redhat.com>
R: Ryan Roberts <ryan.roberts@arm.com>
R: Dev Jain <dev.jain@arm.com>
@@ -17081,7 +17092,7 @@
MEMORY MANAGEMENT - RUST
M: Alice Ryhl <aliceryhl@google.com>
R: Lorenzo Stoakes <ljs@kernel.org>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
L: linux-mm@kvack.org
L: rust-for-linux@vger.kernel.org
S: Maintained
@@ -17095,7 +17106,7 @@
MEMORY MAPPING
M: Andrew Morton <akpm@linux-foundation.org>
-M: Liam R. Howlett <Liam.Howlett@oracle.com>
+M: Liam R. Howlett <liam@infradead.org>
M: Lorenzo Stoakes <ljs@kernel.org>
R: Vlastimil Babka <vbabka@kernel.org>
R: Jann Horn <jannh@google.com>
@@ -17127,7 +17138,7 @@
MEMORY MAPPING - LOCKING
M: Andrew Morton <akpm@linux-foundation.org>
M: Suren Baghdasaryan <surenb@google.com>
-M: Liam R. Howlett <Liam.Howlett@oracle.com>
+M: Liam R. Howlett <liam@infradead.org>
M: Lorenzo Stoakes <ljs@kernel.org>
R: Vlastimil Babka <vbabka@kernel.org>
R: Shakeel Butt <shakeel.butt@linux.dev>
@@ -17142,7 +17153,7 @@
MEMORY MAPPING - MADVISE (MEMORY ADVICE)
M: Andrew Morton <akpm@linux-foundation.org>
-M: Liam R. Howlett <Liam.Howlett@oracle.com>
+M: Liam R. Howlett <liam@infradead.org>
M: Lorenzo Stoakes <ljs@kernel.org>
M: David Hildenbrand <david@kernel.org>
R: Vlastimil Babka <vbabka@kernel.org>
@@ -17330,7 +17341,7 @@
F: sound/soc/atmel
MICROCHIP CSI2DC DRIVER
-M: Eugen Hristev <eugen.hristev@microchip.com>
+M: Eugen Hristev <ehristev@kernel.org>
L: linux-media@vger.kernel.org
S: Supported
F: Documentation/devicetree/bindings/media/microchip,csi2dc.yaml
@@ -17357,7 +17368,7 @@
F: drivers/i2c/busses/i2c-at91.h
MICROCHIP ISC DRIVER
-M: Eugen Hristev <eugen.hristev@microchip.com>
+M: Eugen Hristev <ehristev@kernel.org>
L: linux-media@vger.kernel.org
S: Supported
F: Documentation/devicetree/bindings/media/atmel,isc.yaml
@@ -17369,7 +17380,7 @@
F: include/linux/atmel-isc-media.h
MICROCHIP ISI DRIVER
-M: Eugen Hristev <eugen.hristev@microchip.com>
+M: Eugen Hristev <ehristev@kernel.org>
L: linux-media@vger.kernel.org
S: Supported
F: drivers/media/platform/atmel/atmel-isi.c
@@ -17559,7 +17570,7 @@
F: drivers/gpu/drm/bridge/microchip-lvds.c
MICROCHIP SAMA5D2-COMPATIBLE ADC DRIVER
-M: Eugen Hristev <eugen.hristev@microchip.com>
+M: Eugen Hristev <ehristev@kernel.org>
L: linux-iio@vger.kernel.org
S: Supported
F: Documentation/devicetree/bindings/iio/adc/atmel,sama5d2-adc.yaml
@@ -18238,7 +18249,7 @@
F: include/uapi/linux/mmc/
MULTIPLEXER SUBSYSTEM
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
S: Odd Fixes
F: Documentation/ABI/testing/sysfs-class-mux*
F: Documentation/devicetree/bindings/mux/
@@ -18622,6 +18633,7 @@
X: Documentation/networking/mac80211-injection.rst
X: Documentation/networking/mac80211_hwsim/
X: Documentation/networking/regulatory.rst
+X: include/net/bluetooth/
X: include/net/cfg80211.h
X: include/net/ieee80211_radiotap.h
X: include/net/iw_handler.h
@@ -18672,19 +18684,59 @@
F: tools/testing/selftests/net/ipsec.c
NETWORKING [IPv4/IPv6]
-M: "David S. Miller" <davem@davemloft.net>
M: David Ahern <dsahern@kernel.org>
+M: Ido Schimmel <idosch@nvidia.com>
L: netdev@vger.kernel.org
S: Maintained
-T: git git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git
-F: arch/x86/net/*
-F: include/linux/ip.h
-F: include/linux/ipv6*
+F: Documentation/netlink/specs/rt-addr.yaml
+F: Documentation/netlink/specs/rt-neigh.yaml
+F: Documentation/netlink/specs/rt-route.yaml
+F: Documentation/netlink/specs/rt-rule.yaml
+F: include/linux/inetdevice.h
+F: include/linux/mroute*
+F: include/net/addrconf.h
+F: include/net/arp.h
F: include/net/fib*
+F: include/net/if_inet6.h
+F: include/net/inetpeer.h
F: include/net/ip*
+F: include/net/lwtunnel.h
+F: include/net/ndisc.h
+F: include/net/netns/nexthop.h
+F: include/net/nexthop.h
F: include/net/route.h
-F: net/ipv4/
-F: net/ipv6/
+F: include/uapi/linux/fib_rules.h
+F: include/uapi/linux/in_route.h
+F: include/uapi/linux/mroute*
+F: include/uapi/linux/nexthop.h
+F: net/core/fib*
+F: net/core/lwtunnel.c
+F: net/ipv4/arp.c
+F: net/ipv4/devinet.c
+F: net/ipv4/fib*
+F: net/ipv4/icmp.c
+F: net/ipv4/igmp.c
+F: net/ipv4/inet_fragment.c
+F: net/ipv4/inetpeer.c
+F: net/ipv4/ip*
+F: net/ipv4/metrics.c
+F: net/ipv4/netlink.c
+F: net/ipv4/nexthop.c
+F: net/ipv4/route.c
+F: net/ipv6/addr*
+F: net/ipv6/anycast.c
+F: net/ipv6/exthdrs.c
+F: net/ipv6/exthdrs_core.c
+F: net/ipv6/fib*
+F: net/ipv6/icmp.c
+F: net/ipv6/ip*
+F: net/ipv6/mcast*
+F: net/ipv6/ndisc.c
+F: net/ipv6/output_core.c
+F: net/ipv6/reassembly.c
+F: net/ipv6/route.c
+F: tools/testing/selftests/net/fib*
+F: tools/testing/selftests/net/forwarding/
NETWORKING [LABELED] (NetLabel, Labeled IPsec, SECMARK)
M: Paul Moore <paul@paul-moore.com>
@@ -18819,18 +18871,11 @@
F: drivers/net/net_failover.c
F: include/net/net_failover.h
-NEXTHOP
-M: David Ahern <dsahern@kernel.org>
-L: netdev@vger.kernel.org
-S: Maintained
-F: include/net/netns/nexthop.h
-F: include/net/nexthop.h
-F: include/uapi/linux/nexthop.h
-F: net/ipv4/nexthop.c
-
NFC SUBSYSTEM
-L: netdev@vger.kernel.org
-S: Orphan
+M: David Heidelberg <david+nfc@ixit.cz>
+L: oe-linux-nfc@lists.linux.dev
+S: Maintained
+T: git https://codeberg.org/linux-nfc/linux.git
F: Documentation/devicetree/bindings/net/nfc/
F: drivers/nfc/
F: include/net/nfc/
@@ -18898,7 +18943,8 @@
NIOS2 ARCHITECTURE
M: Dinh Nguyen <dinguyen@kernel.org>
-S: Maintained
+M: Simon Schuster <schuster.simon@siemens-energy.com>
+S: Supported
T: git git://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git
F: arch/nios2/
@@ -19314,7 +19360,7 @@
K: "nxp,tda998x"
NXP TFA9879 DRIVER
-M: Peter Rosin <peda@axentia.se>
+M: Peter Rosin <peda@lysator.liu.se>
L: linux-sound@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/sound/trivial-codec.yaml
@@ -19412,7 +19458,6 @@
F: include/uapi/misc/ocxl.h
OMAP AUDIO SUPPORT
-M: Peter Ujfalusi <peter.ujfalusi@gmail.com>
M: Jarkko Nikula <jarkko.nikula@bitmer.com>
L: linux-sound@vger.kernel.org
L: linux-omap@vger.kernel.org
@@ -20315,13 +20360,14 @@
F: drivers/pci/controller/dwc/pcie-armada8k.c
PCI DRIVER FOR CADENCE PCIE IP
+R: Aksh Garg <a-garg7@ti.com>
L: linux-pci@vger.kernel.org
S: Orphan
F: Documentation/devicetree/bindings/pci/cdns,*
-F: drivers/pci/controller/cadence/*cadence*
+F: drivers/pci/controller/cadence/
PCI DRIVER FOR CIX Sky1
-M: Hans Zhang <hans.zhang@cixtech.com>
+M: Hans Zhang <18255117159@163.com>
L: linux-pci@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/pci/cix,sky1-pcie-*.yaml
@@ -20433,7 +20479,7 @@
F: drivers/pci/controller/plda/pcie-plda.h
PCI DRIVER FOR RENESAS R-CAR
-M: Marek Vasut <marek.vasut+renesas@gmail.com>
+M: Marek Vasut <marek.vasut+renesas@mailbox.org>
M: Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>
L: linux-pci@vger.kernel.org
L: linux-renesas-soc@vger.kernel.org
@@ -20693,15 +20739,13 @@
F: drivers/pci/controller/dwc/pcie-keembay.c
PCIE DRIVER FOR INTEL LGM GW SOC
-M: Chuanhua Lei <lchuanhua@maxlinear.com>
L: linux-pci@vger.kernel.org
-S: Maintained
+S: Orphan
F: Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml
F: drivers/pci/controller/dwc/pcie-intel-gw.c
PCIE DRIVER FOR MEDIATEK
M: Ryder Lee <ryder.lee@mediatek.com>
-M: Jianjun Wang <jianjun.wang@mediatek.com>
L: linux-pci@vger.kernel.org
L: linux-mediatek@lists.infradead.org (moderated for non-subscribers)
S: Supported
@@ -20774,6 +20818,7 @@
S: Odd Fixes
T: git git://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git
F: Documentation/pcmcia/
+F: drivers/net/ethernet/8390/pcnet_cs.c
F: drivers/pcmcia/
F: include/pcmcia/
F: tools/pcmcia/
@@ -21295,7 +21340,7 @@
PRESSURE STALL INFORMATION (PSI)
M: Johannes Weiner <hannes@cmpxchg.org>
M: Suren Baghdasaryan <surenb@google.com>
-R: Peter Ziljstra <peterz@infradead.org>
+R: Peter Zijlstra <peterz@infradead.org>
S: Maintained
F: include/linux/psi*
F: kernel/sched/psi.c
@@ -22906,7 +22951,7 @@
K: riscv
RISC-V IOMMU
-M: Tomasz Jeznach <tjeznach@rivosinc.com>
+M: Tomasz Jeznach <tomasz.jeznach@linux.dev>
L: iommu@lists.linux.dev
L: linux-riscv@lists.infradead.org
S: Maintained
@@ -23369,7 +23414,7 @@
M: Danilo Krummrich <dakr@kernel.org>
R: Lorenzo Stoakes <ljs@kernel.org>
R: Vlastimil Babka <vbabka@kernel.org>
-R: Liam R. Howlett <Liam.Howlett@oracle.com>
+R: Liam R. Howlett <liam@infradead.org>
R: Uladzislau Rezki <urezki@gmail.com>
L: rust-for-linux@vger.kernel.org
S: Maintained
@@ -23521,7 +23566,7 @@
S390 PCI SUBSYSTEM
M: Niklas Schnelle <schnelle@linux.ibm.com>
-M: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
+M: Gerd Bayer <gbayer@linux.ibm.com>
L: linux-s390@vger.kernel.org
S: Supported
F: Documentation/arch/s390/pci.rst
@@ -24075,7 +24120,7 @@
SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) MICROCHIP DRIVER
M: Aubin Constans <aubin.constans@microchip.com>
-R: Eugen Hristev <eugen.hristev@collabora.com>
+R: Eugen Hristev <ehristev@kernel.org>
L: linux-mmc@vger.kernel.org
S: Supported
F: drivers/mmc/host/sdhci-of-at91.c
@@ -24314,7 +24359,7 @@
SHRINKER
M: Andrew Morton <akpm@linux-foundation.org>
M: Dave Chinner <david@fromorbit.com>
-R: Qi Zheng <zhengqi.arch@bytedance.com>
+R: Qi Zheng <qi.zheng@linux.dev>
R: Roman Gushchin <roman.gushchin@linux.dev>
R: Muchun Song <muchun.song@linux.dev>
L: linux-mm@kvack.org
@@ -24616,6 +24661,7 @@
F: fs/smb/client/smbdirect.*
F: fs/smb/smbdirect/
F: fs/smb/server/transport_rdma.*
+F: include/linux/smbdirect.h
SMC91x ETHERNET DRIVER
M: Nicolas Pitre <nico@fluxnic.net>
@@ -24764,6 +24810,7 @@
M: Song Liu <song@kernel.org>
M: Yu Kuai <yukuai@fnnas.com>
R: Li Nan <linan122@huawei.com>
+R: Xiao Ni <xiao@kernel.org>
L: linux-raid@vger.kernel.org
S: Supported
Q: https://patchwork.kernel.org/project/linux-raid/list/
@@ -25018,7 +25065,6 @@
M: Liam Girdwood <lgirdwood@gmail.com>
M: Peter Ujfalusi <peter.ujfalusi@linux.intel.com>
M: Bard Liao <yung-chuan.liao@linux.intel.com>
-M: Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
M: Daniel Baluta <daniel.baluta@nxp.com>
R: Kai Vehmanen <kai.vehmanen@linux.intel.com>
R: Pierre-Louis Bossart <pierre-louis.bossart@linux.dev>
@@ -25533,11 +25579,11 @@
F: drivers/phy/starfive/phy-jh7110-pcie.c
F: drivers/phy/starfive/phy-jh7110-usb.c
-STARFIVE JH8100 EXTERNAL INTERRUPT CONTROLLER DRIVER
+STARFIVE JHB100 EXTERNAL INTERRUPT CONTROLLER DRIVER
M: Changhuang Liang <changhuang.liang@starfivetech.com>
S: Supported
-F: Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
-F: drivers/irqchip/irq-starfive-jh8100-intc.c
+F: Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
+F: drivers/irqchip/irq-starfive-jhb100-intc.c
STATIC BRANCH/CALL
M: Peter Zijlstra <peterz@infradead.org>
@@ -26311,7 +26357,7 @@
F: drivers/irqchip/irq-xtensa-*
TEXAS INSTRUMENTS ASoC DRIVERS
-M: Peter Ujfalusi <peter.ujfalusi@gmail.com>
+M: Sen Wang <sen@ti.com>
L: linux-sound@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml
@@ -26813,12 +26859,6 @@
F: Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml
F: drivers/iio/adc/ti-tsc2046.c
-TI TWL4030 SERIES SOC CODEC DRIVER
-M: Peter Ujfalusi <peter.ujfalusi@gmail.com>
-L: linux-sound@vger.kernel.org
-S: Maintained
-F: sound/soc/codecs/twl4030*
-
TI VPE/CAL DRIVERS
M: Yemike Abhilash Chandra <y-abhilashchandra@ti.com>
L: linux-media@vger.kernel.org
@@ -28825,6 +28865,7 @@
L: x86-cpuid@lists.linux.dev
S: Maintained
W: https://x86-cpuid.org
+F: arch/x86/include/asm/cpuid/leaf_types.h
F: tools/arch/x86/kcpuid/
X86 ENTRY CODE
@@ -28844,6 +28885,7 @@
F: Documentation/admin-guide/hw-vuln/
F: arch/x86/include/asm/nospec-branch.h
F: arch/x86/kernel/cpu/bugs.c
+F: scripts/update-intel-ucode-defs.py
X86 MCE INFRASTRUCTURE
M: Tony Luck <tony.luck@intel.com>
@@ -28908,8 +28950,8 @@
X86 TRUST DOMAIN EXTENSIONS (TDX)
M: Kiryl Shutsemau <kas@kernel.org>
+M: Rick Edgecombe <rick.p.edgecombe@intel.com>
R: Dave Hansen <dave.hansen@linux.intel.com>
-R: Rick Edgecombe <rick.p.edgecombe@intel.com>
L: x86@kernel.org
L: linux-coco@lists.linux.dev
L: kvm@vger.kernel.org
diff --git a/Makefile b/Makefile
index e27c91e..d8da451 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
VERSION = 7
PATCHLEVEL = 1
SUBLEVEL = 0
-EXTRAVERSION = -rc1
+EXTRAVERSION = -rc6
NAME = Baby Opossum Posse
# *DOCUMENTATION*
@@ -486,6 +486,8 @@
-Wclippy::as_ptr_cast_mut \
-Wclippy::as_underscore \
-Wclippy::cast_lossless \
+ -Aclippy::collapsible_if \
+ -Aclippy::collapsible_match \
-Wclippy::ignored_unit_patterns \
-Aclippy::incompatible_msrv \
-Wclippy::mut_mut \
diff --git a/arch/Kconfig b/arch/Kconfig
index e868800..83d362f 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -403,6 +403,10 @@
config ARCH_32BIT_USTAT_F_TINODE
bool
+# Selected by architectures with Total Store Order (TSO)
+config ARCH_MEMORY_ORDER_TSO
+ bool
+
config HAVE_ASM_MODVERSIONS
bool
help
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild
index 483965c..b154b4e 100644
--- a/arch/alpha/include/asm/Kbuild
+++ b/arch/alpha/include/asm/Kbuild
@@ -5,4 +5,5 @@
generic-y += asm-offsets.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += text-patching.h
diff --git a/arch/alpha/kernel/irq.c b/arch/alpha/kernel/irq.c
index c67047c..4a6a8b1 100644
--- a/arch/alpha/kernel/irq.c
+++ b/arch/alpha/kernel/irq.c
@@ -72,16 +72,16 @@ int arch_show_interrupts(struct seq_file *p, int prec)
int j;
#ifdef CONFIG_SMP
- seq_puts(p, "IPI: ");
+ seq_puts(p, " IPI: ");
for_each_online_cpu(j)
seq_printf(p, "%10lu ", cpu_data[j].ipi_count);
seq_putc(p, '\n');
#endif
- seq_puts(p, "PMI: ");
+ seq_puts(p, " PMI: ");
for_each_online_cpu(j)
seq_printf(p, "%10lu ", per_cpu(irq_pmi_count, j));
- seq_puts(p, " Performance Monitoring\n");
- seq_printf(p, "ERR: %10lu\n", irq_err_count);
+ seq_puts(p, " Performance Monitoring\n");
+ seq_printf(p, " ERR: %10lu\n", irq_err_count);
return 0;
}
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index 4c69522e..483caac 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -5,5 +5,6 @@
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
generic-y += parport.h
+generic-y += ring_buffer.h
generic-y += user.h
generic-y += text-patching.h
diff --git a/arch/arm/boot/dts/gemini/gemini-sl93512r.dts b/arch/arm/boot/dts/gemini/gemini-sl93512r.dts
index 4992ec2..341dec9 100644
--- a/arch/arm/boot/dts/gemini/gemini-sl93512r.dts
+++ b/arch/arm/boot/dts/gemini/gemini-sl93512r.dts
@@ -146,7 +146,7 @@ flash@30000000 {
partitions {
compatible = "redboot-fis";
/* Eraseblock at 0xfe0000 */
- fis-index-block = <0x1fc>;
+ fis-index-block = <0x7f>;
};
};
diff --git a/arch/arm/boot/dts/gemini/gemini-sq201.dts b/arch/arm/boot/dts/gemini/gemini-sq201.dts
index f8c6f6e..bfd1e85 100644
--- a/arch/arm/boot/dts/gemini/gemini-sq201.dts
+++ b/arch/arm/boot/dts/gemini/gemini-sq201.dts
@@ -134,7 +134,7 @@ flash@30000000 {
partitions {
compatible = "redboot-fis";
/* Eraseblock at 0xfe0000 */
- fis-index-block = <0x1fc>;
+ fis-index-block = <0x7f>;
};
};
diff --git a/arch/arm/boot/dts/microchip/sam9x7.dtsi b/arch/arm/boot/dts/microchip/sam9x7.dtsi
index d242d7a..c680a50 100644
--- a/arch/arm/boot/dts/microchip/sam9x7.dtsi
+++ b/arch/arm/boot/dts/microchip/sam9x7.dtsi
@@ -990,9 +990,9 @@ gmac: ethernet@f802c000 {
<62 IRQ_TYPE_LEVEL_HIGH 3>, /* Queue 3 */
<63 IRQ_TYPE_LEVEL_HIGH 3>, /* Queue 4 */
<64 IRQ_TYPE_LEVEL_HIGH 3>; /* Queue 5 */
- clocks = <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_GCK 24>, <&pmc PMC_TYPE_GCK 67>;
- clock-names = "hclk", "pclk", "tx_clk", "tsu_clk";
- assigned-clocks = <&pmc PMC_TYPE_GCK 67>;
+ clocks = <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_GCK 24>;
+ clock-names = "hclk", "pclk", "tsu_clk";
+ assigned-clocks = <&pmc PMC_TYPE_GCK 24>;
assigned-clock-rates = <266666666>;
status = "disabled";
};
diff --git a/arch/arm/boot/dts/renesas/r7s72100-genmai.dts b/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
index 3c37565..da552a6 100644
--- a/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
+++ b/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
@@ -34,9 +34,6 @@ flash@18000000 {
clocks = <&mstp9_clks R7S72100_CLK_SPIBSC0>;
power-domains = <&cpg_clocks>;
- #address-cells = <1>;
- #size-cells = <1>;
-
partitions {
compatible = "fixed-partitions";
#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts b/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
index 91178fb..3306bc9 100644
--- a/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
+++ b/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
@@ -36,8 +36,6 @@ flash@18000000 {
power-domains = <&cpg_clocks>;
bank-width = <4>;
device-width = <1>;
- #address-cells = <1>;
- #size-cells = <1>;
partitions {
compatible = "fixed-partitions";
diff --git a/arch/arm/boot/dts/renesas/r7s72100.dtsi b/arch/arm/boot/dts/renesas/r7s72100.dtsi
index 245c26b..6ec57ff 100644
--- a/arch/arm/boot/dts/renesas/r7s72100.dtsi
+++ b/arch/arm/boot/dts/renesas/r7s72100.dtsi
@@ -37,7 +37,7 @@ b_clk: b {
clock-div = <3>;
};
- bsc: bus {
+ bsc: bus@0 {
compatible = "simple-bus";
#address-cells = <1>;
#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7778.dtsi b/arch/arm/boot/dts/renesas/r8a7778.dtsi
index 859dd29..7db456b 100644
--- a/arch/arm/boot/dts/renesas/r8a7778.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7778.dtsi
@@ -40,7 +40,7 @@ aliases {
spi2 = &hspi2;
};
- lbsc: bus {
+ lbsc: bus@0 {
compatible = "simple-bus";
#address-cells = <1>;
#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7779.dtsi b/arch/arm/boot/dts/renesas/r8a7779.dtsi
index e437c22..9e8a7e1 100644
--- a/arch/arm/boot/dts/renesas/r8a7779.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7779.dtsi
@@ -704,7 +704,7 @@ R8A7779_CLK_MMC1 R8A7779_CLK_MMC0
};
};
- lbsc: bus {
+ lbsc: bus@0 {
compatible = "simple-bus";
#address-cells = <1>;
#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7792.dtsi b/arch/arm/boot/dts/renesas/r8a7792.dtsi
index 9e0de69..fbdbcff 100644
--- a/arch/arm/boot/dts/renesas/r8a7792.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7792.dtsi
@@ -86,7 +86,7 @@ extal_clk: extal {
bootph-all;
};
- lbsc: bus {
+ lbsc: bus@0 {
compatible = "simple-bus";
#address-cells = <1>;
#size-cells = <1>;
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 03657ff..decad5f 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -3,6 +3,7 @@
generic-y += extable.h
generic-y += flat.h
generic-y += parport.h
+generic-y += ring_buffer.h
generated-y += mach-types.h
generated-y += unistd-nr.h
diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h
index 311e830..847590d 100644
--- a/arch/arm/include/asm/arch_gicv3.h
+++ b/arch/arm/include/asm/arch_gicv3.h
@@ -7,7 +7,7 @@
#ifndef __ASM_ARCH_GICV3_H
#define __ASM_ARCH_GICV3_H
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
#include <linux/io.h>
#include <linux/io-64-nonatomic-lo-hi.h>
@@ -257,5 +257,5 @@ static inline bool gic_has_relaxed_pmr_sync(void)
return false;
}
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
#endif /* !__ASM_ARCH_GICV3_H */
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 4e8e89a..b5fb469 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -551,8 +551,7 @@ void show_ipi_list(struct seq_file *p, int prec)
if (!ipi_desc[i])
continue;
- seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
- prec >= 4 ? " " : "");
+ seq_printf(p, "%*s%u:", prec - 1, "IPI", i);
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
diff --git a/arch/arm/mach-socfpga/platsmp.c b/arch/arm/mach-socfpga/platsmp.c
index 201191c..349e6c5 100644
--- a/arch/arm/mach-socfpga/platsmp.c
+++ b/arch/arm/mach-socfpga/platsmp.c
@@ -78,6 +78,7 @@ static void __init socfpga_smp_prepare_cpus(unsigned int max_cpus)
}
socfpga_scu_base_addr = of_iomap(np, 0);
+ of_node_put(np);
if (!socfpga_scu_base_addr)
return;
scu_enable(socfpga_scu_base_addr);
diff --git a/arch/arm/mach-versatile/integrator_cp.c b/arch/arm/mach-versatile/integrator_cp.c
index 2ed4ded..03dfb5f 100644
--- a/arch/arm/mach-versatile/integrator_cp.c
+++ b/arch/arm/mach-versatile/integrator_cp.c
@@ -86,14 +86,6 @@ static u64 notrace intcp_read_sched_clock(void)
return val;
}
-static void __init intcp_init_early(void)
-{
- cm_map = syscon_regmap_lookup_by_compatible("arm,core-module-integrator");
- if (IS_ERR(cm_map))
- return;
- sched_clock_register(intcp_read_sched_clock, 32, 24000000);
-}
-
static void __init intcp_init_irq_of(void)
{
cm_init();
@@ -119,6 +111,10 @@ static void __init intcp_init_of(void)
{
struct device_node *cpcon;
+ cm_map = syscon_regmap_lookup_by_compatible("arm,core-module-integrator");
+ if (!IS_ERR(cm_map))
+ sched_clock_register(intcp_read_sched_clock, 32, 24000000);
+
cpcon = of_find_matching_node(NULL, intcp_syscon_match);
if (!cpcon)
return;
@@ -138,7 +134,6 @@ static const char * intcp_dt_board_compat[] = {
DT_MACHINE_START(INTEGRATOR_CP_DT, "ARM Integrator/CP (Device Tree)")
.reserve = integrator_reserve,
.map_io = intcp_map_io,
- .init_early = intcp_init_early,
.init_irq = intcp_init_irq_of,
.init_machine = intcp_init_of,
.dt_compat = intcp_dt_board_compat,
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 7b27ee9..871bd58 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -925,7 +925,6 @@
depends on AEABI && MMU && CPU_V7
default y if ARM_ARCH_TIMER
select HAVE_GENERIC_VDSO
- select GENERIC_TIME_VSYSCALL
select GENERIC_GETTIMEOFDAY
help
Place in the process address space an ELF shared object
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index fe60738..7e331b4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -140,7 +140,6 @@
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL
select GENERIC_GETTIMEOFDAY
select HARDIRQS_SW_RESEND
select HAS_IOPORT
diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi
index 4a7a0ac..7e97361 100644
--- a/arch/arm64/boot/dts/qcom/eliza.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza.dtsi
@@ -843,7 +843,11 @@ ice: crypto@1d88000 {
"qcom,inline-crypto-engine";
reg = <0x0 0x01d88000 0x0 0x18000>;
- clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+ clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
+ <&gcc GCC_UFS_PHY_AHB_CLK>;
+ clock-names = "core",
+ "iface";
+ power-domains = <&gcc GCC_UFS_PHY_GDSC>;
};
tcsr_mutex: hwlock@1f40000 {
diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi
index f23cf81..8243698 100644
--- a/arch/arm64/boot/dts/qcom/glymur.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur.dtsi
@@ -2314,11 +2314,9 @@ usb_mp_qmpphy0: phy@fa3000 {
clocks = <&gcc GCC_USB3_MP_PHY_AUX_CLK>,
<&tcsr TCSR_USB3_0_CLKREF_EN>,
- <&rpmhcc RPMH_CXO_CLK>,
<&gcc GCC_USB3_MP_PHY_COM_AUX_CLK>,
<&gcc GCC_USB3_MP_PHY_PIPE_0_CLK>;
clock-names = "aux",
- "clkref",
"ref",
"com_aux",
"pipe";
@@ -2343,11 +2341,9 @@ usb_mp_qmpphy1: phy@fa5000 {
clocks = <&gcc GCC_USB3_MP_PHY_AUX_CLK>,
<&tcsr TCSR_USB3_1_CLKREF_EN>,
- <&rpmhcc RPMH_CXO_CLK>,
<&gcc GCC_USB3_MP_PHY_COM_AUX_CLK>,
<&gcc GCC_USB3_MP_PHY_PIPE_1_CLK>;
clock-names = "aux",
- "clkref",
"ref",
"com_aux",
"pipe";
@@ -2482,15 +2478,13 @@ usb_1_qmpphy: phy@fde000 {
reg = <0x0 0x00fde000 0x0 0x8000>;
clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>,
- <&rpmhcc RPMH_CXO_CLK>,
+ <&tcsr TCSR_USB4_1_CLKREF_EN>,
<&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>,
- <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>,
- <&tcsr TCSR_USB4_1_CLKREF_EN>;
+ <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>;
clock-names = "aux",
"ref",
"com_aux",
- "usb3_pipe",
- "clkref";
+ "usb3_pipe";
power-domains = <&gcc GCC_USB_1_PHY_GDSC>;
@@ -3750,15 +3744,13 @@ usb_2_qmpphy: phy@88e1000 {
reg = <0x0 0x088e1000 0x0 0x8000>;
clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>,
- <&rpmhcc RPMH_CXO_CLK>,
+ <&tcsr TCSR_USB4_2_CLKREF_EN>,
<&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>,
- <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>,
- <&tcsr TCSR_USB4_2_CLKREF_EN>;
+ <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>;
clock-names = "aux",
"ref",
"com_aux",
- "usb3_pipe",
- "clkref";
+ "usb3_pipe";
power-domains = <&gcc GCC_USB_2_PHY_GDSC>;
diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi
index 4a64a98..a6e463f3 100644
--- a/arch/arm64/boot/dts/qcom/milos.dtsi
+++ b/arch/arm64/boot/dts/qcom/milos.dtsi
@@ -1275,7 +1275,11 @@ ice: crypto@1d88000 {
"qcom,inline-crypto-engine";
reg = <0x0 0x01d88000 0x0 0x18000>;
- clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+ clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
+ <&gcc GCC_UFS_PHY_AHB_CLK>;
+ clock-names = "core",
+ "iface";
+ power-domains = <&gcc UFS_PHY_GDSC>;
};
tcsr_mutex: hwlock@1f40000 {
diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
index 0d9a324..db29173 100644
--- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
@@ -982,12 +982,6 @@ &i2c8 {
status = "okay";
};
-&i2c20 {
- clock-frequency = <400000>;
-
- status = "okay";
-};
-
&lpass_tlmm {
spkr_01_sd_n_active: spkr-01-sd-n-active-state {
pins = "gpio12";
@@ -1308,6 +1302,7 @@ right_tweeter: speaker@0,1 {
&tlmm {
gpio-reserved-ranges = <44 4>, /* SPI11 (TPM) */
<76 4>, /* SPI19 (TZ Protected) */
+ <80 2>, /* I2C20 (Battery SMBus) */
<238 1>; /* UFS Reset */
cam_rgb_default: cam-rgb-default-state {
diff --git a/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso b/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
index 258f8668..90767d7 100644
--- a/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
+++ b/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
@@ -27,7 +27,12 @@ &lvds1 {
status = "okay";
ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
port@1 {
+ reg = <1>;
+
lvds1_out: endpoint {
remote-endpoint = <&panel_in>;
};
diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
index 3e1c989..3ec1b53 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
@@ -699,7 +699,7 @@ scif0: serial@c0700000 {
"renesas,rcar-gen5-scif", "renesas,scif";
reg = <0 0xc0700000 0 0x40>;
interrupts = <GIC_ESPI 10 IRQ_TYPE_LEVEL_HIGH>;
- clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+ clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
clock-names = "fck", "brg_int", "scif_clk";
status = "disabled";
};
@@ -709,7 +709,7 @@ scif1: serial@c0704000 {
"renesas,rcar-gen5-scif", "renesas,scif";
reg = <0 0xc0704000 0 0x40>;
interrupts = <GIC_ESPI 11 IRQ_TYPE_LEVEL_HIGH>;
- clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+ clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
clock-names = "fck", "brg_int", "scif_clk";
status = "disabled";
};
@@ -719,7 +719,7 @@ scif3: serial@c0708000 {
"renesas,rcar-gen5-scif", "renesas,scif";
reg = <0 0xc0708000 0 0x40>;
interrupts = <GIC_ESPI 12 IRQ_TYPE_LEVEL_HIGH>;
- clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+ clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
clock-names = "fck", "brg_int", "scif_clk";
status = "disabled";
};
@@ -729,7 +729,7 @@ scif4: serial@c070c000 {
"renesas,rcar-gen5-scif", "renesas,scif";
reg = <0 0xc070c000 0 0x40>;
interrupts = <GIC_ESPI 13 IRQ_TYPE_LEVEL_HIGH>;
- clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+ clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
clock-names = "fck", "brg_int", "scif_clk";
status = "disabled";
};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index 4052547..7ccddd6 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -1327,6 +1327,7 @@ usb20phyrst: usb20phy-reset@15830000 {
resets = <&cpg 0xaf>;
power-domains = <&cpg>;
#reset-cells = <0>;
+ #mux-state-cells = <1>;
status = "disabled";
};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 9581af5..6f6fe5f 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -1345,6 +1345,7 @@ usb20phyrst: usb20phy-reset@15830000 {
resets = <&cpg 0xaf>;
power-domains = <&cpg>;
#reset-cells = <0>;
+ #mux-state-cells = <1>;
status = "disabled";
};
@@ -1355,6 +1356,7 @@ usb21phyrst: usb21phy-reset@15840000 {
resets = <&cpg 0xaf>;
power-domains = <&cpg>;
#reset-cells = <0>;
+ #mux-state-cells = <1>;
status = "disabled";
};
diff --git a/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi b/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
index 4d2b065..3feffa4 100644
--- a/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
+++ b/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
@@ -46,7 +46,12 @@ &csi2 {
status = "okay";
ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
port@0 {
+ reg = <0>;
+
csi2_in: endpoint {
clock-lanes = <0>;
data-lanes = <1 2>;
diff --git a/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi b/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
index 3670757..f541257 100644
--- a/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
+++ b/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
@@ -26,7 +26,12 @@ &du {
status = "okay";
ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
port@0 {
+ reg = <0>;
+
du_out_rgb: endpoint {
remote-endpoint = <&adv7513_in>;
};
diff --git a/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso b/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
index c83a30a..7807c3f 100644
--- a/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
+++ b/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
@@ -27,7 +27,12 @@ &lvds0 {
status = "okay";
ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
port@1 {
+ reg = <1>;
+
lvds0_out: endpoint {
remote-endpoint = <&panel_in>;
};
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index d905a07..96ce783 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -260,6 +260,7 @@
CONFIG_PCI_ENDPOINT_CONFIGFS=y
CONFIG_PCI_EPF_TEST=m
CONFIG_PCI_PWRCTRL_GENERIC=m
+CONFIG_POWER_SEQUENCING_PCIE_M2=m
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_FW_LOADER_USER_HELPER=y
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index f463a65..cc0702f 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -409,7 +409,7 @@ __AARCH64_INSN_FUNCS(cbz, 0x7F000000, 0x34000000)
__AARCH64_INSN_FUNCS(cbnz, 0x7F000000, 0x35000000)
__AARCH64_INSN_FUNCS(tbz, 0x7F000000, 0x36000000)
__AARCH64_INSN_FUNCS(tbnz, 0x7F000000, 0x37000000)
-__AARCH64_INSN_FUNCS(bcond, 0xFF000010, 0x54000000)
+__AARCH64_INSN_FUNCS(bcond, 0xFF000000, 0x54000000)
__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001)
__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002)
__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003)
diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h
index d4d7451..a8cb5a5 100644
--- a/arch/arm64/include/asm/irqflags.h
+++ b/arch/arm64/include/asm/irqflags.h
@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)
barrier();
}
-static inline void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
{
if (system_uses_irq_prio_masking()) {
__pmr_local_irq_enable();
@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)
barrier();
}
-static inline void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
{
if (system_uses_irq_prio_masking()) {
__pmr_local_irq_disable();
@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)
/*
* Save the current interrupt enable state.
*/
-static inline unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
{
if (system_uses_irq_prio_masking()) {
return __pmr_local_save_flags();
@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)
return flags != GIC_PRIO_IRQON;
}
-static inline bool arch_irqs_disabled_flags(unsigned long flags)
+static __always_inline bool arch_irqs_disabled_flags(unsigned long flags)
{
if (system_uses_irq_prio_masking()) {
return __pmr_irqs_disabled_flags(flags);
@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)
return __pmr_irqs_disabled_flags(__pmr_local_save_flags());
}
-static inline bool arch_irqs_disabled(void)
+static __always_inline bool arch_irqs_disabled(void)
{
if (system_uses_irq_prio_masking()) {
return __pmr_irqs_disabled();
@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)
return flags;
}
-static inline unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
{
if (system_uses_irq_prio_masking()) {
return __pmr_local_irq_save();
@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)
/*
* restore saved IRQ state
*/
-static inline void arch_local_irq_restore(unsigned long flags)
+static __always_inline void arch_local_irq_restore(unsigned long flags)
{
if (system_uses_irq_prio_masking()) {
__pmr_local_irq_restore(flags);
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 74a4f73..229ee79 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -68,7 +68,12 @@
#define KERNEL_SEGMENT_COUNT 5
#if SWAPPER_BLOCK_SIZE > SEGMENT_ALIGN
-#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1)
+/*
+ * KERNEL_SEGMENT_COUNT counts the permanent kernel VMAs. The early mapping
+ * has one additional split, [_text, _stext). Reserve one more page for the
+ * SWAPPER_BLOCK_SIZE-unaligned boundaries.
+ */
+#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 2)
/*
* The initial ID map consists of the kernel image, mapped as two separate
* segments, and may appear misaligned wrt the swapper block size. This means
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 3741444..043495f 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -50,6 +50,9 @@
#include <linux/mm.h>
+#define MARKER(m) \
+ m, __after_##m = m - 1
+
enum __kvm_host_smccc_func {
/* Hypercalls that are unavailable once pKVM has finalised. */
/* __KVM_HOST_SMCCC_FUNC___kvm_hyp_init */
@@ -59,8 +62,10 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config,
+
+ MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM),
+
__KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize,
- __KVM_HOST_SMCCC_FUNC_MIN_PKVM = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize,
/* Hypercalls that are always available and common to [nh]VHE/pKVM. */
__KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
@@ -72,11 +77,20 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,
__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
+ __KVM_HOST_SMCCC_FUNC___tracing_load,
+ __KVM_HOST_SMCCC_FUNC___tracing_unload,
+ __KVM_HOST_SMCCC_FUNC___tracing_enable,
+ __KVM_HOST_SMCCC_FUNC___tracing_swap_reader,
+ __KVM_HOST_SMCCC_FUNC___tracing_update_clock,
+ __KVM_HOST_SMCCC_FUNC___tracing_reset,
+ __KVM_HOST_SMCCC_FUNC___tracing_enable_event,
+ __KVM_HOST_SMCCC_FUNC___tracing_write_event,
__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
- __KVM_HOST_SMCCC_FUNC_MAX_NO_PKVM = __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
+
+ MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY),
/* Hypercalls that are available only when pKVM has finalised. */
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
@@ -100,14 +114,8 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load,
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put,
__KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid,
- __KVM_HOST_SMCCC_FUNC___tracing_load,
- __KVM_HOST_SMCCC_FUNC___tracing_unload,
- __KVM_HOST_SMCCC_FUNC___tracing_enable,
- __KVM_HOST_SMCCC_FUNC___tracing_swap_reader,
- __KVM_HOST_SMCCC_FUNC___tracing_update_clock,
- __KVM_HOST_SMCCC_FUNC___tracing_reset,
- __KVM_HOST_SMCCC_FUNC___tracing_enable_event,
- __KVM_HOST_SMCCC_FUNC___tracing_write_event,
+
+ MARKER(__KVM_HOST_SMCCC_FUNC_MAX)
};
#define DECLARE_KVM_VHE_SYM(sym) extern char sym[]
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 851f617..a49042b 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -450,9 +450,6 @@ struct kvm_vcpu_fault_info {
r = __VNCR_START__ + ((VNCR_ ## r) / 8), \
__after_##r = __MAX__(__before_##r - 1, r)
-#define MARKER(m) \
- m, __after_##m = m - 1
-
enum vcpu_sysreg {
__INVALID_SYSREG__, /* 0 is reserved as an invalid value */
MPIDR_EL1, /* MultiProcessor Affinity Register */
@@ -514,7 +511,6 @@ enum vcpu_sysreg {
ACTLR_EL2, /* Auxiliary Control Register (EL2) */
CPTR_EL2, /* Architectural Feature Trap Register (EL2) */
HACR_EL2, /* Hypervisor Auxiliary Control Register */
- ZCR_EL2, /* SVE Control Register (EL2) */
TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */
TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */
TCR_EL2, /* Translation Control Register (EL2) */
@@ -546,6 +542,7 @@ enum vcpu_sysreg {
SCTLR2_EL2, /* System Control Register 2 (EL2) */
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
+ ZCR_EL2, /* SVE Control Register (EL2) */
/* Any VNCR-capable reg goes after this point */
MARKER(__VNCR_START__),
@@ -1548,7 +1545,7 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature)
#define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f))
#define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f))
-#define kvm_vcpu_initialized(v) vcpu_get_flag(vcpu, VCPU_INITIALIZED)
+#define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED)
int kvm_trng_call(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index 091544e..dc29576 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -23,6 +23,7 @@ static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
{
return TCR_EPD1_MASK | /* disable TTBR1_EL1 */
+ ((tcr & TCR_EL2_DS) ? TCR_DS : 0) |
((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
tcr_el2_ps_to_tcr_el1_ips(tcr) |
(tcr & TCR_EL2_TG0_MASK) |
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index e25d0d1..58200de 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -33,7 +33,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
unsigned long vaddr);
#define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio
-bool tag_clear_highpages(struct page *to, int numpages);
+bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages);
#define __HAVE_ARCH_TAG_CLEAR_HIGHPAGES
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
diff --git a/arch/arm64/include/asm/ring_buffer.h b/arch/arm64/include/asm/ring_buffer.h
new file mode 100644
index 0000000..62316c4
--- /dev/null
+++ b/arch/arm64/include/asm/ring_buffer.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _ASM_ARM64_RING_BUFFER_H
+#define _ASM_ARM64_RING_BUFFER_H
+
+#include <asm/cacheflush.h>
+
+/* Flush D-cache on persistent ring buffer */
+#define arch_ring_buffer_flush_range(start, end) dcache_clean_pop(start, end)
+
+#endif /* _ASM_ARM64_RING_BUFFER_H */
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7365614..7aa08d5 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -844,7 +844,7 @@
#define INIT_SCTLR_EL2_MMU_ON \
(SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \
SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | \
- SCTLR_ELx_ITFSB | SCTLR_EL2_RES1)
+ SCTLR_ELx_ITFSB | SCTLR_ELx_EIS | SCTLR_ELx_EOS | SCTLR_EL2_RES1)
#define INIT_SCTLR_EL2_MMU_OFF \
(SCTLR_EL2_RES1 | ENDIAN_SET_EL2)
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 10869d7..751bd57 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -53,7 +53,8 @@ static inline int tlb_get_level(struct mmu_gather *tlb)
static inline void tlb_flush(struct mmu_gather *tlb)
{
struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
- tlbf_t flags = tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE;
+ tlbf_t flags = (tlb->freed_tables || tlb->unshared_tables) ?
+ TLBF_NONE : TLBF_NOWALKCACHE;
unsigned long stride = tlb_get_unmap_size(tlb);
int tlb_level = tlb_get_level(tlb);
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index cb54335..c7a23f7 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -62,6 +62,13 @@ static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
irqentry_exit_to_kernel_mode_after_preempt(regs, state);
}
+static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs)
+{
+ enter_from_user_mode(regs);
+ mte_disable_tco_entry(current);
+ sme_enter_from_user_mode();
+}
+
/*
* Handle IRQ/context state management when entering from user mode.
* Before this function is called it is not safe to call regular kernel code,
@@ -70,20 +77,30 @@ static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
{
enter_from_user_mode(regs);
+ rseq_note_user_irq_entry();
mte_disable_tco_entry(current);
sme_enter_from_user_mode();
}
+static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs)
+{
+ local_irq_disable();
+ syscall_exit_to_user_mode_prepare(regs);
+ local_daif_mask();
+ sme_exit_to_user_mode();
+ mte_check_tfsr_exit();
+ exit_to_user_mode();
+}
+
/*
* Handle IRQ/context state management when exiting to user mode.
* After this function returns it is not safe to call regular kernel code,
* instrumentable code, or any code which may trigger an exception.
*/
-
static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
{
local_irq_disable();
- exit_to_user_mode_prepare_legacy(regs);
+ irqentry_exit_to_user_mode_prepare(regs);
local_daif_mask();
sme_exit_to_user_mode();
mte_check_tfsr_exit();
@@ -92,7 +109,7 @@ static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
{
- arm64_exit_to_user_mode(regs);
+ arm64_syscall_exit_to_user_mode(regs);
}
/*
@@ -716,12 +733,12 @@ static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
static void noinstr el0_svc(struct pt_regs *regs)
{
- arm64_enter_from_user_mode(regs);
+ arm64_syscall_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
fpsimd_syscall_enter();
local_daif_restore(DAIF_PROCCTX);
do_el0_svc(regs);
- arm64_exit_to_user_mode(regs);
+ arm64_syscall_exit_to_user_mode(regs);
fpsimd_syscall_exit();
}
@@ -868,11 +885,11 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
static void noinstr el0_svc_compat(struct pt_regs *regs)
{
- arm64_enter_from_user_mode(regs);
+ arm64_syscall_enter_from_user_mode(regs);
cortex_a76_erratum_1463225_svc_handler();
local_daif_restore(DAIF_PROCCTX);
do_el0_svc_compat(regs);
- arm64_exit_to_user_mode(regs);
+ arm64_syscall_exit_to_user_mode(regs);
}
static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c
index dac568e..3944ad8 100644
--- a/arch/arm64/kernel/pi/patch-scs.c
+++ b/arch/arm64/kernel/pi/patch-scs.c
@@ -196,9 +196,9 @@ static int scs_handle_fde_frame(const struct eh_frame *frame,
loc += *opcode++ * code_alignment_factor;
loc += (*opcode++ << 8) * code_alignment_factor;
loc += (*opcode++ << 16) * code_alignment_factor;
- loc += (*opcode++ << 24) * code_alignment_factor;
+ loc += ((u64)*opcode++ << 24) * code_alignment_factor;
size -= 4;
- break;
+ break;
case DW_CFA_def_cfa:
case DW_CFA_offset_extended:
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index ba5eab2..4d08598 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -983,8 +983,8 @@ static int sve_set_common(struct task_struct *target,
}
/* Always zero V regs, FPSR, and FPCR */
- memset(¤t->thread.uw.fpsimd_state, 0,
- sizeof(current->thread.uw.fpsimd_state));
+ memset(&target->thread.uw.fpsimd_state, 0,
+ sizeof(target->thread.uw.fpsimd_state));
/* Registers: FPSIMD-only case */
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 08ffc5a..38e6fa2 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -67,6 +67,9 @@ struct rt_sigframe_user_layout {
unsigned long end_offset;
};
+#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16)
+#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16)
+
/*
* Holds any EL0-controlled state that influences unprivileged memory accesses.
* This includes both accesses done in userspace and uaccess done in the kernel.
@@ -74,13 +77,35 @@ struct rt_sigframe_user_layout {
* This state needs to be carefully managed to ensure that it doesn't cause
* uaccess to fail when setting up the signal frame, and the signal handler
* itself also expects a well-defined state when entered.
+ *
+ * The struct should be zero-initialised. Its members should only be accessed
+ * via the accessors below. __valid_fields tracks which of the fields are valid
+ * (have been set to some value).
*/
struct user_access_state {
- u64 por_el0;
+ unsigned int __valid_fields;
+ u64 __por_el0;
};
-#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16)
-#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16)
+#define UA_STATE_HAS_POR_EL0 BIT(0)
+
+static void set_ua_state_por_el0(struct user_access_state *ua_state,
+ u64 por_el0)
+{
+ ua_state->__por_el0 = por_el0;
+ ua_state->__valid_fields |= UA_STATE_HAS_POR_EL0;
+}
+
+static int get_ua_state_por_el0(const struct user_access_state *ua_state,
+ u64 *por_el0)
+{
+ if (ua_state->__valid_fields & UA_STATE_HAS_POR_EL0) {
+ *por_el0 = ua_state->__por_el0;
+ return 0;
+ }
+
+ return -ENOENT;
+}
/*
* Save the user access state into ua_state and reset it to disable any
@@ -94,7 +119,7 @@ static void save_reset_user_access_state(struct user_access_state *ua_state)
for (int pkey = 0; pkey < arch_max_pkey(); pkey++)
por_enable_all |= POR_ELx_PERM_PREP(pkey, POE_RWX);
- ua_state->por_el0 = read_sysreg_s(SYS_POR_EL0);
+ set_ua_state_por_el0(ua_state, read_sysreg_s(SYS_POR_EL0));
write_sysreg_s(por_enable_all, SYS_POR_EL0);
/*
* No ISB required as we can tolerate spurious Overlay faults -
@@ -122,8 +147,10 @@ static void set_handler_user_access_state(void)
*/
static void restore_user_access_state(const struct user_access_state *ua_state)
{
- if (system_supports_poe())
- write_sysreg_s(ua_state->por_el0, SYS_POR_EL0);
+ u64 por_el0;
+
+ if (get_ua_state_por_el0(ua_state, &por_el0) == 0)
+ write_sysreg_s(por_el0, SYS_POR_EL0);
}
static void init_user_layout(struct rt_sigframe_user_layout *user)
@@ -333,11 +360,16 @@ static int restore_fpmr_context(struct user_ctxs *user)
static int preserve_poe_context(struct poe_context __user *ctx,
const struct user_access_state *ua_state)
{
- int err = 0;
+ int err;
+ u64 por_el0;
+
+ err = get_ua_state_por_el0(ua_state, &por_el0);
+ if (WARN_ON_ONCE(err))
+ return err;
__put_user_error(POE_MAGIC, &ctx->head.magic, err);
__put_user_error(sizeof(*ctx), &ctx->head.size, err);
- __put_user_error(ua_state->por_el0, &ctx->por_el0, err);
+ __put_user_error(por_el0, &ctx->por_el0, err);
return err;
}
@@ -353,7 +385,7 @@ static int restore_poe_context(struct user_ctxs *user,
__get_user_error(por_el0, &(user->poe->por_el0), err);
if (!err)
- ua_state->por_el0 = por_el0;
+ set_ua_state_por_el0(ua_state, por_el0);
return err;
}
@@ -1095,7 +1127,7 @@ SYSCALL_DEFINE0(rt_sigreturn)
{
struct pt_regs *regs = current_pt_regs();
struct rt_sigframe __user *frame;
- struct user_access_state ua_state;
+ struct user_access_state ua_state = {};
/* Always make any pending restarted system calls return -EINTR */
current->restart_block.fn = do_no_restart_syscall;
@@ -1507,7 +1539,7 @@ static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
{
struct rt_sigframe_user_layout user;
struct rt_sigframe __user *frame;
- struct user_access_state ua_state;
+ struct user_access_state ua_state = {};
int err = 0;
fpsimd_save_and_flush_current_state();
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 1aa3241..1d0e0e6 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -833,11 +833,10 @@ int arch_show_interrupts(struct seq_file *p, int prec)
unsigned int cpu, i;
for (i = 0; i < MAX_IPI; i++) {
- seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
- prec >= 4 ? " " : "");
+ seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
- seq_printf(p, " %s\n", ipi_types[i]);
+ seq_printf(p, " %s\n", ipi_types[i]);
}
seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 176cbe8..9453321 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -4,6 +4,7 @@
* Author: Christoffer Dall <c.dall@virtualopensystems.com>
*/
+#include <linux/arm-smccc.h>
#include <linux/bug.h>
#include <linux/cpu_pm.h>
#include <linux/errno.h>
@@ -554,8 +555,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
kvm_destroy_mpidr_data(vcpu->kvm);
err = kvm_vgic_vcpu_init(vcpu);
- if (err)
+ if (err) {
+ kvm_vgic_vcpu_destroy(vcpu);
return err;
+ }
err = kvm_share_hyp(vcpu, vcpu + 1);
if (err)
@@ -824,6 +827,10 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
+ irq_lines |= (!irqchip_in_kernel(v->kvm) &&
+ (kvm_timer_should_notify_user(v) ||
+ kvm_pmu_should_notify_user(v)));
+
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
}
@@ -2634,6 +2641,22 @@ static int init_pkvm_host_sve_state(void)
return 0;
}
+static int pkvm_check_sme_dvmsync_fw_call(void)
+{
+ struct arm_smccc_res res;
+
+ if (!cpus_have_final_cap(ARM64_WORKAROUND_4193714))
+ return 0;
+
+ arm_smccc_1_1_smc(ARM_SMCCC_CPU_WORKAROUND_4193714, &res);
+ if (res.a0) {
+ kvm_err("pKVM requires firmware support for C1-Pro erratum 4193714\n");
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
/*
* Finalizes the initialization of hyp mode, once everything else is initialized
* and the initialziation process cannot fail.
@@ -2834,6 +2857,10 @@ static int __init init_hyp_mode(void)
if (err)
goto out_err;
+ err = pkvm_check_sme_dvmsync_fw_call();
+ if (err)
+ goto out_err;
+
err = kvm_hyp_init_protection(hyp_va_bits);
if (err) {
kvm_err("Failed to init hyp memory protection\n");
diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c
index f35b8dd..0622162 100644
--- a/arch/arm64/kvm/config.c
+++ b/arch/arm64/kvm/config.c
@@ -131,7 +131,6 @@ struct reg_feat_map_desc {
}
#define FEAT_SPE ID_AA64DFR0_EL1, PMSVer, IMP
-#define FEAT_SPE_FnE ID_AA64DFR0_EL1, PMSVer, V1P2
#define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP
#define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP
#define FEAT_PMUv3 ID_AA64DFR0_EL1, PMUVer, IMP
@@ -192,7 +191,7 @@ struct reg_feat_map_desc {
#define FEAT_SRMASK ID_AA64MMFR4_EL1, SRMASK, IMP
#define FEAT_PoPS ID_AA64MMFR4_EL1, PoPS, IMP
#define FEAT_PFAR ID_AA64PFR1_EL1, PFAR, IMP
-#define FEAT_Debugv8p9 ID_AA64DFR0_EL1, PMUVer, V3P9
+#define FEAT_Debugv8p9 ID_AA64DFR0_EL1, DebugVer, V8P9
#define FEAT_PMUv3_SS ID_AA64DFR0_EL1, PMSS, IMP
#define FEAT_SEBEP ID_AA64DFR0_EL1, SEBEP, IMP
#define FEAT_EBEP ID_AA64DFR1_EL1, EBEP, IMP
@@ -283,7 +282,7 @@ static bool feat_anerr(struct kvm *kvm)
static bool feat_sme_smps(struct kvm *kvm)
{
/*
- * Revists this if KVM ever supports SME -- this really should
+ * Revisit this if KVM ever supports SME -- this really should
* look at the guest's view of SMIDR_EL1. Funnily enough, this
* is not captured in the JSON file, but only as a note in the
* ARM ARM.
@@ -295,17 +294,27 @@ static bool feat_sme_smps(struct kvm *kvm)
static bool feat_spe_fds(struct kvm *kvm)
{
/*
- * Revists this if KVM ever supports SPE -- this really should
+ * Revisit this if KVM ever supports SPE -- this really should
* look at the guest's view of PMSIDR_EL1.
*/
return (kvm_has_feat(kvm, FEAT_SPEv1p4) &&
(read_sysreg_s(SYS_PMSIDR_EL1) & PMSIDR_EL1_FDS));
}
+static bool feat_spe_fne(struct kvm *kvm)
+{
+ /*
+ * Revisit this if KVM ever supports SPE -- this really should
+ * look at the guest's view of PMSIDR_EL1.
+ */
+ return (kvm_has_feat(kvm, FEAT_SPEv1p2) &&
+ (read_sysreg_s(SYS_PMSIDR_EL1) & PMSIDR_EL1_FnE));
+}
+
static bool feat_trbe_mpam(struct kvm *kvm)
{
/*
- * Revists this if KVM ever supports both MPAM and TRBE --
+ * Revisit this if KVM ever supports both MPAM and TRBE --
* this really should look at the guest's view of TRBIDR_EL1.
*/
return (kvm_has_feat(kvm, FEAT_TRBE) &&
@@ -537,7 +546,7 @@ static const struct reg_bits_to_feat_map hdfgrtr_feat_map[] = {
HDFGRTR_EL2_PMBPTR_EL1 |
HDFGRTR_EL2_PMBLIMITR_EL1,
FEAT_SPE),
- NEEDS_FEAT(HDFGRTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE),
+ NEEDS_FEAT(HDFGRTR_EL2_nPMSNEVFR_EL1, feat_spe_fne),
NEEDS_FEAT(HDFGRTR_EL2_nBRBDATA |
HDFGRTR_EL2_nBRBCTL |
HDFGRTR_EL2_nBRBIDR,
@@ -605,7 +614,7 @@ static const struct reg_bits_to_feat_map hdfgwtr_feat_map[] = {
HDFGWTR_EL2_PMBPTR_EL1 |
HDFGWTR_EL2_PMBLIMITR_EL1,
FEAT_SPE),
- NEEDS_FEAT(HDFGWTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE),
+ NEEDS_FEAT(HDFGWTR_EL2_nPMSNEVFR_EL1, feat_spe_fne),
NEEDS_FEAT(HDFGWTR_EL2_nBRBDATA |
HDFGWTR_EL2_nBRBCTL,
FEAT_BRBE),
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 98b2976..320cd45 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -245,7 +245,7 @@ static inline void __activate_traps_ich_hfgxtr(struct kvm_vcpu *vcpu)
__activate_fgt(hctxt, vcpu, ICH_HFGITR_EL2);
}
-#define __deactivate_fgt(htcxt, vcpu, reg) \
+#define __deactivate_fgt(hctxt, vcpu, reg) \
do { \
write_sysreg_s(ctxt_sys_reg(hctxt, reg), \
SYS_ ## reg); \
@@ -462,11 +462,13 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
{
+ u64 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+
/*
* The vCPU's saved SVE state layout always matches the max VL of the
* vCPU. Start off with the max VL so we can load the SVE state.
*/
- sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
+ sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2);
__sve_restore_state(vcpu_sve_pffr(vcpu),
&vcpu->arch.ctxt.fp_regs.fpsr,
true);
@@ -476,8 +478,10 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* nested guest, as the guest hypervisor could select a smaller VL. Slap
* that into hardware before wrapping up.
*/
- if (is_nested_ctxt(vcpu))
- sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
+ if (is_nested_ctxt(vcpu)) {
+ zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2));
+ sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2);
+ }
write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
}
@@ -501,11 +505,11 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
return;
if (vcpu_has_sve(vcpu)) {
+ zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+
/* A guest hypervisor may restrict the effective max VL. */
if (is_nested_ctxt(vcpu))
- zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
- else
- zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+ zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2));
write_sysreg_el2(zcr_el2, SYS_ZCR);
diff --git a/arch/arm64/kvm/hyp/nvhe/clock.c b/arch/arm64/kvm/hyp/nvhe/clock.c
index 32fc431..a7fc619 100644
--- a/arch/arm64/kvm/hyp/nvhe/clock.c
+++ b/arch/arm64/kvm/hyp/nvhe/clock.c
@@ -35,6 +35,9 @@ void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
struct clock_data *clock = &trace_clock_data;
u64 bank = clock->cur ^ 1;
+ if (!mult || shift >= 64)
+ return;
+
clock->data[bank].mult = mult;
clock->data[bank].shift = shift;
clock->data[bank].epoch_ns = epoch_ns;
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index f337770..9393fe3 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -120,7 +120,7 @@
mov x29, x0
-#ifdef PKVM_DISABLE_STAGE2_ON_PANIC
+#ifdef CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC
/* Ensure host stage-2 is disabled */
mrs x0, hcr_el2
bic x0, x0, #HCR_VM
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 73f2e02..06db299 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -709,6 +709,14 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__kvm_tlb_flush_vmid_range),
HANDLE_FUNC(__kvm_flush_cpu_context),
HANDLE_FUNC(__kvm_timer_set_cntvoff),
+ HANDLE_FUNC(__tracing_load),
+ HANDLE_FUNC(__tracing_unload),
+ HANDLE_FUNC(__tracing_enable),
+ HANDLE_FUNC(__tracing_swap_reader),
+ HANDLE_FUNC(__tracing_update_clock),
+ HANDLE_FUNC(__tracing_reset),
+ HANDLE_FUNC(__tracing_enable_event),
+ HANDLE_FUNC(__tracing_write_event),
HANDLE_FUNC(__vgic_v3_save_aprs),
HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs),
HANDLE_FUNC(__vgic_v5_save_apr),
@@ -735,22 +743,16 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__pkvm_vcpu_load),
HANDLE_FUNC(__pkvm_vcpu_put),
HANDLE_FUNC(__pkvm_tlb_flush_vmid),
- HANDLE_FUNC(__tracing_load),
- HANDLE_FUNC(__tracing_unload),
- HANDLE_FUNC(__tracing_enable),
- HANDLE_FUNC(__tracing_swap_reader),
- HANDLE_FUNC(__tracing_update_clock),
- HANDLE_FUNC(__tracing_reset),
- HANDLE_FUNC(__tracing_enable_event),
- HANDLE_FUNC(__tracing_write_event),
};
static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(unsigned long, id, host_ctxt, 0);
- unsigned long hcall_min = 0, hcall_max = -1;
+ unsigned long hcall_min = 0, hcall_max = __KVM_HOST_SMCCC_FUNC_MAX;
hcall_t hfn;
+ BUILD_BUG_ON(ARRAY_SIZE(host_hcall) != __KVM_HOST_SMCCC_FUNC_MAX);
+
/*
* If pKVM has been initialised then reject any calls to the
* early "privileged" hypercalls. Note that we cannot reject
@@ -763,16 +765,14 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
if (static_branch_unlikely(&kvm_protected_mode_initialized)) {
hcall_min = __KVM_HOST_SMCCC_FUNC_MIN_PKVM;
} else {
- hcall_max = __KVM_HOST_SMCCC_FUNC_MAX_NO_PKVM;
+ hcall_max = __KVM_HOST_SMCCC_FUNC_PKVM_ONLY;
}
id &= ~ARM_SMCCC_CALL_HINTS;
id -= KVM_HOST_SMCCC_ID(0);
- if (unlikely(id < hcall_min || id > hcall_max ||
- id >= ARRAY_SIZE(host_hcall))) {
+ if (unlikely(id < hcall_min || id >= hcall_max))
goto inval;
- }
hfn = host_hcall[id];
if (unlikely(!hfn))
@@ -805,6 +805,10 @@ static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
}
func_id &= ~ARM_SMCCC_CALL_HINTS;
+ if (upper_32_bits(func_id)) {
+ cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
+ goto exit_skip_instr;
+ }
handled = kvm_host_psci_handler(host_ctxt, func_id);
if (!handled)
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 28a471d..25f0462 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -5,6 +5,7 @@
*/
#include <linux/kvm_host.h>
+
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
@@ -14,6 +15,7 @@
#include <hyp/fault.h>
+#include <nvhe/arm-smccc.h>
#include <nvhe/gfp.h>
#include <nvhe/memory.h>
#include <nvhe/mem_protect.h>
@@ -29,6 +31,19 @@ static struct hyp_pool host_s2_pool;
static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm);
#define current_vm (*this_cpu_ptr(&__current_vm))
+static void pkvm_sme_dvmsync_fw_call(void)
+{
+ if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) {
+ struct arm_smccc_res res;
+
+ /*
+ * Ignore the return value. Probing for the workaround
+ * availability took place in init_hyp_mode().
+ */
+ hyp_smccc_1_1_smc(ARM_SMCCC_CPU_WORKAROUND_4193714, &res);
+ }
+}
+
static void guest_lock_component(struct pkvm_hyp_vm *vm)
{
hyp_spin_lock(&vm->lock);
@@ -574,8 +589,14 @@ static int host_stage2_set_owner_metadata_locked(phys_addr_t addr, u64 size,
ret = host_stage2_try(kvm_pgtable_stage2_annotate, &host_mmu.pgt,
addr, size, &host_s2_pool,
KVM_HOST_INVALID_PTE_TYPE_DONATION, annotation);
- if (!ret)
+ if (!ret) {
+ /*
+ * After stage2 maintenance has happened, but before the page
+ * owner has changed.
+ */
+ pkvm_sme_dvmsync_fw_call();
__host_update_page_state(addr, size, PKVM_NOPAGE);
+ }
return ret;
}
@@ -1369,6 +1390,22 @@ int __pkvm_host_reclaim_page_guest(u64 gfn, struct pkvm_hyp_vm *vm)
return ret && ret != -EHWPOISON ? ret : 0;
}
+/*
+ * share/donate install at most one stage-2 leaf (PAGE_SIZE, or one
+ * KVM_PGTABLE_LAST_LEVEL - 1 block for share). kvm_mmu_cache_min_pages()
+ * bounds the worst-case allocation: exact for the PAGE_SIZE leaf,
+ * conservative by one for the block.
+ */
+static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu)
+{
+ struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
+
+ if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu))
+ return -ENOMEM;
+
+ return 0;
+}
+
int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
{
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
@@ -1388,6 +1425,10 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
if (ret)
goto unlock;
+ ret = __guest_check_pgtable_memcache(vcpu);
+ if (ret)
+ goto unlock;
+
meta = host_stage2_encode_gfn_meta(vm, gfn);
WARN_ON(host_stage2_set_owner_metadata_locked(phys, PAGE_SIZE,
PKVM_ID_GUEST, meta));
@@ -1453,6 +1494,10 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu
}
}
+ ret = __guest_check_pgtable_memcache(vcpu);
+ if (ret)
+ goto unlock;
+
for_each_hyp_page(page, phys, size) {
set_host_state(page, PKVM_PAGE_SHARED_OWNED);
page->host_share_guest_count++;
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 7ed96d6..eb1c101 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -266,7 +266,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
if (hyp_vm->kvm.created_vcpus <= vcpu_idx)
goto unlock;
- hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
+ /* Pairs with smp_store_release() in register_hyp_vcpu(). */
+ hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[vcpu_idx]);
if (!hyp_vcpu)
goto unlock;
@@ -751,16 +752,30 @@ static struct pkvm_hyp_vcpu selftest_vcpu = {
struct pkvm_hyp_vcpu *init_selftest_vm(void *virt)
{
struct hyp_page *p = hyp_virt_to_page(virt);
+ unsigned long min_pages, seeded = 0;
int i;
selftest_vm.kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr;
WARN_ON(kvm_guest_prepare_stage2(&selftest_vm, virt));
+ /*
+ * Mirror pkvm_refill_memcache() for the share/donate pre-checks;
+ * the selftest invokes those functions directly and would
+ * otherwise see an empty memcache.
+ */
+ min_pages = kvm_mmu_cache_min_pages(&selftest_vm.kvm.arch.mmu);
+
for (i = 0; i < pkvm_selftest_pages(); i++) {
if (p[i].refcount)
continue;
p[i].refcount = 1;
- hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i]));
+ if (seeded < min_pages) {
+ push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache,
+ hyp_page_to_virt(&p[i]), hyp_virt_to_phys);
+ seeded++;
+ } else {
+ hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i]));
+ }
}
selftest_vm.kvm.arch.pkvm.handle = __pkvm_reserve_vm();
@@ -860,12 +875,30 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
* the page-aligned size of 'struct pkvm_hyp_vcpu'.
* Return 0 on success, negative error code on failure.
*/
+static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm,
+ struct pkvm_hyp_vcpu *hyp_vcpu)
+{
+ unsigned int idx = hyp_vcpu->vcpu.vcpu_idx;
+
+ if (idx >= hyp_vm->kvm.created_vcpus)
+ return -EINVAL;
+
+ if (hyp_vm->vcpus[idx])
+ return -EINVAL;
+
+ /*
+ * Ensure the hyp_vcpu is initialised before publishing it to
+ * the vCPU-load path via 'hyp_vm->vcpus[]'.
+ */
+ smp_store_release(&hyp_vm->vcpus[idx], hyp_vcpu);
+ return 0;
+}
+
int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
unsigned long vcpu_hva)
{
struct pkvm_hyp_vcpu *hyp_vcpu;
struct pkvm_hyp_vm *hyp_vm;
- unsigned int idx;
int ret;
hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
@@ -884,18 +917,11 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
if (ret)
goto unlock;
- idx = hyp_vcpu->vcpu.vcpu_idx;
- if (idx >= hyp_vm->kvm.created_vcpus) {
- ret = -EINVAL;
- goto unlock;
+ ret = register_hyp_vcpu(hyp_vm, hyp_vcpu);
+ if (ret) {
+ unpin_host_vcpu(host_vcpu);
+ unpin_host_sve_state(hyp_vcpu);
}
-
- if (hyp_vm->vcpus[idx]) {
- ret = -EINVAL;
- goto unlock;
- }
-
- hyp_vm->vcpus[idx] = hyp_vcpu;
unlock:
hyp_spin_unlock(&vm_table_lock);
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index d8e5b56..d461981 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -312,10 +312,6 @@ void __noreturn __pkvm_init_finalise(void)
};
pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
- ret = fix_host_ownership();
- if (ret)
- goto out;
-
ret = fix_hyp_pgtable_refcnt();
if (ret)
goto out;
@@ -324,6 +320,10 @@ void __noreturn __pkvm_init_finalise(void)
if (ret)
goto out;
+ ret = fix_host_ownership();
+ if (ret)
+ goto out;
+
ret = hyp_ffa_init(ffa_proxy_pages);
if (ret)
goto out;
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c
index a6ca27b..e7e150a 100644
--- a/arch/arm64/kvm/hyp/nvhe/trace.c
+++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -164,13 +164,16 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
return ret;
}
-static bool hyp_trace_desc_validate(struct hyp_trace_desc *desc, size_t desc_size)
+static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_size)
{
struct ring_buffer_desc *rb_desc;
unsigned int cpu;
size_t nr_bpages;
void *desc_end;
+ if (!is_protected_kvm_enabled())
+ return true;
+
/*
* Both desc_size and bpages_backing_size are untrusted host-provided
* values. We rely on __pkvm_host_donate_hyp() to enforce their validity.
@@ -212,8 +215,10 @@ int __tracing_load(unsigned long desc_hva, size_t desc_size)
if (ret)
return ret;
- if (!hyp_trace_desc_validate(desc, desc_size))
+ if (!hyp_trace_desc_is_valid(desc, desc_size)) {
+ ret = -EINVAL;
goto err_release_desc;
+ }
hyp_spin_lock(&trace_buffer.lock);
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 9db3f11..1e8995a 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -663,7 +663,8 @@ static void __noreturn __hyp_call_panic(u64 spsr, u64 elr, u64 par)
host_ctxt = host_data_ptr(host_ctxt);
vcpu = host_ctxt->__hyp_running_vcpu;
- __deactivate_traps(vcpu);
+ if (vcpu)
+ __deactivate_traps(vcpu);
sysreg_restore_host_state_vhe(host_ctxt);
panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n",
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c
index 8b7f2bf..2411b4c 100644
--- a/arch/arm64/kvm/hyp_trace.c
+++ b/arch/arm64/kvm/hyp_trace.c
@@ -51,8 +51,8 @@ static void __hyp_clock_work(struct work_struct *work)
hyp_clock = container_of(dwork, struct hyp_trace_clock, work);
- ktime_get_snapshot(&snap);
- boot = ktime_to_ns(snap.boot);
+ ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap);
+ boot = ktime_to_ns(snap.systime);
delta_boot = boot - hyp_clock->boot;
delta_cycles = snap.cycles - hyp_clock->cycles;
@@ -118,9 +118,9 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl
hyp_clock->running = false;
}
- ktime_get_snapshot(&snap);
+ ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap);
- hyp_clock->boot = ktime_to_ns(snap.boot);
+ hyp_clock->boot = ktime_to_ns(snap.systime);
hyp_clock->cycles = snap.cycles;
hyp_clock->mult = 0;
@@ -189,7 +189,7 @@ static void hyp_trace_buffer_unshare_hyp(struct hyp_trace_buffer *trace_buffer,
if (cpu > last_cpu)
break;
- __share_page(rb_desc->meta_va);
+ __unshare_page(rb_desc->meta_va);
for (p = 0; p < rb_desc->nr_page_va; p++)
__unshare_page(rb_desc->page_va[p]);
}
@@ -212,14 +212,15 @@ static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer)
}
if (ret) {
- for (p--; p >= 0; p--)
+ while (--p >= 0)
__unshare_page(rb_desc->page_va[p]);
+ __unshare_page(rb_desc->meta_va);
break;
}
}
if (ret)
- hyp_trace_buffer_unshare_hyp(trace_buffer, cpu--);
+ hyp_trace_buffer_unshare_hyp(trace_buffer, --cpu);
return ret;
}
@@ -248,6 +249,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv)
goto err_free_desc;
trace_buffer->desc = desc;
+ trace_buffer->desc_size = desc_size;
ret = hyp_trace_buffer_alloc_bpages_backing(trace_buffer, size);
if (ret)
@@ -297,6 +299,7 @@ static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv)
hyp_trace_buffer_free_bpages_backing(trace_buffer);
free_pages_exact(trace_buffer->desc, trace_buffer->desc_size);
trace_buffer->desc = NULL;
+ trace_buffer->desc_size = 0;
}
static int hyp_trace_enable_tracing(bool enable, void *priv)
diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c
index 58c5fe7..b11b882 100644
--- a/arch/arm64/kvm/hypercalls.c
+++ b/arch/arm64/kvm/hypercalls.c
@@ -28,7 +28,7 @@ static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val)
* system time and counter value must captured at the same
* time to keep consistency and precision.
*/
- ktime_get_snapshot(&systime_snapshot);
+ ktime_get_snapshot_id(CLOCK_REALTIME, &systime_snapshot);
/*
* This is only valid if the current clocksource is the
@@ -61,8 +61,8 @@ static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val)
* in the future (about 292 years from 1970, and at that stage
* nobody will give a damn about it).
*/
- val[0] = upper_32_bits(systime_snapshot.real);
- val[1] = lower_32_bits(systime_snapshot.real);
+ val[0] = upper_32_bits(systime_snapshot.systime);
+ val[1] = lower_32_bits(systime_snapshot.systime);
val[2] = upper_32_bits(cycles);
val[3] = lower_32_bits(cycles);
}
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index d089c10..4da9281 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1576,21 +1576,24 @@ struct kvm_s2_fault_desc {
static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
{
bool write_fault, exec_fault;
+ bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt;
unsigned long mmu_seq;
struct page *page;
struct kvm *kvm = s2fd->vcpu->kvm;
- void *memcache;
+ void *memcache = NULL;
kvm_pfn_t pfn;
gfn_t gfn;
int ret;
- memcache = get_mmu_memcache(s2fd->vcpu);
- ret = topup_mmu_memcache(s2fd->vcpu, memcache);
- if (ret)
- return ret;
+ if (!perm_fault) {
+ memcache = get_mmu_memcache(s2fd->vcpu);
+ ret = topup_mmu_memcache(s2fd->vcpu, memcache);
+ if (ret)
+ return ret;
+ }
if (s2fd->nested)
gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT;
@@ -1631,9 +1634,19 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
goto out_unlock;
}
- ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE,
- __pfn_to_phys(pfn), prot,
- memcache, flags);
+ if (perm_fault) {
+ /*
+ * Drop the SW bits in favour of those stored in the
+ * PTE, which will be preserved.
+ */
+ prot &= ~KVM_NV_GUEST_MAP_SZ;
+ ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa,
+ prot, flags);
+ } else {
+ ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE,
+ __pfn_to_phys(pfn), prot,
+ memcache, flags);
+ }
out_unlock:
kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W);
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 883b6c1..38f672e 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -1834,6 +1834,11 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
resx.res1 = VNCR_EL2_RES1;
set_sysreg_masks(kvm, VNCR_EL2, resx);
+ /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */
+ resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4);
+ resx.res1 = ZCR_ELx_RES1;
+ set_sysreg_masks(kvm, ZCR_EL2, resx);
+
out:
for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
__vcpu_rmw_sys_reg(vcpu, sr, |=, 0);
diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c
index e1860ac..c816db5 100644
--- a/arch/arm64/kvm/pmu-emul.c
+++ b/arch/arm64/kvm/pmu-emul.c
@@ -174,8 +174,8 @@ static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force)
* action is to use PMCR.P, which will reset them to
* 0 (the only use of the 'force' parameter).
*/
- val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32);
- val |= lower_32_bits(val);
+ val = (__vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32)) |
+ lower_32_bits(val);
}
__vcpu_assign_sys_reg(vcpu, reg, val);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 148fc34..fa5c93c 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -2862,21 +2862,16 @@ static bool access_zcr_el2(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
- unsigned int vq;
-
if (guest_hyp_sve_traps_enabled(vcpu)) {
kvm_inject_nested_sve_trap(vcpu);
return false;
}
- if (!p->is_write) {
+ if (!p->is_write)
p->regval = __vcpu_sys_reg(vcpu, ZCR_EL2);
- return true;
- }
+ else
+ __vcpu_assign_sys_reg(vcpu, ZCR_EL2, p->regval);
- vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;
- vq = min(vq, vcpu_sve_max_vq(vcpu));
- __vcpu_assign_sys_reg(vcpu, ZCR_EL2, vq - 1);
return true;
}
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index 2ea9f1c..1d7e5d5 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -2307,6 +2307,10 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
/* dte entry is valid */
offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
+ /* Mimic the MAPD behaviour and reject invalid EID bits. */
+ if (num_eventid_bits > VITS_TYPER_IDBITS)
+ return -EINVAL;
+
if (!vgic_its_check_id(its, baser, id, NULL))
return -EINVAL;
diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v2.c b/arch/arm64/kvm/vgic/vgic-mmio-v2.c
index 406845b..0643e333 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio-v2.c
+++ b/arch/arm64/kvm/vgic/vgic-mmio-v2.c
@@ -91,7 +91,7 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu,
* migration from old kernels to new kernels with legacy
* userspace.
*/
- reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
+ reg = FIELD_GET(GICD_IIDR_REVISION_MASK, val);
switch (reg) {
case KVM_VGIC_IMP_REV_2:
case KVM_VGIC_IMP_REV_3:
diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c
index 89edb84..5913a20 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c
+++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c
@@ -194,7 +194,7 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
return -EINVAL;
- reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
+ reg = FIELD_GET(GICD_IIDR_REVISION_MASK, val);
switch (reg) {
case KVM_VGIC_IMP_REV_2:
case KVM_VGIC_IMP_REV_3:
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 0f3c5c7..7398008 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -1018,7 +1018,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
return vma_alloc_folio(flags, 0, vma, vaddr);
}
-bool tag_clear_highpages(struct page *page, int numpages)
+bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages)
{
/*
* Check if MTE is supported and fall back to clear_highpage().
@@ -1026,13 +1026,16 @@ bool tag_clear_highpages(struct page *page, int numpages)
* post_alloc_hook() will invoke tag_clear_highpages().
*/
if (!system_supports_mte())
- return false;
+ return clear_pages;
/* Newly allocated pages, shouldn't have been tagged yet */
for (int i = 0; i < numpages; i++, page++) {
WARN_ON_ONCE(!try_page_mte_tagging(page));
- mte_zero_clear_page_tags(page_address(page));
+ if (clear_pages)
+ mte_zero_clear_page_tags(page_address(page));
+ else
+ mte_clear_page_tags(page_address(page));
set_page_mte_tagged(page);
}
- return true;
+ return false;
}
diff --git a/arch/csky/include/asm/Kbuild b/arch/csky/include/asm/Kbuild
index 3a5c7f6..7dca0c6 100644
--- a/arch/csky/include/asm/Kbuild
+++ b/arch/csky/include/asm/Kbuild
@@ -9,6 +9,7 @@
generic-y += qrwlock_types.h
generic-y += qspinlock.h
generic-y += parport.h
+generic-y += ring_buffer.h
generic-y += user.h
generic-y += vmlinux.lds.h
generic-y += text-patching.h
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index 1efa1e9..0f887d4 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -5,4 +5,5 @@
generic-y += iomap.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += text-patching.h
diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild
index beb8499..1c7a0db 100644
--- a/arch/loongarch/Kbuild
+++ b/arch/loongarch/Kbuild
@@ -3,7 +3,7 @@
obj-y += net/
obj-y += vdso/
-obj-$(CONFIG_KVM) += kvm/
+obj-$(subst m,y,$(CONFIG_KVM)) += kvm/
# for cleaning
subdir- += boot
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index 3b042db..3f69c5d 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -110,7 +110,6 @@
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL if GENERIC_GETTIMEOFDAY
select GPIOLIB
select HAS_IOPORT
select HAVE_ALIGNED_STRUCT_PAGE if 64BIT
@@ -220,6 +219,7 @@
choice
prompt "Kernel type"
+ default 64BIT # Keep existing behavior
config 32BIT
bool "32-bit kernel"
diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile
index 47516ae..54fcfa1e 100644
--- a/arch/loongarch/Makefile
+++ b/arch/loongarch/Makefile
@@ -55,9 +55,11 @@
ifdef CONFIG_32BIT
tool-archpref = $(32bit-tool-archpref)
UTS_MACHINE := loongarch32
+cflags-y += $(call cc-option,-m32)
else
tool-archpref = $(64bit-tool-archpref)
UTS_MACHINE := loongarch64
+cflags-y += $(call cc-option,-m64)
endif
ifneq ($(SUBARCH),$(ARCH))
diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild
index 9034b58..7e92957 100644
--- a/arch/loongarch/include/asm/Kbuild
+++ b/arch/loongarch/include/asm/Kbuild
@@ -10,5 +10,6 @@
generic-y += user.h
generic-y += ioctl.h
generic-y += mmzone.h
+generic-y += ring_buffer.h
generic-y += statfs.h
generic-y += text-patching.h
diff --git a/arch/loongarch/include/asm/asm-prototypes.h b/arch/loongarch/include/asm/asm-prototypes.h
index 704066b..de0c17f 100644
--- a/arch/loongarch/include/asm/asm-prototypes.h
+++ b/arch/loongarch/include/asm/asm-prototypes.h
@@ -20,3 +20,23 @@ asmlinkage void noinstr __no_stack_protector ret_from_kernel_thread(struct task_
struct pt_regs *regs,
int (*fn)(void *),
void *fn_arg);
+
+struct kvm_run;
+struct kvm_vcpu;
+struct loongarch_fpu;
+
+void kvm_exc_entry(void);
+int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
+
+void kvm_save_fpu(struct loongarch_fpu *fpu);
+void kvm_restore_fpu(struct loongarch_fpu *fpu);
+
+#ifdef CONFIG_CPU_HAS_LSX
+void kvm_save_lsx(struct loongarch_fpu *fpu);
+void kvm_restore_lsx(struct loongarch_fpu *fpu);
+#endif
+
+#ifdef CONFIG_CPU_HAS_LASX
+void kvm_save_lasx(struct loongarch_fpu *fpu);
+void kvm_restore_lasx(struct loongarch_fpu *fpu);
+#endif
diff --git a/arch/loongarch/include/asm/efi.h b/arch/loongarch/include/asm/efi.h
index eddc8e7..1ad764b 100644
--- a/arch/loongarch/include/asm/efi.h
+++ b/arch/loongarch/include/asm/efi.h
@@ -30,6 +30,8 @@ static inline unsigned long efi_get_kimg_min_align(void)
return SZ_2M;
}
-#define EFI_KIMG_PREFERRED_ADDRESS PHYSADDR(VMLINUX_LOAD_ADDRESS)
+unsigned long efi_get_kimg_kaslr_address(void);
+
+#define EFI_KIMG_PREFERRED_ADDRESS efi_get_kimg_kaslr_address()
#endif /* _ASM_LOONGARCH_EFI_H */
diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h
index 130cedbb..776bc48 100644
--- a/arch/loongarch/include/asm/kvm_host.h
+++ b/arch/loongarch/include/asm/kvm_host.h
@@ -87,7 +87,6 @@ struct kvm_context {
struct kvm_world_switch {
int (*exc_entry)(void);
int (*enter_guest)(struct kvm_run *run, struct kvm_vcpu *vcpu);
- unsigned long page_order;
};
#define MAX_PGTABLE_LEVELS 4
@@ -359,8 +358,6 @@ void kvm_exc_entry(void);
int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
extern unsigned long vpid_mask;
-extern const unsigned long kvm_exception_size;
-extern const unsigned long kvm_enter_guest_size;
extern struct kvm_world_switch *kvm_loongarch_ops;
#define SW_GCSR (1 << 0)
diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h
index a1bd6a3..ae937d1 100644
--- a/arch/loongarch/include/asm/linkage.h
+++ b/arch/loongarch/include/asm/linkage.h
@@ -69,7 +69,7 @@
9, 10, 11, 12, 13, 14, 15, 16, \
17, 18, 19, 20, 21, 22, 23, 24, \
25, 26, 27, 28, 29, 30, 31; \
- .cfi_offset \num, SC_REGS + \num * SZREG; \
+ .cfi_offset \num, SC_REGS + \num * 8; \
.endr; \
\
nop; \
diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h
index 0111f0a..acae1c5 100644
--- a/arch/loongarch/include/asm/paravirt.h
+++ b/arch/loongarch/include/asm/paravirt.h
@@ -4,6 +4,12 @@
#ifdef CONFIG_PARAVIRT
+#include <linux/jump_label.h>
+
+DECLARE_STATIC_KEY_FALSE(virt_preempt_key);
+DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key);
+DECLARE_PER_CPU(struct kvm_steal_time, steal_time);
+
int __init pv_ipi_init(void);
int __init pv_time_init(void);
int __init pv_spinlock_init(void);
diff --git a/arch/loongarch/include/asm/qspinlock.h b/arch/loongarch/include/asm/qspinlock.h
index 0ee15b3..fbfc6be 100644
--- a/arch/loongarch/include/asm/qspinlock.h
+++ b/arch/loongarch/include/asm/qspinlock.h
@@ -3,12 +3,9 @@
#define _ASM_LOONGARCH_QSPINLOCK_H
#include <asm/kvm_para.h>
-#include <linux/jump_label.h>
+#include <asm/paravirt.h>
#ifdef CONFIG_PARAVIRT
-DECLARE_STATIC_KEY_FALSE(virt_preempt_key);
-DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key);
-DECLARE_PER_CPU(struct kvm_steal_time, steal_time);
#define virt_spin_lock virt_spin_lock
diff --git a/arch/loongarch/include/asm/vdso/gettimeofday.h b/arch/loongarch/include/asm/vdso/gettimeofday.h
index bae7676..18ba403 100644
--- a/arch/loongarch/include/asm/vdso/gettimeofday.h
+++ b/arch/loongarch/include/asm/vdso/gettimeofday.h
@@ -85,12 +85,6 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
return count;
}
-static inline bool loongarch_vdso_hres_capable(void)
-{
- return true;
-}
-#define __arch_vdso_hres_capable loongarch_vdso_hres_capable
-
#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
#endif /* !__ASSEMBLER__ */
diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c
index 8ba391c..1985ed3 100644
--- a/arch/loongarch/kernel/kprobes.c
+++ b/arch/loongarch/kernel/kprobes.c
@@ -60,16 +60,18 @@ NOKPROBE_SYMBOL(arch_prepare_kprobe);
/* Install breakpoint in text */
void arch_arm_kprobe(struct kprobe *p)
{
- *p->addr = KPROBE_BP_INSN;
- flush_insn_slot(p);
+ u32 insn = KPROBE_BP_INSN;
+
+ larch_insn_text_copy(p->addr, &insn, LOONGARCH_INSN_SIZE);
}
NOKPROBE_SYMBOL(arch_arm_kprobe);
/* Remove breakpoint from text */
void arch_disarm_kprobe(struct kprobe *p)
{
- *p->addr = p->opcode;
- flush_insn_slot(p);
+ u32 insn = p->opcode;
+
+ larch_insn_text_copy(p->addr, &insn, LOONGARCH_INSN_SIZE);
}
NOKPROBE_SYMBOL(arch_disarm_kprobe);
@@ -184,16 +186,16 @@ static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{
switch (kcb->kprobe_status) {
- case KPROBE_HIT_SS:
case KPROBE_HIT_SSDONE:
case KPROBE_HIT_ACTIVE:
kprobes_inc_nmissed_count(p);
setup_singlestep(p, regs, kcb, 1);
break;
+ case KPROBE_HIT_SS:
case KPROBE_REENTER:
pr_warn("Failed to recover from reentered kprobes.\n");
dump_kprobe(p);
- WARN_ON_ONCE(1);
+ BUG();
break;
default:
WARN_ON(1);
diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c
index 16f6a9b..4b61a96 100644
--- a/arch/loongarch/kernel/relocate.c
+++ b/arch/loongarch/kernel/relocate.c
@@ -134,11 +134,23 @@ early_param("nokaslr", nokaslr);
#define KASLR_DISABLED_MESSAGE "KASLR is disabled by %s in %s cmdline.\n"
+/*
+ * Note: strictly-defined KASLR means the kernel's final runtime address
+ * has a random offset from the kernel's load address, which is implemented
+ * in relocate.c; broadly-defined KALSR means the kernel's final runtime
+ * address has a random offset from the kernel's link address (a.k.a.
+ * VMLINUX_LOAD_ADDRESS), which also include the efistlub implementation,
+ * kexec_file implementation and QEMU direct kernel boot. kaslr_disabled()
+ * return true only means strictly-defined KASLR is disabled.
+ */
static inline __init bool kaslr_disabled(void)
{
char *str;
const char *builtin_cmdline = CONFIG_CMDLINE;
+ if (kaslr_offset())
+ return true; /* KASLR is performed during early boot. */
+
str = strstr(builtin_cmdline, "nokaslr");
if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) {
pr_info(KASLR_DISABLED_MESSAGE, "\'nokaslr\'", "built-in");
@@ -210,14 +222,52 @@ static inline void __init *determine_relocation_address(void)
return RELOCATED_KASLR(destination);
}
+static unsigned long __init determine_initrd_address(unsigned long *size)
+{
+ unsigned long start = 0;
+ unsigned long key_length;
+ char *p, *endp, *key = "initrd=";
+
+ key_length = strlen(key);
+ p = strstr(boot_command_line, key);
+
+ if (!p) {
+ key = "initrdmem=";
+ key_length = strlen(key);
+ p = strstr(boot_command_line, key);
+ }
+
+ if (p == boot_command_line || (p > boot_command_line && *(p - 1) == ' ')) {
+ p += key_length;
+ start = memparse(p, &endp);
+ if (*endp == ',')
+ *size = memparse(endp + 1, NULL);
+ }
+
+ return start;
+}
+
static inline int __init relocation_addr_valid(void *location_new)
{
+ unsigned long kernel_start, kernel_size;
+ unsigned long initrd_start, initrd_size = 0;
+
if ((unsigned long)location_new & 0x00000ffff)
return 0; /* Inappropriately aligned new location */
if ((unsigned long)location_new < (unsigned long)_end)
return 0; /* New location overlaps original kernel */
+ initrd_start = determine_initrd_address(&initrd_size);
+ if (initrd_start && initrd_size) {
+ kernel_start = PHYSADDR(location_new);
+ kernel_size = (unsigned long)_end - (unsigned long)_text;
+
+ if (kernel_start < (initrd_start + initrd_size) &&
+ initrd_start < (kernel_start + kernel_size))
+ return 0; /* initrd/initramfs overlaps kernel */
+ }
+
return 1;
}
#endif
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
index 64a048f..5092261 100644
--- a/arch/loongarch/kernel/smp.c
+++ b/arch/loongarch/kernel/smp.c
@@ -88,7 +88,7 @@ void show_ipi_list(struct seq_file *p, int prec)
unsigned int cpu, i;
for (i = 0; i < NR_IPI; i++) {
- seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : "");
+ seq_printf(p, "%*s%u:", prec - 1, "IPI", i);
for_each_online_cpu(cpu)
seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10);
seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]);
diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile
index ae469ed..a4d044d 100644
--- a/arch/loongarch/kvm/Makefile
+++ b/arch/loongarch/kvm/Makefile
@@ -7,11 +7,12 @@
obj-$(CONFIG_KVM) += kvm.o
+obj-y += switch.o
+
kvm-y += exit.o
kvm-y += interrupt.o
kvm-y += main.o
kvm-y += mmu.o
-kvm-y += switch.o
kvm-y += timer.o
kvm-y += tlb.o
kvm-y += vcpu.o
diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c
index da0ad89f..3b95cd0 100644
--- a/arch/loongarch/kvm/exit.c
+++ b/arch/loongarch/kvm/exit.c
@@ -390,6 +390,7 @@ int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
run->mmio.len = 8;
break;
default:
+ ret = EMULATE_FAIL;
break;
}
break;
diff --git a/arch/loongarch/kvm/interrupt.c b/arch/loongarch/kvm/interrupt.c
index 3293095..a18c60d 100644
--- a/arch/loongarch/kvm/interrupt.c
+++ b/arch/loongarch/kvm/interrupt.c
@@ -28,23 +28,29 @@ static unsigned int priority_to_irq[EXCCODE_INT_NUM] = {
static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
{
unsigned int irq = 0;
+ unsigned long old, new;
clear_bit(priority, &vcpu->arch.irq_pending);
if (priority < EXCCODE_INT_NUM)
irq = priority_to_irq[priority];
- if (kvm_guest_has_msgint(&vcpu->arch) && (priority == INT_AVEC)) {
- dmsintc_inject_irq(vcpu);
- set_gcsr_estat(irq);
- return 1;
- }
-
switch (priority) {
+ case INT_AVEC:
+ if (!kvm_guest_has_msgint(&vcpu->arch))
+ break;
+ dmsintc_inject_irq(vcpu);
+ fallthrough;
case INT_TI:
case INT_IPI:
case INT_SWI0:
case INT_SWI1:
+ old = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL);
set_gcsr_estat(irq);
+ new = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL);
+
+ /* Inject TI if TVAL inverted */
+ if (new > old)
+ set_gcsr_estat(CPU_TIMER);
break;
case INT_HWI0 ... INT_HWI7:
@@ -61,22 +67,28 @@ static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
static int kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority)
{
unsigned int irq = 0;
+ unsigned long old, new;
clear_bit(priority, &vcpu->arch.irq_clear);
if (priority < EXCCODE_INT_NUM)
irq = priority_to_irq[priority];
- if (kvm_guest_has_msgint(&vcpu->arch) && (priority == INT_AVEC)) {
- clear_gcsr_estat(irq);
- return 1;
- }
-
switch (priority) {
+ case INT_AVEC:
+ if (!kvm_guest_has_msgint(&vcpu->arch))
+ break;
+ fallthrough;
case INT_TI:
case INT_IPI:
case INT_SWI0:
case INT_SWI1:
+ old = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL);
clear_gcsr_estat(irq);
+ new = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL);
+
+ /* Inject TI if TVAL inverted */
+ if (new > old)
+ set_gcsr_estat(CPU_TIMER);
break;
case INT_HWI0 ... INT_HWI7:
diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c
index 76ebff2..f105a86 100644
--- a/arch/loongarch/kvm/main.c
+++ b/arch/loongarch/kvm/main.c
@@ -348,8 +348,7 @@ void kvm_arch_disable_virtualization_cpu(void)
static int kvm_loongarch_env_init(void)
{
- int cpu, order, ret;
- void *addr;
+ int cpu, ret;
struct kvm_context *context;
vmcs = alloc_percpu(struct kvm_context);
@@ -365,30 +364,8 @@ static int kvm_loongarch_env_init(void)
return -ENOMEM;
}
- /*
- * PGD register is shared between root kernel and kvm hypervisor.
- * So world switch entry should be in DMW area rather than TLB area
- * to avoid page fault reenter.
- *
- * In future if hardware pagetable walking is supported, we won't
- * need to copy world switch code to DMW area.
- */
- order = get_order(kvm_exception_size + kvm_enter_guest_size);
- addr = (void *)__get_free_pages(GFP_KERNEL, order);
- if (!addr) {
- free_percpu(vmcs);
- vmcs = NULL;
- kfree(kvm_loongarch_ops);
- kvm_loongarch_ops = NULL;
- return -ENOMEM;
- }
-
- memcpy(addr, kvm_exc_entry, kvm_exception_size);
- memcpy(addr + kvm_exception_size, kvm_enter_guest, kvm_enter_guest_size);
- flush_icache_range((unsigned long)addr, (unsigned long)addr + kvm_exception_size + kvm_enter_guest_size);
- kvm_loongarch_ops->exc_entry = addr;
- kvm_loongarch_ops->enter_guest = addr + kvm_exception_size;
- kvm_loongarch_ops->page_order = order;
+ kvm_loongarch_ops->exc_entry = (void *)kvm_exc_entry;
+ kvm_loongarch_ops->enter_guest = (void *)kvm_enter_guest;
vpid_mask = read_csr_gstat();
vpid_mask = (vpid_mask & CSR_GSTAT_GIDBIT) >> CSR_GSTAT_GIDBIT_SHIFT;
@@ -428,16 +405,10 @@ static int kvm_loongarch_env_init(void)
static void kvm_loongarch_env_exit(void)
{
- unsigned long addr;
-
if (vmcs)
free_percpu(vmcs);
if (kvm_loongarch_ops) {
- if (kvm_loongarch_ops->exc_entry) {
- addr = (unsigned long)kvm_loongarch_ops->exc_entry;
- free_pages(addr, kvm_loongarch_ops->page_order);
- }
kfree(kvm_loongarch_ops);
}
diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c
index a7fa458..e104897 100644
--- a/arch/loongarch/kvm/mmu.c
+++ b/arch/loongarch/kvm/mmu.c
@@ -95,7 +95,7 @@ static int kvm_flush_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
else
kvm->stat.pages--;
- *pte = ctx->invalid_entry;
+ kvm_set_pte(pte, ctx->invalid_entry);
return 1;
}
diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
index f1768b7..936e4ae 100644
--- a/arch/loongarch/kvm/switch.S
+++ b/arch/loongarch/kvm/switch.S
@@ -4,9 +4,11 @@
*/
#include <linux/linkage.h>
+#include <linux/kvm_types.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/loongarch.h>
+#include <asm/page.h>
#include <asm/regdef.h>
#include <asm/unwind_hints.h>
@@ -100,11 +102,16 @@
* - is still in guest mode, such as pgd table/vmid registers etc,
* - will fix with hw page walk enabled in future
* load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS
+ *
+ * PGD register is shared between root kernel and kvm hypervisor.
+ * So world switch entry should be in DMW area rather than TLB area
+ * to avoid page fault re-enter.
*/
.text
+ .p2align PAGE_SHIFT
.cfi_sections .debug_frame
SYM_CODE_START(kvm_exc_entry)
- UNWIND_HINT_UNDEFINED
+ UNWIND_HINT_END_OF_STACK
csrwr a2, KVM_TEMP_KS
csrrd a2, KVM_VCPU_KS
addi.d a2, a2, KVM_VCPU_ARCH
@@ -190,8 +197,8 @@
kvm_restore_host_gpr a2
jr ra
-SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
SYM_CODE_END(kvm_exc_entry)
+EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry)
/*
* int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
@@ -215,8 +222,8 @@
/* Save kvm_vcpu to kscratch */
csrwr a1, KVM_VCPU_KS
kvm_switch_to_guest
-SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL)
SYM_FUNC_END(kvm_enter_guest)
+EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest)
SYM_FUNC_START(kvm_save_fpu)
fpu_save_csr a0 t1
@@ -224,6 +231,7 @@
fpu_save_cc a0 t1 t2
jr ra
SYM_FUNC_END(kvm_save_fpu)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu)
SYM_FUNC_START(kvm_restore_fpu)
fpu_restore_double a0 t1
@@ -231,6 +239,7 @@
fpu_restore_cc a0 t1 t2
jr ra
SYM_FUNC_END(kvm_restore_fpu)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu)
#ifdef CONFIG_CPU_HAS_LSX
SYM_FUNC_START(kvm_save_lsx)
@@ -239,6 +248,7 @@
lsx_save_data a0 t1
jr ra
SYM_FUNC_END(kvm_save_lsx)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx)
SYM_FUNC_START(kvm_restore_lsx)
lsx_restore_data a0 t1
@@ -246,6 +256,7 @@
fpu_restore_csr a0 t1 t2
jr ra
SYM_FUNC_END(kvm_restore_lsx)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx)
#endif
#ifdef CONFIG_CPU_HAS_LASX
@@ -255,6 +266,7 @@
lasx_save_data a0 t1
jr ra
SYM_FUNC_END(kvm_save_lasx)
+EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx)
SYM_FUNC_START(kvm_restore_lasx)
lasx_restore_data a0 t1
@@ -262,10 +274,8 @@
fpu_restore_csr a0 t1 t2
jr ra
SYM_FUNC_END(kvm_restore_lasx)
+EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx)
#endif
- .section ".rodata"
-SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
-SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
#ifdef CONFIG_CPU_HAS_LBT
STACK_FRAME_NON_STANDARD kvm_restore_fpu
diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c
index 29c2aaba..8356fce 100644
--- a/arch/loongarch/kvm/timer.c
+++ b/arch/loongarch/kvm/timer.c
@@ -96,15 +96,21 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu)
* and set CSR TVAL with -1
*/
write_gcsr_timertick(0);
- __delay(2); /* Wait cycles until timer interrupt injected */
/*
* Writing CSR_TINTCLR_TI to LOONGARCH_CSR_TINTCLR will clear
* timer interrupt, and CSR TVAL keeps unchanged with -1, it
* avoids spurious timer interrupt
*/
- if (!(estat & CPU_TIMER))
+ if (!(estat & CPU_TIMER)) {
+ __delay(2); /* Wait cycles until timer interrupt injected */
+
+ /* Write TVAL with max value if no TI shot */
+ estat = kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT);
+ if (!(estat & CPU_TIMER))
+ write_gcsr_timertick(CSR_TCFG_VAL);
gcsr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR);
+ }
return;
}
diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c
index 8cc5ee1..1317c71 100644
--- a/arch/loongarch/kvm/vm.c
+++ b/arch/loongarch/kvm/vm.c
@@ -125,7 +125,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
r = 1;
break;
case KVM_CAP_NR_VCPUS:
- r = num_online_cpus();
+ r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
break;
case KVM_CAP_MAX_VCPUS:
r = KVM_MAX_VCPUS;
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 3f9ab54..031b39e 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -123,11 +123,7 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
{
unsigned long start_pfn = start >> PAGE_SHIFT;
unsigned long nr_pages = size >> PAGE_SHIFT;
- struct page *page = pfn_to_page(start_pfn);
- /* With altmap the first mapped page is offset from @start */
- if (altmap)
- page += vmem_altmap_offset(altmap);
__remove_pages(start_pfn, nr_pages, altmap);
}
#endif
diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c
index 0dde3dd..b02698a 100644
--- a/arch/loongarch/pci/acpi.c
+++ b/arch/loongarch/pci/acpi.c
@@ -61,11 +61,16 @@ static void acpi_release_root_info(struct acpi_pci_root_info *ci)
static int acpi_prepare_root_resources(struct acpi_pci_root_info *ci)
{
int status;
+ unsigned long long pci_h = 0;
struct resource_entry *entry, *tmp;
struct acpi_device *device = ci->bridge;
status = acpi_pci_probe_root_resources(ci);
if (status > 0) {
+ acpi_evaluate_integer(device->handle, "PCIH", NULL, &pci_h);
+ if (pci_h)
+ return status;
+
resource_list_for_each_entry_safe(entry, tmp, &ci->resources) {
if (entry->res->flags & IORESOURCE_MEM) {
entry->offset = ci->root->mcfg_addr & GENMASK_ULL(63, 40);
diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c
index d233ea2..f33c7ea 100644
--- a/arch/loongarch/pci/pci.c
+++ b/arch/loongarch/pci/pci.c
@@ -132,6 +132,9 @@ static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on)
crtc_reg = regbase;
crtc_offset = 0x400;
break;
+ default:
+ iounmap(regbase);
+ return;
}
for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) {
diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile
index 42aa962..9c9181b 100644
--- a/arch/loongarch/vdso/Makefile
+++ b/arch/loongarch/vdso/Makefile
@@ -12,6 +12,8 @@
ccflags-vdso := \
$(filter -I%,$(KBUILD_CFLAGS)) \
$(filter -E%,$(KBUILD_CFLAGS)) \
+ $(filter -m32,$(KBUILD_CFLAGS)) \
+ $(filter -m64,$(KBUILD_CFLAGS)) \
$(filter -march=%,$(KBUILD_CFLAGS)) \
$(filter -m%-float,$(KBUILD_CFLAGS)) \
$(CLANG_FLAGS) \
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index b282e0d..62543bf3 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -3,5 +3,6 @@
generic-y += extable.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += spinlock.h
generic-y += text-patching.h
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index 7178f99..0030309 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -5,6 +5,7 @@
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
generic-y += parport.h
+generic-y += ring_buffer.h
generic-y += syscalls.h
generic-y += tlb.h
generic-y += user.h
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 4364f3d..323ca08 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -38,7 +38,6 @@
select GENERIC_BUILTIN_DTB if BUILTIN_DTB
select GENERIC_CMOS_UPDATE
select GENERIC_CPU_AUTOPROBE
- select GENERIC_GETTIMEOFDAY
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_ISA_DMA if EISA
@@ -51,7 +50,6 @@
select GENERIC_SCHED_CLOCK if !CAVIUM_OCTEON_SOC
select GENERIC_SMP_IDLE_THREAD
select GENERIC_IDLE_POLL_SETUP
- select GENERIC_TIME_VSYSCALL
select GUP_GET_PXX_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT
select HAS_IOPORT if !NO_IOPORT_MAP || ISA
select HAVE_ARCH_COMPILER_H
@@ -76,7 +74,6 @@
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_TRACER
select HAVE_GCC_PLUGINS
- select HAVE_GENERIC_VDSO
select HAVE_IOREMAP_PROT
select HAVE_IRQ_EXIT_ON_IRQ_STACK
select HAVE_IRQ_TIME_ACCOUNTING
@@ -1136,9 +1133,6 @@
config CSRC_SB1250
bool
-config MIPS_CLOCK_VSYSCALL
- def_bool CSRC_R4K || CLKSRC_MIPS_GIC
-
config GPIO_TXX9
select GPIOLIB
bool
@@ -3170,6 +3164,16 @@
config MIPS_EXTERNAL_TIMER
bool
+config MIPS_GENERIC_GETTIMEOFDAY
+ def_bool y
+ select GENERIC_GETTIMEOFDAY
+ select HAVE_GENERIC_VDSO
+ depends on CSRC_R4K || CLKSRC_MIPS_GIC
+ # GCC (at least up to version 9.2) appears to emit function calls that make use
+ # of the GOT when targeting microMIPS, which we can't use in the VDSO due to
+ # the lack of relocations. As such, we disable the VDSO for microMIPS builds.
+ depends on !(CPU_MICROMIPS && CC_IS_GCC && GCC_VERSION < 90300)
+
menu "CPU Power Management"
if CPU_SUPPORTS_CPUFREQ && MIPS_EXTERNAL_TIMER
@@ -3181,5 +3185,3 @@
endmenu
source "arch/mips/kvm/Kconfig"
-
-source "arch/mips/vdso/Kconfig"
diff --git a/arch/mips/dec/platform.c b/arch/mips/dec/platform.c
index c4fcb8c..723ce16 100644
--- a/arch/mips/dec/platform.c
+++ b/arch/mips/dec/platform.c
@@ -10,6 +10,14 @@
#include <linux/mc146818rtc.h>
#include <linux/platform_device.h>
+#include <asm/bootinfo.h>
+
+#include <asm/dec/interrupts.h>
+#include <asm/dec/ioasic_addrs.h>
+#include <asm/dec/kn01.h>
+#include <asm/dec/kn02.h>
+#include <asm/dec/system.h>
+
static struct resource dec_rtc_resources[] = {
{
.name = "rtc",
@@ -30,11 +38,110 @@ static struct platform_device dec_rtc_device = {
.num_resources = ARRAY_SIZE(dec_rtc_resources),
};
+static struct resource dec_dz_resources[] = {
+ { .name = "dz", .flags = IORESOURCE_MEM, },
+ { .name = "dz", .flags = IORESOURCE_IRQ, },
+};
+
+static struct platform_device dec_dz_device = {
+ .name = "dz",
+ .id = PLATFORM_DEVID_NONE,
+ .resource = dec_dz_resources,
+ .num_resources = ARRAY_SIZE(dec_dz_resources),
+};
+
+static struct platform_device *dec_dz_devices[] __initdata = {
+ &dec_dz_device,
+};
+
+static struct resource dec_zs_resources[][2] = {
+ {
+ { .name = "scc0", .flags = IORESOURCE_MEM, },
+ { .name = "scc0", .flags = IORESOURCE_IRQ, },
+ },
+ {
+ { .name = "scc1", .flags = IORESOURCE_MEM, },
+ { .name = "scc1", .flags = IORESOURCE_IRQ, },
+ },
+};
+
+static struct platform_device dec_zs_device[] = {
+ {
+ .name = "zs",
+ .id = 0,
+ .resource = dec_zs_resources[0],
+ .num_resources = ARRAY_SIZE(dec_zs_resources[0]),
+ },
+ {
+ .name = "zs",
+ .id = 1,
+ .resource = dec_zs_resources[1],
+ .num_resources = ARRAY_SIZE(dec_zs_resources[1]),
+ },
+};
+
static int __init dec_add_devices(void)
{
+ struct platform_device *dec_zs_devices[ARRAY_SIZE(dec_zs_device)];
+ int ret1, ret2, ret3;
+ int num_dz, num_zs;
+ int irq, i;
+
dec_rtc_resources[0].start = RTC_PORT(0);
dec_rtc_resources[0].end = RTC_PORT(0) + dec_kn_slot_size - 1;
- return platform_device_register(&dec_rtc_device);
+
+ i = 0;
+ irq = dec_interrupt[DEC_IRQ_DZ11];
+ if (IS_ENABLED(CONFIG_32BIT) && irq >= 0) {
+ resource_size_t base;
+
+ switch (mips_machtype) {
+ case MACH_DS23100:
+ case MACH_DS5100:
+ base = dec_kn_slot_base + KN01_DZ11;
+ break;
+ default:
+ base = dec_kn_slot_base + KN02_DZ11;
+ break;
+ }
+ dec_dz_device.resource[0].start = base;
+ dec_dz_device.resource[0].end = base + dec_kn_slot_size - 1;
+ dec_dz_device.resource[1].start = irq;
+ dec_dz_device.resource[1].end = irq;
+ i++;
+ }
+ num_dz = i;
+
+ i = 0;
+ irq = dec_interrupt[DEC_IRQ_SCC0];
+ if (irq >= 0) {
+ resource_size_t base = dec_kn_slot_base + IOASIC_SCC0;
+
+ dec_zs_device[i].resource[0].start = base;
+ dec_zs_device[i].resource[0].end = base + dec_kn_slot_size - 1;
+ dec_zs_device[i].resource[1].start = irq;
+ dec_zs_device[i].resource[1].end = irq;
+ dec_zs_devices[i] = &dec_zs_device[i];
+ i++;
+ }
+ irq = dec_interrupt[DEC_IRQ_SCC1];
+ if (irq >= 0) {
+ resource_size_t base = dec_kn_slot_base + IOASIC_SCC1;
+
+ dec_zs_device[i].resource[0].start = base;
+ dec_zs_device[i].resource[0].end = base + dec_kn_slot_size - 1;
+ dec_zs_device[i].resource[1].start = irq;
+ dec_zs_device[i].resource[1].end = irq;
+ dec_zs_devices[i] = &dec_zs_device[i];
+ i++;
+ }
+ num_zs = i;
+
+ ret1 = platform_device_register(&dec_rtc_device);
+ ret2 = IS_ENABLED(CONFIG_32BIT) ?
+ platform_add_devices(dec_dz_devices, num_dz) : 0;
+ ret3 = platform_add_devices(dec_zs_devices, num_zs);
+ return ret1 ? ret1 : ret2 ? ret2 : ret3;
}
device_initcall(dec_add_devices);
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild
index 684569b..9771c3d 100644
--- a/arch/mips/include/asm/Kbuild
+++ b/arch/mips/include/asm/Kbuild
@@ -12,5 +12,6 @@
generic-y += parport.h
generic-y += qrwlock.h
generic-y += qspinlock.h
+generic-y += ring_buffer.h
generic-y += user.h
generic-y += text-patching.h
diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c
index 59eca39..5ce95e1 100644
--- a/arch/mips/kernel/csrc-r4k.c
+++ b/arch/mips/kernel/csrc-r4k.c
@@ -130,7 +130,8 @@ int __init init_r4k_clocksource(void)
* R2 onwards makes the count accessible to user mode so it can be used
* by the VDSO (HWREna is configured by configure_hwrena()).
*/
- if (cpu_has_mips_r2_r6 && rdhwr_count_usable())
+ if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) &&
+ cpu_has_mips_r2_r6 && rdhwr_count_usable())
clocksource_mips.vdso_clock_mode = VDSO_CLOCKMODE_R4K;
clocksource_register_hz(&clocksource_mips, mips_hpt_frequency);
diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c
index 2fa4df3..bd1fc17 100644
--- a/arch/mips/kernel/vdso.c
+++ b/arch/mips/kernel/vdso.c
@@ -129,7 +129,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
* This ensures that when the kernel updates the VDSO data userland
* will observe it without requiring cache invalidations.
*/
- if (cpu_has_dc_aliases) {
+ if (cpu_has_dc_aliases && IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO)) {
base = __ALIGN_MASK(base, shm_align_mask);
base += ((unsigned long)vdso_k_time_data - gic_size) & shm_align_mask;
}
@@ -137,10 +137,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
data_addr = base + gic_size;
vdso_addr = data_addr + VDSO_NR_PAGES * PAGE_SIZE;
- vma = vdso_install_vvar_mapping(mm, data_addr);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto out;
+ if (IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO)) {
+ vma = vdso_install_vvar_mapping(mm, data_addr);
+ if (IS_ERR(vma)) {
+ ret = PTR_ERR(vma);
+ goto out;
+ }
}
/* Map GIC user page. */
diff --git a/arch/mips/vdso/Kconfig b/arch/mips/vdso/Kconfig
deleted file mode 100644
index 7014024..0000000
--- a/arch/mips/vdso/Kconfig
+++ /dev/null
@@ -1,6 +0,0 @@
-# GCC (at least up to version 9.2) appears to emit function calls that make use
-# of the GOT when targeting microMIPS, which we can't use in the VDSO due to
-# the lack of relocations. As such, we disable the VDSO for microMIPS builds.
-
-config MIPS_DISABLE_VDSO
- def_bool CPU_MICROMIPS
diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile
index 69d4593..00d3ba2 100644
--- a/arch/mips/vdso/Makefile
+++ b/arch/mips/vdso/Makefile
@@ -4,7 +4,7 @@
# Include the generic Makefile to check the built vdso.
include $(srctree)/lib/vdso/Makefile.include
-obj-vdso-y := elf.o vgettimeofday.o sigreturn.o
+obj-vdso-y := elf.o sigreturn.o
# Common compiler flags between ABIs.
ccflags-vdso := \
@@ -36,6 +36,7 @@
-D__ASSEMBLY__ -Wa,-gdwarf-2
ifneq ($(c-gettimeofday-y),)
+obj-vdso-y += vgettimeofday.o
CFLAGS_vgettimeofday.o = -include $(c-gettimeofday-y)
# config-n32-o32-env.c prepares the environment to build a 32bit vDSO
@@ -47,10 +48,6 @@
CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE)
-ifdef CONFIG_MIPS_DISABLE_VDSO
- obj-vdso-y := $(filter-out vgettimeofday.o, $(obj-vdso-y))
-endif
-
# VDSO linker flags.
ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \
$(filter -E%,$(KBUILD_CFLAGS)) -shared \
diff --git a/arch/mips/vdso/vdso.lds.S b/arch/mips/vdso/vdso.lds.S
index 5d08be3..05badf3 100644
--- a/arch/mips/vdso/vdso.lds.S
+++ b/arch/mips/vdso/vdso.lds.S
@@ -94,12 +94,10 @@
VERSION
{
LINUX_2.6 {
-#ifndef CONFIG_MIPS_DISABLE_VDSO
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
global:
__vdso_clock_gettime;
-#ifdef CONFIG_MIPS_CLOCK_VSYSCALL
__vdso_gettimeofday;
-#endif
__vdso_clock_getres;
#if _MIPS_SIM != _MIPS_SIM_ABI64
__vdso_clock_gettime64;
diff --git a/arch/mips/vdso/vgettimeofday.c b/arch/mips/vdso/vgettimeofday.c
index 1d23621..00f9fcf 100644
--- a/arch/mips/vdso/vgettimeofday.c
+++ b/arch/mips/vdso/vgettimeofday.c
@@ -18,22 +18,12 @@ int __vdso_clock_gettime(clockid_t clock,
return __cvdso_clock_gettime32(clock, ts);
}
-#ifdef CONFIG_MIPS_CLOCK_VSYSCALL
-
-/*
- * This is behind the ifdef so that we don't provide the symbol when there's no
- * possibility of there being a usable clocksource, because there's nothing we
- * can do without it. When libc fails the symbol lookup it should fall back on
- * the standard syscall path.
- */
int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
}
-#endif /* CONFIG_MIPS_CLOCK_VSYSCALL */
-
int __vdso_clock_getres(clockid_t clock_id,
struct old_timespec32 *res)
{
@@ -59,22 +49,12 @@ int __vdso_clock_gettime(clockid_t clock,
return __cvdso_clock_gettime(clock, ts);
}
-#ifdef CONFIG_MIPS_CLOCK_VSYSCALL
-
-/*
- * This is behind the ifdef so that we don't provide the symbol when there's no
- * possibility of there being a usable clocksource, because there's nothing we
- * can do without it. When libc fails the symbol lookup it should fall back on
- * the standard syscall path.
- */
int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
}
-#endif /* CONFIG_MIPS_CLOCK_VSYSCALL */
-
int __vdso_clock_getres(clockid_t clock_id,
struct __kernel_timespec *res)
{
diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild
index 2800430..0a25309 100644
--- a/arch/nios2/include/asm/Kbuild
+++ b/arch/nios2/include/asm/Kbuild
@@ -5,6 +5,7 @@
generic-y += extable.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += spinlock.h
generic-y += user.h
generic-y += text-patching.h
diff --git a/arch/nios2/include/asm/linkage.h b/arch/nios2/include/asm/linkage.h
index 2113023..c407323 100644
--- a/arch/nios2/include/asm/linkage.h
+++ b/arch/nios2/include/asm/linkage.h
@@ -12,4 +12,6 @@
#define __ALIGN .align 4
#define __ALIGN_STR ".align 4"
+#define _THIS_IP_ ({ unsigned long __ip; asm volatile("nextpc %0" : "=r" (__ip)); __ip; })
+
#endif
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index cef49d6..8aa3462 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -8,4 +8,5 @@
generic-y += spinlock.h
generic-y += qrwlock_types.h
generic-y += qrwlock.h
+generic-y += ring_buffer.h
generic-y += user.h
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index edab2a9..4391783 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -174,15 +174,21 @@
# this hack.
prepare: vdso_prepare
vdso_prepare: prepare0
- $(if $(CONFIG_64BIT),$(Q)$(MAKE) \
- $(build)=arch/parisc/kernel/vdso64 include/generated/vdso64-offsets.h)
- $(if $(CONFIG_PA11)$(CONFIG_COMPAT),$(Q)$(MAKE) \
+ifdef CONFIG_64BIT
+ $(Q)$(MAKE) $(build)=arch/parisc/kernel/vdso64 include/generated/vdso64-offsets.h
+ $(if $(CONFIG_COMPAT),$(Q)$(MAKE) \
$(build)=arch/parisc/kernel/vdso32 include/generated/vdso32-offsets.h)
+else
+ $(Q)$(MAKE) $(build)=arch/parisc/kernel/vdso32 include/generated/vdso32-offsets.h
+endif
endif
-vdso-install-$(CONFIG_PA11) += arch/parisc/kernel/vdso32/vdso32.so
+ifdef CONFIG_64BIT
+vdso-install-y += arch/parisc/kernel/vdso64/vdso64.so
vdso-install-$(CONFIG_COMPAT) += arch/parisc/kernel/vdso32/vdso32.so
-vdso-install-$(CONFIG_64BIT) += arch/parisc/kernel/vdso64/vdso64.so
+else
+vdso-install-y += arch/parisc/kernel/vdso32/vdso32.so
+endif
install: KBUILD_IMAGE := vmlinux
zinstall: KBUILD_IMAGE := vmlinuz
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild
index 4fb596d..d48d158 100644
--- a/arch/parisc/include/asm/Kbuild
+++ b/arch/parisc/include/asm/Kbuild
@@ -4,4 +4,5 @@
generic-y += agp.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += user.h
diff --git a/arch/parisc/include/asm/vdso.h b/arch/parisc/include/asm/vdso.h
index 5501560..e5cca3c 100644
--- a/arch/parisc/include/asm/vdso.h
+++ b/arch/parisc/include/asm/vdso.h
@@ -6,13 +6,14 @@
#ifdef CONFIG_64BIT
#include <generated/vdso64-offsets.h>
+#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name))
#endif
#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT)
#include <generated/vdso32-offsets.h>
-#endif
-
-#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name))
#define VDSO32_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso32_offset_##name))
+#else
+#define VDSO32_SYMBOL(tsk, name) 0UL
+#endif
#endif /* __ASSEMBLER__ */
diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile
index 2f34417..49f937c 100644
--- a/arch/parisc/kernel/Makefile
+++ b/arch/parisc/kernel/Makefile
@@ -46,6 +46,9 @@
# vdso
obj-y += vdso.o
-obj-$(CONFIG_64BIT) += vdso64/
-obj-$(CONFIG_PA11) += vdso32/
+ifdef CONFIG_64BIT
+obj-y += vdso64/
obj-$(CONFIG_COMPAT) += vdso32/
+else
+obj-y += vdso32/
+endif
diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c
index bc47bbe..b52ad70 100644
--- a/arch/parisc/kernel/drivers.c
+++ b/arch/parisc/kernel/drivers.c
@@ -41,9 +41,7 @@
const struct dma_map_ops *hppa_dma_ops __ro_after_init;
EXPORT_SYMBOL(hppa_dma_ops);
-static struct device root = {
- .init_name = "parisc",
-};
+static struct device *root;
static inline int check_dev(struct device *dev)
{
@@ -89,7 +87,7 @@ static int for_each_padev(int (*fn)(struct device *, void *), void * data)
.obj = data,
.fn = fn,
};
- return device_for_each_child(&root, &recurse_data, descend_children);
+ return device_for_each_child(root, &recurse_data, descend_children);
}
/**
@@ -290,7 +288,7 @@ const struct parisc_device *
find_pa_parent_type(const struct parisc_device *padev, int type)
{
const struct device *dev = &padev->dev;
- while (dev != &root) {
+ while (dev != root) {
struct parisc_device *candidate = to_parisc_device(dev);
if (candidate->id.hw_type == type)
return candidate;
@@ -319,7 +317,7 @@ static void get_node_path(struct device *dev, struct hardware_path *path)
dev = dev->parent;
}
- while (dev != &root) {
+ while (dev != root) {
if (dev_is_pci(dev)) {
unsigned int devfn = to_pci_dev(dev)->devfn;
path->bc[i--] = PCI_SLOT(devfn) | (PCI_FUNC(devfn)<< 5);
@@ -482,7 +480,7 @@ static struct parisc_device * __init alloc_tree_node(
static struct parisc_device *create_parisc_device(struct hardware_path *modpath)
{
int i;
- struct device *parent = &root;
+ struct device *parent = root;
for (i = 0; i < 6; i++) {
if (modpath->bc[i] == -1)
continue;
@@ -755,7 +753,7 @@ parse_tree_node(struct device *parent, int index, struct hardware_path *modpath)
struct device *hwpath_to_device(struct hardware_path *modpath)
{
int i;
- struct device *parent = &root;
+ struct device *parent = root;
for (i = 0; i < 6; i++) {
if (modpath->bc[i] == -1)
continue;
@@ -880,7 +878,7 @@ void __init walk_central_bus(void)
{
walk_native_bus(CENTRAL_BUS_ADDR,
CENTRAL_BUS_ADDR + (MAX_NATIVE_DEVICES * NATIVE_DEVICE_OFFSET),
- &root);
+ root);
}
static __init void print_parisc_device(struct parisc_device *dev)
@@ -907,9 +905,10 @@ void __init init_parisc_bus(void)
{
if (bus_register(&parisc_bus_type))
panic("Could not register PA-RISC bus type\n");
- if (device_register(&root))
+
+ root = root_device_register("parisc");
+ if (IS_ERR(root))
panic("Could not register PA-RISC root device\n");
- get_device(&root);
}
static __init void qemu_header(void)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index e93df95..c99fd83 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -213,7 +213,6 @@
select GENERIC_IRQ_SHOW_LEVEL
select GENERIC_PCI_IOMAP if PCI
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL
select HAS_IOPORT if PCI
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP
diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug
index f15e592..e8718bc 100644
--- a/arch/powerpc/Kconfig.debug
+++ b/arch/powerpc/Kconfig.debug
@@ -83,11 +83,10 @@
depends on DEBUG_KERNEL
config GUEST_STATE_BUFFER_TEST
- def_tristate n
+ def_tristate KUNIT_ALL_TESTS
prompt "Enable Guest State Buffer unit tests"
depends on KUNIT
depends on KVM_BOOK3S_HV_POSSIBLE
- default KUNIT_ALL_TESTS
help
The Guest State Buffer is a data format specified in the PAPR.
It is by hcalls to communicate the state of L2 guests between
diff --git a/arch/powerpc/configs/amigaone_defconfig b/arch/powerpc/configs/amigaone_defconfig
index 69ef3dc..7a51539 100644
--- a/arch/powerpc/configs/amigaone_defconfig
+++ b/arch/powerpc/configs/amigaone_defconfig
@@ -76,7 +76,6 @@
# CONFIG_HW_RANDOM is not set
# CONFIG_HWMON is not set
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_TILEBLITTING=y
CONFIG_FB_RADEON=y
CONFIG_FB_3DFX=y
diff --git a/arch/powerpc/configs/chrp32_defconfig b/arch/powerpc/configs/chrp32_defconfig
index b799c95..66eae5b 100644
--- a/arch/powerpc/configs/chrp32_defconfig
+++ b/arch/powerpc/configs/chrp32_defconfig
@@ -76,7 +76,6 @@
CONFIG_NVRAM=y
# CONFIG_HWMON is not set
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_OF=y
CONFIG_FB_MATROX=y
CONFIG_FB_MATROX_MILLENIUM=y
diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig
index 04bbb37..5ca1676 100644
--- a/arch/powerpc/configs/g5_defconfig
+++ b/arch/powerpc/configs/g5_defconfig
@@ -85,6 +85,8 @@
CONFIG_MAC_EMUMOUSEBTN=y
CONFIG_WINDFARM=y
CONFIG_WINDFARM_PM81=y
+CONFIG_WINDFARM_PM72=y
+CONFIG_WINDFARM_RM31=y
CONFIG_WINDFARM_PM91=y
CONFIG_WINDFARM_PM112=y
CONFIG_WINDFARM_PM121=y
@@ -121,7 +123,6 @@
CONFIG_AGP=m
CONFIG_AGP_UNINORTH=m
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_TILEBLITTING=y
CONFIG_FB_OF=y
CONFIG_FB_NVIDIA=y
diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig
index 8bbf51b3..89bcbeb 100644
--- a/arch/powerpc/configs/pasemi_defconfig
+++ b/arch/powerpc/configs/pasemi_defconfig
@@ -98,7 +98,6 @@
CONFIG_SENSORS_LM90=y
CONFIG_DRM=y
CONFIG_DRM_RADEON=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_TILEBLITTING=y
CONFIG_FB_VGA16=y
CONFIG_FB_NVIDIA=y
diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig
index cc98024..5d32c27 100644
--- a/arch/powerpc/configs/powernv_defconfig
+++ b/arch/powerpc/configs/powernv_defconfig
@@ -196,7 +196,6 @@
# CONFIG_PTP_1588_CLOCK is not set
CONFIG_DRM=y
CONFIG_DRM_AST=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_OF=y
CONFIG_FB_MATROX=m
CONFIG_FB_MATROX_MILLENIUM=y
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 3bf518e..6316ca4 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -249,7 +249,6 @@
CONFIG_I2C_AMD8111=y
CONFIG_I2C_PASEMI=y
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_OF=y
CONFIG_FB_MATROX=y
CONFIG_FB_MATROX_MILLENIUM=y
diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig
index 0fd49f6..20cc17d 100644
--- a/arch/powerpc/configs/ppc64e_defconfig
+++ b/arch/powerpc/configs/ppc64e_defconfig
@@ -118,7 +118,6 @@
CONFIG_I2C_CHARDEV=y
CONFIG_I2C_AMD8111=y
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
CONFIG_FB_OF=y
CONFIG_FB_MATROX=y
CONFIG_FB_MATROX_MILLENIUM=y
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index ccabc6e..eda1fec 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -393,6 +393,7 @@
CONFIG_TUN=m
CONFIG_VETH=m
CONFIG_VIRTIO_NET=m
+CONFIG_EL3=m
CONFIG_VORTEX=m
CONFIG_TYPHOON=m
CONFIG_ADAPTEC_STARFIRE=m
diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig
index ff1bed4..005536e 100644
--- a/arch/powerpc/configs/skiroot_defconfig
+++ b/arch/powerpc/configs/skiroot_defconfig
@@ -214,7 +214,6 @@
CONFIG_DRM=m
CONFIG_DRM_AST=m
CONFIG_FB=y
-CONFIG_FIRMWARE_EDID=y
# CONFIG_VGA_CONSOLE is not set
CONFIG_FRAMEBUFFER_CONSOLE=y
CONFIG_LOGO=y
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index 2e23533..805b5ae 100644
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -5,4 +5,5 @@
generic-y += agp.h
generic-y += mcs_spinlock.h
generic-y += qrwlock.h
+generic-y += ring_buffer.h
generic-y += early_ioremap.h
diff --git a/arch/powerpc/include/asm/pmac_low_i2c.h b/arch/powerpc/include/asm/pmac_low_i2c.h
index 21bd729..fead8fa 100644
--- a/arch/powerpc/include/asm/pmac_low_i2c.h
+++ b/arch/powerpc/include/asm/pmac_low_i2c.h
@@ -79,10 +79,6 @@ extern int pmac_i2c_match_adapter(struct device_node *dev,
struct i2c_adapter *adapter);
-/* (legacy) Locking functions exposed to i2c-keywest */
-extern int pmac_low_i2c_lock(struct device_node *np);
-extern int pmac_low_i2c_unlock(struct device_node *np);
-
/* Access functions for platform code */
extern int pmac_i2c_open(struct pmac_i2c_bus *bus, int polled);
extern void pmac_i2c_close(struct pmac_i2c_bus *bus);
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 66ed5fe..44ec416 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -135,6 +135,13 @@ struct cpumask *cpu_coregroup_mask(int cpu);
const struct cpumask *cpu_die_mask(int cpu);
int cpu_die_id(int cpu);
+/*
+ * Points to where the LLC is. On power9 this will point at CACHE
+ * domain, On others it will point to SMT domain. In all cases
+ * cpu_l2_cache_mask points to where LLC is
+ */
+#define arch_llc_mask(cpu) cpu_l2_cache_mask(cpu)
+
#ifdef CONFIG_PPC64
#include <asm/smp.h>
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 4bbeb86..3460d1a 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -376,6 +376,47 @@ void vtime_task_switch(struct task_struct *prev)
acct->starttime = acct0->starttime;
}
}
+
+#ifdef CONFIG_NO_HZ_COMMON
+/**
+ * vtime_reset - Fast forward vtime entry clocks
+ *
+ * Called from dynticks idle IRQ entry to fast-forward the clocks to current time
+ * so that the IRQ time is still accounted by vtime while nohz cputime is paused.
+ */
+void vtime_reset(void)
+{
+ struct cpu_accounting_data *acct = get_accounting(current);
+
+ acct->starttime = mftb();
+#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
+ acct->startspurr = read_spurr(acct->starttime);
+#endif
+}
+
+/**
+ * vtime_dyntick_start - Inform vtime about entry to idle-dynticks
+ *
+ * Called when idle enters in dyntick mode. The idle cputime that elapsed so far
+ * is accumulated and the tick subsystem takes over the idle cputime accounting.
+ */
+void vtime_dyntick_start(void)
+{
+ vtime_account_idle(current);
+}
+
+/**
+ * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks
+ *
+ * Called when idle exits from dyntick mode. The vtime entry clocks are
+ * fast-forward to current time so that idle accounting restarts elapsing from
+ * now.
+ */
+void vtime_dyntick_stop(void)
+{
+ vtime_reset();
+}
+#endif /* CONFIG_NO_HZ_COMMON */
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
void __no_kcsan __delay(unsigned long loops)
@@ -458,6 +499,10 @@ DEFINE_PER_CPU(u8, irq_work_pending);
#endif /* 32 vs 64 bit */
+/*
+ * Must be called with preemption disabled since it updates
+ * per-CPU irq_work state and programs the local CPU decrementer.
+ */
void arch_irq_work_raise(void)
{
/*
@@ -471,10 +516,8 @@ void arch_irq_work_raise(void)
* which could get tangled up if we're messing with the same state
* here.
*/
- preempt_disable();
set_irq_work_pending_flag();
set_dec(1);
- preempt_enable();
}
static void set_dec_or_work(u64 val)
diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile
index 8834dfe..368759f 100644
--- a/arch/powerpc/kernel/vdso/Makefile
+++ b/arch/powerpc/kernel/vdso/Makefile
@@ -62,6 +62,12 @@
# 32-bit one. clang validates the values passed to these arguments during
# parsing, even when -fno-stack-protector is passed afterwards.
CC32FLAGSREMOVE += -mstack-protector-guard%
+# ftrace is disabled for the vdso but arch/powerpc/Makefile adds this define to
+# KBUILD_CPPFLAGS, which enables use of the 'patchable_function_entry'
+# attribute in the 'inline' define via 'notrace'. This attribute is not
+# supported for the powerpcle target, resulting in many instances of
+# -Wunknown-attributes.
+CC32FLAGSREMOVE += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
endif
LD32FLAGS := -Wl,-soname=linux-vdso32.so.1
AS32FLAGS := -D__VDSO32__
diff --git a/arch/powerpc/kexec/Makefile b/arch/powerpc/kexec/Makefile
index 470eb04..ec7a0ee 100644
--- a/arch/powerpc/kexec/Makefile
+++ b/arch/powerpc/kexec/Makefile
@@ -16,4 +16,4 @@
KCOV_INSTRUMENT_core_$(BITS).o := n
UBSAN_SANITIZE_core_$(BITS).o := n
KASAN_SANITIZE_core.o := n
-KASAN_SANITIZE_core_$(BITS) := n
+KASAN_SANITIZE_core_$(BITS).o := n
diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c
index 554b248..57e897b 100644
--- a/arch/powerpc/lib/vmx-helper.c
+++ b/arch/powerpc/lib/vmx-helper.c
@@ -52,7 +52,14 @@ int exit_vmx_usercopy(void)
}
EXPORT_SYMBOL(exit_vmx_usercopy);
-int enter_vmx_ops(void)
+/*
+ * Can be called from kexec copy_page() path with MMU off. The kexec
+ * code sets preempt_count to HARDIRQ_OFFSET so we return early here.
+ * Since in_interrupt() is always inline, __no_sanitize_address on this
+ * function is sufficient to avoid KASAN shadow memory accesses in real
+ * mode.
+ */
+int __no_sanitize_address enter_vmx_ops(void)
{
if (in_interrupt())
return 0;
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 8b00814..2e6adf5 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -2242,6 +2242,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
const u64 last_period = event->hw.last_period;
s64 prev, delta, left;
int record = 0;
+ int mark_event = regs->dsisr & MMCRA_SAMPLE_ENABLE;
if (event->hw.state & PERF_HES_STOPPED) {
write_pmc(event->hw.idx, 0);
@@ -2304,9 +2305,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
* In ISA v3.0 and before values "0" and "7" are considered reserved.
* In ISA v3.1, value "7" has been used to indicate "larx/stcx".
* Drop the sample if "type" has reserved values for this field with a
- * ISA version check.
+ * ISA version check for marked events.
*/
- if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&
+ if (mark_event && event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&
ppmu->get_mem_data_src) {
val = (regs->dar & SIER_TYPE_MASK) >> SIER_TYPE_SHIFT;
if (val == 0 || (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31))) {
diff --git a/arch/powerpc/perf/hv-gpci.c b/arch/powerpc/perf/hv-gpci.c
index 5cac2cf..10c82cf 100644
--- a/arch/powerpc/perf/hv-gpci.c
+++ b/arch/powerpc/perf/hv-gpci.c
@@ -210,7 +210,7 @@ static ssize_t processor_bus_topology_show(struct device *dev, struct device_att
0, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
@@ -244,12 +244,14 @@ static ssize_t processor_bus_topology_show(struct device *dev, struct device_att
starting_index, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
}
+out_success:
+ put_cpu_var(hv_gpci_reqb);
return n;
out:
@@ -278,7 +280,7 @@ static ssize_t processor_config_show(struct device *dev, struct device_attribute
0, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
@@ -312,12 +314,14 @@ static ssize_t processor_config_show(struct device *dev, struct device_attribute
starting_index, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
}
+out_success:
+ put_cpu_var(hv_gpci_reqb);
return n;
out:
@@ -346,7 +350,7 @@ static ssize_t affinity_domain_via_virtual_processor_show(struct device *dev,
0, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
@@ -382,12 +386,14 @@ static ssize_t affinity_domain_via_virtual_processor_show(struct device *dev,
starting_index, secondary_index, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
}
+out_success:
+ put_cpu_var(hv_gpci_reqb);
return n;
out:
@@ -416,7 +422,7 @@ static ssize_t affinity_domain_via_domain_show(struct device *dev, struct device
0, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
@@ -448,12 +454,14 @@ static ssize_t affinity_domain_via_domain_show(struct device *dev, struct device
starting_index, 0, buf, &n, arg);
if (!ret)
- return n;
+ goto out_success;
if (ret != H_PARAMETER)
goto out;
}
+out_success:
+ put_cpu_var(hv_gpci_reqb);
return n;
out:
diff --git a/arch/powerpc/platforms/44x/warp.c b/arch/powerpc/platforms/44x/warp.c
index a5001d3..6f674f8 100644
--- a/arch/powerpc/platforms/44x/warp.c
+++ b/arch/powerpc/platforms/44x/warp.c
@@ -293,6 +293,8 @@ static int pika_dtm_thread(void __iomem *fpga)
schedule_timeout(HZ);
}
+ put_device(&client->dev);
+
return 0;
}
diff --git a/arch/powerpc/platforms/82xx/km82xx.c b/arch/powerpc/platforms/82xx/km82xx.c
index 99f0f0f..4ad2235 100644
--- a/arch/powerpc/platforms/82xx/km82xx.c
+++ b/arch/powerpc/platforms/82xx/km82xx.c
@@ -27,8 +27,8 @@
static void __init km82xx_pic_init(void)
{
- struct device_node *np __free(device_node);
- np = of_find_compatible_node(NULL, NULL, "fsl,pq2-pic");
+ struct device_node *np __free(device_node) = of_find_compatible_node(NULL,
+ NULL, "fsl,pq2-pic");
if (!np) {
pr_err("PIC init: can not find cpm-pic node\n");
diff --git a/arch/powerpc/platforms/8xx/cpm1.c b/arch/powerpc/platforms/8xx/cpm1.c
index 7433be7..f00734f 100644
--- a/arch/powerpc/platforms/8xx/cpm1.c
+++ b/arch/powerpc/platforms/8xx/cpm1.c
@@ -477,7 +477,7 @@ int cpm1_gpiochip_add16(struct device *dev)
struct device_node *np = dev->of_node;
struct cpm1_gpio16_chip *cpm1_gc;
struct gpio_chip *gc;
- u16 mask;
+ u32 mask;
cpm1_gc = devm_kzalloc(dev, sizeof(*cpm1_gc), GFP_KERNEL);
if (!cpm1_gc)
@@ -485,7 +485,7 @@ int cpm1_gpiochip_add16(struct device *dev)
spin_lock_init(&cpm1_gc->lock);
- if (!of_property_read_u16(np, "fsl,cpm1-gpio-irq-mask", &mask)) {
+ if (!of_property_read_u32(np, "fsl,cpm1-gpio-irq-mask", &mask)) {
int i, j;
for (i = 0, j = 0; i < 16; i++)
diff --git a/arch/powerpc/platforms/pasemi/pci.c b/arch/powerpc/platforms/pasemi/pci.c
index 60f990a..2df9552 100644
--- a/arch/powerpc/platforms/pasemi/pci.c
+++ b/arch/powerpc/platforms/pasemi/pci.c
@@ -272,13 +272,12 @@ void __init pas_pci_init(void)
{
struct device_node *root = of_find_node_by_path("/");
struct device_node *np;
- int res;
pci_set_flags(PCI_SCAN_ALL_PCIE_DEVS);
np = of_find_compatible_node(root, NULL, "pasemi,rootbus");
if (np) {
- res = pas_add_bridge(np);
+ pas_add_bridge(np);
of_node_put(np);
}
of_node_put(root);
diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c
index 73b7f4e..da72a30 100644
--- a/arch/powerpc/platforms/powermac/low_i2c.c
+++ b/arch/powerpc/platforms/powermac/low_i2c.c
@@ -1058,40 +1058,6 @@ int pmac_i2c_match_adapter(struct device_node *dev, struct i2c_adapter *adapter)
}
EXPORT_SYMBOL_GPL(pmac_i2c_match_adapter);
-int pmac_low_i2c_lock(struct device_node *np)
-{
- struct pmac_i2c_bus *bus, *found = NULL;
-
- list_for_each_entry(bus, &pmac_i2c_busses, link) {
- if (np == bus->controller) {
- found = bus;
- break;
- }
- }
- if (!found)
- return -ENODEV;
- return pmac_i2c_open(bus, 0);
-}
-EXPORT_SYMBOL_GPL(pmac_low_i2c_lock);
-
-int pmac_low_i2c_unlock(struct device_node *np)
-{
- struct pmac_i2c_bus *bus, *found = NULL;
-
- list_for_each_entry(bus, &pmac_i2c_busses, link) {
- if (np == bus->controller) {
- found = bus;
- break;
- }
- }
- if (!found)
- return -ENODEV;
- pmac_i2c_close(bus);
- return 0;
-}
-EXPORT_SYMBOL_GPL(pmac_low_i2c_unlock);
-
-
int pmac_i2c_open(struct pmac_i2c_bus *bus, int polled)
{
int rc;
diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c
index 12c4737..9109c21 100644
--- a/arch/powerpc/platforms/ps3/device-init.c
+++ b/arch/powerpc/platforms/ps3/device-init.c
@@ -950,8 +950,6 @@ static int __init ps3_start_probe_thread(enum ps3_bus_type bus_type)
static int __init ps3_register_devices(void)
{
- int result;
-
if (!firmware_has_feature(FW_FEATURE_PS3_LV1))
return -ENODEV;
@@ -959,7 +957,7 @@ static int __init ps3_register_devices(void)
/* ps3_repository_dump_bus_info(); */
- result = ps3_start_probe_thread(PS3_BUS_TYPE_STORAGE);
+ ps3_start_probe_thread(PS3_BUS_TYPE_STORAGE);
ps3_register_vuart_devices();
diff --git a/arch/powerpc/platforms/pseries/htmdump.c b/arch/powerpc/platforms/pseries/htmdump.c
index 742ec52..489a80e 100644
--- a/arch/powerpc/platforms/pseries/htmdump.c
+++ b/arch/powerpc/platforms/pseries/htmdump.c
@@ -16,6 +16,7 @@ static void *htm_buf;
static void *htm_status_buf;
static void *htm_info_buf;
static void *htm_caps_buf;
+static void *htm_mem_buf;
static u32 nodeindex;
static u32 nodalchipindex;
static u32 coreindexonchip;
@@ -86,7 +87,7 @@ static ssize_t htm_return_check(long rc)
static ssize_t htmdump_read(struct file *filp, char __user *ubuf,
size_t count, loff_t *ppos)
{
- void *htm_buf = filp->private_data;
+ void *htm_buf_data = filp->private_data;
unsigned long page, read_size, available;
loff_t offset;
long rc, ret;
@@ -100,7 +101,7 @@ static ssize_t htmdump_read(struct file *filp, char __user *ubuf,
* - last three values are address, size and offset
*/
rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
- htmtype, H_HTM_OP_DUMP_DATA, virt_to_phys(htm_buf),
+ htmtype, H_HTM_OP_DUMP_DATA, virt_to_phys(htm_buf_data),
PAGE_SIZE, page);
ret = htm_return_check(rc);
@@ -112,7 +113,61 @@ static ssize_t htmdump_read(struct file *filp, char __user *ubuf,
available = PAGE_SIZE;
read_size = min(count, available);
*ppos += read_size;
- return simple_read_from_buffer(ubuf, count, &offset, htm_buf, available);
+ return simple_read_from_buffer(ubuf, count, &offset, htm_buf_data, available);
+}
+
+static ssize_t htmsystem_mem_read(struct file *filp, char __user *ubuf,
+ size_t count, loff_t *ppos)
+{
+ void *htm_mem_data = filp->private_data;
+ long rc, ret;
+ u64 *num_entries;
+ u64 to_copy = 0;
+ loff_t offset = 0;
+ u64 mem_offset = 0;
+
+ /*
+ * Invoke H_HTM call with:
+ * - operation as htm status (H_HTM_OP_STATUS)
+ * - last three values as addr, size and offset. "offset"
+ * is value from output buffer header that points to next
+ * entry to dump. 0 is the first entry to dump. next entry
+ * is read from the output bufferbyte offset 0x8.
+ *
+ * When first time hcall is invoked, mem_offset should be
+ * zero because zero is the first entry.
+ * In the next hcall, offset of next entry to read from is
+ * picked from output buffer header itself. So don't fill
+ * mem_offset for first read.
+ *
+ * If there is no further data to read in next iteration,
+ * offset value from output buffer header will point to -1.
+ */
+ if (*ppos) {
+ mem_offset = *(u64 *)(htm_mem_data + 0x8);
+ if (mem_offset == -1)
+ return 0;
+ }
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_DUMP_SYSMEM_CONF, virt_to_phys(htm_mem_data),
+ PAGE_SIZE, be64_to_cpu(mem_offset));
+ ret = htm_return_check(rc);
+ if (ret <= 0) {
+ pr_debug("H_HTM hcall returned for op: H_HTM_OP_DUMP_SYSMEM_CONF with hcall returning %ld\n", ret);
+ return ret;
+ }
+
+ /*
+ * HTM system mem buffer, start of buffer + 0x10 gives the
+ * number of HTM entries in the buffer.
+ * So total count to copy is:
+ * 32 bytes (for first 5 fields) + (number of HTM entries * entry size)
+ */
+ num_entries = htm_mem_data + 0x10;
+ to_copy = 32 + (be64_to_cpu(*num_entries) * 32);
+
+ *ppos += to_copy;
+ return simple_read_from_buffer(ubuf, count, &offset, htm_mem_data, to_copy);
}
static const struct file_operations htmdump_fops = {
@@ -121,6 +176,12 @@ static const struct file_operations htmdump_fops = {
.open = simple_open,
};
+static const struct file_operations htmsystem_mem_fops = {
+ .llseek = NULL,
+ .read = htmsystem_mem_read,
+ .open = simple_open,
+};
+
static int htmconfigure_set(void *data, u64 val)
{
long rc, ret;
@@ -226,20 +287,31 @@ static int htmstart_get(void *data, u64 *val)
static ssize_t htmstatus_read(struct file *filp, char __user *ubuf,
size_t count, loff_t *ppos)
{
- void *htm_status_buf = filp->private_data;
+ void *htm_status_data = filp->private_data;
long rc, ret;
u64 *num_entries;
u64 to_copy;
int htmstatus_flag;
+ loff_t offset = 0;
+ u64 status_offset = 0;
/*
* Invoke H_HTM call with:
* - operation as htm status (H_HTM_OP_STATUS)
- * - last three values as addr, size and offset
+ * - last three values as addr, size and offset.
+ * "offset" is value from output buffer header
+ * that points to next entry to dump. 0 is the first
+ * entry to dump. next entry is read from the output
+ * bufferbyte offset 0x8.
*/
+ if (*ppos) {
+ status_offset = *(u64 *)(htm_status_data + 0x8);
+ if (status_offset == -1)
+ return 0;
+ }
rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
- htmtype, H_HTM_OP_STATUS, virt_to_phys(htm_status_buf),
- PAGE_SIZE, 0);
+ htmtype, H_HTM_OP_STATUS, virt_to_phys(htm_status_data),
+ PAGE_SIZE, be64_to_cpu(status_offset));
ret = htm_return_check(rc);
if (ret <= 0) {
@@ -255,13 +327,15 @@ static ssize_t htmstatus_read(struct file *filp, char __user *ubuf,
* So total count to copy is:
* 32 bytes (for first 7 fields) + (number of HTM entries * entry size)
*/
- num_entries = htm_status_buf + 0x10;
+ num_entries = htm_status_data + 0x10;
if (htmtype == 0x2)
htmstatus_flag = 0x8;
else
htmstatus_flag = 0x6;
to_copy = 32 + (be64_to_cpu(*num_entries) * htmstatus_flag);
- return simple_read_from_buffer(ubuf, count, ppos, htm_status_buf, to_copy);
+ *ppos += to_copy;
+
+ return simple_read_from_buffer(ubuf, count, &offset, htm_status_data, to_copy);
}
static const struct file_operations htmstatus_fops = {
@@ -273,19 +347,30 @@ static const struct file_operations htmstatus_fops = {
static ssize_t htminfo_read(struct file *filp, char __user *ubuf,
size_t count, loff_t *ppos)
{
- void *htm_info_buf = filp->private_data;
+ void *htm_info_data = filp->private_data;
long rc, ret;
u64 *num_entries;
u64 to_copy;
+ loff_t offset = 0;
+ u64 info_offset = 0;
/*
* Invoke H_HTM call with:
* - operation as htm status (H_HTM_OP_STATUS)
* - last three values as addr, size and offset
+ * "offset" is value from output buffer header
+ * that points to next entry to dump. 0 is the first
+ * entry to dump. next entry is read from the output
+ * bufferbyte offset 0x8.
*/
+ if (*ppos) {
+ info_offset = *(u64 *)(htm_info_data + 0x8);
+ if (info_offset == -1)
+ return 0;
+ }
rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
- htmtype, H_HTM_OP_DUMP_SYSPROC_CONF, virt_to_phys(htm_info_buf),
- PAGE_SIZE, 0);
+ htmtype, H_HTM_OP_DUMP_SYSPROC_CONF, virt_to_phys(htm_info_data),
+ PAGE_SIZE, be64_to_cpu(info_offset));
ret = htm_return_check(rc);
if (ret <= 0) {
@@ -301,15 +386,17 @@ static ssize_t htminfo_read(struct file *filp, char __user *ubuf,
* So total count to copy is:
* 32 bytes (for first 5 fields) + (number of HTM entries * entry size)
*/
- num_entries = htm_info_buf + 0x10;
+ num_entries = htm_info_data + 0x10;
to_copy = 32 + (be64_to_cpu(*num_entries) * 16);
- return simple_read_from_buffer(ubuf, count, ppos, htm_info_buf, to_copy);
+
+ *ppos += to_copy;
+ return simple_read_from_buffer(ubuf, count, &offset, htm_info_data, to_copy);
}
static ssize_t htmcaps_read(struct file *filp, char __user *ubuf,
size_t count, loff_t *ppos)
{
- void *htm_caps_buf = filp->private_data;
+ void *htm_caps_data = filp->private_data;
long rc, ret;
/*
@@ -319,7 +406,7 @@ static ssize_t htmcaps_read(struct file *filp, char __user *ubuf,
* and zero
*/
rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
- htmtype, H_HTM_OP_CAPABILITIES, virt_to_phys(htm_caps_buf),
+ htmtype, H_HTM_OP_CAPABILITIES, virt_to_phys(htm_caps_data),
0x80, 0);
ret = htm_return_check(rc);
@@ -328,7 +415,7 @@ static ssize_t htmcaps_read(struct file *filp, char __user *ubuf,
return ret;
}
- return simple_read_from_buffer(ubuf, count, ppos, htm_caps_buf, 0x80);
+ return simple_read_from_buffer(ubuf, count, ppos, htm_caps_data, 0x80);
}
static const struct file_operations htminfo_fops = {
@@ -457,9 +544,17 @@ static int htmdump_init_debugfs(void)
return -ENOMEM;
}
+ /* Memory to present HTM system memory configuration */
+ htm_mem_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!htm_mem_buf) {
+ pr_err("Failed to allocate htm mem buf\n");
+ return -ENOMEM;
+ }
+
debugfs_create_file("htmstatus", 0400, htmdump_debugfs_dir, htm_status_buf, &htmstatus_fops);
debugfs_create_file("htminfo", 0400, htmdump_debugfs_dir, htm_info_buf, &htminfo_fops);
debugfs_create_file("htmcaps", 0400, htmdump_debugfs_dir, htm_caps_buf, &htmcaps_fops);
+ debugfs_create_file("htmsystem_mem", 0400, htmdump_debugfs_dir, htm_mem_buf, &htmsystem_mem_fops);
return 0;
}
@@ -482,6 +577,10 @@ static void __exit htmdump_exit(void)
{
debugfs_remove_recursive(htmdump_debugfs_dir);
kfree(htm_buf);
+ kfree(htm_status_buf);
+ kfree(htm_info_buf);
+ kfree(htm_caps_buf);
+ kfree(htm_mem_buf);
}
module_init(htmdump_init);
diff --git a/arch/powerpc/platforms/pseries/papr-hvpipe.c b/arch/powerpc/platforms/pseries/papr-hvpipe.c
index 14ae480..0c40bdd 100644
--- a/arch/powerpc/platforms/pseries/papr-hvpipe.c
+++ b/arch/powerpc/platforms/pseries/papr-hvpipe.c
@@ -190,33 +190,34 @@ static int hvpipe_rtas_recv_msg(char __user *buf, int size)
return -ENOMEM;
}
- ret = rtas_ibm_receive_hvpipe_msg(work_area, &srcID,
- &bytes_written);
- if (!ret) {
- /*
- * Recv HVPIPE RTAS is successful.
- * When releasing FD or no one is waiting on the
- * specific source, issue recv HVPIPE RTAS call
- * so that pipe is not blocked - this func is called
- * with NULL buf.
- */
- if (buf) {
- if (size < bytes_written) {
- pr_err("Received the payload size = %d, but the buffer size = %d\n",
- bytes_written, size);
- bytes_written = size;
- }
- ret = copy_to_user(buf,
- rtas_work_area_raw_buf(work_area),
- bytes_written);
- if (!ret)
- ret = bytes_written;
- }
- } else {
- pr_err("ibm,receive-hvpipe-msg failed with %d\n",
- ret);
+ /*
+ * Recv HVPIPE RTAS is successful.
+ * When releasing FD or no one is waiting on the
+ * specific source, issue recv HVPIPE RTAS call
+ * so that pipe is not blocked - this func is called
+ * with NULL buf.
+ */
+ ret = rtas_ibm_receive_hvpipe_msg(work_area, &srcID, &bytes_written);
+ if (ret) {
+ pr_err("ibm,receive-hvpipe-msg failed with %d\n", ret);
+ goto out;
}
+ if (!buf)
+ goto out;
+
+ if (size < bytes_written) {
+ pr_err("Received the payload size = %d, but the buffer size = %d\n",
+ bytes_written, size);
+ bytes_written = size;
+ }
+
+ if (copy_to_user(buf, rtas_work_area_raw_buf(work_area), bytes_written))
+ ret = -EFAULT;
+ else
+ ret = bytes_written;
+
+out:
rtas_work_area_free(work_area);
return ret;
}
@@ -327,8 +328,8 @@ static ssize_t papr_hvpipe_handle_read(struct file *file,
{
struct hvpipe_source_info *src_info = file->private_data;
- struct papr_hvpipe_hdr hdr;
- long ret;
+ struct papr_hvpipe_hdr hdr = {};
+ ssize_t ret = 0;
/*
* Return -ENXIO during migration
@@ -376,7 +377,7 @@ static ssize_t papr_hvpipe_handle_read(struct file *file,
ret = copy_to_user(buf, &hdr, HVPIPE_HDR_LEN);
if (ret)
- return ret;
+ return -EFAULT;
/*
* Message event has payload, so get the payload with
@@ -385,19 +386,23 @@ static ssize_t papr_hvpipe_handle_read(struct file *file,
if (hdr.flags & HVPIPE_MSG_AVAILABLE) {
ret = hvpipe_rtas_recv_msg(buf + HVPIPE_HDR_LEN,
size - HVPIPE_HDR_LEN);
- if (ret > 0) {
+ /*
+ * Always clear MSG_AVAILABLE once the RTAS call has drained
+ * the message, regardless of whether copy_to_user succeeded.
+ */
+ if (ret >= 0 || ret == -EFAULT)
src_info->hvpipe_status &= ~HVPIPE_MSG_AVAILABLE;
- ret += HVPIPE_HDR_LEN;
- }
} else if (hdr.flags & HVPIPE_LOST_CONNECTION) {
/*
* Hypervisor is closing the pipe for the specific
* source. So notify user space.
*/
src_info->hvpipe_status &= ~HVPIPE_LOST_CONNECTION;
- ret = HVPIPE_HDR_LEN;
}
+ if (ret >= 0)
+ ret += HVPIPE_HDR_LEN;
+
return ret;
}
@@ -444,16 +449,18 @@ static int papr_hvpipe_handle_release(struct inode *inode,
struct file *file)
{
struct hvpipe_source_info *src_info;
+ unsigned long flags;
/*
* Hold the lock, remove source from src_list, reset the
* hvpipe status and release the lock to prevent any race
* with message event IRQ.
*/
- spin_lock(&hvpipe_src_list_lock);
+ spin_lock_irqsave(&hvpipe_src_list_lock, flags);
src_info = file->private_data;
list_del(&src_info->list);
file->private_data = NULL;
+ spin_unlock_irqrestore(&hvpipe_src_list_lock, flags);
/*
* If the pipe for this specific source has any pending
* payload, issue recv HVPIPE RTAS so that pipe will not
@@ -461,10 +468,8 @@ static int papr_hvpipe_handle_release(struct inode *inode,
*/
if (src_info->hvpipe_status & HVPIPE_MSG_AVAILABLE) {
src_info->hvpipe_status = 0;
- spin_unlock(&hvpipe_src_list_lock);
hvpipe_rtas_recv_msg(NULL, 0);
- } else
- spin_unlock(&hvpipe_src_list_lock);
+ }
kfree(src_info);
return 0;
@@ -479,50 +484,53 @@ static const struct file_operations papr_hvpipe_handle_ops = {
static int papr_hvpipe_dev_create_handle(u32 srcID)
{
- struct hvpipe_source_info *src_info __free(kfree) = NULL;
-
- spin_lock(&hvpipe_src_list_lock);
- /*
- * Do not allow more than one process communicates with
- * each source.
- */
- src_info = hvpipe_find_source(srcID);
- if (src_info) {
- spin_unlock(&hvpipe_src_list_lock);
- pr_err("pid(%d) is already using the source(%d)\n",
- src_info->tsk->pid, srcID);
- return -EALREADY;
- }
- spin_unlock(&hvpipe_src_list_lock);
+ struct hvpipe_source_info *src_info;
+ int fd;
+ unsigned long flags;
src_info = kzalloc_obj(*src_info, GFP_KERNEL_ACCOUNT);
if (!src_info)
return -ENOMEM;
src_info->srcID = srcID;
- src_info->tsk = current;
init_waitqueue_head(&src_info->recv_wqh);
- FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
- anon_inode_getfile("[papr-hvpipe]", &papr_hvpipe_handle_ops,
- (void *)src_info, O_RDWR));
- if (fdf.err)
- return fdf.err;
-
- retain_and_null_ptr(src_info);
- spin_lock(&hvpipe_src_list_lock);
/*
- * If two processes are executing ioctl() for the same
- * source ID concurrently, prevent the second process to
- * acquire FD.
+ * Do not allow more than one process communicates with
+ * each source.
*/
+ spin_lock_irqsave(&hvpipe_src_list_lock, flags);
if (hvpipe_find_source(srcID)) {
- spin_unlock(&hvpipe_src_list_lock);
+ spin_unlock_irqrestore(&hvpipe_src_list_lock, flags);
+ pr_err("pid(%s:%d) could not get the source(%d)\n",
+ current->comm, task_pid_nr(current), srcID);
+ kfree(src_info);
return -EALREADY;
}
list_add(&src_info->list, &hvpipe_src_list);
- spin_unlock(&hvpipe_src_list_lock);
- return fd_publish(fdf);
+ spin_unlock_irqrestore(&hvpipe_src_list_lock, flags);
+
+ fd = FD_ADD(O_RDONLY | O_CLOEXEC,
+ anon_inode_getfile("[papr-hvpipe]", &papr_hvpipe_handle_ops,
+ (void *)src_info, O_RDWR));
+ if (fd < 0) {
+ spin_lock_irqsave(&hvpipe_src_list_lock, flags);
+ list_del(&src_info->list);
+ spin_unlock_irqrestore(&hvpipe_src_list_lock, flags);
+ /*
+ * if we fail to add FD, that means no userspace program is
+ * polling. In that case if there is a msg pending because the
+ * interrupt was fired after the src_info was added to the
+ * global list, then let's consume it here, to unblock the
+ * hvpipe
+ */
+ if (src_info->hvpipe_status & HVPIPE_MSG_AVAILABLE)
+ hvpipe_rtas_recv_msg(NULL, 0);
+ kfree(src_info);
+ return fd;
+ }
+
+ return fd;
}
/*
@@ -685,20 +693,19 @@ static int __init enable_hvpipe_IRQ(void)
struct device_node *np;
hvpipe_check_exception_token = rtas_function_token(RTAS_FN_CHECK_EXCEPTION);
- if (hvpipe_check_exception_token == RTAS_UNKNOWN_SERVICE)
+ if (hvpipe_check_exception_token == RTAS_UNKNOWN_SERVICE)
return -ENODEV;
/* hvpipe events */
np = of_find_node_by_path("/event-sources/ibm,hvpipe-msg-events");
- if (np != NULL) {
- request_event_sources_irqs(np, hvpipe_event_interrupt,
- "HPIPE_EVENT");
- of_node_put(np);
- } else {
- pr_err("Can not enable hvpipe event IRQ\n");
+ if (!np) {
+ pr_err("No device node found, could not enable hvpipe event IRQ\n");
return -ENODEV;
}
+ request_event_sources_irqs(np, hvpipe_event_interrupt, "HPIPE_EVENT");
+ of_node_put(np);
+
return 0;
}
@@ -775,23 +782,29 @@ static int __init papr_hvpipe_init(void)
}
ret = enable_hvpipe_IRQ();
- if (!ret) {
- ret = set_hvpipe_sys_param(1);
- if (!ret)
- ret = misc_register(&papr_hvpipe_dev);
- }
+ if (ret)
+ goto out_wq;
- if (!ret) {
- pr_info("hvpipe feature is enabled\n");
- hvpipe_feature = true;
- return 0;
- }
+ ret = misc_register(&papr_hvpipe_dev);
+ if (ret)
+ goto out_wq;
- pr_err("hvpipe feature is not enabled %d\n", ret);
+ ret = set_hvpipe_sys_param(1);
+ if (ret)
+ goto out_misc;
+
+ pr_info("hvpipe feature is enabled\n");
+ hvpipe_feature = true;
+ return 0;
+
+out_misc:
+ misc_deregister(&papr_hvpipe_dev);
+out_wq:
destroy_workqueue(papr_hvpipe_wq);
out:
kfree(papr_hvpipe_work);
papr_hvpipe_work = NULL;
+ pr_err("hvpipe feature is not enabled %d\n", ret);
return ret;
}
machine_device_initcall(pseries, papr_hvpipe_init);
diff --git a/arch/powerpc/platforms/pseries/papr-hvpipe.h b/arch/powerpc/platforms/pseries/papr-hvpipe.h
index c343f42..4bdf7bb 100644
--- a/arch/powerpc/platforms/pseries/papr-hvpipe.h
+++ b/arch/powerpc/platforms/pseries/papr-hvpipe.h
@@ -21,7 +21,6 @@ struct hvpipe_source_info {
u32 srcID;
u32 hvpipe_status;
wait_queue_head_t recv_wqh; /* wake up poll() waitq */
- struct task_struct *tsk;
};
/*
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index d235396..195ebc2 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -123,7 +123,6 @@
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL if GENERIC_GETTIMEOFDAY
select HARDIRQS_SW_RESEND
select HAS_IOPORT if MMU
select HAVE_ALIGNED_STRUCT_PAGE
@@ -937,6 +936,28 @@
help
Enable detecting support for vector misaligned loads and stores.
+config RISCV_SBI_FWFT_DELEGATE_MISALIGNED
+ bool "Request firmware delegation of unaligned access exceptions"
+ depends on RISCV_SBI
+ depends on NONPORTABLE
+ help
+ Use SBI FWFT to request delegation of load address misaligned and
+ store address misaligned exceptions, if possible, and prefer Linux
+ kernel emulation of these accesses to firmware emulation.
+
+ Unfortunately, Linux's emulation is still incomplete. Namely, it
+ currently does not handle vector instructions and KVM guest accesses.
+ On platforms where these accesses would have been handled by firmware,
+ enabling this causes unexpected kernel oopses, userspaces crashes and
+ KVM guest crashes. If you are sure that these are not a problem for
+ your platform, you can say Y here, which may improve performance.
+
+ Saying N here will not worsen emulation support for unaligned accesses
+ even in the case where the firmware also has incomplete support. It
+ simply keeps the firmware's emulation enabled.
+
+ If you don't know what to do here, say N.
+
choice
prompt "Unaligned Accesses Support"
default RISCV_PROBE_UNALIGNED_ACCESS
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
index 2d14e92..9078e5b 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
@@ -101,16 +101,6 @@ &ccc_nw {
status = "okay";
};
-&i2c0 {
- pinctrl-names = "default";
- pinctrl-0 = <&i2c0_fabric>;
-};
-
-&i2c1 {
- pinctrl-names = "default";
- pinctrl-0 = <&i2c1_mssio>;
-};
-
&mmuart1 {
pinctrl-names = "default";
pinctrl-0 = <&uart1_fabric>;
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
index 8afedec..636493f 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
@@ -14,6 +14,16 @@ / {
"microchip,mpfs";
};
+&i2c0 {
+ pinctrl-names = "default";
+ pinctrl-0 = <&i2c0_fabric>;
+};
+
+&i2c1 {
+ pinctrl-names = "default";
+ pinctrl-0 = <&i2c1_mssio>;
+};
+
&syscontroller {
microchip,bitstream-flash = <&sys_ctrl_flash>;
};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
index 556aa96..6fadce8 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
@@ -11,3 +11,22 @@ / {
"microchip,mpfs-icicle-kit",
"microchip,mpfs";
};
+
+&i2c0 {
+ pinctrl-names = "default";
+ pinctrl-0 = <&i2c0_fabric>;
+};
+
+/*
+ * Due to silicon errata, routing via MSS IOs doesn't work on ES devices.
+ * Instead, i2c1, appearing on B1/C1, which are normally MSS IOs, is routed
+ * via the fabric and back to B1/C1 via "fabric-test" functionality.
+ * This is done silently by Libero, so the iomux0 setting for i2c1 has to
+ * be fabric IO, despite tooling etc saying that MSS IOs are used.
+ *
+ * See Section 3.3 of https://ww1.microchip.com/downloads/aemDocuments/documents/FPGA/ProductDocuments/Errata/polarfiresoc/microsemi_polarfire_soc_fpga_egineering_samples_errata_er0219_v1.pdf
+ */
+&i2c1 {
+ pinctrl-names = "default";
+ pinctrl-0 = <&i2c1_fabric>;
+};
diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
index 8cfe803..a7a1c09 100644
--- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
@@ -135,29 +135,6 @@ &tdm_ext {
clock-frequency = <49152000>;
};
-&camss {
- assigned-clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>,
- <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>;
- assigned-clock-rates = <49500000>, <198000000>;
-
- ports {
- #address-cells = <1>;
- #size-cells = <0>;
-
- port@0 {
- reg = <0>;
- };
-
- port@1 {
- reg = <1>;
-
- camss_from_csi2rx: endpoint {
- remote-endpoint = <&csi2rx_to_camss>;
- };
- };
- };
-};
-
&csi2rx {
assigned-clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>;
assigned-clock-rates = <297000000>;
@@ -175,9 +152,7 @@ port@0 {
port@1 {
reg = <1>;
- csi2rx_to_camss: endpoint {
- remote-endpoint = <&camss_from_csi2rx>;
- };
+ /* remote CAMSS endpoint */
};
};
};
diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi
index 6e56e9d2..9c3e459 100644
--- a/arch/riscv/boot/dts/starfive/jh7110.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi
@@ -1199,34 +1199,6 @@ csi_phy: phy@19820000 {
#phy-cells = <0>;
};
- camss: isp@19840000 {
- compatible = "starfive,jh7110-camss";
- reg = <0x0 0x19840000 0x0 0x10000>,
- <0x0 0x19870000 0x0 0x30000>;
- reg-names = "syscon", "isp";
- clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>,
- <&ispcrg JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C>,
- <&ispcrg JH7110_ISPCLK_DVP_INV>,
- <&ispcrg JH7110_ISPCLK_VIN_P_AXI_WR>,
- <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>,
- <&syscrg JH7110_SYSCLK_ISP_TOP_CORE>,
- <&syscrg JH7110_SYSCLK_ISP_TOP_AXI>;
- clock-names = "apb_func", "wrapper_clk_c", "dvp_inv",
- "axiwr", "mipi_rx0_pxl", "ispcore_2x",
- "isp_axi";
- resets = <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_P>,
- <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_C>,
- <&ispcrg JH7110_ISPRST_VIN_P_AXI_RD>,
- <&ispcrg JH7110_ISPRST_VIN_P_AXI_WR>,
- <&syscrg JH7110_SYSRST_ISP_TOP>,
- <&syscrg JH7110_SYSRST_ISP_TOP_AXI>;
- reset-names = "wrapper_p", "wrapper_c", "axird",
- "axiwr", "isp_top_n", "isp_top_axi";
- power-domains = <&pwrc JH7110_PD_ISP>;
- interrupts = <92>, <87>, <90>, <88>;
- status = "disabled";
- };
-
voutcrg: clock-controller@295c0000 {
compatible = "starfive,jh7110-voutcrg";
reg = <0x0 0x295c0000 0x0 0x10000>;
diff --git a/arch/riscv/errata/mips/errata.c b/arch/riscv/errata/mips/errata.c
index e984a81..2c3dc22 100644
--- a/arch/riscv/errata/mips/errata.c
+++ b/arch/riscv/errata/mips/errata.c
@@ -57,7 +57,7 @@ void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
}
tmp = (1U << alt->patch_id);
- if (cpu_req_errata && tmp) {
+ if (cpu_req_errata & tmp) {
mutex_lock(&text_mutex);
patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt),
alt->alt_len);
diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild
index bd5fc94..7721b63 100644
--- a/arch/riscv/include/asm/Kbuild
+++ b/arch/riscv/include/asm/Kbuild
@@ -14,5 +14,6 @@
generic-y += qrwlock.h
generic-y += qrwlock_types.h
generic-y += qspinlock.h
+generic-y += ring_buffer.h
generic-y += user.h
generic-y += vmlinux.lds.h
diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h
index ac80216..226289c 100644
--- a/arch/riscv/include/asm/syscall_wrapper.h
+++ b/arch/riscv/include/asm/syscall_wrapper.h
@@ -32,6 +32,10 @@ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *);
__diag_push(); \
__diag_ignore(GCC, 8, "-Wattribute-alias", \
"Type aliasing is used to sanitize syscall arguments"); \
+ __diag_ignore(clang, 23, "-Wunknown-warning-option", \
+ "Avoid breaking versions without -Wattribute-alias"); \
+ __diag_ignore(clang, 23, "-Wattribute-alias", \
+ "Type aliasing is used to sanitize syscall arguments"); \
static long __se_##prefix##name(ulong, ulong, ulong, ulong, ulong, ulong, \
ulong) \
__attribute__((alias(__stringify(___se_##prefix##name)))); \
diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h
index 9ec08fa..61cb3cb 100644
--- a/arch/riscv/include/asm/vdso/gettimeofday.h
+++ b/arch/riscv/include/asm/vdso/gettimeofday.h
@@ -9,12 +9,6 @@
#include <asm/csr.h>
#include <uapi/linux/time.h>
-/*
- * 32-bit land is lacking generic time vsyscalls as well as the legacy 32-bit
- * time syscalls like gettimeofday. Skip these definitions since on 32-bit.
- */
-#ifdef CONFIG_GENERIC_TIME_VSYSCALL
-
#define VDSO_HAS_CLOCK_GETRES 1
static __always_inline
@@ -66,8 +60,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
return ret;
}
-#endif /* CONFIG_GENERIC_TIME_VSYSCALL */
-
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
const struct vdso_time_data *vd)
{
diff --git a/arch/riscv/kernel/compat_signal.c b/arch/riscv/kernel/compat_signal.c
index 6ec4e34..cf3eb33 100644
--- a/arch/riscv/kernel/compat_signal.c
+++ b/arch/riscv/kernel/compat_signal.c
@@ -107,6 +107,8 @@ static long compat_restore_sigcontext(struct pt_regs *regs,
/* sc_regs is structured the same as the start of pt_regs */
err = __copy_from_user(&cregs, &sc->sc_regs, sizeof(sc->sc_regs));
+ if (unlikely(err))
+ return err;
cregs_to_regs(&cregs, regs);
diff --git a/arch/riscv/kernel/copy-unaligned.S b/arch/riscv/kernel/copy-unaligned.S
index 2b3d939..90f3549 100644
--- a/arch/riscv/kernel/copy-unaligned.S
+++ b/arch/riscv/kernel/copy-unaligned.S
@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) 2023 Rivos Inc. */
+#include <linux/cfi_types.h>
#include <linux/linkage.h>
#include <asm/asm.h>
@@ -9,7 +10,7 @@
/* void __riscv_copy_words_unaligned(void *, const void *, size_t) */
/* Performs a memcpy without aligning buffers, using word loads and stores. */
/* Note: The size is truncated to a multiple of 8 * SZREG */
-SYM_FUNC_START(__riscv_copy_words_unaligned)
+SYM_TYPED_FUNC_START(__riscv_copy_words_unaligned)
andi a4, a2, ~((8*SZREG)-1)
beqz a4, 2f
add a3, a1, a4
@@ -41,7 +42,7 @@
/* void __riscv_copy_bytes_unaligned(void *, const void *, size_t) */
/* Performs a memcpy without aligning buffers, using only byte accesses. */
/* Note: The size is truncated to a multiple of 8 */
-SYM_FUNC_START(__riscv_copy_bytes_unaligned)
+SYM_TYPED_FUNC_START(__riscv_copy_bytes_unaligned)
andi a4, a2, ~(8-1)
beqz a4, 2f
add a3, a1, a4
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 1734f9a..f46aa56 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -896,10 +896,8 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap)
* CPU cores with the ratified spec will contain non-zero
* marchid.
*/
- if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) {
- this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v];
+ if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0)
clear_bit(RISCV_ISA_EXT_v, source_isa);
- }
riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
@@ -1104,16 +1102,16 @@ early_param("riscv_isa_fallback", riscv_isa_fallback_setup);
void __init riscv_fill_hwcap(void)
{
char print_str[NUM_ALPHA_EXTS + 1];
- unsigned long isa2hwcap[26] = {0};
+ unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0};
int i, j;
- isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;
- isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;
- isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;
- isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;
- isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;
- isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;
- isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;
+ isa2hwcap[RISCV_ISA_EXT_i] = COMPAT_HWCAP_ISA_I;
+ isa2hwcap[RISCV_ISA_EXT_m] = COMPAT_HWCAP_ISA_M;
+ isa2hwcap[RISCV_ISA_EXT_a] = COMPAT_HWCAP_ISA_A;
+ isa2hwcap[RISCV_ISA_EXT_f] = COMPAT_HWCAP_ISA_F;
+ isa2hwcap[RISCV_ISA_EXT_d] = COMPAT_HWCAP_ISA_D;
+ isa2hwcap[RISCV_ISA_EXT_c] = COMPAT_HWCAP_ISA_C;
+ isa2hwcap[RISCV_ISA_EXT_v] = COMPAT_HWCAP_ISA_V;
if (!acpi_disabled) {
riscv_fill_hwcap_from_isa_string(isa2hwcap);
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 93de2e7..793bcee 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -577,8 +577,8 @@ static int compat_riscv_gpr_set(struct task_struct *target,
struct compat_user_regs_struct cregs;
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &cregs, 0, -1);
-
- cregs_to_regs(&cregs, task_pt_regs(target));
+ if (!ret)
+ cregs_to_regs(&cregs, task_pt_regs(target));
return ret;
}
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 5ed5095..fa66f9c 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -226,8 +226,7 @@ void show_ipi_stats(struct seq_file *p, int prec)
unsigned int cpu, i;
for (i = 0; i < IPI_MAX; i++) {
- seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
- prec >= 4 ? " " : "");
+ seq_printf(p, "%*s%u:", prec - 1, "IPI", i);
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
seq_printf(p, " %s\n", ipi_names[i]);
diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c
index 2a27d3f..81b7682 100644
--- a/arch/riscv/kernel/traps_misaligned.c
+++ b/arch/riscv/kernel/traps_misaligned.c
@@ -584,7 +584,7 @@ static int cpu_online_check_unaligned_access_emulated(unsigned int cpu)
static bool misaligned_traps_delegated;
-#ifdef CONFIG_RISCV_SBI
+#if defined(CONFIG_RISCV_SBI_FWFT_DELEGATE_MISALIGNED)
static int cpu_online_sbi_unaligned_setup(unsigned int cpu)
{
diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c
index 6eaa0d9..cbfb4e4 100644
--- a/arch/riscv/kernel/usercfi.c
+++ b/arch/riscv/kernel/usercfi.c
@@ -109,15 +109,16 @@ void set_indir_lp_lock(struct task_struct *task, bool lock)
task->thread_info.user_cfi_state.ufcfi_locked = lock;
}
/*
- * If size is 0, then to be compatible with regular stack we want it to be as big as
- * regular stack. Else PAGE_ALIGN it and return back
+ * The shadow stack only stores the return address and not any variables
+ * this should be more than sufficient for most applications.
+ * Else PAGE_ALIGN it and return back
*/
static unsigned long calc_shstk_size(unsigned long size)
{
if (size)
return PAGE_ALIGN(size);
- return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G));
+ return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 2, SZ_2G));
}
/*
diff --git a/arch/riscv/kernel/vec-copy-unaligned.S b/arch/riscv/kernel/vec-copy-unaligned.S
index 7ce4de6..361039f 100644
--- a/arch/riscv/kernel/vec-copy-unaligned.S
+++ b/arch/riscv/kernel/vec-copy-unaligned.S
@@ -2,6 +2,7 @@
/* Copyright (C) 2024 Rivos Inc. */
#include <linux/args.h>
+#include <linux/cfi_types.h>
#include <linux/linkage.h>
#include <asm/asm.h>
@@ -16,7 +17,7 @@
/* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */
/* Performs a memcpy without aligning buffers, using word loads and stores. */
/* Note: The size is truncated to a multiple of WORD_EEW */
-SYM_FUNC_START(__riscv_copy_vec_words_unaligned)
+SYM_TYPED_FUNC_START(__riscv_copy_vec_words_unaligned)
andi a4, a2, ~(WORD_EEW-1)
beqz a4, 2f
add a3, a1, a4
@@ -38,7 +39,7 @@
/* void __riscv_copy_vec_bytes_unaligned(void *, const void *, size_t) */
/* Performs a memcpy without aligning buffers, using only byte accesses. */
/* Note: The size is truncated to a multiple of 8 */
-SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned)
+SYM_TYPED_FUNC_START(__riscv_copy_vec_bytes_unaligned)
andi a4, a2, ~(8-1)
beqz a4, 2f
add a3, a1, a4
diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c
index 4d89b94..f09f925 100644
--- a/arch/riscv/kvm/vcpu_insn.c
+++ b/arch/riscv/kvm/vcpu_insn.c
@@ -415,7 +415,6 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
shift = 8 * (sizeof(ulong) - len);
} else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
len = 1;
- shift = 8 * (sizeof(ulong) - len);
#ifdef CONFIG_64BIT
} else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
len = 8;
@@ -649,22 +648,22 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
case 1:
data8 = *((u8 *)run->mmio.data);
SET_RD(insn, &vcpu->arch.guest_context,
- (ulong)data8 << shift >> shift);
+ (long)((ulong)data8 << shift) >> shift);
break;
case 2:
data16 = *((u16 *)run->mmio.data);
SET_RD(insn, &vcpu->arch.guest_context,
- (ulong)data16 << shift >> shift);
+ (long)((ulong)data16 << shift) >> shift);
break;
case 4:
data32 = *((u32 *)run->mmio.data);
SET_RD(insn, &vcpu->arch.guest_context,
- (ulong)data32 << shift >> shift);
+ (long)((ulong)data32 << shift) >> shift);
break;
case 8:
data64 = *((u64 *)run->mmio.data);
SET_RD(insn, &vcpu->arch.guest_context,
- (ulong)data64 << shift >> shift);
+ (long)((ulong)data64 << shift) >> shift);
break;
default:
return -EOPNOTSUPP;
diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c
index a935ed9..bb46dcb 100644
--- a/arch/riscv/kvm/vcpu_pmu.c
+++ b/arch/riscv/kvm/vcpu_pmu.c
@@ -453,8 +453,10 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s
}
kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC);
- if (!kvpmu->sdata)
- return -ENOMEM;
+ if (!kvpmu->sdata) {
+ sbiret = SBI_ERR_FAILURE;
+ goto out;
+ }
/* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */
if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) {
@@ -499,8 +501,10 @@ int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low
}
einfo = kzalloc(shmem_size, GFP_KERNEL);
- if (!einfo)
- return -ENOMEM;
+ if (!einfo) {
+ ret = SBI_ERR_FAILURE;
+ goto out;
+ }
ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size);
if (ret) {
diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c
index 3b83470..60e5029 100644
--- a/arch/riscv/kvm/vcpu_sbi_sta.c
+++ b/arch/riscv/kvm/vcpu_sbi_sta.c
@@ -46,7 +46,7 @@ void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu)
gfn = shmem >> PAGE_SHIFT;
hva = kvm_vcpu_gfn_to_hva(vcpu, gfn);
- if (WARN_ON(kvm_is_error_hva(hva))) {
+ if (kvm_is_error_hva(hva)) {
vcpu->arch.sta.shmem = INVALID_GPA;
return;
}
diff --git a/arch/riscv/kvm/vcpu_sbi_v01.c b/arch/riscv/kvm/vcpu_sbi_v01.c
index 188d5ea..c9c323d 100644
--- a/arch/riscv/kvm/vcpu_sbi_v01.c
+++ b/arch/riscv/kvm/vcpu_sbi_v01.c
@@ -55,6 +55,8 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,
for_each_set_bit(i, &hmask, BITS_PER_LONG) {
rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i);
+ if (!rvcpu)
+ continue;
ret = kvm_riscv_vcpu_set_interrupt(rvcpu, IRQ_VS_SOFT);
if (ret < 0)
break;
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index decd7df4..fa8d2f6 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -792,6 +792,27 @@ static void __init set_mmap_rnd_bits_max(void)
mmap_rnd_bits_max = MMAP_VA_BITS - PAGE_SHIFT - 3;
}
+static bool __init is_vaddr_valid(unsigned long va)
+{
+ unsigned long up = 0;
+
+ switch (satp_mode) {
+ case SATP_MODE_39:
+ up = 1UL << 38;
+ break;
+ case SATP_MODE_48:
+ up = 1UL << 47;
+ break;
+ case SATP_MODE_57:
+ up = 1UL << 56;
+ break;
+ default:
+ return false;
+ }
+
+ return (va < up) || (va >= (ULONG_MAX - up + 1));
+}
+
/*
* There is a simple way to determine if 4-level is supported by the
* underlying hardware: establish 1:1 mapping in 4-level page table mode
@@ -833,6 +854,9 @@ static __init void set_satp_mode(uintptr_t dtb_pa)
set_satp_mode_pmd + PMD_SIZE,
PMD_SIZE, PAGE_KERNEL_EXEC);
retry:
+ if (!is_vaddr_valid(set_satp_mode_pmd))
+ goto out;
+
create_pgd_mapping(early_pg_dir,
set_satp_mode_pmd,
pgtable_l5_enabled ?
@@ -855,6 +879,7 @@ static __init void set_satp_mode(uintptr_t dtb_pa)
disable_pgtable_l4();
}
+out:
memset(early_pg_dir, 0, PAGE_SIZE);
memset(early_p4d, 0, PAGE_SIZE);
memset(early_pud, 0, PAGE_SIZE);
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index ecbcbb7..2a5e784 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -177,7 +177,6 @@
select GENERIC_ENTRY
select GENERIC_GETTIMEOFDAY
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL
select GENERIC_IOREMAP if PCI
select HAVE_ALIGNED_STRUCT_PAGE
select HAVE_ARCH_AUDITSYSCALL
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index c28f9a7..730c90b 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -56,6 +56,10 @@
CONFIG_CHSC_SCH=y
CONFIG_VFIO_CCW=m
CONFIG_VFIO_AP=m
+CONFIG_VFIO_DEVICE_CDEV=y
+CONFIG_IOMMUFD_DRIVER=y
+CONFIG_IOMMUFD_DRIVER_CORE=y
+CONFIG_IOMMUFD=y
CONFIG_CMM=m
CONFIG_APPLDATA_BASE=y
CONFIG_S390_HYPFS_FS=y
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index d89c988..dd5fc14 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -54,6 +54,10 @@
CONFIG_CHSC_SCH=y
CONFIG_VFIO_CCW=m
CONFIG_VFIO_AP=m
+CONFIG_VFIO_DEVICE_CDEV=y
+CONFIG_IOMMUFD_DRIVER=y
+CONFIG_IOMMUFD_DRIVER_CORE=y
+CONFIG_IOMMUFD=y
CONFIG_CMM=m
CONFIG_APPLDATA_BASE=y
CONFIG_S390_HYPFS_FS=y
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild
index 80bad7d..0c1fc47 100644
--- a/arch/s390/include/asm/Kbuild
+++ b/arch/s390/include/asm/Kbuild
@@ -7,3 +7,4 @@
generic-y += asm-offsets.h
generic-y += mcs_spinlock.h
generic-y += mmzone.h
+generic-y += ring_buffer.h
diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h
index 59017fd..50a270e 100644
--- a/arch/s390/include/asm/bug.h
+++ b/arch/s390/include/asm/bug.h
@@ -12,12 +12,11 @@
#if defined(CONFIG_BUG) && defined(CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS)
#ifdef CONFIG_DEBUG_BUGVERBOSE
-#define __BUG_ENTRY_VERBOSE(format, file, line) \
- " .long " format " - . # bug_entry::format\n" \
+#define __BUG_ENTRY_VERBOSE(file, line) \
" .long " file " - . # bug_entry::file\n" \
" .short " line " # bug_entry::line\n"
#else
-#define __BUG_ENTRY_VERBOSE(format, file, line)
+#define __BUG_ENTRY_VERBOSE(file, line)
#endif
#ifdef CONFIG_DEBUG_BUGVERBOSE_DETAILED
@@ -28,9 +27,10 @@
#define __BUG_ENTRY(format, file, line, flags, size) \
" .section __bug_table,\"aw\"\n" \
- "1: .long 0b - . # bug_entry::bug_addr\n" \
- __BUG_ENTRY_VERBOSE(format, file, line) \
- " .short "flags" # bug_entry::flags\n" \
+ "1: .long 0b - . # bug_entry::bug_addr\n"\
+ " .long " format " - . # bug_entry::format\n" \
+ __BUG_ENTRY_VERBOSE(file, line) \
+ " .short "flags" # bug_entry::flags\n" \
" .org 1b+"size"\n" \
" .previous"
diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h
index 32536ee..e4ad09a 100644
--- a/arch/s390/include/asm/idle.h
+++ b/arch/s390/include/asm/idle.h
@@ -8,10 +8,12 @@
#ifndef _S390_IDLE_H
#define _S390_IDLE_H
+#include <linux/percpu-defs.h>
#include <linux/types.h>
#include <linux/device.h>
struct s390_idle_data {
+ bool idle_dyntick;
unsigned long idle_count;
unsigned long idle_time;
unsigned long clock_idle_enter;
diff --git a/arch/s390/include/asm/linkage.h b/arch/s390/include/asm/linkage.h
index df3fb7d..1b3ac55 100644
--- a/arch/s390/include/asm/linkage.h
+++ b/arch/s390/include/asm/linkage.h
@@ -7,4 +7,6 @@
#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT, 0x07
#define __ALIGN_STR __stringify(__ALIGN)
+#define _THIS_IP_ ({ unsigned long __ip; asm volatile("larl %0, ." : "=d" (__ip)); __ip; })
+
#endif
diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c
index 31430e9..7650f2a 100644
--- a/arch/s390/kernel/debug.c
+++ b/arch/s390/kernel/debug.c
@@ -1414,6 +1414,9 @@ static inline char *debug_get_user_string(const char __user *user_buf,
{
char *buffer;
+ if (!user_len)
+ return ERR_PTR(-EINVAL);
+
buffer = memdup_user_nul(user_buf, user_len);
if (IS_ERR(buffer))
return buffer;
@@ -1584,6 +1587,11 @@ static int debug_input_flush_fn(debug_info_t *id, struct debug_view *view,
char input_buf[1];
int rc = user_len;
+ if (!user_len) {
+ rc = -EINVAL;
+ goto out;
+ }
+
if (user_len > 0x10000)
user_len = 0x10000;
if (*offset != 0) {
diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c
index 1f1b06b..4685d7c 100644
--- a/arch/s390/kernel/idle.c
+++ b/arch/s390/kernel/idle.c
@@ -31,7 +31,10 @@ void account_idle_time_irq(void)
/* Account time spent with enabled wait psw loaded as idle time. */
__atomic64_add(idle_time, &idle->idle_time);
__atomic64_add_const(1, &idle->idle_count);
- account_idle_time(cputime_to_nsecs(idle_time));
+
+ /* Dyntick idle time accounted by nohz/scheduler */
+ if (!idle->idle_dyntick)
+ account_idle_time(cputime_to_nsecs(idle_time));
}
void noinstr arch_cpu_idle(void)
diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c
index 86f71a3..cdb8006 100644
--- a/arch/s390/kernel/perf_pai.c
+++ b/arch/s390/kernel/perf_pai.c
@@ -186,6 +186,13 @@ static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset)
return page[nr];
}
+static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v)
+{
+ if (offset)
+ nr += offset / sizeof(*page);
+ page[nr] = v;
+}
+
/* Read the counter values. Return value from location in CMP. For base
* event xxx_ALL sum up all events. Returns counter value.
*/
@@ -551,6 +558,8 @@ static void paicrypt_del(struct perf_event *event, int flags)
/* Create raw data and save it in buffer. Calculate the delta for each
* counter between this invocation and the last invocation.
* Returns number of bytes copied.
+ * After reading from PAI counter page, save the read value to the old
+ * page to calculate PAI counter deltas.
* Saves only entries with positive counter difference of the form
* 2 bytes: Number of counter
* 8 bytes: Value of counter
@@ -562,16 +571,22 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
int i, outidx = 0;
for (i = 1; i <= pp->num_avail; i++) {
- u64 val = 0, val_old = 0;
+ u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0;
if (!exclude_kernel) {
- val += pai_getctr(page, i, pp->kernel_offset);
- val_old += pai_getctr(page_old, i, pp->kernel_offset);
+ val_k = pai_getctr(page, i, pp->kernel_offset);
+ val_old_k = pai_getctr(page_old, i, pp->kernel_offset);
+ if (val_k != val_old_k)
+ pai_setctr(page_old, i, pp->kernel_offset, val_k);
}
if (!exclude_user) {
- val += pai_getctr(page, i, 0);
- val_old += pai_getctr(page_old, i, 0);
+ val = pai_getctr(page, i, 0);
+ val_old = pai_getctr(page_old, i, 0);
+ if (val != val_old)
+ pai_setctr(page_old, i, 0, val);
}
+ val += val_k;
+ val_old += val_old_k;
if (val >= val_old)
val -= val_old;
else
@@ -602,8 +617,6 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
struct perf_event *event)
{
- int idx = PAI_PMU_IDX(event);
- struct pai_pmu *pp = &pai_pmu[idx];
struct perf_sample_data data;
struct perf_raw_record raw;
struct pt_regs regs;
@@ -634,8 +647,6 @@ static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
overflow = perf_event_overflow(event, &data, ®s);
perf_event_update_userpage(event);
- /* Save crypto counter lowcore page after reading event data. */
- memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size);
return overflow;
}
@@ -651,7 +662,7 @@ static void pai_have_sample(struct perf_event *event, struct pai_map *cpump)
rawsize = pai_copy(cpump->save, cpump->area, pp,
(unsigned long *)PAI_SAVE_AREA(event),
event->attr.exclude_user,
- event->attr.exclude_kernel);
+ !pp->kernel_offset ? true : event->attr.exclude_kernel);
if (rawsize) /* No incremented counters */
pai_push_sample(rawsize, cpump, event);
}
diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c
index 1913a55..1377c6f 100644
--- a/arch/s390/kernel/topology.c
+++ b/arch/s390/kernel/topology.c
@@ -192,17 +192,21 @@ static void tl_to_masks(struct sysinfo_15_1_x *info)
end = (union topology_entry *)((unsigned long)info + info->length);
while (tle < end) {
switch (tle->nl) {
+ /*
+ * Adjust drawer_id, book_id, and socked_id so they match the
+ * numbering scheme of e.g. the hardware management console.
+ */
case 3:
drawer = drawer->next;
- drawer->id = tle->container.id;
+ drawer->id = tle->container.id - 1;
break;
case 2:
book = book->next;
- book->id = tle->container.id;
+ book->id = tle->container.id - 1;
break;
case 1:
socket = socket->next;
- socket->id = tle->container.id;
+ socket->id = tle->container.id - 1;
break;
case 0:
add_cpus_to_mask(&tle->cpu, drawer, book, socket);
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index bf48744..d1102a6 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -17,6 +17,7 @@
#include <asm/vtimer.h>
#include <asm/vtime.h>
#include <asm/cpu_mf.h>
+#include <asm/idle.h>
#include <asm/smp.h>
#include "entry.h"
@@ -110,6 +111,16 @@ static void account_system_index_scaled(struct task_struct *p, u64 cputime,
account_system_index_time(p, cputime_to_nsecs(cputime), index);
}
+static inline void vtime_reset_last_update(struct lowcore *lc)
+{
+ asm volatile(
+ " stpt %0\n" /* Store current cpu timer value */
+ " stckf %1" /* Store current tod clock value */
+ : "=Q" (lc->last_update_timer),
+ "=Q" (lc->last_update_clock)
+ : : "cc");
+}
+
/*
* Update process times based on virtual cpu times stored by entry.S
* to the lowcore fields user_timer, system_timer & steal_clock.
@@ -121,17 +132,16 @@ static int do_account_vtime(struct task_struct *tsk)
timer = lc->last_update_timer;
clock = lc->last_update_clock;
- asm volatile(
- " stpt %0\n" /* Store current cpu timer value */
- " stckf %1" /* Store current tod clock value */
- : "=Q" (lc->last_update_timer),
- "=Q" (lc->last_update_clock)
- : : "cc");
+
+ vtime_reset_last_update(lc);
+
clock = lc->last_update_clock - clock;
timer -= lc->last_update_timer;
if (hardirq_count())
lc->hardirq_timer += timer;
+ else if (in_serving_softirq())
+ lc->softirq_timer += timer;
else
lc->system_timer += timer;
@@ -231,14 +241,63 @@ EXPORT_SYMBOL_GPL(vtime_account_kernel);
void vtime_account_softirq(struct task_struct *tsk)
{
- get_lowcore()->softirq_timer += vtime_delta();
+ if (!__this_cpu_read(s390_idle.idle_dyntick))
+ get_lowcore()->softirq_timer += vtime_delta();
+ else
+ vtime_flush(tsk);
}
void vtime_account_hardirq(struct task_struct *tsk)
{
- get_lowcore()->hardirq_timer += vtime_delta();
+ if (!__this_cpu_read(s390_idle.idle_dyntick)) {
+ get_lowcore()->hardirq_timer += vtime_delta();
+ } else {
+ /*
+ * In dynticks mode, the idle cputime is accounted by the nohz
+ * subsystem. Therefore the s390 timer/clocks are reset on IRQ
+ * entry and steal time must be accounted now.
+ */
+ vtime_flush(tsk);
+ }
}
+#ifdef CONFIG_NO_HZ_COMMON
+/**
+ * vtime_reset - Fast forward vtime entry clocks
+ *
+ * Called from dynticks idle IRQ entry to fast-forward the clocks to current time
+ * so that the IRQ time is still accounted by vtime while nohz cputime is paused.
+ */
+void vtime_reset(void)
+{
+ vtime_reset_last_update(get_lowcore());
+}
+
+/**
+ * vtime_dyntick_start - Inform vtime about entry to idle-dynticks
+ *
+ * Called when idle enters in dyntick mode. The idle cputime that elapsed so far
+ * is flushed and the tick subsystem takes over the idle cputime accounting.
+ */
+void vtime_dyntick_start(void)
+{
+ __this_cpu_write(s390_idle.idle_dyntick, true);
+ vtime_flush(current);
+}
+
+/**
+ * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks
+ *
+ * Called when idle exits from dyntick mode. The vtime entry clocks are
+ * fast-forward to current time and idle accounting resumes.
+ */
+void vtime_dyntick_stop(void)
+{
+ vtime_reset_last_update(get_lowcore());
+ __this_cpu_write(s390_idle.idle_dyntick, false);
+}
+#endif /* CONFIG_NO_HZ_COMMON */
+
/*
* Sorted add to a list. List is linear searched until first bigger
* element is found.
diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c
index 7b8d70f..4a41c02 100644
--- a/arch/s390/kvm/dat.c
+++ b/arch/s390/kvm/dat.c
@@ -267,6 +267,7 @@ static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t g
/* No need to take locks as the page table is not installed yet. */
pgste_init.prefix_notif = old.s.fc1.prefix_notif;
pgste_init.vsie_notif = old.s.fc1.vsie_notif;
+ pgste_init.vsie_gmem = old.s.fc1.vsie_notif;
pgste_init.pcl = uses_skeys && init.h.i;
dat_init_pgstes(pt, pgste_init.val);
} else {
diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h
index 8f8278c4..873e13a 100644
--- a/arch/s390/kvm/dat.h
+++ b/arch/s390/kvm/dat.h
@@ -145,7 +145,8 @@ union pgste {
unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */
unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
unsigned long vsie_notif : 1; /* Referenced in a shadow table */
- unsigned long : 5;
+ unsigned long vsie_gmem : 1; /* Contains nested guest memory */
+ unsigned long : 4;
unsigned long : 8;
};
struct {
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index b07accd..4f8d5592c9 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -1445,6 +1445,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
} else {
pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false);
pgste.vsie_notif = 1;
+ pgste.vsie_gmem = 1;
}
pgste_set_unlock(ptep_h, pgste);
if (rc)
diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c
index 3c26e35..957126a 100644
--- a/arch/s390/kvm/gmap.c
+++ b/arch/s390/kvm/gmap.c
@@ -125,7 +125,7 @@ struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit)
int gmap_set_limit(struct gmap *gmap, gfn_t limit)
{
- struct kvm_s390_mmu_cache *mc;
+ struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
int rc, type;
type = gmap_limit_to_type(limit);
@@ -142,7 +142,6 @@ int gmap_set_limit(struct gmap *gmap, gfn_t limit)
rc = dat_set_asce_limit(mc, &gmap->asce, type);
} while (rc == -ENOMEM);
- kvm_s390_free_mmu_cache(mc);
return 0;
}
@@ -822,8 +821,8 @@ int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t
int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count)
{
- struct kvm_s390_mmu_cache *mc;
- int rc;
+ struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
+ int rc = 0;
mc = kvm_s390_new_mmu_cache();
if (!mc)
@@ -1026,13 +1025,15 @@ int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level)
int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
kvm_pfn_t pfn, int level, bool wr)
{
+ unsigned long bitmask;
union crste *crstep;
union pgste pgste;
union pte *ptep;
union pte pte;
int flags, rc;
- KVM_BUG_ON(!is_shadow(sg), sg->kvm);
+ if (KVM_BUG_ON(!is_shadow(sg) || level <= TABLE_TYPE_PAGE_TABLE, sg->kvm))
+ return -EINVAL;
lockdep_assert_held(&sg->parent->children_lock);
flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0);
@@ -1041,8 +1042,9 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf
if (rc)
return rc;
if (level <= TABLE_TYPE_REGION1) {
+ bitmask = -1UL << (8 + 11 * level);
scoped_guard(spinlock, &sg->host_to_rmap_lock)
- rc = gmap_insert_rmap(sg, p_gfn, r_gfn, level);
+ rc = gmap_insert_rmap(sg, p_gfn, r_gfn & bitmask, level);
}
if (rc)
return rc;
@@ -1143,8 +1145,10 @@ void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn)
}
scoped_guard(spinlock, &sg->host_to_rmap_lock)
head = radix_tree_delete(&sg->host_to_rmap, gfn);
- gmap_for_each_rmap_safe(rmap, rnext, head)
+ gmap_for_each_rmap_safe(rmap, rnext, head) {
gmap_unshadow_level(sg, rmap->r_gfn, rmap->level);
+ kfree(rmap);
+ }
}
}
diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h
index 96ee139..742e42a 100644
--- a/arch/s390/kvm/gmap.h
+++ b/arch/s390/kvm/gmap.h
@@ -167,6 +167,36 @@ static inline bool gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end)
return _gmap_unmap_prefix(gmap, gfn, end, false);
}
+/**
+ * pte_needs_unshadow() -- Check if the pte operations triggers unshadowing.
+ * @oldpte: the previous value for the guest pte.
+ * @newpte: the new pte being set.
+ * @pgste: the pgste for the pte entry.
+ *
+ * If the pgste.vsie_notif bit is not set, return false: the page is not
+ * involved in vsie and thus should not trigger an unshadow operation.
+ *
+ * If the pgste.vsie_gmem bit is set, this pte represents shadowed guest
+ * memory. The access rights on g3's memory should be synchronized with g1's
+ * and g2's. Therefore unshadowing is triggered if the new and old pte
+ * differ in protection, or if the new pte is invalid.
+ *
+ * If the pgste.vsie_gmem bit is not set, this pte maps the g2 dat tables
+ * for g3. If the entry becomes writable or absent, it becomes impossible to
+ * guarantee that the shadow mapping will match g2's mapping. In that case,
+ * trigger an unshadow event.
+ *
+ * Return: true if an unshadow event should be triggered, otherwise false.
+ */
+static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union pgste pgste)
+{
+ if (!pgste.vsie_notif)
+ return false;
+ if (pgste.vsie_gmem)
+ return (oldpte.h.p != newpte.h.p) || newpte.h.i;
+ return !newpte.h.p || !newpte.s.pr;
+}
+
static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte,
union pgste pgste, gfn_t gfn, bool needs_lock)
{
@@ -180,8 +210,9 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un
pgste.prefix_notif = 0;
gmap_unmap_prefix(gmap, gfn, gfn + 1);
}
- if (pgste.vsie_notif && (ptep->h.p != newpte.h.p || newpte.h.i)) {
+ if (pte_needs_unshadow(*ptep, newpte, pgste)) {
pgste.vsie_notif = 0;
+ pgste.vsie_gmem = 0;
if (needs_lock)
gmap_handle_vsie_unshadow_event(gmap, gfn);
else
@@ -189,6 +220,7 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un
}
if (!ptep->s.d && newpte.s.d && !newpte.s.s)
SetPageDirty(pfn_to_page(newpte.h.pfra));
+ pgste.zero = 0;
return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap));
}
@@ -198,6 +230,30 @@ static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, uni
return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true);
}
+/**
+ * crste_needs_unshadow() -- Check if the crste operations triggers unshadowing.
+ * @oldcrste: the previous value for the crste.
+ * @newcrste: the new value for the crste.
+ *
+ * If the old crste did not have the vsie_notif bit set, return false: the
+ * page is not involved in vsie and thus should not trigger an unshadow
+ * operation. Conversely, if the bit is set, it can only be g3 memory, since
+ * dat tables are never mapped using large pages.
+ *
+ * Similar to the pgste.vsie_gmem case of pte_needs_unshadow(), if the
+ * protection bit is changing or the new page is invalid, trigger an
+ * unshadow event. Also trigger an unshadow event if the new crste does not
+ * have the vsie_notif bit set.
+ *
+ * Return: true if an unshadow event should be triggered, otherwise false.
+ */
+static inline bool crste_needs_unshadow(union crste oldcrste, union crste newcrste)
+{
+ if (!oldcrste.s.fc1.vsie_notif)
+ return false;
+ return (newcrste.h.p != oldcrste.h.p) || newcrste.h.i || !newcrste.s.fc1.vsie_notif;
+}
+
static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
union crste oldcrste, union crste newcrste,
gfn_t gfn, bool needs_lock)
@@ -216,8 +272,7 @@ static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, unio
newcrste.s.fc1.prefix_notif = 0;
gmap_unmap_prefix(gmap, gfn, gfn + align);
}
- if (crste_leaf(oldcrste) && oldcrste.s.fc1.vsie_notif &&
- (newcrste.h.p || newcrste.h.i || !newcrste.s.fc1.vsie_notif)) {
+ if (crste_leaf(oldcrste) && crste_needs_unshadow(oldcrste, newcrste)) {
newcrste.s.fc1.vsie_notif = 0;
if (needs_lock)
gmap_handle_vsie_unshadow_event(gmap, gfn);
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 07f59c3..3bcdbbb 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -3310,8 +3310,7 @@ static void aen_host_forward(unsigned long si)
struct zpci_gaite *gaite;
struct kvm *kvm;
- gaite = (struct zpci_gaite *)aift->gait +
- (si * sizeof(struct zpci_gaite));
+ gaite = aift->gait + si;
if (gaite->count == 0)
return;
if (gaite->aisb != 0)
diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c
index 86d93e8..5b075c3 100644
--- a/arch/s390/kvm/pci.c
+++ b/arch/s390/kvm/pci.c
@@ -166,7 +166,7 @@ static int kvm_zpci_set_airq(struct zpci_dev *zdev)
fib.fmt0.noi = airq_iv_end(zdev->aibv);
fib.fmt0.aibv = virt_to_phys(zdev->aibv->vector);
fib.fmt0.aibvo = 0;
- fib.fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8);
+ fib.fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8;
fib.fmt0.aisbo = zdev->aisb & 63;
fib.gd = zdev->gisa;
@@ -290,8 +290,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib,
phys_to_virt(fib->fmt0.aibv));
spin_lock_irq(&aift->gait_lock);
- gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb *
- sizeof(struct zpci_gaite));
+ gaite = aift->gait + zdev->aisb;
/* If assist not requested, host will get all alerts */
if (assist)
@@ -309,7 +308,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib,
/* Update guest FIB for re-issue */
fib->fmt0.aisbo = zdev->aisb & 63;
- fib->fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8);
+ fib->fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8;
fib->fmt0.isc = gisc;
/* Save some guest fib values in the host for later use */
@@ -357,8 +356,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force)
if (zdev->kzdev->fib.fmt0.aibv == 0)
goto out;
spin_lock_irq(&aift->gait_lock);
- gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb *
- sizeof(struct zpci_gaite));
+ gaite = aift->gait + zdev->aisb;
isc = gaite->gisc;
gaite->count--;
if (gaite->count == 0) {
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 191cc53..028aeb9 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -438,7 +438,7 @@ void do_secure_storage_access(struct pt_regs *regs)
panic("Unexpected PGM 0x3d with TEID bit 61=0");
}
if (is_kernel_fault(regs)) {
- folio = phys_to_folio(addr);
+ folio = virt_to_folio((void *)addr);
if (unlikely(!folio_try_get(folio)))
return;
rc = uv_convert_from_secure(folio_to_phys(folio));
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index 4d3f10e..f0403d3 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -3,4 +3,5 @@
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
generic-y += parport.h
+generic-y += ring_buffer.h
generic-y += text-patching.h
diff --git a/arch/sh/include/asm/setup.h b/arch/sh/include/asm/setup.h
index 63c9efc..8488f76 100644
--- a/arch/sh/include/asm/setup.h
+++ b/arch/sh/include/asm/setup.h
@@ -7,7 +7,7 @@
/*
* This is set up by the setup-routine at boot-time
*/
-extern unsigned char *boot_params_page;
+extern unsigned char boot_params_page[];
#define PARAM boot_params_page
#define MOUNT_ROOT_RDONLY (*(unsigned long *) (PARAM+0x000))
diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c
index 9022d8a..03c39b5 100644
--- a/arch/sh/kernel/irq.c
+++ b/arch/sh/kernel/irq.c
@@ -46,7 +46,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%*s:", prec, "NMI");
for_each_online_cpu(j)
seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat.__nmi_count, j), 10);
- seq_printf(p, " Non-maskable interrupts\n");
+ seq_printf(p, " Non-maskable interrupts\n");
seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index a6b787e..f83d506 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -103,7 +103,6 @@
select HAVE_REGS_AND_STACK_ACCESS_API
select ARCH_USE_QUEUED_RWLOCKS
select ARCH_USE_QUEUED_SPINLOCKS
- select GENERIC_TIME_VSYSCALL
select ARCH_HAS_PTE_SPECIAL
select PCI_DOMAINS if PCI
select ARCH_HAS_GIGANTIC_PAGE
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 17ee8a2..49c6bb3 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -4,4 +4,5 @@
generic-y += agp.h
generic-y += kvm_para.h
generic-y += mcs_spinlock.h
+generic-y += ring_buffer.h
generic-y += text-patching.h
diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c
index 5210991..22db727 100644
--- a/arch/sparc/kernel/irq_32.c
+++ b/arch/sparc/kernel/irq_32.c
@@ -199,19 +199,19 @@ int arch_show_interrupts(struct seq_file *p, int prec)
int j;
#ifdef CONFIG_SMP
- seq_printf(p, "RES:");
+ seq_printf(p, "%*s:", prec, "RES");
for_each_online_cpu(j)
seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_resched_count, 10);
- seq_printf(p, " IPI rescheduling interrupts\n");
- seq_printf(p, "CAL:");
+ seq_printf(p, " IPI rescheduling interrupts\n");
+ seq_printf(p, "%*s:", prec, "CAL");
for_each_online_cpu(j)
seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_call_count, 10);
- seq_printf(p, " IPI function call interrupts\n");
+ seq_printf(p, " IPI function call interrupts\n");
#endif
- seq_printf(p, "NMI:");
+ seq_printf(p, "%*s:", prec, "NMI");
for_each_online_cpu(j)
seq_put_decimal_ull_width(p, " ", cpu_data(j).counter, 10);
- seq_printf(p, " Non-maskable interrupts\n");
+ seq_printf(p, " Non-maskable interrupts\n");
return 0;
}
diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c
index c5466a9..3f55c69 100644
--- a/arch/sparc/kernel/irq_64.c
+++ b/arch/sparc/kernel/irq_64.c
@@ -303,10 +303,10 @@ int arch_show_interrupts(struct seq_file *p, int prec)
{
int j;
- seq_printf(p, "NMI:");
+ seq_printf(p, "%*s:", prec, "NMI");
for_each_online_cpu(j)
seq_put_decimal_ull_width(p, " ", cpu_data(j).__nmi_count, 10);
- seq_printf(p, " Non-maskable interrupts\n");
+ seq_printf(p, " Non-maskable interrupts\n");
return 0;
}
diff --git a/arch/um/Makefile b/arch/um/Makefile
index 721b652..937639e 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -71,7 +71,8 @@
-D_FILE_OFFSET_BITS=64 -idirafter $(srctree)/include \
-idirafter $(objtree)/include -D__KERNEL__ -D__UM_HOST__ \
-include $(srctree)/include/linux/compiler-version.h \
- -include $(srctree)/include/linux/kconfig.h
+ -include $(srctree)/include/linux/kconfig.h \
+ -idirafter $(ARCH_DIR)/include/generated
#This will adjust *FLAGS accordingly to the platform.
include $(srctree)/$(ARCH_DIR)/Makefile-os-Linux
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild
index 1b9b82b..8fdc0bd 100644
--- a/arch/um/include/asm/Kbuild
+++ b/arch/um/include/asm/Kbuild
@@ -16,7 +16,9 @@
generic-y += module.lds.h
generic-y += parport.h
generic-y += percpu.h
+generic-y += percpu_types.h
generic-y += preempt.h
+generic-y += ring_buffer.h
generic-y += runtime-const.h
generic-y += softirq_stack.h
generic-y += switch_to.h
diff --git a/arch/um/kernel/irq.c b/arch/um/kernel/irq.c
index 5929d49..ddfd6e9 100644
--- a/arch/um/kernel/irq.c
+++ b/arch/um/kernel/irq.c
@@ -716,12 +716,12 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%*s: ", prec, "RES");
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_stats(cpu)->irq_resched_count);
- seq_puts(p, " Rescheduling interrupts\n");
+ seq_puts(p, " Rescheduling interrupts\n");
seq_printf(p, "%*s: ", prec, "CAL");
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_stats(cpu)->irq_call_count);
- seq_puts(p, " Function call interrupts\n");
+ seq_puts(p, " Function call interrupts\n");
#endif
return 0;
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index f3f7cb0..4abb296 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -113,6 +113,7 @@
select ARCH_HAS_ZONE_DMA_SET if EXPERT
select ARCH_HAVE_NMI_SAFE_CMPXCHG
select ARCH_HAVE_EXTRA_ELF_NOTES
+ select ARCH_MEMORY_ORDER_TSO
select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI
select ARCH_MIGHT_HAVE_PC_PARPORT
@@ -180,7 +181,6 @@
select GENERIC_IRQ_SHOW
select GENERIC_PENDING_IRQ if SMP
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL
select GENERIC_GETTIMEOFDAY
select GENERIC_VDSO_OVERFLOW_PROTECT
select GUP_GET_PXX_LOW_HIGH if X86_PAE
@@ -238,6 +238,7 @@
select HAVE_EFFICIENT_UNALIGNED_ACCESS
select HAVE_EISA if X86_32
select HAVE_EXIT_THREAD
+ select HAVE_FUTEX_ROBUST_UNLOCK
select HAVE_GENERIC_TIF_BITS
select HAVE_GUP_FAST
select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE
@@ -683,7 +684,6 @@
depends on X86_32
depends on X86_EXTENDED_PLATFORM
depends on X86_PLATFORM_DEVICES
- depends on X86_TSC
depends on PCI
depends on PCI_GOANY
depends on X86_IO_APIC
@@ -695,17 +695,6 @@
Say Y here if you have a Quark based system such as the Arduino
compatible Intel Galileo.
-config X86_RDC321X
- bool "RDC R-321x SoC"
- depends on X86_32
- depends on X86_EXTENDED_PLATFORM
- select M486
- select X86_REBOOTFIXUPS
- help
- This option is needed for RDC R-321x system-on-chip, also known
- as R-8610-(G).
- If you don't have one of these chips, you should say N here.
-
config X86_INTEL_LPSS
bool "Intel Low Power Subsystem Support"
depends on X86 && ACPI && PCI
@@ -1644,33 +1633,6 @@
Set whether the default state of memory_corruption_check is
on or off.
-config MATH_EMULATION
- bool
- depends on MODIFY_LDT_SYSCALL
- prompt "Math emulation" if X86_32 && (M486SX || MELAN)
- help
- Linux can emulate a math coprocessor (used for floating point
- operations) if you don't have one. 486DX and Pentium processors have
- a math coprocessor built in, 486SX and 386 do not, unless you added
- a 487DX or 387, respectively. (The messages during boot time can
- give you some hints here ["man dmesg"].) Everyone needs either a
- coprocessor or this emulation.
-
- If you don't have a math coprocessor, you need to say Y here; if you
- say Y here even though you have a coprocessor, the coprocessor will
- be used nevertheless. (This behavior can be changed with the kernel
- command line option "no387", which comes handy if your coprocessor
- is broken. Try "man bootparam" or see the documentation of your boot
- loader (lilo or loadlin) about how to pass options to the kernel at
- boot time.) This means that it is a good idea to say Y here if you
- intend to use this kernel on different machines.
-
- More information about the internals of the Linux math coprocessor
- emulation can be found in <file:arch/x86/math-emu/README>.
-
- If you are not sure, say Y; apart from resulting in a 66 KB bigger
- kernel, it won't hurt.
-
config MTRR
def_bool y
prompt "MTRR (Memory Type Range Register) support" if EXPERT
@@ -2437,10 +2399,6 @@
def_bool n
select CALL_PADDING
-config PREFIX_SYMBOLS
- def_bool y
- depends on CALL_PADDING && !CFI
-
menuconfig CPU_MITIGATIONS
bool "Mitigations for CPU vulnerabilities"
default y
@@ -2780,18 +2738,16 @@
1) make sure that you have enough swap space and that it is
enabled.
2) pass the "idle=poll" option to the kernel
- 3) switch on floating point emulation in the kernel and pass
- the "no387" option to the kernel
- 4) pass the "floppy=nodma" option to the kernel
- 5) pass the "mem=4M" option to the kernel (thereby disabling
+ 3) pass the "floppy=nodma" option to the kernel
+ 4) pass the "mem=4M" option to the kernel (thereby disabling
all but the first 4 MB of RAM)
- 6) make sure that the CPU is not over clocked.
- 7) read the sig11 FAQ at <http://www.bitwizard.nl/sig11/>
- 8) disable the cache from your BIOS settings
- 9) install a fan for the video card or exchange video RAM
- 10) install a better fan for the CPU
- 11) exchange RAM chips
- 12) exchange the motherboard.
+ 5) make sure that the CPU is not over clocked.
+ 6) read the sig11 FAQ at <http://www.bitwizard.nl/sig11/>
+ 7) disable the cache from your BIOS settings
+ 8) install a fan for the video card or exchange video RAM
+ 9) install a better fan for the CPU
+ 10) exchange RAM chips
+ 11) exchange the motherboard.
To compile this driver as a module, choose M here: the
module will be called apm.
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index d7ba921..22f0309 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -32,8 +32,6 @@
- "Athlon" for the AMD K7 family (Athlon/Duron/Thunderbird).
- "Crusoe" for the Transmeta Crusoe series.
- "Efficeon" for the Transmeta Efficeon series.
- - "Winchip-C6" for original IDT Winchip.
- - "Winchip-2" for IDT Winchips with 3dNow! capabilities.
- "AMD Elan" for the 32-bit AMD Elan embedded CPU.
- "GeodeGX1" for Geode GX1 (Cyrix MediaGX).
- "Geode GX/LX" For AMD Geode GX and LX processors.
@@ -45,14 +43,6 @@
See each option's help text for additional details. If you don't know
what to do, choose "Pentium-Pro".
-config M586
- bool "586/K5/5x86/6x86/6x86MX"
- depends on X86_32
- help
- Select this for an 586 or 686 series processor such as the AMD K5,
- the Cyrix 5x86, 6x86 and 6x86MX. This choice does not
- assume the RDTSC (Read Time Stamp Counter) instruction.
-
config M586TSC
bool "Pentium-Classic"
depends on X86_32
@@ -155,24 +145,6 @@
help
Select this for a Transmeta Efficeon processor.
-config MWINCHIPC6
- bool "Winchip-C6"
- depends on X86_32
- help
- Select this for an IDT Winchip C6 chip. Linux and GCC
- treat this chip as a 586TSC with some extended instructions
- and alignment requirements.
-
-config MWINCHIP3D
- bool "Winchip-2/Winchip-2A/Winchip-3"
- depends on X86_32
- help
- Select this for an IDT Winchip-2, 2A or 3. Linux and GCC
- treat this chip as a 586TSC with some extended instructions
- and alignment requirements. Also enable out of order memory
- stores for this CPU, which can increase performance of some
- operations.
-
config MGEODEGX1
bool "GeodeGX1"
depends on X86_32
@@ -270,19 +242,15 @@
default "7" if MPENTIUM4
default "6" if MK7 || MPENTIUMM || MATOM || MVIAC7 || X86_GENERIC || X86_64
default "4" if MGEODEGX1
- default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
+ default "5" if MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MGEODE_LX
config X86_F00F_BUG
def_bool y
depends on M586MMX || M586TSC || M586
-config X86_INVD_BUG
- def_bool y
- depends on M486SX || M486
-
config X86_ALIGNMENT_16
def_bool y
- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1
+ depends on MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1
config X86_INTEL_USERCOPY
def_bool y
@@ -290,11 +258,10 @@
config X86_USE_PPRO_CHECKSUM
def_bool y
- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM
+ depends on MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM
config X86_TSC
def_bool y
- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MATOM) || X86_64
config X86_HAVE_PAE
def_bool y
@@ -318,7 +285,7 @@
config X86_DEBUGCTLMSR
def_bool y
- depends on !(MK6 || MWINCHIPC6 || MWINCHIP3D || MCYRIXIII || M586MMX || M586TSC || M586) && !UML
+ depends on !(MK6 || MCYRIXIII || M586MMX || M586TSC || M586) && !UML
config IA32_FEAT_CTL
def_bool y
@@ -334,10 +301,6 @@
This lets you choose what x86 vendor support code your kernel
will include.
-config BROADCAST_TLB_FLUSH
- def_bool y
- depends on CPU_SUP_AMD && 64BIT
-
config CPU_SUP_INTEL
default y
bool "Support Intel processors" if PROCESSOR_SELECT
@@ -419,20 +382,6 @@
If unsure, say N.
-config CPU_SUP_UMC_32
- default y
- bool "Support UMC processors" if PROCESSOR_SELECT
- depends on M486SX || M486 || (EXPERT && !64BIT)
- help
- This enables detection, tunings and quirks for UMC processors
-
- You need this enabled if you want your kernel to run on a
- UMC CPU. Disabling this option on other types of CPUs
- makes the kernel a tiny bit smaller. Disabling it on a UMC
- CPU might render the kernel unbootable.
-
- If unsure, say N.
-
config CPU_SUP_ZHAOXIN
default y
bool "Support Zhaoxin processors" if PROCESSOR_SELECT
@@ -458,3 +407,7 @@
makes the kernel a tiny bit smaller.
If unsure, say N.
+
+config BROADCAST_TLB_FLUSH
+ def_bool y
+ depends on CPU_SUP_AMD && 64BIT
diff --git a/arch/x86/Kconfig.cpufeatures b/arch/x86/Kconfig.cpufeatures
index 532cbc2..89cbf8f 100644
--- a/arch/x86/Kconfig.cpufeatures
+++ b/arch/x86/Kconfig.cpufeatures
@@ -70,7 +70,6 @@
config X86_REQUIRED_FEATURE_FPU
def_bool y
- depends on !MATH_EMULATION
config X86_REQUIRED_FEATURE_PAE
def_bool y
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 46fec0b..96bfa13 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -276,7 +276,6 @@
libs-y += arch/x86/lib/
# drivers-y are linked after core-y
-drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/
drivers-$(CONFIG_PCI) += arch/x86/pci/
# suspend and hibernation support
diff --git a/arch/x86/Makefile_32.cpu b/arch/x86/Makefile_32.cpu
index 7c9898c..c5aa169 100644
--- a/arch/x86/Makefile_32.cpu
+++ b/arch/x86/Makefile_32.cpu
@@ -10,7 +10,6 @@
align := -falign-functions=0 -falign-jumps=0 -falign-loops=0
endif
-cflags-$(CONFIG_M586) += -march=i586
cflags-$(CONFIG_M586TSC) += -march=i586
cflags-$(CONFIG_M586MMX) += -march=pentium-mmx
cflags-$(CONFIG_M686) += -march=i686
@@ -24,16 +23,11 @@
cflags-$(CONFIG_MK7) += -march=athlon
cflags-$(CONFIG_MCRUSOE) += -march=i686 $(align)
cflags-$(CONFIG_MEFFICEON) += -march=i686 $(call tune,pentium3) $(align)
-cflags-$(CONFIG_MWINCHIPC6) += $(call cc-option,-march=winchip-c6,-march=i586)
-cflags-$(CONFIG_MWINCHIP3D) += $(call cc-option,-march=winchip2,-march=i586)
cflags-$(CONFIG_MCYRIXIII) += $(call cc-option,-march=c3,-march=i486) $(align)
cflags-$(CONFIG_MVIAC3_2) += $(call cc-option,-march=c3-2,-march=i686)
cflags-$(CONFIG_MVIAC7) += -march=i686
cflags-$(CONFIG_MATOM) += -march=atom
-# AMD Elan support
-cflags-$(CONFIG_MELAN) += -march=i486
-
# Geode GX1 support
cflags-$(CONFIG_MGEODEGX1) += -march=pentium-mmx
cflags-$(CONFIG_MGEODE_LX) += $(call cc-option,-march=geode,-march=pentium-mmx)
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index 3b0948a..8e4bf53 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -219,7 +219,8 @@ static void parse_gb_huge_pages(char *param, char *val)
if (!strcmp(param, "hugepages") && gbpage_sz) {
p = val;
- max_gb_huge_pages = simple_strtoull(p, &p, 0);
+ if (boot_kstrtoul(p, 0, &max_gb_huge_pages))
+ warn("Failed to parse hugepages= boot parameter\n");
return;
}
}
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 0e89e19..1b2fb35 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -2,6 +2,7 @@
#include "misc.h"
#include <asm/bootparam.h>
#include <asm/bootparam_utils.h>
+#include <asm/cpuid/api.h>
#include <asm/e820/types.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index b76a7c9..c07a2c3 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -43,6 +43,7 @@
#include <asm/init.h>
#include <asm/setup.h>
#include <asm/sections.h>
+#include <asm/cpuid/api.h>
#include <asm/coco.h>
#include <asm/sev.h>
diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c
index b25c6a9..ac0f900 100644
--- a/arch/x86/boot/string.c
+++ b/arch/x86/boot/string.c
@@ -289,6 +289,9 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
unsigned long long _res;
unsigned int rv;
+ if (s[0] == '+')
+ s++;
+
s = _parse_integer_fixup_radix(s, &base);
rv = _parse_integer(s, base, &_res);
if (rv & KSTRTOX_OVERFLOW)
@@ -304,35 +307,12 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
return 0;
}
-/**
- * kstrtoull - convert a string to an unsigned long long
- * @s: The start of the string. The string must be null-terminated, and may also
- * include a single newline before its terminating null. The first character
- * may also be a plus sign, but not a minus sign.
- * @base: The number base to use. The maximum supported base is 16. If base is
- * given as 0, then the base of the string is automatically detected with the
- * conventional semantics - If it begins with 0x the number will be parsed as a
- * hexadecimal (case insensitive), if it otherwise begins with 0, it will be
- * parsed as an octal number. Otherwise it will be parsed as a decimal.
- * @res: Where to write the result of the conversion on success.
- *
- * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
- * Used as a replacement for the obsolete simple_strtoull. Return code must
- * be checked.
- */
-int kstrtoull(const char *s, unsigned int base, unsigned long long *res)
-{
- if (s[0] == '+')
- s++;
- return _kstrtoull(s, base, res);
-}
-
static int _kstrtoul(const char *s, unsigned int base, unsigned long *res)
{
unsigned long long tmp;
int rv;
- rv = kstrtoull(s, base, &tmp);
+ rv = _kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (unsigned long)tmp)
@@ -364,7 +344,7 @@ int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res)
*/
if (sizeof(unsigned long) == sizeof(unsigned long long) &&
__alignof__(unsigned long) == __alignof__(unsigned long long))
- return kstrtoull(s, base, (unsigned long long *)res);
+ return _kstrtoull(s, base, (unsigned long long *)res);
else
return _kstrtoul(s, base, res);
}
diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h
index a5b05eb..4092bf2 100644
--- a/arch/x86/boot/string.h
+++ b/arch/x86/boot/string.h
@@ -28,6 +28,5 @@ extern unsigned long long simple_strtoull(const char *cp, char **endp,
unsigned int base);
long simple_strtol(const char *cp, char **endp, unsigned int base);
-int kstrtoull(const char *s, unsigned int base, unsigned long long *res);
int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res);
#endif /* BOOT_STRING_H */
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index 7ed3da9..ecd77d3 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -367,17 +367,13 @@ static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long
local_irq_save(flags);
- if (sev_cfg.ghcbs_initialized)
- ghcb = __sev_get_ghcb(&state);
- else
- ghcb = boot_ghcb;
+ ghcb = __sev_get_ghcb(&state);
/* Invoke the hypervisor to perform the page state changes */
if (!ghcb || vmgexit_psc(ghcb, data))
sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
- if (sev_cfg.ghcbs_initialized)
- __sev_put_ghcb(&state);
+ __sev_put_ghcb(&state);
local_irq_restore(flags);
diff --git a/arch/x86/coco/sev/internal.h b/arch/x86/coco/sev/internal.h
index b1d0c66a..b9632c0 100644
--- a/arch/x86/coco/sev/internal.h
+++ b/arch/x86/coco/sev/internal.h
@@ -70,6 +70,9 @@ void svsm_pval_pages(struct snp_psc_desc *desc);
int svsm_perform_call_protocol(struct svsm_call *call);
bool snp_svsm_vtpm_probe(void);
+noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code);
+noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code);
+
static inline u64 sev_es_rd_ghcb_msr(void)
{
return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB);
diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c
index 5afd663..e1e03f1 100644
--- a/arch/x86/coco/sev/noinstr.c
+++ b/arch/x86/coco/sev/noinstr.c
@@ -121,8 +121,10 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
WARN_ON(!irqs_disabled());
- if (!sev_cfg.ghcbs_initialized)
+ if (!sev_cfg.ghcbs_initialized) {
+ state->ghcb = NULL;
return boot_ghcb;
+ }
data = this_cpu_read(runtime_data);
ghcb = &data->ghcb_page;
diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c
index 2acf4a7..916d62c 100644
--- a/arch/x86/coco/sev/svsm.c
+++ b/arch/x86/coco/sev/svsm.c
@@ -74,20 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call)
flags = native_local_irq_save();
- if (sev_cfg.ghcbs_initialized)
- ghcb = __sev_get_ghcb(&state);
- else if (boot_ghcb)
- ghcb = boot_ghcb;
- else
- ghcb = NULL;
+ ghcb = __sev_get_ghcb(&state);
do {
ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
: __pi_svsm_perform_msr_protocol(call);
} while (ret == -EAGAIN);
- if (sev_cfg.ghcbs_initialized)
- __sev_put_ghcb(&state);
+ __sev_put_ghcb(&state);
native_local_irq_restore(flags);
diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c
index d98b5c0..96b62b4 100644
--- a/arch/x86/coco/sev/vc-handle.c
+++ b/arch/x86/coco/sev/vc-handle.c
@@ -954,7 +954,7 @@ static __always_inline bool vc_is_db(unsigned long error_code)
* Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
* and will panic when an error happens.
*/
-DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
+noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
{
irqentry_state_t irq_state;
@@ -1006,7 +1006,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
* Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
* and will kill the current task with SIGBUS when an error happens.
*/
-DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
+noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
{
/*
* Handle #DB before calling into !noinstr code to avoid recursive #DB.
@@ -1032,6 +1032,14 @@ DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
irqentry_exit_to_user_mode(regs);
}
+DEFINE_IDTENTRY_RAW_ERRORCODE(exc_vmm_communication)
+{
+ if (user_mode(regs))
+ return user_exc_vmm_communication(regs, error_code);
+ else
+ return kernel_exc_vmm_communication(regs, error_code);
+}
+
bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
{
unsigned long exit_code = regs->orig_ax;
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 186915a..29b6f1e 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -14,6 +14,7 @@
#include <asm/ia32.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
+#include <asm/cpuid/api.h>
#include <asm/paravirt_types.h>
#include <asm/pgtable.h>
#include <asm/set_memory.h>
diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
index 72cae8e..83b4762 100644
--- a/arch/x86/entry/Makefile
+++ b/arch/x86/entry/Makefile
@@ -13,7 +13,7 @@
CFLAGS_syscall_32.o += -fno-stack-protector
CFLAGS_syscall_64.o += -fno-stack-protector
-obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o
+obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o
obj-y += vdso/
obj-y += vsyscall/
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
new file mode 100644
index 0000000..14cd43d
--- /dev/null
+++ b/arch/x86/entry/common.c
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/entry-common.h>
+#include <linux/kvm_types.h>
+#include <linux/hrtimer_rearm.h>
+#include <asm/fred.h>
+#include <asm/desc.h>
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+/*
+ * On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as
+ * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
+ * x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs
+ * to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
+ * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
+ * the VM-Exit is held pending until it's unblocked in the host.
+ */
+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
+{
+ if (event_type == EVENT_TYPE_EXTINT) {
+#ifdef CONFIG_X86_64
+ /*
+ * Use FRED dispatch, even when running IDT. The dispatch
+ * tables are kept in sync between FRED and IDT, and the FRED
+ * dispatch works well with CFI.
+ */
+ fred_entry_from_kvm(event_type, vector);
+#else
+ idt_entry_from_kvm(vector);
+#endif
+ /*
+ * Strictly speaking, only the NMI path requires noinstr.
+ */
+ instrumentation_begin();
+ /*
+ * KVM/VMX will dispatch from IRQ-disabled but for a context
+ * that will have IRQs-enabled. This confuses the entry code
+ * and it will not have reprogrammed the timer. Do so now.
+ */
+ hrtimer_rearm_deferred();
+ instrumentation_end();
+
+ return;
+ }
+
+ WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);
+
+#ifdef CONFIG_X86_64
+ if (cpu_feature_enabled(X86_FEATURE_FRED))
+ return fred_entry_from_kvm(event_type, vector);
+#endif
+
+ /*
+ * Notably, we must use IDT dispatch for NMI when running in IDT mode.
+ * The FRED NMI context is significantly different and will not work
+ * right (specifically FRED fixed the NMI recursion issue).
+ */
+ idt_do_nmi_irqoff();
+}
+EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);
+#endif
diff --git a/arch/x86/entry/entry.S b/arch/x86/entry/entry.S
index 6ba2b3a..2bc217b 100644
--- a/arch/x86/entry/entry.S
+++ b/arch/x86/entry/entry.S
@@ -75,3 +75,51 @@
#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
EXPORT_SYMBOL(__ref_stack_chk_guard);
#endif
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+.macro IDT_DO_EVENT_IRQOFF call_insn call_target
+ /*
+ * Unconditionally create a stack frame, getting the correct RSP on the
+ * stack (for x86-64) would take two instructions anyways, and RBP can
+ * be used to restore RSP to make objtool happy (see below).
+ */
+ push %_ASM_BP
+ mov %_ASM_SP, %_ASM_BP
+
+#ifdef CONFIG_X86_64
+ /*
+ * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
+ * creating the synthetic interrupt stack frame for the IRQ/NMI.
+ */
+ and $-16, %rsp
+ push $__KERNEL_DS
+ push %rbp
+#endif
+ pushf
+ push $__KERNEL_CS
+ \call_insn \call_target
+
+ /*
+ * "Restore" RSP from RBP, even though IRET has already unwound RSP to
+ * the correct value. objtool doesn't know the callee will IRET and,
+ * without the explicit restore, thinks the stack is getting walloped.
+ * Using an unwind hint is problematic due to x86-64's dynamic alignment.
+ */
+ leave
+ RET
+.endm
+
+#ifndef CONFIG_X86_64
+.pushsection .text, "ax"
+SYM_FUNC_START(idt_do_interrupt_irqoff)
+ IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
+SYM_FUNC_END(idt_do_interrupt_irqoff)
+.popsection
+#endif
+
+.pushsection .noinstr.text, "ax"
+SYM_FUNC_START(idt_do_nmi_irqoff)
+ IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
+SYM_FUNC_END(idt_do_nmi_irqoff)
+.popsection
+#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 42447b1..c6d9965 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -492,7 +492,7 @@
movq %rsp, %rdi /* pt_regs pointer */
- call kernel_\cfunc
+ call \cfunc
/*
* No need to switch back to the IST stack. The current stack is either
@@ -503,7 +503,7 @@
/* Switch to the regular task stack */
.Lfrom_usermode_switch_stack_\@:
- idtentry_body user_\cfunc, has_error_code=1
+ idtentry_body \cfunc, has_error_code=1
_ASM_NOKPROBE(\asmsym)
SYM_CODE_END(\asmsym)
diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S
index 894f7f1..0d2768a 100644
--- a/arch/x86/entry/entry_64_fred.S
+++ b/arch/x86/entry/entry_64_fred.S
@@ -147,5 +147,4 @@
RET
SYM_FUNC_END(asm_fred_entry_from_kvm)
-EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
#endif
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index fbe2d10..fb3594dd 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -177,16 +177,6 @@ static noinstr void fred_extint(struct pt_regs *regs)
}
}
-#ifdef CONFIG_AMD_MEM_ENCRYPT
-noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
-{
- if (user_mode(regs))
- return user_exc_vmm_communication(regs, error_code);
- else
- return kernel_exc_vmm_communication(regs, error_code);
-}
-#endif
-
static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
{
/* Optimize for #PF. That's the only exception which matters performance wise */
diff --git a/arch/x86/entry/vdso/common/vfutex.c b/arch/x86/entry/vdso/common/vfutex.c
new file mode 100644
index 0000000..454f059
--- /dev/null
+++ b/arch/x86/entry/vdso/common/vfutex.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <vdso/futex.h>
+
+/*
+ * Assembly template for the try unlock functions. The basic functionality is:
+ *
+ * mov esi, %eax Move the TID into EAX
+ * xor %ecx, %ecx Clear ECX
+ * lock_cmpxchgl %ecx, (%rdi) Attempt the TID -> 0 transition
+ * .Lcs_start: Start of the critical section
+ * jnz .Lcs_end If cmpxchl failed jump to the end
+ * .Lcs_success: Start of the success section
+ * movq %rcx, (%rdx) Set the pending op pointer to 0
+ * .Lcs_end: End of the critical section
+ *
+ * .Lcs_start and .Lcs_end establish the critical section range. .Lcs_success is
+ * technically not required, but there for illustration, debugging and testing.
+ *
+ * When CONFIG_COMPAT is enabled then the 64-bit VDSO provides two functions.
+ * One for the regular 64-bit sized pending operation pointer and one for a
+ * 32-bit sized pointer to support gaming emulators.
+ *
+ * The 32-bit VDSO provides only the one for 32-bit sized pointers.
+ */
+#define __stringify_1(x...) #x
+#define __stringify(x...) __stringify_1(x)
+
+#define LABEL(prefix, which) __stringify(prefix##_try_unlock_cs_##which:)
+
+#define JNZ_END(prefix) "jnz " __stringify(prefix) "_try_unlock_cs_end\n"
+
+#define CLEAR_POPQ "movq %[zero], %a[pop]\n"
+#define CLEAR_POPL "movl %k[zero], %a[pop]\n"
+
+#define futex_robust_try_unlock(prefix, clear_pop, __lock, __tid, __pop)\
+({ \
+ asm volatile ( \
+ " \n" \
+ " lock cmpxchgl %k[zero], %a[lock] \n" \
+ " \n" \
+ LABEL(prefix, start) \
+ " \n" \
+ JNZ_END(prefix) \
+ " \n" \
+ LABEL(prefix, success) \
+ " \n" \
+ clear_pop \
+ " \n" \
+ LABEL(prefix, end) \
+ : [tid] "+&a" (__tid) \
+ : [lock] "D" (__lock), \
+ [pop] "d" (__pop), \
+ [zero] "r" (0UL) \
+ : "memory" \
+ ); \
+ __tid; \
+})
+
+#ifdef __x86_64__
+__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop)
+{
+ return futex_robust_try_unlock(__futex_list64, CLEAR_POPQ, lock, tid, pop);
+}
+#endif /* __x86_64__ */
+
+#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
+__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop)
+{
+ return futex_robust_try_unlock(__futex_list32, CLEAR_POPL, lock, tid, pop);
+}
+#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
diff --git a/arch/x86/entry/vdso/vdso32/Makefile b/arch/x86/entry/vdso/vdso32/Makefile
index ded4fc6..ab4b1f6 100644
--- a/arch/x86/entry/vdso/vdso32/Makefile
+++ b/arch/x86/entry/vdso/vdso32/Makefile
@@ -7,8 +7,9 @@
vdsos-y := 32
# Files to link into the vDSO:
-vobjs-y := note.o vclock_gettime.o vgetcpu.o
-vobjs-y += system_call.o sigreturn.o
+vobjs-y := note.o vclock_gettime.o vgetcpu.o
+vobjs-y += system_call.o sigreturn.o
+vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO32 -m32 -mregparm=0
diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
index 55554f8..cee8f7f 100644
--- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S
+++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
@@ -30,6 +30,9 @@
__vdso_clock_gettime64;
__vdso_clock_getres_time64;
__vdso_getcpu;
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list32_try_unlock;
+#endif
};
LINUX_2.5 {
diff --git a/arch/x86/entry/vdso/vdso32/vfutex.c b/arch/x86/entry/vdso/vdso32/vfutex.c
new file mode 100644
index 0000000..940a6ee
--- /dev/null
+++ b/arch/x86/entry/vdso/vdso32/vfutex.c
@@ -0,0 +1 @@
+#include "common/vfutex.c"
diff --git a/arch/x86/entry/vdso/vdso64/Makefile b/arch/x86/entry/vdso/vdso64/Makefile
index bfffaf1..7c07900 100644
--- a/arch/x86/entry/vdso/vdso64/Makefile
+++ b/arch/x86/entry/vdso/vdso64/Makefile
@@ -8,9 +8,10 @@
vdsos-$(CONFIG_X86_X32_ABI) += x32
# Files to link into the vDSO:
-vobjs-y := note.o vclock_gettime.o vgetcpu.o
-vobjs-y += vgetrandom.o vgetrandom-chacha.o
-vobjs-$(CONFIG_X86_SGX) += vsgx.o
+vobjs-y := note.o vclock_gettime.o vgetcpu.o
+vobjs-y += vgetrandom.o vgetrandom-chacha.o
+vobjs-$(CONFIG_X86_SGX) += vsgx.o
+vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small
diff --git a/arch/x86/entry/vdso/vdso64/vdso64.lds.S b/arch/x86/entry/vdso/vdso64/vdso64.lds.S
index 5ce3f2b..4a72122 100644
--- a/arch/x86/entry/vdso/vdso64/vdso64.lds.S
+++ b/arch/x86/entry/vdso/vdso64/vdso64.lds.S
@@ -32,6 +32,13 @@
#endif
getrandom;
__vdso_getrandom;
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list64_try_unlock;
+#ifdef CONFIG_COMPAT
+ __vdso_futex_robust_list32_try_unlock;
+#endif
+#endif
local: *;
};
}
diff --git a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
index 3dbd20c..b917dc6 100644
--- a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
+++ b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
@@ -22,6 +22,13 @@
__vdso_getcpu;
__vdso_time;
__vdso_clock_getres;
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list64_try_unlock;
+#ifdef CONFIG_COMPAT
+ __vdso_futex_robust_list32_try_unlock;
+#endif
+#endif
local: *;
};
}
diff --git a/arch/x86/entry/vdso/vdso64/vfutex.c b/arch/x86/entry/vdso/vdso64/vfutex.c
new file mode 100644
index 0000000..940a6ee
--- /dev/null
+++ b/arch/x86/entry/vdso/vdso64/vfutex.c
@@ -0,0 +1 @@
+#include "common/vfutex.c"
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index a6bfcc8..18dfd80 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -6,6 +6,7 @@
*/
#include <linux/mm.h>
#include <linux/err.h>
+#include <linux/futex.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/slab.h>
@@ -73,6 +74,31 @@ static void vdso_fix_landing(const struct vdso_image *image,
regs->ip = new_vma->vm_start + ipoffset;
}
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+static void vdso_futex_robust_unlock_update_ips(void)
+{
+ const struct vdso_image *image = current->mm->context.vdso_image;
+ unsigned long vdso = (unsigned long) current->mm->context.vdso;
+ struct futex_mm_data *fd = ¤t->mm->futex;
+ unsigned int idx = 0;
+
+ futex_reset_cs_ranges(fd);
+
+#ifdef CONFIG_X86_64
+ futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list64_try_unlock_cs_start,
+ vdso + image->sym___futex_list64_try_unlock_cs_end, false);
+ idx++;
+#endif /* CONFIG_X86_64 */
+
+#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
+ futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list32_try_unlock_cs_start,
+ vdso + image->sym___futex_list32_try_unlock_cs_end, true);
+#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
+}
+#else
+static inline void vdso_futex_robust_unlock_update_ips(void) { }
+#endif
+
static int vdso_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
@@ -80,6 +106,7 @@ static int vdso_mremap(const struct vm_special_mapping *sm,
vdso_fix_landing(image, new_vma);
current->mm->context.vdso = (void __user *)new_vma->vm_start;
+ vdso_futex_robust_unlock_update_ips();
return 0;
}
@@ -178,13 +205,15 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
do_munmap(mm, text_start, image->size, NULL);
- do_munmap(mm, addr, image->size, NULL);
+ do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
goto up_fail;
}
current->mm->context.vdso = (void __user *)text_start;
current->mm->context.vdso_image = image;
+ vdso_futex_robust_unlock_update_ips();
+
up_fail:
mmap_write_unlock(mm);
return ret;
diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c
index 0c92ed5..6a4dfc9 100644
--- a/arch/x86/events/amd/core.c
+++ b/arch/x86/events/amd/core.c
@@ -8,8 +8,10 @@
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/jiffies.h>
+
#include <asm/apicdef.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/nmi.h>
@@ -1032,7 +1034,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
* Unmasking the LVTPC is not required as the Mask (M) bit of the LVT
* PMI entry is not set by the local APIC when a PMC overflow occurs
*/
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
done:
cpuc->enabled = pmu_enabled;
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index e0bd505..3531f9c 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -15,6 +15,7 @@
#include <linux/sched/clock.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include "../perf_event.h"
@@ -1600,7 +1601,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
if (handled)
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
perf_sample_event_took(sched_clock() - stamp);
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index d24da37..5b437dc 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/perf_event.h>
+
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/perf_event.h>
diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c
index dad4279..744dffa 100644
--- a/arch/x86/events/amd/power.c
+++ b/arch/x86/events/amd/power.c
@@ -10,8 +10,11 @@
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
+
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
+
#include "../perf_event.h"
/* Event code: LSB 8 bits, passed in attr->config any other bit is reserved. */
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index dd956cf..05cff39 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -16,6 +16,7 @@
#include <linux/smp.h>
#include <asm/perf_event.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#define NUM_COUNTERS_NB 4
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 810ab21..d1af33d 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -1294,13 +1294,16 @@ int x86_perf_rdpmc_index(struct perf_event *event)
return event->hw.event_base_rdpmc;
}
-static inline int match_prev_assignment(struct hw_perf_event *hwc,
+static inline int match_prev_assignment(struct perf_event *event,
struct cpu_hw_events *cpuc,
int i)
{
+ struct hw_perf_event *hwc = &event->hw;
+
return hwc->idx == cpuc->assign[i] &&
- hwc->last_cpu == smp_processor_id() &&
- hwc->last_tag == cpuc->tags[i];
+ hwc->last_cpu == smp_processor_id() &&
+ hwc->last_tag == cpuc->tags[i] &&
+ !is_acr_event_group(event);
}
static void x86_pmu_start(struct perf_event *event, int flags);
@@ -1346,7 +1349,7 @@ static void x86_pmu_enable(struct pmu *pmu)
* - no other event has used the counter since
*/
if (hwc->idx == -1 ||
- match_prev_assignment(hwc, cpuc, i))
+ match_prev_assignment(event, cpuc, i))
continue;
/*
@@ -1367,7 +1370,7 @@ static void x86_pmu_enable(struct pmu *pmu)
event = cpuc->event_list[i];
hwc = &event->hw;
- if (!match_prev_assignment(hwc, cpuc, i))
+ if (!match_prev_assignment(event, cpuc, i))
x86_assign_hw_event(event, cpuc, i);
else if (i < n_running)
continue;
@@ -1750,7 +1753,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
}
if (handled)
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
return handled;
}
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index d9488ad..9a46351 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -17,6 +17,7 @@
#include <linux/kvm_host.h>
#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
#include <asm/debugreg.h>
#include <asm/hardirq.h>
#include <asm/intel-family.h>
@@ -215,39 +216,47 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly =
static struct event_constraint intel_grt_event_constraints[] __read_mostly = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_skt_event_constraints[] __read_mostly = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */
FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */
FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_arw_event_constraints[] __read_mostly = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */
FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */
FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */
+ FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */
INTEL_UEVENT_CONSTRAINT(0x01b7, 0x1),
INTEL_UEVENT_CONSTRAINT(0x02b7, 0x2),
INTEL_UEVENT_CONSTRAINT(0x04b7, 0x4),
INTEL_UEVENT_CONSTRAINT(0x08b7, 0x8),
- INTEL_UEVENT_CONSTRAINT(0x01d4, 0x1),
- INTEL_UEVENT_CONSTRAINT(0x02d4, 0x2),
- INTEL_UEVENT_CONSTRAINT(0x04d4, 0x4),
- INTEL_UEVENT_CONSTRAINT(0x08d4, 0x8),
INTEL_UEVENT_CONSTRAINT(0x0175, 0x1),
INTEL_UEVENT_CONSTRAINT(0x0275, 0x2),
INTEL_UEVENT_CONSTRAINT(0x21d3, 0x1),
@@ -310,10 +319,11 @@ static struct extra_reg intel_skl_extra_regs[] __read_mostly = {
static struct event_constraint intel_icl_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* old INST_RETIRED.PREC_DIST */
- FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
- FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -355,11 +365,12 @@ static struct extra_reg intel_glc_extra_regs[] __read_mostly = {
static struct event_constraint intel_glc_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
- FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
- FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -379,9 +390,13 @@ static struct event_constraint intel_glc_event_constraints[] = {
INTEL_UEVENT_CONSTRAINT(0x01a3, 0xf),
INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf),
+ INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf),
+ INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf),
INTEL_UEVENT_CONSTRAINT(0x08a3, 0xf),
+ INTEL_UEVENT_CONSTRAINT(0x0ca3, 0xf),
INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1),
INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1),
+ INTEL_UEVENT_CONSTRAINT(0x01cd, 0xfe),
INTEL_UEVENT_CONSTRAINT(0x02cd, 0x1),
INTEL_EVENT_CONSTRAINT(0xce, 0x1),
INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf),
@@ -407,11 +422,12 @@ static struct extra_reg intel_rwc_extra_regs[] __read_mostly = {
static struct event_constraint intel_lnc_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
- FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
- FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -423,8 +439,6 @@ static struct event_constraint intel_lnc_event_constraints[] = {
INTEL_EVENT_CONSTRAINT(0x20, 0xf),
- INTEL_UEVENT_CONSTRAINT(0x012a, 0xf),
- INTEL_UEVENT_CONSTRAINT(0x012b, 0xf),
INTEL_UEVENT_CONSTRAINT(0x0148, 0x4),
INTEL_UEVENT_CONSTRAINT(0x0175, 0x4),
@@ -435,15 +449,14 @@ static struct event_constraint intel_lnc_event_constraints[] = {
INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4),
INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1),
INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1),
- INTEL_UEVENT_CONSTRAINT(0x10a4, 0x1),
+ INTEL_UEVENT_CONSTRAINT(0x10a4, 0x8),
INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8),
INTEL_UEVENT_CONSTRAINT(0x01cd, 0x3fc),
INTEL_UEVENT_CONSTRAINT(0x02cd, 0x3),
+ INTEL_UEVENT_CONSTRAINT(0x87d0, 0x3ff),
INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf),
- INTEL_UEVENT_CONSTRAINT(0x00e0, 0xf),
-
EVENT_CONSTRAINT_END
};
@@ -460,11 +473,12 @@ static struct extra_reg intel_lnc_extra_regs[] __read_mostly = {
static struct event_constraint intel_pnc_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
- FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
- FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
+ FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
- FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -706,6 +720,80 @@ static __initconst const u64 glc_hw_cache_event_ids
},
};
+/* ADL P-core (Golden cove) specific event code. */
+static __initconst const u64 adl_glc_hw_cache_event_ids
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x81d0,
+ [ C(RESULT_MISS) ] = 0xe124,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x82d0,
+ },
+ },
+ [ C(L1I ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_MISS) ] = 0xe424,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = -1,
+ },
+ },
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x12a,
+ [ C(RESULT_MISS) ] = 0x12a,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x12a,
+ [ C(RESULT_MISS) ] = 0x12a,
+ },
+ },
+ [ C(DTLB) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x81d0,
+ [ C(RESULT_MISS) ] = 0xe12,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x82d0,
+ [ C(RESULT_MISS) ] = 0xe13,
+ },
+ },
+ [ C(ITLB) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = 0xe11,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = -1,
+ },
+ [ C(OP_PREFETCH) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = -1,
+ },
+ },
+ [ C(BPU ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x4c4,
+ [ C(RESULT_MISS) ] = 0x4c5,
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = -1,
+ },
+ [ C(OP_PREFETCH) ] = {
+ [ C(RESULT_ACCESS) ] = -1,
+ [ C(RESULT_MISS) ] = -1,
+ },
+ },
+};
+
static __initconst const u64 glc_hw_cache_extra_regs
[PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX]
@@ -713,18 +801,71 @@ static __initconst const u64 glc_hw_cache_extra_regs
{
[ C(LL ) ] = {
[ C(OP_READ) ] = {
- [ C(RESULT_ACCESS) ] = 0x10001,
- [ C(RESULT_MISS) ] = 0x3fbfc00001,
+ [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
},
[ C(OP_WRITE) ] = {
- [ C(RESULT_ACCESS) ] = 0x3f3ffc0002,
- [ C(RESULT_MISS) ] = 0x3f3fc00002,
+ [ C(RESULT_ACCESS) ] = 0x3f3ffc0002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3f3fc00002, /* OCR.DEMAND_RFO.L3_MISS */
},
},
[ C(NODE) ] = {
[ C(OP_READ) ] = {
- [ C(RESULT_ACCESS) ] = 0x10c000001,
- [ C(RESULT_MISS) ] = 0x3fb3000001,
+ [ C(RESULT_ACCESS) ] = 0x104000001, /* OCR.DEMAND_DATA_RD.LOCAL_DRAM */
+ [ C(RESULT_MISS) ] = 0x730000001, /* OCR.DEMAND_DATA_RD.REMOTE_DRAM */
+ },
+ },
+};
+
+/* ADL P-core (Golden cove) specific extra regs value. */
+static __initconst const u64 adl_glc_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3fbfc00002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+static __initconst const u64 lnc_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x9E7FA000001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x9E7FA000002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+/* ARL specific lioncove hw_cache_extra_regs[] variant. */
+static __initconst const u64 arl_lnc_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFE7F8000001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFE7F8000002, /* OCR.DEMAND_RFO.L3_MISS */
},
},
};
@@ -815,12 +956,29 @@ static __initconst const u64 pnc_hw_cache_extra_regs
{
[ C(LL ) ] = {
[ C(OP_READ) ] = {
- [ C(RESULT_ACCESS) ] = 0x4000000000000001,
- [ C(RESULT_MISS) ] = 0xFFFFF000000001,
+ [ C(RESULT_ACCESS) ] = 0x4000000000000001, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFFFFF000000001, /* OMR.DEMAND_DATA_RD.L3_MISS */
},
[ C(OP_WRITE) ] = {
- [ C(RESULT_ACCESS) ] = 0x4000000000000002,
- [ C(RESULT_MISS) ] = 0xFFFFF000000002,
+ [ C(RESULT_ACCESS) ] = 0x4000000000000002, /* OMR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFFFFF000000002, /* OMR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+static __initconst const u64 cyc_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x4000000000000001, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFF03F000000001, /* OMR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x4000000000000002, /* OMR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0xFF03F000000002, /* OMR.DEMAND_RFO.L3_MISS */
},
},
};
@@ -1019,6 +1177,41 @@ static __initconst const u64 skl_hw_cache_extra_regs
},
};
+static __initconst const u64 snc_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL ) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x3F3FFC0002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [ C(RESULT_MISS) ] = 0x3F3FC00002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ [ C(OP_PREFETCH) ] = {
+ [ C(RESULT_ACCESS) ] = 0x0,
+ [ C(RESULT_MISS) ] = 0x0,
+ },
+ },
+ [ C(NODE) ] = {
+ [ C(OP_READ) ] = {
+ [ C(RESULT_ACCESS) ] = 0x104000001, /* OCR.DEMAND_DATA_RD.LOCAL_DRAM */
+ [ C(RESULT_MISS) ] = 0x730000001, /* OCR.DEMAND_DATA_RD.REMOTE_DRAM */
+ },
+ [ C(OP_WRITE) ] = {
+ [ C(RESULT_ACCESS) ] = 0x104000002, /* OCR.DEMAND_RFO.LOCAL_DRAM */
+ [ C(RESULT_MISS) ] = 0x730000002, /* OCR.DEMAND_RFO.REMOTE_DRAM */
+ },
+ [ C(OP_PREFETCH) ] = {
+ [ C(RESULT_ACCESS) ] = 0x0,
+ [ C(RESULT_MISS) ] = 0x0,
+ },
+ },
+};
+
#define SNB_DMND_DATA_RD (1ULL << 0)
#define SNB_DMND_RFO (1ULL << 1)
#define SNB_DMND_IFETCH (1ULL << 2)
@@ -2342,22 +2535,82 @@ static __initconst const u64 tnt_hw_cache_extra_regs
},
};
+static __initconst const u64 grt_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x3F84400001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x3F84400002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+static __initconst const u64 cmt_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x3fbfc00002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+static __initconst const u64 skt_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x13FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x13FBFC00002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
+static __initconst const u64 dkt_hw_cache_extra_regs
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+ [C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x33FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0x33FBFC00002, /* OCR.DEMAND_RFO.L3_MISS */
+ },
+ },
+};
+
static __initconst const u64 arw_hw_cache_extra_regs
[PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX]
[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
[C(LL)] = {
[C(OP_READ)] = {
- [C(RESULT_ACCESS)] = 0x4000000000000001,
- [C(RESULT_MISS)] = 0xFFFFF000000001,
+ [C(RESULT_ACCESS)] = 0x4000000000000009, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0xFF03F000000009, /* OMR.DEMAND_DATA_RD.L3_MISS */
},
[C(OP_WRITE)] = {
- [C(RESULT_ACCESS)] = 0x4000000000000002,
- [C(RESULT_MISS)] = 0xFFFFF000000002,
- },
- [C(OP_PREFETCH)] = {
- [C(RESULT_ACCESS)] = 0x0,
- [C(RESULT_MISS)] = 0x0,
+ [C(RESULT_ACCESS)] = 0x400000000000000A, /* OMR.DEMAND_RFO.ANY_RESPONSE */
+ [C(RESULT_MISS)] = 0xFF03F00000000A, /* OMR.DEMAND_RFO.L3_MISS */
},
},
};
@@ -2392,9 +2645,12 @@ static struct attribute *grt_mem_attrs[] = {
};
static struct extra_reg intel_grt_extra_regs[] __read_mostly = {
- /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
- INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
- INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
+ /*
+ * Must define OFFCORE_RSP_X first, see intel_fixup_er().
+ * Bit 63 only valid on OFFCORE_RSP_0 MSR.
+ */
+ INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x8003f03fffffffffull, RSP_0),
+ INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3f03fffffffffull, RSP_1),
INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0),
EVENT_EXTRA_END
};
@@ -2426,10 +2682,6 @@ static struct extra_reg intel_arw_extra_regs[] __read_mostly = {
INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1),
INTEL_UEVENT_EXTRA_REG(0x04b7, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2),
INTEL_UEVENT_EXTRA_REG(0x08b7, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3),
- INTEL_UEVENT_EXTRA_REG(0x01d4, MSR_OMR_0, 0xc0ffffffffffffffull, OMR_0),
- INTEL_UEVENT_EXTRA_REG(0x02d4, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1),
- INTEL_UEVENT_EXTRA_REG(0x04d4, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2),
- INTEL_UEVENT_EXTRA_REG(0x08d4, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3),
INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0),
INTEL_UEVENT_EXTRA_REG(0x0127, MSR_SNOOP_RSP_0, 0xffffffffffffffffull, SNOOP_0),
INTEL_UEVENT_EXTRA_REG(0x0227, MSR_SNOOP_RSP_1, 0xffffffffffffffffull, SNOOP_1),
@@ -3118,11 +3370,11 @@ static void intel_pmu_enable_fixed(struct perf_event *event)
intel_set_masks(event, idx);
/*
- * Enable IRQ generation (0x8), if not PEBS,
- * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
- * if requested:
+ * Enable IRQ generation (0x8), if not PEBS or self-reloaded
+ * ACR event, and enable ring-3 counting (0x2) and ring-0
+ * counting (0x1) if requested:
*/
- if (!event->attr.precise_ip)
+ if (!event->attr.precise_ip && !is_acr_self_reload_event(event))
bits |= INTEL_FIXED_0_ENABLE_PMI;
if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
bits |= INTEL_FIXED_0_USER;
@@ -3169,10 +3421,10 @@ static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)
wrmsrl(msr_b + msr_offset, mask);
cpuc->acr_cfg_b[idx] = mask;
}
- /* Only need to update the reload value when there is a valid config value. */
- if (mask && cpuc->acr_cfg_c[idx] != reload) {
+ /* Only update CFG_C reload when ACR is actively enabled (mask != 0) */
+ if (mask && ((cpuc->cfg_c_val[idx] & ARCH_PEBS_RELOAD) != reload)) {
wrmsrl(msr_c + msr_offset, reload);
- cpuc->acr_cfg_c[idx] = reload;
+ cpuc->cfg_c_val[idx] = reload;
}
}
@@ -3198,14 +3450,15 @@ static void intel_pmu_enable_event_ext(struct perf_event *event)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
- union arch_pebs_index old, new;
- struct arch_pebs_cap cap;
u64 ext = 0;
- cap = hybrid(cpuc->pmu, arch_pebs_cap);
+ if (is_acr_event_group(event))
+ ext |= (-hwc->sample_period) & ARCH_PEBS_RELOAD;
if (event->attr.precise_ip) {
u64 pebs_data_cfg = intel_get_arch_pebs_data_config(event);
+ struct arch_pebs_cap cap = hybrid(cpuc->pmu, arch_pebs_cap);
+ union arch_pebs_index old, new;
ext |= ARCH_PEBS_EN;
if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD)
@@ -3306,6 +3559,15 @@ static void intel_pmu_enable_event(struct perf_event *event)
intel_set_masks(event, idx);
static_call_cond(intel_pmu_enable_acr_event)(event);
static_call_cond(intel_pmu_enable_event_ext)(event);
+ /*
+ * For self-reloaded ACR event, don't enable PMI since
+ * HW won't set overflow bit in GLOBAL_STATUS. Otherwise,
+ * the PMI would be recognized as a suspicious NMI.
+ */
+ if (is_acr_self_reload_event(event))
+ hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
+ else if (!event->attr.precise_ip)
+ hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
__x86_pmu_enable_event(hwc, enable_mask);
break;
case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
@@ -3332,23 +3594,41 @@ static void intel_pmu_enable_event(struct perf_event *event)
static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc)
{
struct perf_event *event, *leader;
- int i, j, idx;
+ int i, j, k, bit, idx;
+ /*
+ * FIXME: ACR mask parsing relies on cpuc->event_list[] (active events only).
+ * Disabling an ACR event causes bit-shifting errors in the acr_mask of
+ * remaining group members. As ACR sampling requires all events to be active,
+ * this limitation is acceptable for now. Revisit if independent event toggling
+ * is required.
+ */
for (i = 0; i < cpuc->n_events; i++) {
leader = cpuc->event_list[i];
if (!is_acr_event_group(leader))
continue;
- /* The ACR events must be contiguous. */
+ /* Find the last event of the ACR group. */
for (j = i; j < cpuc->n_events; j++) {
event = cpuc->event_list[j];
if (event->group_leader != leader->group_leader)
break;
- for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
- if (i + idx >= cpuc->n_events ||
- !is_acr_event_group(cpuc->event_list[i + idx]))
- return;
- __set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1);
+ }
+
+ /*
+ * Translate the user-space ACR mask (attr.config2) into the physical
+ * counter bitmask (hw.config1) for each ACR event in the group.
+ * NOTE: ACR event contiguity is guaranteed by intel_pmu_hw_config().
+ */
+ for (k = i; k < j; k++) {
+ event = cpuc->event_list[k];
+ event->hw.config1 = 0;
+ for_each_set_bit(bit, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {
+ idx = i + bit;
+ /* Event index of ACR group must locate in [i, j). */
+ if (idx >= j || !is_acr_event_group(cpuc->event_list[idx]))
+ continue;
+ __set_bit(cpuc->assign[idx], (unsigned long *)&event->hw.config1);
}
}
i = j - 1;
@@ -3504,7 +3784,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
int bit;
int handled = 0;
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
/*
* Ignore a range of extra bits in status that do not indicate
@@ -7463,6 +7743,15 @@ static __always_inline void intel_pmu_init_glc(struct pmu *pmu)
intel_pmu_ref_cycles_ext();
}
+static __always_inline void intel_pmu_init_glc_hybrid(struct pmu *pmu)
+{
+ intel_pmu_init_glc(pmu);
+
+ /* ADL has different extra MSR values from Server for the L3 or node OCR/OMR events. */
+ memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids));
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs), adl_glc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+}
+
static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
{
x86_pmu.mid_ack = true;
@@ -7475,7 +7764,7 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
memcpy(hybrid_var(pmu, hw_cache_event_ids), glp_hw_cache_event_ids, sizeof(hw_cache_event_ids));
- memcpy(hybrid_var(pmu, hw_cache_extra_regs), tnt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs), grt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
hybrid_var(pmu, hw_cache_event_ids)[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
hybrid(pmu, event_constraints) = intel_grt_event_constraints;
hybrid(pmu, pebs_constraints) = intel_grt_pebs_event_constraints;
@@ -7484,12 +7773,24 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
intel_pmu_ref_cycles_ext();
}
+static __always_inline void intel_pmu_init_cmt(struct pmu *pmu)
+{
+ intel_pmu_init_grt(pmu);
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+ cmt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+ hybrid(pmu, pebs_constraints) = intel_cmt_pebs_event_constraints;
+ hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
+}
+
static __always_inline void intel_pmu_init_lnc(struct pmu *pmu)
{
intel_pmu_init_glc(pmu);
hybrid(pmu, event_constraints) = intel_lnc_event_constraints;
hybrid(pmu, pebs_constraints) = intel_lnc_pebs_event_constraints;
hybrid(pmu, extra_regs) = intel_lnc_extra_regs;
+
+ memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids));
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs), lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
}
static __always_inline void intel_pmu_init_pnc(struct pmu *pmu)
@@ -7504,16 +7805,44 @@ static __always_inline void intel_pmu_init_pnc(struct pmu *pmu)
hybrid(pmu, event_constraints) = intel_pnc_event_constraints;
hybrid(pmu, pebs_constraints) = intel_pnc_pebs_event_constraints;
hybrid(pmu, extra_regs) = intel_pnc_extra_regs;
+ static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
+}
+
+static __always_inline void intel_pmu_init_cyc(struct pmu *pmu)
+{
+ intel_pmu_init_pnc(pmu);
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+ cyc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
}
static __always_inline void intel_pmu_init_skt(struct pmu *pmu)
{
- intel_pmu_init_grt(pmu);
+ intel_pmu_init_cmt(pmu);
hybrid(pmu, event_constraints) = intel_skt_event_constraints;
- hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+ skt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
}
+/* Hybrid client variant. */
+static __always_inline void intel_pmu_init_dkt_hybrid(struct pmu *pmu)
+{
+ intel_pmu_init_skt(pmu);
+ hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints;
+}
+
+/*
+ * Darkmont is used by the CWF and PTL E-cores, but their L3 OCR
+ * events require different extra MSR values. Keep a separate init
+ * function for the non-hybrid server variant.
+ */
+static __always_inline void intel_pmu_init_dkt(struct pmu *pmu)
+{
+ intel_pmu_init_dkt_hybrid(pmu);
+ memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+ dkt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+}
+
static __always_inline void intel_pmu_init_arw(struct pmu *pmu)
{
intel_pmu_init_grt(pmu);
@@ -7522,7 +7851,7 @@ static __always_inline void intel_pmu_init_arw(struct pmu *pmu)
memcpy(hybrid_var(pmu, hw_cache_extra_regs),
arw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
hybrid(pmu, event_constraints) = intel_arw_event_constraints;
- hybrid(pmu, pebs_constraints) = intel_arw_pebs_event_constraints;
+ hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints;
hybrid(pmu, extra_regs) = intel_arw_extra_regs;
static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
}
@@ -7836,8 +8165,7 @@ __init int intel_pmu_init(void)
case INTEL_ATOM_CRESTMONT:
case INTEL_ATOM_CRESTMONT_X:
- intel_pmu_init_grt(NULL);
- x86_pmu.extra_regs = intel_cmt_extra_regs;
+ intel_pmu_init_cmt(NULL);
intel_pmu_pebs_data_source_cmt();
x86_pmu.pebs_latency_data = cmt_latency_data;
x86_pmu.get_event_constraints = cmt_get_event_constraints;
@@ -7849,7 +8177,7 @@ __init int intel_pmu_init(void)
break;
case INTEL_ATOM_DARKMONT_X:
- intel_pmu_init_skt(NULL);
+ intel_pmu_init_dkt(NULL);
intel_pmu_pebs_data_source_cmt();
x86_pmu.pebs_latency_data = cmt_latency_data;
x86_pmu.get_event_constraints = cmt_get_event_constraints;
@@ -8125,17 +8453,19 @@ __init int intel_pmu_init(void)
case INTEL_ICELAKE_X:
case INTEL_ICELAKE_D:
+ memcpy(hw_cache_extra_regs, snc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
x86_pmu.pebs_ept = 1;
pmem = true;
- fallthrough;
+ goto snc_common;
case INTEL_ICELAKE_L:
case INTEL_ICELAKE:
case INTEL_TIGERLAKE_L:
case INTEL_TIGERLAKE:
case INTEL_ROCKETLAKE:
+ memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+ snc_common:
x86_pmu.late_ack = true;
memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
- memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
intel_pmu_lbr_init_skl();
@@ -8231,7 +8561,7 @@ __init int intel_pmu_init(void)
/* Initialize big core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
- intel_pmu_init_glc(&pmu->pmu);
+ intel_pmu_init_glc_hybrid(&pmu->pmu);
if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) {
pmu->cntr_mask64 <<= 2;
pmu->cntr_mask64 |= 0x3;
@@ -8288,13 +8618,12 @@ __init int intel_pmu_init(void)
/* Initialize big core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
- intel_pmu_init_glc(&pmu->pmu);
+ intel_pmu_init_glc_hybrid(&pmu->pmu);
pmu->extra_regs = intel_rwc_extra_regs;
/* Initialize Atom core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
- intel_pmu_init_grt(&pmu->pmu);
- pmu->extra_regs = intel_cmt_extra_regs;
+ intel_pmu_init_cmt(&pmu->pmu);
intel_pmu_pebs_data_source_mtl();
pr_cont("Meteorlake Hybrid events, ");
@@ -8305,16 +8634,50 @@ __init int intel_pmu_init(void)
case INTEL_WILDCATLAKE_L:
pr_cont("Pantherlake Hybrid events, ");
name = "pantherlake_hybrid";
+
+ intel_pmu_init_hybrid(hybrid_big_small);
+
+ /* Initialize big core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+ intel_pmu_init_lnc(&pmu->pmu);
+ /* Initialize Atom core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+ intel_pmu_init_dkt_hybrid(&pmu->pmu);
+
+ goto lnl_common;
+
+ case INTEL_ARROWLAKE:
+ pr_cont("Arrowlake Hybrid events, ");
+ name = "arrowlake_hybrid";
+
+ intel_pmu_init_hybrid(hybrid_big_small);
+
+ /* Initialize big core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+ intel_pmu_init_lnc(&pmu->pmu);
+ memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs),
+ arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+ /* Initialize Atom core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+ intel_pmu_init_skt(&pmu->pmu);
+
goto lnl_common;
case INTEL_LUNARLAKE_M:
- case INTEL_ARROWLAKE:
pr_cont("Lunarlake Hybrid events, ");
name = "lunarlake_hybrid";
- lnl_common:
intel_pmu_init_hybrid(hybrid_big_small);
+ /* Initialize big core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+ intel_pmu_init_lnc(&pmu->pmu);
+ /* Initialize Atom core specific PerfMon capabilities.*/
+ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+ intel_pmu_init_skt(&pmu->pmu);
+
+ lnl_common:
+
x86_pmu.pebs_latency_data = lnl_latency_data;
x86_pmu.get_event_constraints = mtl_get_event_constraints;
x86_pmu.hw_config = adl_hw_config;
@@ -8325,14 +8688,6 @@ __init int intel_pmu_init(void)
extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
- /* Initialize big core specific PerfMon capabilities.*/
- pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
- intel_pmu_init_lnc(&pmu->pmu);
-
- /* Initialize Atom core specific PerfMon capabilities.*/
- pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
- intel_pmu_init_skt(&pmu->pmu);
-
intel_pmu_pebs_data_source_lnl();
break;
@@ -8352,6 +8707,8 @@ __init int intel_pmu_init(void)
/* Initialize big core specific PerfMon capabilities. */
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
intel_pmu_init_lnc(&pmu->pmu);
+ memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs),
+ arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
/* Initialize Atom core specific PerfMon capabilities. */
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
@@ -8359,8 +8716,7 @@ __init int intel_pmu_init(void)
/* Initialize Lower Power Atom specific PerfMon capabilities. */
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX];
- intel_pmu_init_grt(&pmu->pmu);
- pmu->extra_regs = intel_cmt_extra_regs;
+ intel_pmu_init_cmt(&pmu->pmu);
intel_pmu_pebs_data_source_arl_h();
pr_cont("ArrowLake-H Hybrid events, ");
@@ -8385,7 +8741,7 @@ __init int intel_pmu_init(void)
/* Initialize big core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
- intel_pmu_init_pnc(&pmu->pmu);
+ intel_pmu_init_cyc(&pmu->pmu);
/* Initialize Atom core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 7f0d515..cb72af9 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -1292,18 +1292,21 @@ struct event_constraint intel_glm_pebs_event_constraints[] = {
struct event_constraint intel_grt_pebs_event_constraints[] = {
/* Allow all events as PEBS with no flags */
INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
- INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf),
+ INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f),
EVENT_CONSTRAINT_END
};
-struct event_constraint intel_arw_pebs_event_constraints[] = {
+struct event_constraint intel_cmt_pebs_event_constraints[] = {
+ /* Allow all events as PEBS with no flags */
+ INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
+ INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
+ EVENT_CONSTRAINT_END
+};
+
+struct event_constraint intel_dkt_pebs_event_constraints[] = {
/* Allow all events as PEBS with no flags */
INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff),
INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
- INTEL_FLAGS_UEVENT_CONSTRAINT(0x01d4, 0x1),
- INTEL_FLAGS_UEVENT_CONSTRAINT(0x02d4, 0x2),
- INTEL_FLAGS_UEVENT_CONSTRAINT(0x04d4, 0x4),
- INTEL_FLAGS_UEVENT_CONSTRAINT(0x08d4, 0x8),
EVENT_CONSTRAINT_END
};
@@ -1500,6 +1503,13 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */
+
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1), /* TOPDOWN.BAD_SPEC_SLOTS */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1), /* TOPDOWN.BR_MISPREDICT_SLOTS */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8), /* TOPDOWN.MEMORY_BOUND_SLOTS */
+
INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc),
INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
@@ -1509,6 +1519,7 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
+ INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff), /* MEM_INST_RETIRED.ANY */
INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c
index e614baf..e887adc 100644
--- a/arch/x86/events/intel/knc.c
+++ b/arch/x86/events/intel/knc.c
@@ -238,7 +238,7 @@ static int knc_pmu_handle_irq(struct pt_regs *regs)
goto done;
}
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
struct perf_event *event = cpuc->events[bit];
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index 72f2adc..cae2e02 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -4,6 +4,7 @@
#include <linux/types.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/perf_event.h>
#include <asm/msr.h>
diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c
index 02bfdb7..5368dc3 100644
--- a/arch/x86/events/intel/p4.c
+++ b/arch/x86/events/intel/p4.c
@@ -1077,7 +1077,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
}
if (handled)
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
/*
* When dealing with the unmasking of the LVTPC on P4 perf hw, it has
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index fad87d3..eae24bb 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -137,6 +137,16 @@ static inline bool is_acr_event_group(struct perf_event *event)
return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR);
}
+static inline bool is_acr_self_reload_event(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (hwc->idx < 0)
+ return false;
+
+ return test_bit(hwc->idx, (unsigned long *)&hwc->config1);
+}
+
struct amd_nb {
int nb_id; /* NorthBridge id */
int refcnt; /* reference count */
@@ -312,10 +322,8 @@ struct cpu_hw_events {
u64 fixed_ctrl_val;
u64 active_fixed_ctrl_val;
- /* Intel ACR configuration */
+ /* Intel ACR/arch-PEBS configuration */
u64 acr_cfg_b[X86_PMC_IDX_MAX];
- u64 acr_cfg_c[X86_PMC_IDX_MAX];
- /* Cached CFG_C values */
u64 cfg_c_val[X86_PMC_IDX_MAX];
/*
@@ -1702,7 +1710,9 @@ extern struct event_constraint intel_glp_pebs_event_constraints[];
extern struct event_constraint intel_grt_pebs_event_constraints[];
-extern struct event_constraint intel_arw_pebs_event_constraints[];
+extern struct event_constraint intel_cmt_pebs_event_constraints[];
+
+extern struct event_constraint intel_dkt_pebs_event_constraints[];
extern struct event_constraint intel_nehalem_pebs_event_constraints[];
diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c
index 4bdfcf0..e506f67 100644
--- a/arch/x86/events/zhaoxin/core.c
+++ b/arch/x86/events/zhaoxin/core.c
@@ -13,6 +13,7 @@
#include <linux/nmi.h>
#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
#include <asm/hardirq.h>
#include <asm/apic.h>
#include <asm/msr.h>
@@ -373,7 +374,7 @@ static int zhaoxin_pmu_handle_irq(struct pt_regs *regs)
else
zhaoxin_pmu_ack_status(status);
- inc_irq_stat(apic_perf_irqs);
+ inc_perf_irq_stat();
/*
* CondChgd bit 63 doesn't mean any overflow status. Ignore
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index 323adc9..55a8b6d 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -219,7 +219,7 @@ static inline bool hv_reenlightenment_available(void)
DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
{
apic_eoi();
- inc_irq_stat(irq_hv_reenlightenment_count);
+ inc_irq_stat(HYPERV_REENLIGHTENMENT);
schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
}
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 4566000..078fd2c 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -14,3 +14,4 @@
generic-y += fprobe.h
generic-y += mcs_spinlock.h
generic-y += mmzone.h
+generic-y += ring_buffer.h
diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h
index fab1119..db42b47 100644
--- a/arch/x86/include/asm/acrn.h
+++ b/arch/x86/include/asm/acrn.h
@@ -2,6 +2,8 @@
#ifndef _ASM_X86_ACRN_H
#define _ASM_X86_ACRN_H
+#include <asm/cpuid/api.h>
+
/*
* This CPUID returns feature bitmaps in EAX.
* Guest VM uses this to detect the appropriate feature bit.
diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
index 80c1696..23ab054 100644
--- a/arch/x86/include/asm/bug.h
+++ b/arch/x86/include/asm/bug.h
@@ -153,7 +153,7 @@ struct arch_va_list {
struct sysv_va_list args;
};
extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs);
-#endif /* __ASSEMBLER__ */
+static __always_inline __printf(1, 2) void __WARN_validate_printf(const char *fmt, ...) { }
#define __WARN_bug_entry(flags, format) ({ \
struct bug_entry *bug; \
@@ -172,6 +172,7 @@ extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs);
#define __WARN_print_arg(flags, format, arg...) \
do { \
int __flags = (flags) | BUGFLAG_WARNING | BUGFLAG_ARGS ; \
+ __WARN_validate_printf(format, ## arg); \
static_call_mod(WARN_trap)(__WARN_bug_entry(__flags, format), ## arg); \
asm (""); /* inhibit tail-call optimization */ \
} while (0)
@@ -187,6 +188,7 @@ do { \
} \
__ret_warn_on; \
})
+#endif /* __ASSEMBLER__ */
#endif /* HAVE_ARCH_BUG_FORMAT_ARGS */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 3ddc1d3..90680f9 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -125,7 +125,6 @@ static __always_inline bool _static_cpu_has(u16 bit)
#define cpu_has_bug(c, bit) cpu_has(c, (bit))
#define set_cpu_bug(c, bit) set_cpu_cap(c, (bit))
-#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit))
#define static_cpu_has_bug(bit) static_cpu_has((bit))
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit))
diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h
index 44fa82e..82eddfa 100644
--- a/arch/x86/include/asm/cpuid/api.h
+++ b/arch/x86/include/asm/cpuid/api.h
@@ -7,6 +7,7 @@
#include <linux/build_bug.h>
#include <linux/types.h>
+#include <asm/processor.h>
#include <asm/string.h>
/*
@@ -131,12 +132,12 @@ static inline void __cpuid_read(u32 leaf, u32 subleaf, u32 *regs)
__cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX);
}
-#define cpuid_subleaf(leaf, subleaf, regs) { \
+#define cpuid_read_subleaf(leaf, subleaf, regs) { \
static_assert(sizeof(*(regs)) == 16); \
__cpuid_read(leaf, subleaf, (u32 *)(regs)); \
}
-#define cpuid_leaf(leaf, regs) { \
+#define cpuid_read(leaf, regs) { \
static_assert(sizeof(*(regs)) == 16); \
__cpuid_read(leaf, 0, (u32 *)(regs)); \
}
@@ -228,7 +229,7 @@ static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves)
*/
static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs)
{
- cpuid_leaf(0x2, regs);
+ cpuid_read(0x2, regs);
/*
* All Intel CPUs must report an iteration count of 1. In case
@@ -289,4 +290,250 @@ static inline bool cpuid_amd_hygon_has_l3_cache(void)
return cpuid_edx(0x80000006);
}
+/*
+ * 'struct cpuid_leaves' accessors (without sanity checks):
+ *
+ * For internal use by the CPUID parser.
+ */
+
+/* Return constified pointers for all call-site APIs */
+#define __const_ptr(_ptr) \
+ ((const __typeof__(*(_ptr)) *)(_ptr))
+
+#define __cpuid_leaves_subleaf(_leaves, _leaf, _subleaf) \
+ __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf)[0])
+
+#define __cpuid_leaves_subleaf_n(_leaves, _leaf, _index) \
+ __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## n)[_index])
+
+#define __cpuid_leaves_subleaf_info(_leaves, _leaf, _subleaf) \
+ __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf ## _ ## info))
+
+/*
+ * 'struct cpuid_table' accessors (with sanity checks):
+ *
+ * For internal use by the CPUID parser.
+ */
+
+#define __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf) \
+ __cpuid_leaves_subleaf_info(&((_table)->leaves), _leaf, _subleaf)->nr_entries
+
+#define __cpuid_table_subleaf_range_size(_table, _leaf) \
+ ARRAY_SIZE((_table)->leaves.leaf_ ## _leaf ## _n)
+
+#define __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf) \
+ (((_subleaf) < (__cpuid_leaf_first_subleaf(_leaf))) || \
+ ((_subleaf) > (__cpuid_leaf_first_subleaf(_leaf) + \
+ __cpuid_table_subleaf_range_size(_table, _leaf) - 1)))
+
+/* Return NULL if the parser did not fill that leaf. Check cpuid_subleaf(). */
+#define __cpuid_table_subleaf(_table, _leaf, _subleaf) \
+({ \
+ unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf); \
+ \
+ (____f != 1) ? NULL : __cpuid_leaves_subleaf(&((_table)->leaves), _leaf, _subleaf); \
+})
+
+/*
+ * Return NULL if the CPUID parser did not fill this leaf, or if the given
+ * dynamic subleaf value is out of range. Check cpuid_subleaf_n().
+ */
+#define __cpuid_table_subleaf_n(_table, _leaf, _subleaf) \
+({ \
+ unsigned int ____i = (_subleaf) - __cpuid_leaf_first_subleaf(_leaf); \
+ unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, n); \
+ \
+ /* CPUID parser might not have filled the entire subleaf range */ \
+ ((____i >= ____f) || __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf)) ? \
+ NULL : __cpuid_leaves_subleaf_n(&((_table)->leaves), _leaf, ____i); \
+})
+
+/*
+ * Compile-time checks for leaves with a subleaf range:
+ */
+
+#define __cpuid_assert_subleaf_range(_cpuinfo, _leaf) \
+ static_assert(__cpuid_table_subleaf_range_size(&(_cpuinfo)->cpuid, _leaf) > 1)
+
+#define __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf) \
+ BUILD_BUG_ON(__builtin_constant_p(_subleaf) && \
+ __cpuid_table_invalid_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf))
+
+/*
+ * CPUID Parser Call-site APIs
+ *
+ * Call sites should use below APIs instead of invoking direct CPUID queries.
+ *
+ * Benefits include:
+ *
+ * - Return CPUID output as typed C structures that are auto-generated from a
+ * centralized database (see <asm/cpuid/leaf_types.h). Such data types have a
+ * full C99 bitfield layout per CPUID leaf/subleaf combination. Call sites
+ * can thus avoid doing ugly and cryptic bitwise operations on raw CPUID data.
+ *
+ * - Return cached, per-CPU, CPUID output. Below APIs do not invoke any CPUID
+ * queries, thus avoiding their side effects like serialization and VM exits.
+ * Call-site-specific hard coded constants and macros for caching CPUID query
+ * outputs can also be avoided.
+ *
+ * - Return sanitized CPUID data. Below APIs return NULL if the given CPUID
+ * leaf/subleaf input is not supported by hardware, or if the hardware CPUID
+ * output was deemed invalid by the CPUID parser. This centralizes all CPUID
+ * data sanitization in one place (the kernel's CPUID parser.)
+ *
+ * - A centralized global view of system CPUID data. Below APIs will reflect
+ * any kernel-enforced feature masking or overrides, unlike ad hoc parsing of
+ * raw CPUID output by drivers and individual call sites.
+ */
+
+/*
+ * Call-site APIs for CPUID leaves with a single subleaf:
+ */
+
+/**
+ * cpuid_subleaf() - Access parsed CPUID
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x7, 0xf
+ * @_subleaf: CPUID subleaf, in compile-time decimal format; e.g. 0, 1, 3
+ *
+ * Returns a pointer to parsed CPUID output, from the CPUID table inside
+ * @_cpuinfo, as a <cpuid/leaf_types.h> data type: 'struct leaf_0xM_N', where
+ * 0xM is the token provided at @_leaf, and N is the token provided at
+ * @_subleaf; e.g. struct leaf_0x7_0.
+ *
+ * Returns NULL if the requested CPUID @_leaf/@_subleaf query output is not
+ * present at the parsed CPUID table inside @_cpuinfo. This can happen if:
+ *
+ * - The CPUID table inside @_cpuinfo has not yet been populated.
+ * - The CPUID table inside @_cpuinfo was populated, but the CPU does not
+ * implement the requested CPUID @_leaf/@_subleaf combination.
+ * - The CPUID table inside @_cpuinfo was populated, but the kernel's CPUID
+ * parser has predetermined that the requested CPUID @_leaf/@_subleaf
+ * hardware output is invalid or unsupported.
+ *
+ * Example usage::
+ *
+ * const struct leaf_0x7_0 *l7_0 = cpuid_subleaf(c, 0x7, 0);
+ * if (!l7_0) {
+ * // Handle error
+ * }
+ *
+ * const struct leaf_0x7_1 *l7_1 = cpuid_subleaf(c, 0x7, 1);
+ * if (!l7_1) {
+ * // Handle error
+ * }
+ */
+#define cpuid_subleaf(_cpuinfo, _leaf, _subleaf) \
+ __cpuid_table_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf) \
+
+/**
+ * cpuid_leaf() - Access parsed CPUID data
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x0, 0x2, 0x80000000
+ *
+ * Similar to cpuid_subleaf(), but with a CPUID subleaf = 0.
+ *
+ * Example usage::
+ *
+ * const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0);
+ * if (!l0) {
+ * // Handle error
+ * }
+ *
+ * const struct leaf_0x80000000_0 *el0 = cpuid_leaf(c, 0x80000000);
+ * if (!el0) {
+ * // Handle error
+ * }
+ */
+#define cpuid_leaf(_cpuinfo, _leaf) \
+ cpuid_subleaf(_cpuinfo, _leaf, 0)
+
+/**
+ * cpuid_leaf_raw() - Access parsed CPUID data in raw format
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format
+ *
+ * Similar to cpuid_leaf(), but returns a raw 'struct cpuid_regs' pointer to
+ * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
+ */
+#define cpuid_leaf_raw(_cpuinfo, _leaf) \
+ ((const struct cpuid_regs *)(cpuid_leaf(_cpuinfo, _leaf)))
+
+/*
+ * Call-site APIs for CPUID leaves with a subleaf range:
+ */
+
+/**
+ * cpuid_subleaf_n() - Access parsed CPUID data for leaf with a subleaf range
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller
+ * than cpuid_subleaf_count(@_cpuinfo, @_leaf).
+ *
+ * Build-time errors will be emitted in the following cases:
+ *
+ * - @_leaf has no subleaf range. Leaves with a subleaf range have an '_n' type
+ * suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
+ *
+ * - @_subleaf is known at compile-time but is out of range.
+ *
+ * Example usage::
+ *
+ * const struct leaf_0x4_n *l4;
+ *
+ * for (int i = 0; i < cpuid_subleaf_count(c, 0x4); i++) {
+ * l4 = cpuid_subleaf_n(c, 0x4, i);
+ * if (!l4) {
+ * // Handle error
+ * }
+ * ...
+ * }
+ *
+ * Beside the standard error situations detailed at cpuid_subleaf(), this
+ * macro will also return NULL if @_subleaf is out of the leaf's subleaf range.
+ */
+#define cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf) \
+({ \
+ __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \
+ __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf); \
+ __cpuid_table_subleaf_n(&(_cpuinfo)->cpuid, _leaf, _subleaf); \
+})
+
+/**
+ * cpuid_subleaf_n_raw() - Access parsed CPUID data for leaf with subleaf range
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller
+ * than cpuid_subleaf_count(@_cpuinfo, @_leaf).
+ *
+ * Similar to cpuid_subleaf_n(), but returns a raw 'struct cpuid_regs' pointer to
+ * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
+ */
+#define cpuid_subleaf_n_raw(_cpuinfo, _leaf, _subleaf) \
+ ((const struct cpuid_regs *)cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf))
+
+/**
+ * cpuid_subleaf_count() - Number of filled subleaves for @_leaf
+ * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ *
+ * Return the number of subleaves filled by the CPUID parser for @_leaf.
+ *
+ * @_leaf must have subleaf range. Leaves with a subleaf range have an '_n' type
+ * suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
+ */
+#define cpuid_subleaf_count(_cpuinfo, _leaf) \
+({ \
+ __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \
+ __cpuid_table_nr_filled_subleaves(&(_cpuinfo)->cpuid, _leaf, n); \
+})
+
+/*
+ * CPUID parser exported APIs:
+ */
+
+void cpuid_scan_cpu(struct cpuinfo_x86 *c);
+void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf);
+void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end);
+
#endif /* _ASM_X86_CPUID_API_H */
diff --git a/arch/x86/include/asm/cpuid/leaf_types.h b/arch/x86/include/asm/cpuid/leaf_types.h
new file mode 100644
index 0000000..222d2d2
--- /dev/null
+++ b/arch/x86/include/asm/cpuid/leaf_types.h
@@ -0,0 +1,2350 @@
+/* SPDX-License-Identifier: MIT */
+/* Generator: x86-cpuid-db v3.1 */
+
+/*
+ * Auto-generated file.
+ * Please submit all updates and bugfixes to https://x86-cpuid.org
+ */
+
+#ifndef _ASM_X86_CPUID_LEAF_TYPES
+#define _ASM_X86_CPUID_LEAF_TYPES
+
+#include <linux/types.h>
+
+/*
+ * Leaf 0x0
+ * Maximum standard leaf + CPU vendor string
+ */
+
+struct leaf_0x0_0 {
+ // eax
+ u32 max_std_leaf : 32; // Highest standard CPUID leaf
+ // ebx
+ u32 cpu_vendorid_0 : 32; // CPU vendor ID string bytes 0 - 3
+ // ecx
+ u32 cpu_vendorid_2 : 32; // CPU vendor ID string bytes 8 - 11
+ // edx
+ u32 cpu_vendorid_1 : 32; // CPU vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x1
+ * CPU FMS (Family/Model/Stepping) + standard feature flags
+ */
+
+struct leaf_0x1_0 {
+ // eax
+ u32 stepping : 4, // Stepping ID
+ base_model : 4, // Base CPU model ID
+ base_family_id : 4, // Base CPU family ID
+ cpu_type : 2, // CPU type
+ : 2, // Reserved
+ ext_model : 4, // Extended CPU model ID
+ ext_family : 8, // Extended CPU family ID
+ : 4; // Reserved
+ // ebx
+ u32 brand_id : 8, // Brand index
+ clflush_size : 8, // CLFLUSH instruction cache line size
+ n_logical_cpu : 8, // Logical CPU count
+ local_apic_id : 8; // Initial local APIC physical ID
+ // ecx
+ u32 sse3 : 1, // Streaming SIMD Extensions 3 (SSE3)
+ pclmulqdq : 1, // PCLMULQDQ instruction support
+ dtes64 : 1, // 64-bit DS save area
+ monitor : 1, // MONITOR/MWAIT support
+ dscpl : 1, // CPL Qualified Debug Store
+ vmx : 1, // Virtual Machine Extensions
+ smx : 1, // Safer Mode Extensions
+ est : 1, // Enhanced Intel SpeedStep
+ tm2 : 1, // Thermal Monitor 2
+ ssse3 : 1, // Supplemental SSE3
+ cntxt_id : 1, // L1 Context ID
+ sdbg : 1, // Silicon Debug
+ fma : 1, // FMA extensions using YMM state
+ cx16 : 1, // CMPXCHG16B instruction support
+ xtpr_update : 1, // xTPR Update Control
+ pdcm : 1, // Perfmon and Debug Capability
+ : 1, // Reserved
+ pcid : 1, // Process-context identifiers
+ dca : 1, // Direct Cache Access
+ sse4_1 : 1, // SSE4.1
+ sse4_2 : 1, // SSE4.2
+ x2apic : 1, // X2APIC support
+ movbe : 1, // MOVBE instruction support
+ popcnt : 1, // POPCNT instruction support
+ tsc_deadline_timer : 1, // APIC timer one-shot operation
+ aes : 1, // AES instructions
+ xsave : 1, // XSAVE (and related instructions) support
+ osxsave : 1, // XSAVE (and related instructions) are enabled by OS
+ avx : 1, // AVX instructions support
+ f16c : 1, // Half-precision floating-point conversion support
+ rdrand : 1, // RDRAND instruction support
+ guest_status : 1; // System is running as guest; (para-)virtualized system
+ // edx
+ u32 fpu : 1, // Floating-Point Unit on-chip (x87)
+ vme : 1, // Virtual-8086 Mode Extensions
+ de : 1, // Debugging Extensions
+ pse : 1, // Page Size Extension
+ tsc : 1, // Time Stamp Counter
+ msr : 1, // Model-Specific Registers (RDMSR and WRMSR support)
+ pae : 1, // Physical Address Extensions
+ mce : 1, // Machine Check Exception
+ cx8 : 1, // CMPXCHG8B instruction
+ apic : 1, // APIC on-chip
+ : 1, // Reserved
+ sep : 1, // SYSENTER, SYSEXIT, and associated MSRs
+ mtrr : 1, // Memory Type Range Registers
+ pge : 1, // Page Global Extensions
+ mca : 1, // Machine Check Architecture
+ cmov : 1, // Conditional Move Instruction
+ pat : 1, // Page Attribute Table
+ pse36 : 1, // Page Size Extension (36-bit)
+ psn : 1, // Processor Serial Number
+ clflush : 1, // CLFLUSH instruction
+ : 1, // Reserved
+ ds : 1, // Debug Store
+ acpi : 1, // Thermal monitor and clock control
+ mmx : 1, // MMX instructions
+ fxsr : 1, // FXSAVE and FXRSTOR instructions
+ sse : 1, // SSE instructions
+ sse2 : 1, // SSE2 instructions
+ selfsnoop : 1, // Self Snoop
+ htt : 1, // Hyper-threading
+ tm : 1, // Thermal Monitor
+ ia64 : 1, // Legacy IA-64 (Itanium) support bit, now reserved
+ pbe : 1; // Pending Break Enable
+};
+
+/*
+ * Leaf 0x2
+ * Intel cache and TLB information one-byte descriptors
+ */
+
+struct leaf_0x2_0 {
+ // eax
+ u32 iteration_count : 8, // Number of times this leaf must be queried
+ desc1 : 8, // Descriptor #1
+ desc2 : 8, // Descriptor #2
+ desc3 : 7, // Descriptor #3
+ eax_invalid : 1; // Descriptors 1-3 are invalid if set
+ // ebx
+ u32 desc4 : 8, // Descriptor #4
+ desc5 : 8, // Descriptor #5
+ desc6 : 8, // Descriptor #6
+ desc7 : 7, // Descriptor #7
+ ebx_invalid : 1; // Descriptors 4-7 are invalid if set
+ // ecx
+ u32 desc8 : 8, // Descriptor #8
+ desc9 : 8, // Descriptor #9
+ desc10 : 8, // Descriptor #10
+ desc11 : 7, // Descriptor #11
+ ecx_invalid : 1; // Descriptors 8-11 are invalid if set
+ // edx
+ u32 desc12 : 8, // Descriptor #12
+ desc13 : 8, // Descriptor #13
+ desc14 : 8, // Descriptor #14
+ desc15 : 7, // Descriptor #15
+ edx_invalid : 1; // Descriptors 12-15 are invalid if set
+};
+
+/*
+ * Leaf 0x4
+ * Intel deterministic cache parameters
+ */
+
+struct leaf_0x4_n {
+ // eax
+ u32 cache_type : 5, // Cache type field
+ cache_level : 3, // Cache level (1-based)
+ cache_self_init : 1, // Self-initializing cache level
+ fully_associative : 1, // Fully-associative cache
+ : 4, // Reserved
+ num_threads_sharing : 12, // Number logical CPUs sharing this cache
+ num_cores_on_die : 6; // Number of cores in the physical package
+ // ebx
+ u32 cache_linesize : 12, // System coherency line size (0-based)
+ cache_npartitions : 10, // Physical line partitions (0-based)
+ cache_nways : 10; // Ways of associativity (0-based)
+ // ecx
+ u32 cache_nsets : 31, // Cache number of sets (0-based)
+ : 1; // Reserved
+ // edx
+ u32 wbinvd_rll_no_guarantee : 1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches
+ ll_inclusive : 1, // Cache is inclusive of Lower-Level caches
+ complex_indexing : 1, // Not a direct-mapped cache (complex function)
+ : 29; // Reserved
+};
+
+#define LEAF_0x4_SUBLEAF_N_FIRST 0
+#define LEAF_0x4_SUBLEAF_N_LAST 31
+
+/*
+ * Leaf 0x5
+ * MONITOR/MWAIT instructions
+ */
+
+struct leaf_0x5_0 {
+ // eax
+ u32 min_mon_size : 16, // Smallest monitor-line size, in bytes
+ : 16; // Reserved
+ // ebx
+ u32 max_mon_size : 16, // Largest monitor-line size, in bytes
+ : 16; // Reserved
+ // ecx
+ u32 mwait_ext : 1, // MONITOR/MWAIT extensions
+ mwait_irq_break : 1, // Interrupts as a break event for MWAIT
+ : 30; // Reserved
+ // edx
+ u32 n_c0_substates : 4, // Number of C0 sub C-states
+ n_c1_substates : 4, // Number of C1 sub C-states
+ n_c2_substates : 4, // Number of C2 sub C-states
+ n_c3_substates : 4, // Number of C3 sub C-states
+ n_c4_substates : 4, // Number of C4 sub C-states
+ n_c5_substates : 4, // Number of C5 sub C-states
+ n_c6_substates : 4, // Number of C6 sub C-states
+ n_c7_substates : 4; // Number of C7 sub C-states
+};
+
+/*
+ * Leaf 0x6
+ * Thermal and power management
+ */
+
+struct leaf_0x6_0 {
+ // eax
+ u32 digital_temp : 1, // Digital temperature sensor
+ turbo_boost : 1, // Intel Turbo Boost
+ lapic_timer_always_on : 1, // Always-Running APIC Timer (not affected by p-state)
+ : 1, // Reserved
+ power_limit_event : 1, // Power Limit Notification (PLN) event
+ ecmd : 1, // Clock modulation duty cycle extension
+ package_thermal : 1, // Package thermal management
+ hwp_base_regs : 1, // HWP (Hardware P-states) base registers
+ hwp_notify : 1, // HWP notification (IA32_HWP_INTERRUPT MSR)
+ hwp_activity_window : 1, // HWP activity window (IA32_HWP_REQUEST[bits 41:32])
+ hwp_energy_perf_pr : 1, // HWP Energy Performance Preference
+ hwp_package_req : 1, // HWP Package Level Request
+ : 1, // Reserved
+ hdc_base_regs : 1, // HDC base registers
+ turbo_boost_3_0 : 1, // Intel Turbo Boost Max 3.0
+ hwp_capabilities : 1, // HWP Highest Performance change
+ hwp_peci_override : 1, // HWP PECI override
+ hwp_flexible : 1, // Flexible HWP
+ hwp_fast : 1, // IA32_HWP_REQUEST MSR fast access mode
+ hw_feedback : 1, // HW_FEEDBACK MSRs
+ hwp_ignore_idle : 1, // Ignoring idle logical CPU HWP request is supported
+ : 1, // Reserved
+ hwp_ctl : 1, // IA32_HWP_CTL MSR
+ thread_director : 1, // Intel thread director
+ therm_interrupt_bit25 : 1, // IA32_THERM_INTERRUPT MSR bit 25
+ : 7; // Reserved
+ // ebx
+ u32 n_therm_thresholds : 4, // Digital thermometer thresholds
+ : 28; // Reserved
+ // ecx
+ u32 aperf_mperf : 1, // MPERF/APERF MSRs (effective frequency interface)
+ : 2, // Reserved
+ energy_perf_bias : 1, // IA32_ENERGY_PERF_BIAS MSR
+ : 4, // Reserved
+ hw_feedback_nclasses : 8, // Number of Intel Thread Director classes
+ : 16; // Reserved
+ // edx
+ u32 perfcap_reporting : 1, // Performance capability reporting
+ encap_reporting : 1, // Energy efficiency capability reporting
+ : 6, // Reserved
+ feedback_sz : 4, // Feedback interface structure size, in 4K pages
+ : 4, // Reserved
+ this_lcpu_hwfdbk_idx : 16; // This logical CPU hardware feedback interface index
+};
+
+/*
+ * Leaf 0x7
+ * Extended CPU features
+ */
+
+struct leaf_0x7_0 {
+ // eax
+ u32 leaf7_n_subleaves : 32; // Number of leaf 0x7 subleaves
+ // ebx
+ u32 fsgsbase : 1, // FSBASE/GSBASE read/write
+ tsc_adjust : 1, // IA32_TSC_ADJUST MSR
+ sgx : 1, // Intel SGX (Software Guard Extensions)
+ bmi1 : 1, // Bit manipulation extensions group 1
+ hle : 1, // Hardware Lock Elision
+ avx2 : 1, // AVX2 instruction set
+ fdp_excptn_only : 1, // FPU Data Pointer updated only on x87 exceptions
+ smep : 1, // Supervisor Mode Execution Protection
+ bmi2 : 1, // Bit manipulation extensions group 2
+ erms : 1, // Enhanced REP MOVSB/STOSB
+ invpcid : 1, // INVPCID instruction (Invalidate Processor Context ID)
+ rtm : 1, // Intel restricted transactional memory
+ pqm : 1, // Intel RDT-CMT / AMD Platform-QoS cache monitoring
+ zero_fcs_fds : 1, // Deprecated FPU CS/DS (stored as zero)
+ mpx : 1, // Intel memory protection extensions
+ rdt_a : 1, // Intel RDT / AMD Platform-QoS Enforcement
+ avx512f : 1, // AVX-512 foundation instructions
+ avx512dq : 1, // AVX-512 double/quadword instructions
+ rdseed : 1, // RDSEED instruction
+ adx : 1, // ADCX/ADOX instructions
+ smap : 1, // Supervisor mode access prevention
+ avx512ifma : 1, // AVX-512 integer fused multiply add
+ : 1, // Reserved
+ clflushopt : 1, // CLFLUSHOPT instruction
+ clwb : 1, // CLWB instruction
+ intel_pt : 1, // Intel processor trace
+ avx512pf : 1, // AVX-512 prefetch instructions
+ avx512er : 1, // AVX-512 exponent/reciprocal instructions
+ avx512cd : 1, // AVX-512 conflict detection instructions
+ sha : 1, // SHA/SHA256 instructions
+ avx512bw : 1, // AVX-512 byte/word instructions
+ avx512vl : 1; // AVX-512 VL (128/256 vector length) extensions
+ // ecx
+ u32 prefetchwt1 : 1, // PREFETCHWT1 (Intel Xeon Phi only)
+ avx512vbmi : 1, // AVX-512 Vector byte manipulation instructions
+ umip : 1, // User mode instruction protection
+ pku : 1, // Protection keys for user-space
+ ospke : 1, // OS protection keys enable
+ waitpkg : 1, // WAITPKG instructions
+ avx512_vbmi2 : 1, // AVX-512 vector byte manipulation instructions group 2
+ cet_ss : 1, // CET shadow stack features
+ gfni : 1, // Galois field new instructions
+ vaes : 1, // Vector AES instructions
+ vpclmulqdq : 1, // VPCLMULQDQ 256-bit instruction
+ avx512_vnni : 1, // Vector neural network instructions
+ avx512_bitalg : 1, // AVX-512 bitwise algorithms
+ tme : 1, // Intel total memory encryption
+ avx512_vpopcntdq : 1, // AVX-512: POPCNT for vectors of DWORD/QWORD
+ : 1, // Reserved
+ la57 : 1, // 57-bit linear addresses (five-level paging)
+ mawau_val_lm : 5, // BNDLDX/BNDSTX MAWAU value in 64-bit mode
+ rdpid : 1, // RDPID instruction
+ key_locker : 1, // Intel key locker
+ bus_lock_detect : 1, // OS bus-lock detection
+ cldemote : 1, // CLDEMOTE instruction
+ : 1, // Reserved
+ movdiri : 1, // MOVDIRI instruction
+ movdir64b : 1, // MOVDIR64B instruction
+ enqcmd : 1, // Enqueue stores (ENQCMD{,S})
+ sgx_lc : 1, // Intel SGX launch configuration
+ pks : 1; // Protection keys for supervisor-mode pages
+ // edx
+ u32 : 1, // Reserved
+ sgx_keys : 1, // Intel SGX attestation services
+ avx512_4vnniw : 1, // AVX-512 neural network instructions
+ avx512_4fmaps : 1, // AVX-512 multiply accumulation single precision
+ fsrm : 1, // Fast short REP MOVSB
+ uintr : 1, // User interrupts
+ : 2, // Reserved
+ avx512_vp2intersect : 1, // VP2INTERSECT{D,Q} instructions
+ srbds_ctrl : 1, // SRBDS mitigation MSR
+ md_clear : 1, // VERW MD_CLEAR microcode
+ rtm_always_abort : 1, // XBEGIN (RTM transaction) always aborts
+ : 1, // Reserved
+ tsx_force_abort : 1, // MSR TSX_FORCE_ABORT, RTM_ABORT bit
+ serialize : 1, // SERIALIZE instruction
+ hybrid_cpu : 1, // The CPU is identified as a 'hybrid part'
+ tsxldtrk : 1, // TSX suspend/resume load address tracking
+ : 1, // Reserved
+ pconfig : 1, // PCONFIG instruction
+ arch_lbr : 1, // Intel architectural LBRs
+ cet_ibt : 1, // CET indirect branch tracking
+ : 1, // Reserved
+ amx_bf16 : 1, // AMX-BF16: tile bfloat16
+ avx512_fp16 : 1, // AVX-512 FP16 instructions
+ amx_tile : 1, // AMX-TILE: tile architecture
+ amx_int8 : 1, // AMX-INT8: tile 8-bit integer
+ spec_ctrl : 1, // Speculation Control (IBRS/IBPB: indirect branch restrictions)
+ intel_stibp : 1, // Single thread indirect branch predictors
+ flush_l1d : 1, // FLUSH L1D cache: IA32_FLUSH_CMD MSR
+ arch_capabilities : 1, // Intel IA32_ARCH_CAPABILITIES MSR
+ core_capabilities : 1, // IA32_CORE_CAPABILITIES MSR
+ spec_ctrl_ssbd : 1; // Speculative store bypass disable
+};
+
+struct leaf_0x7_1 {
+ // eax
+ u32 : 4, // Reserved
+ avx_vnni : 1, // AVX-VNNI instructions
+ avx512_bf16 : 1, // AVX-512 bfloat16 instructions
+ lass : 1, // Linear address space separation
+ cmpccxadd : 1, // CMPccXADD instructions
+ arch_perfmon_ext : 1, // ArchPerfmonExt: leaf 0x23
+ : 1, // Reserved
+ fzrm : 1, // Fast zero-length REP MOVSB
+ fsrs : 1, // Fast short REP STOSB
+ fsrc : 1, // Fast Short REP CMPSB/SCASB
+ : 4, // Reserved
+ fred : 1, // FRED: Flexible return and event delivery transitions
+ lkgs : 1, // LKGS: Load 'kernel' (userspace) GS
+ wrmsrns : 1, // WRMSRNS instruction (WRMSR-non-serializing)
+ nmi_src : 1, // NMI-source reporting with FRED event data
+ amx_fp16 : 1, // AMX-FP16: FP16 tile operations
+ hreset : 1, // HRESET (Thread director history reset)
+ avx_ifma : 1, // Integer fused multiply add
+ : 2, // Reserved
+ lam : 1, // Linear address masking
+ rd_wr_msrlist : 1, // RDMSRLIST/WRMSRLIST instructions
+ : 4; // Reserved
+ // ebx
+ u32 intel_ppin : 1, // Protected processor inventory number (PPIN{,_CTL} MSRs)
+ : 31; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 4, // Reserved
+ avx_vnni_int8 : 1, // AVX-VNNI-INT8 instructions
+ avx_ne_convert : 1, // AVX-NE-CONVERT instructions
+ : 2, // Reserved
+ amx_complex : 1, // AMX-COMPLEX instructions (starting from Granite Rapids)
+ : 5, // Reserved
+ prefetchit_0_1 : 1, // PREFETCHIT0/1 instructions
+ : 3, // Reserved
+ cet_sss : 1, // CET supervisor shadow stacks safe to use
+ : 13; // Reserved
+};
+
+struct leaf_0x7_2 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 intel_psfd : 1, // Intel predictive store forward disable
+ ipred_ctrl : 1, // MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S}
+ rrsba_ctrl : 1, // MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S}
+ ddp_ctrl : 1, // MSR bit IA32_SPEC_CTRL.DDPD_U
+ bhi_ctrl : 1, // MSR bit IA32_SPEC_CTRL.BHI_DIS_S
+ mcdt_no : 1, // MCDT mitigation not needed
+ uclock_disable : 1, // UC-lock disable
+ : 25; // Reserved
+};
+
+/*
+ * Leaf 0x9
+ * Intel DCA (Direct Cache Access)
+ */
+
+struct leaf_0x9_0 {
+ // eax
+ u32 dca_enabled_in_bios : 1, // DCA is enabled in BIOS
+ : 31; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0xa
+ * Intel PMU (Performance Monitoring Unit)
+ */
+
+struct leaf_0xa_0 {
+ // eax
+ u32 pmu_version : 8, // Performance monitoring unit version ID
+ num_counters_gp : 8, // Number of general-purpose PMU counters per logical CPU
+ bit_width_gp : 8, // Bitwidth of PMU general-purpose counters
+ events_mask_len : 8; // Length of CPUID(0xa).EBX bit vector
+ // ebx
+ u32 no_core_cycle : 1, // Core cycle event not available
+ no_instruction_retired : 1, // Instruction retired event not available
+ no_reference_cycles : 1, // Reference cycles event not available
+ no_llc_reference : 1, // LLC-reference event not available
+ no_llc_misses : 1, // LLC-misses event not available
+ no_br_insn_retired : 1, // Branch instruction retired event not available
+ no_br_misses_retired : 1, // Branch mispredict retired event not available
+ no_topdown_slots : 1, // Topdown slots event not available
+ no_backend_bound : 1, // Topdown backend bound not available
+ no_bad_speculation : 1, // Topdown bad speculation not available
+ no_frontend_bound : 1, // Topdown frontend bound not available
+ no_retiring : 1, // Topdown retiring not available
+ no_lbr_inserts : 1, // LBR inserts not available
+ : 19; // Reserved
+ // ecx
+ u32 pmu_fcounters_bitmap : 32; // Fixed-function PMU counters support bitmap
+ // edx
+ u32 num_counters_fixed : 5, // Number of fixed PMU counters
+ bitwidth_fixed : 8, // Bitwidth of PMU fixed counters
+ : 2, // Reserved
+ anythread_deprecation : 1, // AnyThread mode deprecation
+ : 16; // Reserved
+};
+
+/*
+ * Leaf 0xb
+ * CPU extended topology v1
+ */
+
+struct leaf_0xb_n {
+ // eax
+ u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive)
+ : 27; // Reserved
+ // ebx
+ u32 domain_lcpus_count : 16, // Logical CPUs count across all instances of this domain
+ : 16; // Reserved
+ // ecx
+ u32 domain_nr : 8, // This domain level (subleaf ID)
+ domain_type : 8, // This domain type
+ : 16; // Reserved
+ // edx
+ u32 x2apic_id : 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0xb_SUBLEAF_N_FIRST 0
+#define LEAF_0xb_SUBLEAF_N_LAST 1
+
+/*
+ * Leaf 0xd
+ * CPU extended state
+ */
+
+struct leaf_0xd_0 {
+ // eax
+ u32 xcr0_x87 : 1, // XCR0.X87
+ xcr0_sse : 1, // XCR0.SSE
+ xcr0_avx : 1, // XCR0.AVX
+ xcr0_mpx_bndregs : 1, // XCR0.BNDREGS: MPX BND0-BND3 registers
+ xcr0_mpx_bndcsr : 1, // XCR0.BNDCSR: MPX BNDCFGU/BNDSTATUS registers
+ xcr0_avx512_opmask : 1, // XCR0.OPMASK: AVX-512 k0-k7 registers
+ xcr0_avx512_zmm_hi256 : 1, // XCR0.ZMM_Hi256: AVX-512 ZMM0->ZMM7/15 registers
+ xcr0_avx512_hi16_zmm : 1, // XCR0.HI16_ZMM: AVX-512 ZMM16->ZMM31 registers
+ : 1, // Reserved
+ xcr0_pkru : 1, // XCR0.PKRU: XSAVE PKRU registers
+ : 1, // Reserved
+ xcr0_cet_u : 1, // XCR0.CET_U: CET user state
+ xcr0_cet_s : 1, // XCR0.CET_S: CET supervisor state
+ : 4, // Reserved
+ xcr0_tileconfig : 1, // XCR0.TILECONFIG: AMX can manage TILECONFIG
+ xcr0_tiledata : 1, // XCR0.TILEDATA: AMX can manage TILEDATA
+ : 13; // Reserved
+ // ebx
+ u32 xsave_sz_xcr0 : 32; // XSAVE/XRSTOR area byte size, for XCR0 enabled features
+ // ecx
+ u32 xsave_sz_max : 32; // XSAVE/XRSTOR area max byte size, all CPU features
+ // edx
+ u32 : 30, // Reserved
+ xcr0_lwp : 1, // AMD XCR0.LWP: Light-weight Profiling
+ : 1; // Reserved
+};
+
+struct leaf_0xd_1 {
+ // eax
+ u32 xsaveopt : 1, // XSAVEOPT instruction
+ xsavec : 1, // XSAVEC instruction
+ xgetbv1 : 1, // XGETBV instruction with ECX = 1
+ xsaves : 1, // XSAVES/XRSTORS instructions (and XSS MSR)
+ xfd : 1, // Extended feature disable
+ : 27; // Reserved
+ // ebx
+ u32 xsave_sz_xcr0_xss : 32; // XSAVES/XSAVEC area byte size, for XCR0|XSS enabled features
+ // ecx
+ u32 : 8, // Reserved
+ xss_pt : 1, // PT state
+ : 1, // Reserved
+ xss_pasid : 1, // PASID state
+ xss_cet_u : 1, // CET user state
+ xss_cet_s : 1, // CET supervisor state
+ xss_hdc : 1, // HDC state
+ xss_uintr : 1, // UINTR state
+ xss_lbr : 1, // LBR state
+ xss_hwp : 1, // HWP state
+ : 15; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0xd_n {
+ // eax
+ u32 xsave_sz : 32; // Subleaf-N feature save area size, in bytes
+ // ebx
+ u32 xsave_offset : 32; // Subleaf-N feature save area offset, in bytes
+ // ecx
+ u32 is_xss_bit : 1, // Subleaf N describes an XSS bit (otherwise XCR0)
+ compacted_xsave_64byte_aligned : 1, // When compacted, subleaf-N XSAVE area is 64-byte aligned
+ : 30; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+#define LEAF_0xd_SUBLEAF_N_FIRST 2
+#define LEAF_0xd_SUBLEAF_N_LAST 63
+
+/*
+ * Leaf 0xf
+ * Intel RDT / AMD PQoS resource monitoring
+ */
+
+struct leaf_0xf_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 core_rmid_max : 32; // RMID max within this core (0-based)
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 1, // Reserved
+ llc_qos_mon : 1, // LLC QoS-monitoring
+ : 30; // Reserved
+};
+
+struct leaf_0xf_1 {
+ // eax
+ u32 l3c_qm_bitwidth : 8, // L3 QoS-monitoring counter bitwidth (24-based)
+ l3c_qm_overflow_bit : 1, // QM_CTR MSR bit 61 is an overflow bit
+ io_rdt_cmt : 1, // non-CPU agent supporting Intel RDT CMT present
+ io_rdt_mbm : 1, // non-CPU agent supporting Intel RDT MBM present
+ : 21; // Reserved
+ // ebx
+ u32 l3c_qm_conver_factor : 32; // QM_CTR MSR conversion factor to bytes
+ // ecx
+ u32 l3c_qm_rmid_max : 32; // L3 QoS-monitoring max RMID
+ // edx
+ u32 l3c_qm_occupancy : 1, // L3 QoS occupancy monitoring
+ l3c_qm_mbm_total : 1, // L3 QoS total bandwidth monitoring
+ l3c_qm_mbm_local : 1, // L3 QoS local bandwidth monitoring
+ : 29; // Reserved
+};
+
+/*
+ * Leaf 0x10
+ * Intel RDT / AMD PQoS allocation
+ */
+
+struct leaf_0x10_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 : 1, // Reserved
+ cat_l3 : 1, // L3 Cache Allocation Technology
+ cat_l2 : 1, // L2 Cache Allocation Technology
+ mba : 1, // Memory Bandwidth Allocation
+ : 28; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x10_n {
+ // eax
+ u32 cat_cbm_len : 5, // L3/L2_CAT capacity bitmask length, minus-one notation
+ : 27; // Reserved
+ // ebx
+ u32 cat_units_bitmap : 32; // L3/L2_CAT allocation units bitmap
+ // ecx
+ u32 : 1, // Reserved
+ l3_cat_cos_infreq_updates : 1, // L3_CAT COS updates should be infrequent
+ cat_cdp_supported : 1, // L3/L2_CAT Code and Data Prioritization
+ cat_sparse_1s : 1, // L3/L2_CAT non-contiguous 1s value
+ : 28; // Reserved
+ // edx
+ u32 cat_cos_max : 16, // L3/L2_CAT max Class of Service
+ : 16; // Reserved
+};
+
+#define LEAF_0x10_SUBLEAF_N_FIRST 1
+#define LEAF_0x10_SUBLEAF_N_LAST 2
+
+struct leaf_0x10_3 {
+ // eax
+ u32 mba_max_delay : 12, // Max MBA throttling value; minus-one notation
+ : 20; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 mba_per_thread : 1, // Per-thread MBA controls
+ : 1, // Reserved
+ mba_delay_linear : 1, // Delay values are linear
+ : 29; // Reserved
+ // edx
+ u32 mba_cos_max : 16, // MBA max Class of Service
+ : 16; // Reserved
+};
+
+/*
+ * Leaf 0x12
+ * Intel SGX (Software Guard Extensions)
+ */
+
+struct leaf_0x12_0 {
+ // eax
+ u32 sgx1 : 1, // SGX1 leaf functions
+ sgx2 : 1, // SGX2 leaf functions
+ : 3, // Reserved
+ enclv_leaves : 1, // ENCLV leaves
+ encls_leaves : 1, // ENCLS leaves
+ enclu_everifyreport2 : 1, // ENCLU leaf EVERIFYREPORT2
+ : 2, // Reserved
+ encls_eupdatesvn : 1, // ENCLS leaf EUPDATESVN
+ enclu_edeccssa : 1, // ENCLU leaf EDECCSSA
+ : 20; // Reserved
+ // ebx
+ u32 miscselect_exinfo : 1, // SSA.MISC frame: Enclave #PF and #GP reporting
+ miscselect_cpinfo : 1, // SSA.MISC frame: Enclave #CP reporting
+ : 30; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 max_enclave_sz_not64 : 8, // Maximum enclave size in non-64-bit mode (log2)
+ max_enclave_sz_64 : 8, // Maximum enclave size in 64-bit mode (log2)
+ : 16; // Reserved
+};
+
+struct leaf_0x12_1 {
+ // eax
+ u32 secs_attr_init : 1, // Enclave initialized by EINIT
+ secs_attr_debug : 1, // Enclave permits debugger read/write
+ secs_attr_mode64bit : 1, // Enclave runs in 64-bit mode
+ : 1, // Reserved
+ secs_attr_provisionkey : 1, // Provisioning key
+ secs_attr_einittoken_key : 1, // EINIT token key
+ secs_attr_cet : 1, // CET attributes
+ secs_attr_kss : 1, // Key Separation and Sharing
+ : 2, // Reserved
+ secs_attr_aexnotify : 1, // Enclave threads: AEX notifications
+ : 21; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 xfrm_x87 : 1, // Enclave XFRM.X87
+ xfrm_sse : 1, // Enclave XFRM.SSE
+ xfrm_avx : 1, // Enclave XFRM.AVX
+ xfrm_mpx_bndregs : 1, // Enclave XFRM.BNDREGS (MPX BND0-BND3 registers)
+ xfrm_mpx_bndcsr : 1, // Enclave XFRM.BNDCSR (MPX BNDCFGU/BNDSTATUS registers)
+ xfrm_avx512_opmask : 1, // Enclave XFRM.OPMASK (AVX-512 k0-k7 registers)
+ xfrm_avx512_zmm_hi256 : 1, // Enclave XFRM.ZMM_Hi256 (AVX-512 ZMM0->ZMM7/15 registers)
+ xfrm_avx512_hi16_zmm : 1, // Enclave XFRM.HI16_ZMM (AVX-512 ZMM16->ZMM31 registers)
+ : 1, // Reserved
+ xfrm_pkru : 1, // Enclave XFRM.PKRU (XSAVE PKRU registers)
+ : 7, // Reserved
+ xfrm_tileconfig : 1, // Enclave XFRM.TILECONFIG (AMX can manage TILECONFIG)
+ xfrm_tiledata : 1, // Enclave XFRM.TILEDATA (AMX can manage TILEDATA)
+ : 13; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x12_n {
+ // eax
+ u32 subleaf_type : 4, // Subleaf type
+ : 8, // Reserved
+ epc_sec_base_addr_0 : 20; // EPC section base address, bits[12:31]
+ // ebx
+ u32 epc_sec_base_addr_1 : 20, // EPC section base address, bits[32:51]
+ : 12; // Reserved
+ // ecx
+ u32 epc_sec_type : 4, // EPC section type / property encoding
+ : 8, // Reserved
+ epc_sec_size_0 : 20; // EPC section size, bits[12:31]
+ // edx
+ u32 epc_sec_size_1 : 20, // EPC section size, bits[32:51]
+ : 12; // Reserved
+};
+
+#define LEAF_0x12_SUBLEAF_N_FIRST 2
+#define LEAF_0x12_SUBLEAF_N_LAST 31
+
+/*
+ * Leaf 0x14
+ * Intel Processor Trace
+ */
+
+struct leaf_0x14_0 {
+ // eax
+ u32 pt_max_subleaf : 32; // Maximum leaf 0x14 subleaf
+ // ebx
+ u32 cr3_filtering : 1, // IA32_RTIT_CR3_MATCH is accessible
+ psb_cyc : 1, // Configurable PSB and cycle-accurate mode
+ ip_filtering : 1, // IP/TraceStop filtering; Warm-reset PT MSRs preservation
+ mtc_timing : 1, // MTC timing packet; COFI-based packets suppression
+ ptwrite : 1, // PTWRITE instruction
+ power_event_trace : 1, // Power Event Trace
+ psb_pmi_preserve : 1, // PSB and PMI preservation
+ event_trace : 1, // Event Trace packet generation
+ tnt_disable : 1, // TNT packet generation disable
+ : 23; // Reserved
+ // ecx
+ u32 topa_output : 1, // ToPA output scheme
+ topa_multiple_entries : 1, // ToPA tables can hold multiple entries
+ single_range_output : 1, // Single-range output
+ trace_transport_output : 1, // Trace Transport subsystem output
+ : 27, // Reserved
+ ip_payloads_lip : 1; // IP payloads have LIP values (CS base included)
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x14_1 {
+ // eax
+ u32 num_address_ranges : 3, // Number of configurable address ranges
+ : 13, // Reserved
+ mtc_periods_bmp : 16; // MTC period encodings bitmap
+ // ebx
+ u32 cycle_thresholds_bmp : 16, // Cycle Threshold encodings bitmap
+ psb_periods_bmp : 16; // Configurable PSB frequency encodings bitmap
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x15
+ * Intel TSC (Time Stamp Counter)
+ */
+
+struct leaf_0x15_0 {
+ // eax
+ u32 tsc_denominator : 32; // Denominator of the TSC/'core crystal clock' ratio
+ // ebx
+ u32 tsc_numerator : 32; // Numerator of the TSC/'core crystal clock' ratio
+ // ecx
+ u32 cpu_crystal_hz : 32; // Core crystal clock nominal frequency, in Hz
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x16
+ * Intel processor frequency
+ */
+
+struct leaf_0x16_0 {
+ // eax
+ u32 cpu_base_mhz : 16, // Processor base frequency, in MHz
+ : 16; // Reserved
+ // ebx
+ u32 cpu_max_mhz : 16, // Processor max frequency, in MHz
+ : 16; // Reserved
+ // ecx
+ u32 bus_mhz : 16, // Bus reference frequency, in MHz
+ : 16; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x17
+ * Intel SoC vendor attributes
+ */
+
+struct leaf_0x17_0 {
+ // eax
+ u32 soc_max_subleaf : 32; // Maximum leaf 0x17 subleaf
+ // ebx
+ u32 soc_vendor_id : 16, // SoC vendor ID
+ is_vendor_scheme : 1, // Assigned by industry enumeration scheme (not Intel)
+ : 15; // Reserved
+ // ecx
+ u32 soc_proj_id : 32; // SoC project ID, assigned by vendor
+ // edx
+ u32 soc_stepping_id : 32; // SoC project stepping ID, assigned by vendor
+};
+
+struct leaf_0x17_n {
+ // eax
+ u32 vendor_brand_a : 32; // Vendor Brand ID string, bytes subleaf_nr * (0 -> 3)
+ // ebx
+ u32 vendor_brand_b : 32; // Vendor Brand ID string, bytes subleaf_nr * (4 -> 7)
+ // ecx
+ u32 vendor_brand_c : 32; // Vendor Brand ID string, bytes subleaf_nr * (8 -> 11)
+ // edx
+ u32 vendor_brand_d : 32; // Vendor Brand ID string, bytes subleaf_nr * (12 -> 15)
+};
+
+#define LEAF_0x17_SUBLEAF_N_FIRST 1
+#define LEAF_0x17_SUBLEAF_N_LAST 3
+
+/*
+ * Leaf 0x18
+ * Intel deterministic address translation (TLB) parameters
+ */
+
+struct leaf_0x18_n {
+ // eax
+ u32 tlb_max_subleaf : 32; // Maximum leaf 0x18 subleaf
+ // ebx
+ u32 tlb_4k_page : 1, // TLB supports 4KB-page entries
+ tlb_2m_page : 1, // TLB supports 2MB-page entries
+ tlb_4m_page : 1, // TLB supports 4MB-page entries
+ tlb_1g_page : 1, // TLB supports 1GB-page entries
+ : 4, // Reserved
+ hard_partitioning : 3, // Partitioning between logical CPUs
+ : 5, // Reserved
+ n_way_associative : 16; // Ways of associativity
+ // ecx
+ u32 n_sets : 32; // Number of sets
+ // edx
+ u32 tlb_type : 5, // Translation cache type (TLB type)
+ tlb_cache_level : 3, // Translation cache level (1-based)
+ is_fully_associative : 1, // Fully-associative
+ : 5, // Reserved
+ tlb_max_addressable_ids : 12, // Max number of addressable IDs - 1
+ : 6; // Reserved
+};
+
+#define LEAF_0x18_SUBLEAF_N_FIRST 0
+#define LEAF_0x18_SUBLEAF_N_LAST 31
+
+/*
+ * Leaf 0x19
+ * Intel key locker
+ */
+
+struct leaf_0x19_0 {
+ // eax
+ u32 kl_cpl0_only : 1, // CPL0-only key locker restriction
+ kl_no_encrypt : 1, // No-encrypt key locker restriction
+ kl_no_decrypt : 1, // No-decrypt key locker restriction
+ : 29; // Reserved
+ // ebx
+ u32 aes_keylocker : 1, // AES key locker instructions
+ : 1, // Reserved
+ aes_keylocker_wide : 1, // AES wide key locker instructions
+ : 1, // Reserved
+ kl_msr_iwkey : 1, // Key locker MSRs and IWKEY backups
+ : 27; // Reserved
+ // ecx
+ u32 loadiwkey_no_backup : 1, // LOADIWKEY NoBackup parameter
+ iwkey_rand : 1, // IWKEY randomization
+ : 30; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x1a
+ * Intel hybrid CPUs identification (e.g. Atom, Core)
+ */
+
+struct leaf_0x1a_0 {
+ // eax
+ u32 core_native_model : 24, // This core's native model ID
+ core_type : 8; // This core's type
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x1b
+ * Intel PCONFIG (Platform configuration)
+ */
+
+struct leaf_0x1b_n {
+ // eax
+ u32 pconfig_subleaf_type : 12, // CPUID 0x1b subleaf type
+ : 20; // Reserved
+ // ebx
+ u32 pconfig_target_id_x : 32; // A supported PCONFIG target ID
+ // ecx
+ u32 pconfig_target_id_y : 32; // A supported PCONFIG target ID
+ // edx
+ u32 pconfig_target_id_z : 32; // A supported PCONFIG target ID
+};
+
+#define LEAF_0x1b_SUBLEAF_N_FIRST 0
+#define LEAF_0x1b_SUBLEAF_N_LAST 31
+
+/*
+ * Leaf 0x1c
+ * Intel LBR (Last Branch Record)
+ */
+
+struct leaf_0x1c_0 {
+ // eax
+ u32 lbr_depth_mask : 8, // Max LBR stack depth bitmask
+ : 22, // Reserved
+ lbr_deep_c_reset : 1, // LBRs may be cleared on MWAIT C-state > C1
+ lbr_ip_is_lip : 1; // LBR IP contain Last IP (otherwise effective IP)
+ // ebx
+ u32 lbr_cpl : 1, // CPL filtering
+ lbr_branch_filter : 1, // Branch filtering
+ lbr_call_stack : 1, // Call-stack mode
+ : 29; // Reserved
+ // ecx
+ u32 lbr_mispredict : 1, // Branch misprediction bit
+ lbr_timed_lbr : 1, // Timed LBRs (CPU cycles since last LBR entry)
+ lbr_branch_type : 1, // Branch type field
+ : 13, // Reserved
+ lbr_events_gpc_bmp : 4, // PMU-events logging support
+ : 12; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x1d
+ * Intel AMX (Advanced Matrix Extensions) tile information
+ */
+
+struct leaf_0x1d_0 {
+ // eax
+ u32 amx_max_palette : 32; // Highest palette ID / subleaf ID
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x1d_1 {
+ // eax
+ u32 amx_palette_size : 16, // AMX palette total tiles size, in bytes
+ amx_tile_size : 16; // AMX single tile's size, in bytes
+ // ebx
+ u32 amx_tile_row_size : 16, // AMX tile single row's size, in bytes
+ amx_palette_nr_tiles : 16; // AMX palette number of tiles
+ // ecx
+ u32 amx_tile_nr_rows : 16, // AMX tile max number of rows
+ : 16; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x1e
+ * Intel TMUL (Tile-matrix Multiply)
+ */
+
+struct leaf_0x1e_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 tmul_maxk : 8, // TMUL unit maximum height, K (rows or columns)
+ tmul_maxn : 16, // TMUL unit maximum SIMD dimension, N (column bytes)
+ : 8; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x1f
+ * Intel extended topology v2
+ */
+
+struct leaf_0x1f_n {
+ // eax
+ u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive)
+ : 27; // Reserved
+ // ebx
+ u32 domain_lcpus_count : 16, // Logical CPUs count across all instances of this domain
+ : 16; // Reserved
+ // ecx
+ u32 domain_level : 8, // This domain level (subleaf ID)
+ domain_type : 8, // This domain type
+ : 16; // Reserved
+ // edx
+ u32 x2apic_id : 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0x1f_SUBLEAF_N_FIRST 0
+#define LEAF_0x1f_SUBLEAF_N_LAST 5
+
+/*
+ * Leaf 0x20
+ * Intel HRESET (History Reset)
+ */
+
+struct leaf_0x20_0 {
+ // eax
+ u32 hreset_nr_subleaves : 32; // CPUID 0x20 max subleaf + 1
+ // ebx
+ u32 hreset_thread_director : 1, // Intel thread director HRESET
+ : 31; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x21
+ * Intel TD (Trust Domain)
+ */
+
+struct leaf_0x21_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 tdx_vendorid_0 : 32; // TDX vendor ID string bytes 0 - 3
+ // ecx
+ u32 tdx_vendorid_2 : 32; // TDX vendor ID string bytes 8 - 11
+ // edx
+ u32 tdx_vendorid_1 : 32; // TDX vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x23
+ * Intel Architectural Performance Monitoring Extended (ArchPerfmonExt)
+ */
+
+struct leaf_0x23_0 {
+ // eax
+ u32 subleaf_0 : 1, // Subleaf 0, this subleaf
+ counters_subleaf : 1, // Subleaf 1, PMU counter bitmaps
+ acr_subleaf : 1, // Subleaf 2, Auto Counter Reload bitmaps
+ events_subleaf : 1, // Subleaf 3, PMU event bitmaps
+ pebs_caps_subleaf : 1, // Subleaf 4, PEBS capabilities
+ pebs_subleaf : 1, // Subleaf 5, Arch PEBS bitmaps
+ : 26; // Reserved
+ // ebx
+ u32 unitmask2 : 1, // IA32_PERFEVTSELx MSRs UnitMask2 bit
+ eq : 1, // IA32_PERFEVTSELx MSRs EQ bit
+ rdpmc_user_disable : 1, // RDPMC userspace disable
+ : 29; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x23_1 {
+ // eax
+ u32 gp_counters : 32; // Bitmap of general-purpose PMU counters
+ // ebx
+ u32 fixed_counters : 32; // Bitmap of fixed PMU counters
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x23_2 {
+ // eax
+ u32 acr_gp_reload : 32; // Bitmap of general-purpose counters that can be reloaded
+ // ebx
+ u32 acr_fixed_reload : 32; // Bitmap of fixed counters that can be reloaded
+ // ecx
+ u32 acr_gp_trigger : 32; // Bitmap of general-purpose counters that can trigger reloads
+ // edx
+ u32 acr_fixed_trigger : 32; // Bitmap of fixed counters that can trigger reloads
+};
+
+struct leaf_0x23_3 {
+ // eax
+ u32 core_cycles_evt : 1, // Core cycles event
+ insn_retired_evt : 1, // Instructions retired event
+ ref_cycles_evt : 1, // Reference cycles event
+ llc_refs_evt : 1, // Last-level cache references event
+ llc_misses_evt : 1, // Last-level cache misses event
+ br_insn_ret_evt : 1, // Branch instruction retired event
+ br_mispr_evt : 1, // Branch mispredict retired event
+ td_slots_evt : 1, // Topdown slots event
+ td_backend_bound_evt : 1, // Topdown backend bound event
+ td_bad_spec_evt : 1, // Topdown bad speculation event
+ td_frontend_bound_evt : 1, // Topdown frontend bound event
+ td_retiring_evt : 1, // Topdown retiring event
+ : 20; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x23_4 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 : 3, // Reserved
+ allow_in_record : 1, // ALLOW_IN_RECORD bit in MSRs
+ counters_gp : 1, // Counters group sub-group general-purpose counters
+ counters_fixed : 1, // Counters group sub-group fixed-function counters
+ counters_metrics : 1, // Counters group sub-group performance metrics
+ : 1, // Reserved
+ lbr : 2, // LBR group
+ : 6, // Reserved
+ xer : 8, // XER group
+ : 5, // Reserved
+ gpr : 1, // GPR group
+ aux : 1, // AUX group
+ : 1; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x23_5 {
+ // eax
+ u32 pebs_gp : 32; // Architectural PEBS general-purpose counters
+ // ebx
+ u32 pebs_pdist_gp : 32; // Architectural PEBS PDIST general-purpose counters
+ // ecx
+ u32 pebs_fixed : 32; // Architectural PEBS fixed counters
+ // edx
+ u32 pebs_pdist_fixed : 32; // Architectural PEBS PDIST fixed counters
+};
+
+/*
+ * Leaf 0x40000000
+ * Maximum hypervisor leaf + hypervisor vendor string
+ */
+
+struct leaf_0x40000000_0 {
+ // eax
+ u32 max_hyp_leaf : 32; // Maximum hypervisor leaf
+ // ebx
+ u32 hypervisor_id_0 : 32; // Hypervisor ID string bytes 0 - 3
+ // ecx
+ u32 hypervisor_id_1 : 32; // Hypervisor ID string bytes 4 - 7
+ // edx
+ u32 hypervisor_id_2 : 32; // Hypervisor ID string bytes 8 - 11
+};
+
+/*
+ * Leaf 0x4c780001
+ * Linux-defined synthetic feature flags
+ */
+
+struct leaf_0x4c780001_0 {
+ // eax
+ u32 cxmmx : 1, // Cyrix MMX extensions
+ k6_mtrr : 1, // AMD K6 nonstandard MTRRs
+ cyrix_arr : 1, // Cyrix ARRs (= MTRRs)
+ centaur_mcr : 1, // Centaur MCRs (= MTRRs)
+ k8 : 1, // Opteron, Athlon64
+ zen5 : 1, // CPU based on Zen5 micro-architecture
+ zen6 : 1, // CPU based on Zen6 micro-architecture
+ : 1, // Reserved
+ constant_tsc : 1, // TSC ticks at a constant rate
+ up : 1, // SMP kernel running on UP
+ art : 1, // Always running timer (ART)
+ arch_perfmon : 1, // Intel Architectural PerfMon
+ pebs : 1, // Precise-Event Based Sampling
+ bts : 1, // Branch Trace Store
+ syscall32 : 1, // SYSCALL in IA32 userspace
+ sysenter32 : 1, // SYSENTER in IA32 userspace
+ rep_good : 1, // REP microcode works well
+ amd_lbr_v2 : 1, // AMD Last Branch Record Extension version 2
+ clear_cpu_buf : 1, // Clear CPU buffers using VERW
+ acc_power : 1, // AMD Accumulated Power Mechanism
+ nopl : 1, // The NOPL instructions
+ always : 1, // Always-present feature
+ xtopology : 1, // CPU topology enumeration extensions
+ tsc_reliable : 1, // TSC is known to be reliable
+ nonstop_tsc : 1, // TSC does not stop in C states
+ cpuid : 1, // CPU has the CPUID instruction
+ extd_apicid : 1, // Extended APIC ID (8 bits)
+ amd_dcm : 1, // AMD multi-node processor
+ aperfmperf : 1, // APERF/MPERF MSRs: P-State hardware coordination feedback
+ rapl : 1, // AMD/Hygon RAPL interface
+ nonstop_tsc_s3 : 1, // TSC does not stop in S3 state
+ tsc_known_freq : 1; // TSC has known frequency
+ // ebx
+ u32 ring3mwait : 1, // Ring 3 MONITOR/MWAIT instructions
+ cpuid_fault : 1, // Intel CPUID faulting
+ cpb : 1, // AMD Core Performance Boost
+ epb : 1, // IA32_ENERGY_PERF_BIAS support
+ cat_l3 : 1, // Cache Allocation Technology L3
+ cat_l2 : 1, // Cache Allocation Technology L2
+ cdp_l3 : 1, // Code and Data Prioritization L3
+ tdx_host_platform : 1, // Platform supports being a TDX host
+ hw_pstate : 1, // AMD Hardware P-state control
+ proc_feedback : 1, // AMD Processor Feedback Interface
+ xcompacted : 1, // Use compacted XSTATE (XSAVES or XSAVEC)
+ pti : 1, // Kernel Page Table Isolation enabled
+ kernel_ibrs : 1, // Set/clear IBRS on kernel entry/exit
+ rsb_vmexit : 1, // Fill RSB on VM-Exit
+ intel_ppin : 1, // Intel Processor Inventory Number
+ cdp_l2 : 1, // Code and Data Prioritization L2
+ msr_spec_ctrl : 1, // MSR SPEC_CTRL is implemented
+ ssbd : 1, // Speculative Store Bypass Disable
+ mba : 1, // Memory Bandwidth Allocation
+ rsb_ctxsw : 1, // Fill RSB on context switches
+ perfmon_v2 : 1, // AMD Performance Monitoring Version 2
+ : 1, // Reserved
+ use_ibrs_fw : 1, // Use IBRS during runtime firmware calls
+ ss_bypass_disable : 1, // Disable Speculative Store Bypass
+ ls_cfg_ssbd : 1, // AMD SSBD implementation via LS_CFG MSR
+ ibrs : 1, // Indirect Branch Restricted Speculation
+ ibpb : 1, // Indirect Branch Prediction Barrier (without RSB flush guarantee)
+ stibp : 1, // Single Thread Indirect Branch Predictors
+ zen : 1, // Generic flag for all Zen and newer
+ l1tf_pteinv : 1, // L1TF workaround PTE inversion
+ ibrs_enhanced : 1, // Enhanced IBRS
+ msr_ia32_feat_ctl : 1; // MSR IA32_FEAT_CTL configured
+ // ecx
+ u32 tpr_shadow : 1, // Intel TPR Shadow
+ flexpriority : 1, // Intel FlexPriority
+ ept : 1, // Intel Extended Page Table
+ vpid : 1, // Intel Virtual Processor ID
+ coherency_sfw_no : 1, // SNP cache coherency software workaround not needed
+ : 10, // Reserved
+ vmmcall : 1, // Prefer VMMCALL to VMCALL
+ xenpv : 1, // Xen paravirtual guest
+ ept_ad : 1, // Intel Extended Page Table access-dirty bit
+ vmcall : 1, // Hypervisor supports the VMCALL instruction
+ vmw_vmmcall : 1, // VMware prefers the VMMCALL instruction
+ pvunlock : 1, // PV unlock function
+ vcpupreempt : 1, // PV vcpu_is_preempted function
+ tdx_guest : 1, // Intel Trust Domain Extensions Guest
+ : 9; // Reserved
+ // edx
+ u32 cqm_llc : 1, // LLC QoS
+ cqm_occup_llc : 1, // LLC occupancy monitoring
+ cqm_mbm_total : 1, // LLC Total MBM monitoring
+ cqm_mbm_local : 1, // LLC Local MBM monitoring
+ fence_swapgs_user : 1, // LFENCE in user entry SWAPGS path
+ fence_swapgs_kernel : 1, // LFENCE in kernel entry SWAPGS path
+ split_lock_detect : 1, // #AC for split lock
+ per_thread_mba : 1, // Per-thread Memory Bandwidth Allocation
+ sgx1 : 1, // SGX Basic
+ sgx2 : 1, // SGX Enclave Dynamic Memory Management (EDMM)
+ entry_ibpb : 1, // Issue an IBPB on kernel entry
+ rrsba_ctrl : 1, // RET prediction control
+ retpoline : 1, // Generic Retpoline mitigation for Spectre variant 2
+ retpoline_lfence : 1, // Use LFENCE for Spectre variant 2
+ rethunk : 1, // Use Return THUNK
+ unret : 1, // AMD BTB untrain return
+ use_ibpb_fw : 1, // Use IBPB during runtime firmware calls
+ rsb_vmexit_lite : 1, // Fill RSB on VM exit when EIBRS is enabled
+ sgx_edeccssa : 1, // SGX EDECCSSA user leaf function
+ call_depth : 1, // Call depth tracking for RSB stuffing
+ msr_tsx_ctrl : 1, // MSR IA32_TSX_CTRL (Intel) implemented
+ smba : 1, // Slow Memory Bandwidth Allocation
+ bmec : 1, // Bandwidth Monitoring Event Configuration
+ user_shstk : 1, // Shadow stack support for user mode applications
+ srso : 1, // AMD BTB untrain RETs
+ srso_alias : 1, // AMD BTB untrain RETs through aliasing
+ ibpb_on_vmexit : 1, // Issue an IBPB only on VMEXIT
+ apic_msrs_fence : 1, // IA32_TSC_DEADLINE and X2APIC MSRs need fencing
+ zen2 : 1, // CPU based on Zen2 microarchitecture
+ zen3 : 1, // CPU based on Zen3 microarchitecture
+ zen4 : 1, // CPU based on Zen4 microarchitecture
+ zen1 : 1; // CPU based on Zen1 microarchitecture
+};
+
+struct leaf_0x4c780001_1 {
+ // eax
+ u32 overflow_recov : 1, // MCA overflow recovery support
+ succor : 1, // Uncorrectable error containment and recovery
+ : 1, // Reserved
+ smca : 1, // Scalable MCA
+ : 28; // Reserved
+ // ebx
+ u32 amd_lbr_pmc_freeze : 1, // AMD LBR and PMC Freeze
+ clear_bhb_loop : 1, // Clear branch history at SYSCALL entry using SW loop
+ bhi_ctrl : 1, // BHI_DIS_S HW control available
+ clear_bhb_hw : 1, // BHI_DIS_S HW control enabled
+ clear_bhb_vmexit : 1, // Clear branch history at VMEXIT using SW loop
+ amd_fast_cppc : 1, // AMD fast Collaborative Processor Performance Control
+ amd_htr_cores : 1, // Heterogeneous Core Topology
+ amd_workload_class : 1, // Workload Classification
+ prefer_ymm : 1, // Avoid ZMM registers due to downclocking
+ apx : 1, // Advanced Performance Extensions
+ indirect_thunk_its : 1, // Use thunk for indirect branches in lower half of cache line
+ tsa_sq_no : 1, // AMD CPU not vulnerable to TSA-SQ
+ tsa_l1_no : 1, // AMD CPU not vulnerable to TSA-L1
+ clear_cpu_buf_vm : 1, // Clear CPU buffers using VERW before VMRUN
+ ibpb_exit_to_user : 1, // Use IBPB on exit-to-userspace, see VMSCAPE bug
+ : 17; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x4c780002
+ * Linux-defined synthetic CPU bug flags
+ */
+
+struct leaf_0x4c780002_0 {
+ // eax
+ u32 f00f : 1, // Intel F00F
+ fdiv : 1, // FPU FDIV
+ coma : 1, // Cyrix 6x86 coma
+ amd_tlb_mmatch : 1, // AMD Erratum 383
+ amd_apic_c1e : 1, // AMD Erratum 400
+ bug_11ap : 1, // Bad local APIC aka 11AP
+ fxsave_leak : 1, // FXSAVE leaks FOP/FIP/FOP
+ clflush_monitor : 1, // AAI65, CLFLUSH required before MONITOR
+ sysret_ss_attrs : 1, // SYSRET does not fix up SS attributes
+ espfix : 1, // IRET to 16-bit SS corrupts ESP/RSP high bits (x86-32)
+ null_seg : 1, // Setting a selector to NULL preserves the base
+ swapgs_fence : 1, // SWAPGS without input dep on GS
+ monitor : 1, // IPI required to wake up remote CPU
+ amd_e400 : 1, // CPU is among the affected by Erratum 400
+ cpu_meltdown : 1, // CPU affected by meltdown; needs kernel page table isolation
+ spectre_v1 : 1, // CPU affected by Spectre variant 1 with conditional branches
+ spectre_v2 : 1, // CPU affected by Spectre variant 2 with indirect branches
+ spec_store_bypass : 1, // CPU affected by speculative store bypass attack
+ l1tf : 1, // CPU affected by L1 Terminal Fault
+ mds : 1, // CPU affected by Microarchitectural data sampling
+ msbds_only : 1, // Microarchitectural data sampling: CPU only affected by the MSBDS variant
+ swapgs : 1, // CPU affected by speculation through SWAPGS
+ taa : 1, // CPU is affected by TSX Async Abort (TAA)
+ itlb_multihit : 1, // CPU may incur MCE during certain page attribute changes
+ srbds : 1, // CPU may leak RNG bits if not mitigated
+ mmio_stale_data : 1, // CPU affected by Processor MMIO Stale Data vulnerabilities
+ : 1, // Reserved
+ retbleed : 1, // CPU affected by Retbleed
+ eibrs_pbrsb : 1, // EIBRS is vulnerable to Post Barrier RSB Predictions
+ smt_rsb : 1, // CPU vulnerable to Cross-Thread Return Address Predictions
+ gds : 1, // CPU affected by Gather Data Sampling
+ tdx_pw_mce : 1; // CPU may incur #MC if non-TD software does partial write to TDX private memory
+ // ebx
+ u32 srso : 1, // AMD SRSO bug
+ div0 : 1, // AMD DIV0 speculation bug
+ rfds : 1, // CPU vulnerable to Register File Data Sampling
+ bhi : 1, // CPU affected by Branch History Injection
+ ibpb_no_ret : 1, // IBPB omits return target predictions
+ spectre_v2_user : 1, // CPU affected by Spectre variant 2 between user processes
+ old_microcode : 1, // CPU has old microcode; it must be vulnerable to something
+ its : 1, // CPU affected by Indirect Target Selection
+ its_native_only : 1, // CPU affected by ITS; VMX is not affected
+ tsa : 1, // CPU affected by Transient Scheduler Attacks
+ vmscape : 1, // CPU affected by VMSCAPE attacks from guests
+ : 21; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000000
+ * Maximum extended leaf + CPU vendor string
+ */
+
+struct leaf_0x80000000_0 {
+ // eax
+ u32 max_ext_leaf : 32; // Maximum extended CPUID leaf
+ // ebx
+ u32 cpu_vendorid_0 : 32; // Vendor ID string bytes 0 - 3
+ // ecx
+ u32 cpu_vendorid_2 : 32; // Vendor ID string bytes 8 - 11
+ // edx
+ u32 cpu_vendorid_1 : 32; // Vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x80000001
+ * Extended CPU features
+ */
+
+struct leaf_0x80000001_0 {
+ // eax
+ u32 e_stepping_id : 4, // Stepping ID
+ e_base_model : 4, // Base processor model
+ e_base_family : 4, // Base processor family
+ e_base_type : 2, // Base processor type (Transmeta)
+ : 2, // Reserved
+ e_ext_model : 4, // Extended processor model
+ e_ext_family : 8, // Extended processor family
+ : 4; // Reserved
+ // ebx
+ u32 brand_id : 16, // Brand ID
+ : 12, // Reserved
+ pkg_type : 4; // Package type
+ // ecx
+ u32 lahf_lm : 1, // LAHF and SAHF in 64-bit mode
+ cmp_legacy : 1, // Multi-processing legacy mode (No HT)
+ svm : 1, // Secure Virtual Machine
+ extapic : 1, // Extended APIC space
+ cr8_legacy : 1, // LOCK MOV CR0 means MOV CR8
+ lzcnt_abm : 1, // LZCNT advanced bit manipulation
+ sse4a : 1, // SSE4A support
+ misaligned_sse : 1, // Misaligned SSE mode
+ _3dnow_prefetch : 1, // 3DNow PREFETCH/PREFETCHW support
+ osvw : 1, // OS visible workaround
+ ibs : 1, // Instruction based sampling
+ xop : 1, // XOP: extended operation (AVX instructions)
+ skinit : 1, // SKINIT/STGI support
+ wdt : 1, // Watchdog timer support
+ : 1, // Reserved
+ lwp : 1, // Lightweight profiling
+ fma4 : 1, // 4-operand FMA instruction
+ tce : 1, // Translation cache extension
+ : 1, // Reserved
+ nodeid_msr : 1, // NodeId MSR (0xc001100c)
+ : 1, // Reserved
+ tbm : 1, // Trailing bit manipulations
+ topoext : 1, // Topology Extensions (leaf 0x8000001d)
+ perfctr_core : 1, // Core performance counter extensions
+ perfctr_nb : 1, // NB/DF performance counter extensions
+ : 1, // Reserved
+ data_bp_ext : 1, // Data access breakpoint extension
+ perf_tsc : 1, // Performance time-stamp counter
+ perfctr_llc : 1, // LLC (L3) performance counter extensions
+ mwaitx : 1, // MWAITX/MONITORX support
+ addr_mask_ext : 1, // Breakpoint address mask extension (to bit 31)
+ : 1; // Reserved
+ // edx
+ u32 e_fpu : 1, // Floating-Point Unit on-chip (x87)
+ e_vme : 1, // Virtual-8086 Mode Extensions
+ e_de : 1, // Debugging Extensions
+ e_pse : 1, // Page Size Extension
+ e_tsc : 1, // Time Stamp Counter
+ e_msr : 1, // Model-Specific Registers (RDMSR and WRMSR support)
+ pae : 1, // Physical Address Extensions
+ mce : 1, // Machine Check Exception
+ cx8 : 1, // CMPXCHG8B instruction
+ apic : 1, // APIC on-chip
+ : 1, // Reserved
+ syscall : 1, // SYSCALL and SYSRET instructions
+ mtrr : 1, // Memory Type Range Registers
+ pge : 1, // Page Global Extensions
+ mca : 1, // Machine Check Architecture
+ cmov : 1, // Conditional Move Instruction
+ pat : 1, // Page Attribute Table
+ pse36 : 1, // Page Size Extension (36-bit)
+ : 1, // Reserved
+ obsolete_mp_bit : 1, // Out-of-spec AMD Multiprocessing bit
+ nx : 1, // No-execute page protection
+ : 1, // Reserved
+ mmxext : 1, // AMD MMX extensions
+ e_mmx : 1, // MMX instructions
+ e_fxsr : 1, // FXSAVE and FXRSTOR instructions
+ fxsr_opt : 1, // FXSAVE and FXRSTOR optimizations
+ page1gb : 1, // 1-GB large page support
+ rdtscp : 1, // RDTSCP instruction
+ : 1, // Reserved
+ lm : 1, // Long mode (x86-64, 64-bit support)
+ _3dnowext : 1, // AMD 3DNow extensions
+ _3dnow : 1; // 3DNow instructions
+};
+
+/*
+ * Leaf 0x80000002
+ * CPU brand ID string, bytes 0 - 15
+ */
+
+struct leaf_0x80000002_0 {
+ // eax
+ u32 cpu_brandid_0 : 32; // CPU brand ID string, bytes 0 - 3
+ // ebx
+ u32 cpu_brandid_1 : 32; // CPU brand ID string, bytes 4 - 7
+ // ecx
+ u32 cpu_brandid_2 : 32; // CPU brand ID string, bytes 8 - 11
+ // edx
+ u32 cpu_brandid_3 : 32; // CPU brand ID string, bytes 12 - 15
+};
+
+/*
+ * Leaf 0x80000003
+ * CPU brand ID string, bytes 16 - 31
+ */
+
+struct leaf_0x80000003_0 {
+ // eax
+ u32 cpu_brandid_4 : 32; // CPU brand ID string bytes, 16 - 19
+ // ebx
+ u32 cpu_brandid_5 : 32; // CPU brand ID string bytes, 20 - 23
+ // ecx
+ u32 cpu_brandid_6 : 32; // CPU brand ID string bytes, 24 - 27
+ // edx
+ u32 cpu_brandid_7 : 32; // CPU brand ID string bytes, 28 - 31
+};
+
+/*
+ * Leaf 0x80000004
+ * CPU brand ID string, bytes 32 - 47
+ */
+
+struct leaf_0x80000004_0 {
+ // eax
+ u32 cpu_brandid_8 : 32; // CPU brand ID string, bytes 32 - 35
+ // ebx
+ u32 cpu_brandid_9 : 32; // CPU brand ID string, bytes 36 - 39
+ // ecx
+ u32 cpu_brandid_10 : 32; // CPU brand ID string, bytes 40 - 43
+ // edx
+ u32 cpu_brandid_11 : 32; // CPU brand ID string, bytes 44 - 47
+};
+
+/*
+ * Leaf 0x80000005
+ * AMD/Transmeta L1 cache and TLB
+ */
+
+struct leaf_0x80000005_0 {
+ // eax
+ u32 l1_itlb_2m_4m_nentries : 8, // L1 ITLB #entries, 2M and 4M pages
+ l1_itlb_2m_4m_assoc : 8, // L1 ITLB associativity, 2M and 4M pages
+ l1_dtlb_2m_4m_nentries : 8, // L1 DTLB #entries, 2M and 4M pages
+ l1_dtlb_2m_4m_assoc : 8; // L1 DTLB associativity, 2M and 4M pages
+ // ebx
+ u32 l1_itlb_4k_nentries : 8, // L1 ITLB #entries, 4K pages
+ l1_itlb_4k_assoc : 8, // L1 ITLB associativity, 4K pages
+ l1_dtlb_4k_nentries : 8, // L1 DTLB #entries, 4K pages
+ l1_dtlb_4k_assoc : 8; // L1 DTLB associativity, 4K pages
+ // ecx
+ u32 l1_dcache_line_size : 8, // L1 dcache line size, in bytes
+ l1_dcache_nlines : 8, // L1 dcache lines per tag
+ l1_dcache_assoc : 8, // L1 dcache associativity
+ l1_dcache_size_kb : 8; // L1 dcache size, in KB
+ // edx
+ u32 l1_icache_line_size : 8, // L1 icache line size, in bytes
+ l1_icache_nlines : 8, // L1 icache lines per tag
+ l1_icache_assoc : 8, // L1 icache associativity
+ l1_icache_size_kb : 8; // L1 icache size, in KB
+};
+
+/*
+ * Leaf 0x80000006
+ * (Mostly AMD) L2/L3 cache and TLB
+ */
+
+struct leaf_0x80000006_0 {
+ // eax
+ u32 l2_itlb_2m_4m_nentries : 12, // L2 iTLB #entries, 2M and 4M pages
+ l2_itlb_2m_4m_assoc : 4, // L2 iTLB associativity, 2M and 4M pages
+ l2_dtlb_2m_4m_nentries : 12, // L2 dTLB #entries, 2M and 4M pages
+ l2_dtlb_2m_4m_assoc : 4; // L2 dTLB associativity, 2M and 4M pages
+ // ebx
+ u32 l2_itlb_4k_nentries : 12, // L2 iTLB #entries, 4K pages
+ l2_itlb_4k_assoc : 4, // L2 iTLB associativity, 4K pages
+ l2_dtlb_4k_nentries : 12, // L2 dTLB #entries, 4K pages
+ l2_dtlb_4k_assoc : 4; // L2 dTLB associativity, 4K pages
+ // ecx
+ u32 l2_line_size : 8, // L2 cache line size, in bytes
+ l2_nlines : 4, // L2 cache number of lines per tag
+ l2_assoc : 4, // L2 cache associativity
+ l2_size_kb : 16; // L2 cache size, in KB
+ // edx
+ u32 l3_line_size : 8, // L3 cache line size, in bytes
+ l3_nlines : 4, // L3 cache number of lines per tag
+ l3_assoc : 4, // L3 cache associativity
+ : 2, // Reserved
+ l3_size_range : 14; // L3 cache size range
+};
+
+/*
+ * Leaf 0x80000007
+ * CPU power management (mostly AMD) and AMD RAS
+ */
+
+struct leaf_0x80000007_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 mca_overflow_recovery : 1, // MCA overflow conditions not fatal
+ succor : 1, // Software containment of uncorrectable errors
+ hw_assert : 1, // Hardware assert MSRs
+ scalable_mca : 1, // Scalable MCA (MCAX MSRs)
+ : 28; // Reserved
+ // ecx
+ u32 cpu_pwr_sample_ratio : 32; // CPU power sample time ratio
+ // edx
+ u32 digital_temp : 1, // Digital temperature sensor
+ powernow_freq_id : 1, // PowerNOW! frequency scaling
+ powernow_volt_id : 1, // PowerNOW! voltage scaling
+ thermal_trip : 1, // THERMTRIP (Thermal Trip)
+ hw_thermal_control : 1, // Hardware thermal control
+ sw_thermal_control : 1, // Software thermal control
+ _100mhz_steps : 1, // 100 MHz multiplier control
+ hw_pstate : 1, // Hardware P-state control
+ constant_tsc : 1, // TSC ticks at constant rate across all P and C states
+ core_perf_boost : 1, // Core performance boost
+ eff_freq_ro : 1, // Read-only effective frequency interface
+ proc_feedback : 1, // Processor feedback interface (deprecated)
+ proc_power_reporting : 1, // Processor power reporting interface
+ connected_standby : 1, // CPU Connected Standby support
+ rapl_interface : 1, // Runtime Average Power Limit interface
+ : 17; // Reserved
+};
+
+/*
+ * Leaf 0x80000008
+ * CPU capacity parameters and extended feature flags (mostly AMD)
+ */
+
+struct leaf_0x80000008_0 {
+ // eax
+ u32 phys_addr_bits : 8, // Max physical address bits
+ virt_addr_bits : 8, // Max virtual address bits
+ guest_phys_addr_bits : 8, // Max nested-paging guest physical address bits
+ : 8; // Reserved
+ // ebx
+ u32 clzero : 1, // CLZERO instruction
+ insn_retired_perf : 1, // Instruction retired counter MSR
+ xsave_err_ptr : 1, // XSAVE/XRSTOR always saves/restores FPU error pointers
+ invlpgb : 1, // INVLPGB broadcasts a TLB invalidate
+ rdpru : 1, // RDPRU (Read Processor Register at User level)
+ : 1, // Reserved
+ mba : 1, // Memory Bandwidth Allocation (AMD bit)
+ : 1, // Reserved
+ mcommit : 1, // MCOMMIT instruction
+ wbnoinvd : 1, // WBNOINVD instruction
+ : 2, // Reserved
+ ibpb : 1, // Indirect Branch Prediction Barrier
+ wbinvd_int : 1, // Interruptible WBINVD/WBNOINVD
+ ibrs : 1, // Indirect Branch Restricted Speculation
+ stibp : 1, // Single Thread Indirect Branch Prediction mode
+ ibrs_always_on : 1, // IBRS always-on preferred
+ stibp_always_on : 1, // STIBP always-on preferred
+ ibrs_fast : 1, // IBRS is preferred over software solution
+ ibrs_same_mode : 1, // IBRS provides same mode protection
+ no_efer_lmsle : 1, // Long-Mode Segment Limit Enable unsupported
+ tlb_flush_nested : 1, // INVLPGB RAX[5] bit can be set
+ : 1, // Reserved
+ amd_ppin : 1, // Protected Processor Inventory Number
+ amd_ssbd : 1, // Speculative Store Bypass Disable
+ virt_ssbd : 1, // virtualized SSBD (Speculative Store Bypass Disable)
+ amd_ssb_no : 1, // SSBD is not needed (fixed in hardware)
+ cppc : 1, // Collaborative Processor Performance Control
+ amd_psfd : 1, // Predictive Store Forward Disable
+ btc_no : 1, // CPU not affected by Branch Type Confusion
+ ibpb_ret : 1, // IBPB clears RSB/RAS too
+ branch_sampling : 1; // Branch Sampling
+ // ecx
+ u32 cpu_nthreads : 8, // Number of physical threads - 1
+ : 4, // Reserved
+ apicid_coreid_len : 4, // Number of thread core ID bits (shift) in APIC ID
+ perf_tsc_len : 2, // Performance time-stamp counter size
+ : 14; // Reserved
+ // edx
+ u32 invlpgb_max_pages : 16, // INVLPGB maximum page count
+ rdpru_max_reg_id : 16; // RDPRU max register ID (ECX input)
+};
+
+/*
+ * Leaf 0x8000000a
+ * AMD SVM (Secure Virtual Machine)
+ */
+
+struct leaf_0x8000000a_0 {
+ // eax
+ u32 svm_version : 8, // SVM revision number
+ : 24; // Reserved
+ // ebx
+ u32 svm_nasid : 32; // Number of address space identifiers (ASID)
+ // ecx
+ u32 : 4, // Reserved
+ pml : 1, // Page Modification Logging (PML)
+ : 27; // Reserved
+ // edx
+ u32 nested_pt : 1, // Nested paging
+ lbr_virt : 1, // LBR virtualization
+ svm_lock : 1, // SVM lock
+ nrip_save : 1, // NRIP save support on #VMEXIT
+ tsc_rate_msr : 1, // MSR-based TSC rate control
+ vmcb_clean : 1, // VMCB clean bits support
+ flush_by_asid : 1, // Flush by ASID + Extended VMCB TLB_Control
+ decode_assists : 1, // Decode Assists support
+ : 2, // Reserved
+ pause_filter : 1, // Pause intercept filter
+ : 1, // Reserved
+ pf_threshold : 1, // Pause filter threshold
+ avic : 1, // Advanced virtual interrupt controller
+ : 1, // Reserved
+ v_vmsave_vmload : 1, // Virtual VMSAVE/VMLOAD (nested virtualization)
+ v_gif : 1, // Virtualize the Global Interrupt Flag
+ gmet : 1, // Guest mode execution trap
+ x2avic : 1, // Virtual x2APIC
+ sss_check : 1, // Supervisor Shadow Stack restrictions
+ v_spec_ctrl : 1, // Virtual SPEC_CTRL
+ ro_gpt : 1, // Read-Only guest page table support
+ : 1, // Reserved
+ h_mce_override : 1, // Host MCE override
+ tlbsync_int : 1, // TLBSYNC intercept + INVLPGB/TLBSYNC in VMCB
+ nmi_virt : 1, // NMI virtualization
+ ibs_virt : 1, // IBS Virtualization
+ ext_lvt_off_chg : 1, // Extended LVT offset fault change
+ svme_addr_chk : 1, // Guest SVME address check
+ : 3; // Reserved
+};
+
+/*
+ * Leaf 0x80000019
+ * AMD TLB characteristics for 1GB pages
+ */
+
+struct leaf_0x80000019_0 {
+ // eax
+ u32 l1_itlb_1g_nentries : 12, // L1 iTLB #entries, 1G pages
+ l1_itlb_1g_assoc : 4, // L1 iTLB associativity, 1G pages
+ l1_dtlb_1g_nentries : 12, // L1 dTLB #entries, 1G pages
+ l1_dtlb_1g_assoc : 4; // L1 dTLB associativity, 1G pages
+ // ebx
+ u32 l2_itlb_1g_nentries : 12, // L2 iTLB #entries, 1G pages
+ l2_itlb_1g_assoc : 4, // L2 iTLB associativity, 1G pages
+ l2_dtlb_1g_nentries : 12, // L2 dTLB #entries, 1G pages
+ l2_dtlb_1g_assoc : 4; // L2 dTLB associativity, 1G pages
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001a
+ * AMD instruction optimizations
+ */
+
+struct leaf_0x8000001a_0 {
+ // eax
+ u32 fp_128 : 1, // Internal FP/SIMD exec data path is 128-bits wide
+ movu_preferred : 1, // SSE: MOVU* better than MOVL*/MOVH*
+ fp_256 : 1, // Internal FP/SSE exec data path is 256-bits wide
+ : 29; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001b
+ * AMD IBS (Instruction-Based Sampling)
+ */
+
+struct leaf_0x8000001b_0 {
+ // eax
+ u32 ibs_flags : 1, // IBS feature flags
+ ibs_fetch_sampling : 1, // IBS fetch sampling
+ ibs_op_sampling : 1, // IBS execution sampling
+ ibs_rdwr_op_counter : 1, // IBS read/write of op counter
+ ibs_op_count : 1, // IBS OP counting mode
+ ibs_branch_target : 1, // IBS branch target address reporting
+ ibs_op_counters_ext : 1, // IBS IbsOpCurCnt/IbsOpMaxCnt extend by 7 bits
+ ibs_rip_invalid_chk : 1, // IBS invalid RIP indication
+ ibs_op_branch_fuse : 1, // IBS fused branch micro-op indication
+ ibs_fetch_ctl_ext : 1, // IBS Fetch Control Extended MSR
+ ibs_op_data_4 : 1, // IBS op data 4 MSR
+ ibs_l3_miss_filter : 1, // IBS L3-miss filtering (Zen4+)
+ : 20; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001c
+ * AMD LWP (Lightweight Profiling)
+ */
+
+struct leaf_0x8000001c_0 {
+ // eax
+ u32 os_lwp_avail : 1, // OS: LWP is available to application programs
+ os_lwpval : 1, // OS: LWPVAL instruction
+ os_lwp_ire : 1, // OS: Instructions Retired Event
+ os_lwp_bre : 1, // OS: Branch Retired Event
+ os_lwp_dme : 1, // OS: Dcache Miss Event
+ os_lwp_cnh : 1, // OS: CPU Clocks Not Halted event
+ os_lwp_rnh : 1, // OS: CPU Reference clocks Not Halted event
+ : 22, // Reserved
+ os_lwp_cont : 1, // OS: LWP sampling in continuous mode
+ os_lwp_ptsc : 1, // OS: Performance Time Stamp Counter in event records
+ os_lwp_int : 1; // OS: Interrupt on threshold overflow
+ // ebx
+ u32 lwp_lwpcb_sz : 8, // Control Block size, in quadwords
+ lwp_event_sz : 8, // Event record size, in bytes
+ lwp_max_events : 8, // Max EventID supported
+ lwp_event_offset : 8; // Control Block events area offset
+ // ecx
+ u32 lwp_latency_max : 5, // Cache latency counters number of bits
+ lwp_data_addr : 1, // Cache miss events report data cache address
+ lwp_latency_rnd : 3, // Cache latency rounding amount
+ lwp_version : 7, // LWP version
+ lwp_buf_min_sz : 8, // LWP event ring buffer min size, 32 event record units
+ : 4, // Reserved
+ lwp_branch_predict : 1, // Branches Retired events can be filtered
+ lwp_ip_filtering : 1, // IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP)
+ lwp_cache_levels : 1, // Cache-related events: filter by cache level
+ lwp_cache_latency : 1; // Cache-related events: filter by latency
+ // edx
+ u32 hw_lwp_avail : 1, // HW: LWP available
+ hw_lwpval : 1, // HW: LWPVAL available
+ hw_lwp_ire : 1, // HW: Instructions Retired Event
+ hw_lwp_bre : 1, // HW: Branch Retired Event
+ hw_lwp_dme : 1, // HW: Dcache Miss Event
+ hw_lwp_cnh : 1, // HW: Clocks Not Halted event
+ hw_lwp_rnh : 1, // HW: Reference clocks Not Halted event
+ : 22, // Reserved
+ hw_lwp_cont : 1, // HW: LWP sampling in continuous mode
+ hw_lwp_ptsc : 1, // HW: Performance Time Stamp Counter in event records
+ hw_lwp_int : 1; // HW: Interrupt on threshold overflow
+};
+
+/*
+ * Leaf 0x8000001d
+ * AMD deterministic cache parameters
+ */
+
+struct leaf_0x8000001d_n {
+ // eax
+ u32 cache_type : 5, // Cache type field
+ cache_level : 3, // Cache level (1-based)
+ cache_self_init : 1, // Self-initializing cache level
+ fully_associative : 1, // Fully-associative cache
+ : 4, // Reserved
+ num_threads_sharing : 12, // Number of logical CPUs sharing cache
+ : 6; // Reserved
+ // ebx
+ u32 cache_linesize : 12, // System coherency line size (0-based)
+ cache_npartitions : 10, // Physical line partitions (0-based)
+ cache_nways : 10; // Ways of associativity (0-based)
+ // ecx
+ u32 cache_nsets : 31, // Cache number of sets (0-based)
+ : 1; // Reserved
+ // edx
+ u32 wbinvd_rll_no_guarantee : 1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches
+ ll_inclusive : 1, // Cache is inclusive of Lower-Level caches
+ : 30; // Reserved
+};
+
+#define LEAF_0x8000001d_SUBLEAF_N_FIRST 0
+#define LEAF_0x8000001d_SUBLEAF_N_LAST 31
+
+/*
+ * Leaf 0x8000001e
+ * AMD CPU topology
+ */
+
+struct leaf_0x8000001e_0 {
+ // eax
+ u32 ext_apic_id : 32; // Extended APIC ID
+ // ebx
+ u32 core_id : 8, // Unique per-socket logical core unit ID
+ core_nthreads : 8, // #Threads per core (zero-based)
+ : 16; // Reserved
+ // ecx
+ u32 node_id : 8, // Node (die) ID of invoking logical CPU
+ nnodes_per_socket : 3, // #nodes in invoking logical CPU's package/socket
+ : 21; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001f
+ * AMD encrypted memory capabilities (SME/SEV)
+ */
+
+struct leaf_0x8000001f_0 {
+ // eax
+ u32 sme : 1, // Secure Memory Encryption
+ sev : 1, // Secure Encrypted Virtualization
+ vm_page_flush : 1, // VM Page Flush MSR
+ sev_encrypted_state : 1, // SEV Encrypted State
+ sev_nested_paging : 1, // SEV secure nested paging
+ vm_permission_levels : 1, // VMPL
+ rpmquery : 1, // RPMQUERY instruction
+ vmpl_sss : 1, // VMPL supervisor shadow stack
+ secure_tsc : 1, // Secure TSC
+ virt_tsc_aux : 1, // Hardware virtualizes TSC_AUX
+ sme_coherent : 1, // Cache coherency enforcement across encryption domains
+ req_64bit_hypervisor : 1, // SEV guest mandates 64-bit hypervisor
+ restricted_injection : 1, // Restricted Injection supported
+ alternate_injection : 1, // Alternate Injection supported
+ debug_swap : 1, // SEV-ES: Full debug state swap
+ disallow_host_ibs : 1, // SEV-ES: Disallowing IBS use by the host
+ virt_transparent_enc : 1, // Virtual Transparent Encryption
+ vmgexit_parameter : 1, // SEV_FEATURES: VmgexitParameter
+ virt_tom_msr : 1, // Virtual TOM MSR
+ virt_ibs : 1, // SEV-ES guests: IBS state virtualization
+ : 4, // Reserved
+ vmsa_reg_protection : 1, // VMSA register protection
+ smt_protection : 1, // SMT protection
+ : 2, // Reserved
+ svsm_page_msr : 1, // SVSM communication page MSR
+ nested_virt_snp_msr : 1, // VIRT_RMPUPDATE/VIRT_PSMASH MSRs
+ : 2; // Reserved
+ // ebx
+ u32 pte_cbit_pos : 6, // PTE bit number to enable memory encryption
+ phys_addr_reduction_nbits : 6, // Reduction of phys address space in bits
+ vmpl_count : 4, // Number of VM permission levels (VMPL)
+ : 16; // Reserved
+ // ecx
+ u32 enc_guests_max : 32; // Max number of simultaneous encrypted guests
+ // edx
+ u32 min_sev_asid_no_sev_es : 32; // Minimum ASID for SEV-enabled SEV-ES-disabled guest
+};
+
+/*
+ * Leaf 0x80000020
+ * AMD PQoS (Platform QoS) extended features
+ */
+
+struct leaf_0x80000020_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 : 1, // Reserved
+ mba : 1, // Memory Bandwidth Allocation support
+ smba : 1, // Slow Memory Bandwidth Allocation support
+ bmec : 1, // Bandwidth Monitoring Event Configuration support
+ l3rr : 1, // L3 Range Reservation support
+ abmc : 1, // Assignable Bandwidth Monitoring Counters
+ sdciae : 1, // Smart Data Cache Injection (SDCI) Allocation Enforcement
+ : 25; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+struct leaf_0x80000020_1 {
+ // eax
+ u32 mba_limit_len : 32; // MBA enforcement limit size
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 mba_cos_max : 32; // MBA max Class of Service number (zero-based)
+};
+
+struct leaf_0x80000020_2 {
+ // eax
+ u32 smba_limit_len : 32; // SMBA enforcement limit size
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 smba_cos_max : 32; // SMBA max Class of Service number (zero-based)
+};
+
+struct leaf_0x80000020_3 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 bmec_num_events : 8, // BMEC number of bandwidth events available
+ : 24; // Reserved
+ // ecx
+ u32 bmec_local_reads : 1, // Local NUMA reads can be tracked
+ bmec_remote_reads : 1, // Remote NUMA reads can be tracked
+ bmec_local_nontemp_wr : 1, // Local NUMA non-temporal writes can be tracked
+ bmec_remote_nontemp_wr : 1, // Remote NUMA non-temporal writes can be tracked
+ bmec_local_slow_mem_rd : 1, // Local NUMA slow-memory reads can be tracked
+ bmec_remote_slow_mem_rd : 1, // Remote NUMA slow-memory reads can be tracked
+ bmec_all_dirty_victims : 1, // Dirty QoS victims to all types of memory can be tracked
+ : 25; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000021
+ * AMD extended CPU features 2
+ */
+
+struct leaf_0x80000021_0 {
+ // eax
+ u32 no_nested_data_bp : 1, // No nested data breakpoints
+ fsgs_non_serializing : 1, // WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing
+ lfence_serializing : 1, // LFENCE always serializing / synchronizes RDTSC
+ smm_page_cfg_lock : 1, // SMM paging configuration lock
+ : 2, // Reserved
+ null_sel_clr_base : 1, // Null selector clears base
+ upper_addr_ignore : 1, // EFER MSR Upper Address Ignore
+ auto_ibrs : 1, // EFER MSR Automatic IBRS
+ no_smm_ctl_msr : 1, // SMM_CTL MSR not available
+ fsrs : 1, // Fast Short REP STOSB
+ fsrc : 1, // Fast Short REP CMPSB
+ : 1, // Reserved
+ prefetch_ctl_msr : 1, // Prefetch control MSR
+ : 2, // Reserved
+ opcode_reclaim : 1, // Reserves opcode space
+ user_cpuid_disable : 1, // #GP when executing CPUID at CPL > 0
+ epsf : 1, // Enhanced Predictive Store Forwarding
+ : 3, // Reserved
+ wl_feedback : 1, // Workload-based heuristic feedback to OS
+ : 1, // Reserved
+ eraps : 1, // Enhanced Return Address Predictor Security
+ : 2, // Reserved
+ sbpb : 1, // Selective Branch Predictor Barrier
+ ibpb_brtype : 1, // Branch predictions flushed from CPU branch predictor
+ srso_no : 1, // No SRSO vulnerability
+ srso_uk_no : 1, // No SRSO at user-kernel boundary
+ srso_msr_fix : 1; // MSR BP_CFG[BpSpecReduce] SRSO mitigation
+ // ebx
+ u32 microcode_patch_size : 16, // Microcode patch size, in 16-byte units
+ rap_size : 8, // Return Address Predictor size
+ : 8; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000022
+ * AMD extended performance monitoring
+ */
+
+struct leaf_0x80000022_0 {
+ // eax
+ u32 perfmon_v2 : 1, // Performance monitoring v2
+ lbr_v2 : 1, // Last Branch Record v2 extensions (LBR Stack)
+ lbr_pmc_freeze : 1, // Freezing core performance counters / LBR Stack
+ : 29; // Reserved
+ // ebx
+ u32 n_pmc_core : 4, // Number of core performance counters
+ lbr_v2_stack_size : 6, // Number of LBR stack entries
+ n_pmc_northbridge : 6, // Number of northbridge performance counters
+ n_pmc_umc : 6, // Number of UMC performance counters
+ : 10; // Reserved
+ // ecx
+ u32 active_umc_bitmask : 32; // Active UMCs bitmask
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000023
+ * AMD multi-key encrypted memory
+ */
+
+struct leaf_0x80000023_0 {
+ // eax
+ u32 mem_hmk_mode : 1, // MEM-HMK encryption mode
+ : 31; // Reserved
+ // ebx
+ u32 mem_hmk_avail_keys : 16, // Total number of available encryption keys
+ : 16; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000026
+ * AMD extended CPU topology
+ */
+
+struct leaf_0x80000026_n {
+ // eax
+ u32 x2apic_id_shift : 5, // Bit width of this level (previous levels inclusive)
+ : 24, // Reserved
+ core_has_pwreff_ranking : 1, // This core has a power efficiency ranking
+ domain_has_hybrid_cores : 1, // This domain level has hybrid (E, P) cores
+ domain_core_count_asymm : 1; // The 'Core' domain has asymmetric cores count
+ // ebx
+ u32 domain_lcpus_count : 16, // Number of logical CPUs at this domain instance
+ core_pwreff_ranking : 8, // This core's static power efficiency ranking
+ core_native_model_id : 4, // This core's native model ID
+ core_type : 4; // This core's type
+ // ecx
+ u32 domain_level : 8, // This domain level (subleaf ID)
+ domain_type : 8, // This domain type
+ : 16; // Reserved
+ // edx
+ u32 x2apic_id : 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0x80000026_SUBLEAF_N_FIRST 0
+#define LEAF_0x80000026_SUBLEAF_N_LAST 3
+
+/*
+ * Leaf 0x80860000
+ * Maximum Transmeta leaf + CPU vendor string
+ */
+
+struct leaf_0x80860000_0 {
+ // eax
+ u32 max_tra_leaf : 32; // Maximum Transmeta leaf
+ // ebx
+ u32 cpu_vendorid_0 : 32; // Transmeta vendor ID string bytes 0 - 3
+ // ecx
+ u32 cpu_vendorid_2 : 32; // Transmeta vendor ID string bytes 8 - 11
+ // edx
+ u32 cpu_vendorid_1 : 32; // Transmeta vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x80860001
+ * Transmeta extended CPU features
+ */
+
+struct leaf_0x80860001_0 {
+ // eax
+ u32 stepping : 4, // Stepping ID
+ base_model : 4, // Base CPU model ID
+ base_family_id : 4, // Base CPU family ID
+ cpu_type : 2, // CPU type
+ : 18; // Reserved
+ // ebx
+ u32 cpu_rev_mask_minor : 8, // CPU revision ID, mask minor
+ cpu_rev_mask_major : 8, // CPU revision ID, mask major
+ cpu_rev_minor : 8, // CPU revision ID, minor
+ cpu_rev_major : 8; // CPU revision ID, major
+ // ecx
+ u32 cpu_base_mhz : 32; // CPU nominal frequency, in MHz
+ // edx
+ u32 recovery : 1, // Recovery CMS is active (after bad flush)
+ longrun : 1, // LongRun power management capabilities
+ : 1, // Reserved
+ lrti : 1, // LongRun Table Interface
+ : 28; // Reserved
+};
+
+/*
+ * Leaf 0x80860002
+ * Transmeta CMS (Code Morphing Software)
+ */
+
+struct leaf_0x80860002_0 {
+ // eax
+ u32 cpu_rev_id : 32; // CPU revision ID
+ // ebx
+ u32 cms_rev_mask_2 : 8, // CMS revision ID, mask component 2
+ cms_rev_mask_1 : 8, // CMS revision ID, mask component 1
+ cms_rev_minor : 8, // CMS revision ID, minor
+ cms_rev_major : 8; // CMS revision ID, major
+ // ecx
+ u32 cms_rev_mask_3 : 32; // CMS revision ID, mask component 3
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0x80860003
+ * Transmeta CPU information string, bytes 0 - 15
+ */
+
+struct leaf_0x80860003_0 {
+ // eax
+ u32 cpu_info_0 : 32; // CPU info string bytes 0 - 3
+ // ebx
+ u32 cpu_info_1 : 32; // CPU info string bytes 4 - 7
+ // ecx
+ u32 cpu_info_2 : 32; // CPU info string bytes 8 - 11
+ // edx
+ u32 cpu_info_3 : 32; // CPU info string bytes 12 - 15
+};
+
+/*
+ * Leaf 0x80860004
+ * Transmeta CPU information string, bytes 16 - 31
+ */
+
+struct leaf_0x80860004_0 {
+ // eax
+ u32 cpu_info_4 : 32; // CPU info string bytes 16 - 19
+ // ebx
+ u32 cpu_info_5 : 32; // CPU info string bytes 20 - 23
+ // ecx
+ u32 cpu_info_6 : 32; // CPU info string bytes 24 - 27
+ // edx
+ u32 cpu_info_7 : 32; // CPU info string bytes 28 - 31
+};
+
+/*
+ * Leaf 0x80860005
+ * Transmeta CPU information string, bytes 32 - 47
+ */
+
+struct leaf_0x80860005_0 {
+ // eax
+ u32 cpu_info_8 : 32; // CPU info string bytes 32 - 35
+ // ebx
+ u32 cpu_info_9 : 32; // CPU info string bytes 36 - 39
+ // ecx
+ u32 cpu_info_10 : 32; // CPU info string bytes 40 - 43
+ // edx
+ u32 cpu_info_11 : 32; // CPU info string bytes 44 - 47
+};
+
+/*
+ * Leaf 0x80860006
+ * Transmeta CPU information string, bytes 48 - 63
+ */
+
+struct leaf_0x80860006_0 {
+ // eax
+ u32 cpu_info_12 : 32; // CPU info string bytes 48 - 51
+ // ebx
+ u32 cpu_info_13 : 32; // CPU info string bytes 52 - 55
+ // ecx
+ u32 cpu_info_14 : 32; // CPU info string bytes 56 - 59
+ // edx
+ u32 cpu_info_15 : 32; // CPU info string bytes 60 - 63
+};
+
+/*
+ * Leaf 0x80860007
+ * Transmeta live CPU information
+ */
+
+struct leaf_0x80860007_0 {
+ // eax
+ u32 cpu_cur_mhz : 32; // Current CPU frequency, in MHz
+ // ebx
+ u32 cpu_cur_voltage : 32; // Current CPU voltage, in millivolts
+ // ecx
+ u32 cpu_cur_perf_pctg : 32; // Current CPU performance percentage, 0 - 100
+ // edx
+ u32 cpu_cur_gate_delay : 32; // Current CPU gate delay, in femtoseconds
+};
+
+/*
+ * Leaf 0xc0000000
+ * Maximum Centaur/Zhaoxin leaf
+ */
+
+struct leaf_0xc0000000_0 {
+ // eax
+ u32 max_cntr_leaf : 32; // Maximum Centaur/Zhaoxin leaf
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 : 32; // Reserved
+};
+
+/*
+ * Leaf 0xc0000001
+ * Centaur/Zhaoxin extended CPU features
+ */
+
+struct leaf_0xc0000001_0 {
+ // eax
+ u32 : 32; // Reserved
+ // ebx
+ u32 : 32; // Reserved
+ // ecx
+ u32 : 32; // Reserved
+ // edx
+ u32 ccs_sm2 : 1, // CCS SM2 instructions
+ ccs_sm2_en : 1, // CCS SM2 enabled
+ rng : 1, // Random Number Generator
+ rng_en : 1, // RNG enabled
+ ccs_sm3_sm4 : 1, // CCS SM3 and SM4 instructions
+ ccs_sm3_sm4_en : 1, // CCS SM3/SM4 enabled
+ ace : 1, // Advanced Cryptography Engine
+ ace_en : 1, // ACE enabled
+ ace2 : 1, // Advanced Cryptography Engine v2
+ ace2_en : 1, // ACE v2 enabled
+ phe : 1, // PadLock Hash Engine
+ phe_en : 1, // PHE enabled
+ pmm : 1, // PadLock Montgomery Multiplier
+ pmm_en : 1, // PMM enabled
+ : 2, // Reserved
+ parallax : 1, // Parallax auto adjust processor voltage
+ parallax_en : 1, // Parallax enabled
+ : 2, // Reserved
+ tm3 : 1, // Thermal Monitor v3
+ tm3_en : 1, // TM v3 enabled
+ : 3, // Reserved
+ phe2 : 1, // PadLock Hash Engine v2 (SHA384/SHA512)
+ phe2_en : 1, // PHE v2 enabled
+ rsa : 1, // RSA instructions (XMODEXP/MONTMUL2)
+ rsa_en : 1, // RSA instructions enabled
+ : 3; // Reserved
+};
+
+#endif /* _ASM_X86_CPUID_LEAF_TYPES */
diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h
index 8a00364..3d0e611 100644
--- a/arch/x86/include/asm/cpuid/types.h
+++ b/arch/x86/include/asm/cpuid/types.h
@@ -5,6 +5,8 @@
#include <linux/build_bug.h>
#include <linux/types.h>
+#include <asm/cpuid/leaf_types.h>
+
/*
* Types for raw CPUID access:
*/
@@ -30,6 +32,12 @@ enum cpuid_regs_idx {
#define CPUID_LEAF_FREQ 0x16
#define CPUID_LEAF_TILE 0x1d
+#define CPUID_RANGE(idx) ((idx) & 0xffff0000)
+#define CPUID_RANGE_MAX(idx) (CPUID_RANGE(idx) + 0xffff)
+
+#define CPUID_BASE_START 0x00000000
+#define CPUID_BASE_END CPUID_RANGE_MAX(CPUID_BASE_START)
+
/*
* Types for CPUID(0x2) parsing:
*/
@@ -124,4 +132,94 @@ extern const struct leaf_0x2_table cpuid_0x2_table[256];
*/
#define TLB_0x63_2M_4M_ENTRIES 32
+/*
+ * Types for centralized CPUID tables:
+ *
+ * For internal use by the CPUID parser.
+ */
+
+/**
+ * struct leaf_parse_info - CPUID query parse info
+ * @nr_entries: Number of valid entries filled by the CPUID parser
+ */
+struct leaf_parse_info {
+ unsigned int nr_entries;
+};
+
+/**
+ * __CPUID_LEAF() - Define a CPUID output and parse info entry
+ * @_name: Struct type name of the CPUID leaf/subleaf (e.g. 'leaf_0x7_0'). Such
+ * types are defined at <cpuid/leaf_types.h> and follow the leaf_0xM_N
+ * format, where 0xM is the leaf and N is the subleaf.
+ * @_count: Number of storage entries to allocate for this leaf/subleaf.
+ *
+ * For a given leaf/subleaf, define an array of CPUID storage entries and an associated
+ * query info structure.
+ *
+ * Use an array of storage entries to accommodate CPUID leaves with multiple subleaves
+ * having the same output format. This is common for hierarchical enumeration; e.g.,
+ * CPUID(0x4), CPUID(0x12), and CPUID(0x8000001d).
+ */
+#define __CPUID_LEAF(_name, _count) \
+ struct _name _name[_count]; \
+ struct leaf_parse_info _name##_info
+
+/**
+ * CPUID_LEAF() - Define a 'struct cpuid_leaves' storage entry
+ * @_leaf: Leaf number, in compile-time 0xN format
+ * @_subleaf: Subleaf number, in compile-time decimal format
+ *
+ * Convenience wrapper around __CPUID_LEAF().
+ */
+#define CPUID_LEAF(_leaf, _subleaf) \
+ __CPUID_LEAF(leaf_ ## _leaf ## _ ## _subleaf, 1)
+
+#define __cpuid_leaf_first_subleaf(_l) \
+ LEAF_ ## _l ## _ ## SUBLEAF_N_FIRST
+#define __cpuid_leaf_last_subleaf(_l) \
+ LEAF_ ## _l ## _ ## SUBLEAF_N_LAST
+
+#define __cpuid_leaf_subleaf_count_min(_l) 2
+#define __cpuid_leaf_subleaf_count_max(_l) \
+ (__cpuid_leaf_last_subleaf(_l) - __cpuid_leaf_first_subleaf(_l) + 1)
+
+/**
+ * CPUID_LEAF_N() - Define a 'struct cpuid_leaves' storage entry
+ * @_leaf: Leaf number, in compile-time 0xN format
+ * @_count: Number of storage entries to allocate for that leaf. It must not exceed
+ * the limits defined at <cpuid/leaf_types.h>.
+ *
+ * Convenience wrapper around __CPUID_LEAF().
+ */
+#define CPUID_LEAF_N(_leaf, _count) \
+ static_assert(_count >= __cpuid_leaf_subleaf_count_min(_leaf)); \
+ static_assert(_count <= __cpuid_leaf_subleaf_count_max(_leaf)); \
+ __CPUID_LEAF(leaf_ ## _leaf ## _ ## n, _count)
+
+/*
+ * struct cpuid_leaves - Parsed CPUID data
+ */
+struct cpuid_leaves {
+ /* Leaf Subleaf number (or max number of subleaves) */
+ CPUID_LEAF ( 0x0, 0 );
+ CPUID_LEAF ( 0x1, 0 );
+};
+
+/*
+ * Types for centralized CPUID tables:
+ *
+ * For external use.
+ */
+
+/**
+ * struct cpuid_table - Per-CPU CPUID data repository
+ * @leaves: Parsed CPUID queries output and their metadata
+ *
+ * This is to be embedded inside 'struct cpuinfo_x86' to provide parsed and
+ * sanitized CPUID data per CPU.
+ */
+struct cpuid_table {
+ struct cpuid_leaves leaves;
+};
+
#endif /* _ASM_X86_CPUID_TYPES_H */
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index ec95fe4..00aeae8 100644
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);
extern void idt_setup_apic_and_irq_gates(void);
extern bool idt_is_f00f_address(unsigned long address);
+extern void idt_do_interrupt_irqoff(unsigned long address);
+extern void idt_do_nmi_irqoff(void);
+extern void idt_entry_from_kvm(unsigned int vector);
+
#ifdef CONFIG_X86_64
extern void idt_setup_early_pf(void);
#else
diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h
index 7e6b9314..2f2ce8a 100644
--- a/arch/x86/include/asm/desc_defs.h
+++ b/arch/x86/include/asm/desc_defs.h
@@ -145,7 +145,7 @@ struct gate_struct {
typedef struct gate_struct gate_desc;
#ifndef _SETUP
-static inline unsigned long gate_offset(const gate_desc *g)
+static __always_inline unsigned long gate_offset(const gate_desc *g)
{
#ifdef CONFIG_X86_64
return g->offset_low | ((unsigned long)g->offset_middle << 16) |
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
deleted file mode 100644
index 7c0a52c..0000000
--- a/arch/x86/include/asm/device.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_DEVICE_H
-#define _ASM_X86_DEVICE_H
-
-struct dev_archdata {
-};
-
-struct pdev_archdata {
-};
-
-#endif /* _ASM_X86_DEVICE_H */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index dc8fe13..be58b7f 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -137,7 +137,8 @@ extern void __init efi_dump_pagetable(void);
extern void __init efi_apply_memmap_quirks(void);
extern int __init efi_reuse_config(u64 tables, int nr_tables);
extern void efi_delete_dummy_variable(void);
-extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr);
+extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr,
+ const struct pt_regs *regs);
extern void efi_unmap_boot_services(void);
void arch_efi_call_virt_setup(void);
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index 7535131..eca24b5 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void)
}
#define arch_exit_to_user_mode arch_exit_to_user_mode
+extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
+
#endif
diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h
index cd6f194..90c63fe 100644
--- a/arch/x86/include/asm/fpu/api.h
+++ b/arch/x86/include/asm/fpu/api.h
@@ -119,12 +119,6 @@ extern void fpu__init_system(void);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);
-#ifdef CONFIG_MATH_EMULATION
-extern void fpstate_init_soft(struct swregs_state *soft);
-#else
-static inline void fpstate_init_soft(struct swregs_state *soft) {}
-#endif
-
/* State tracking */
DECLARE_PER_CPU(bool, kernel_fpu_allowed);
DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h
index 2bb6567..18a2f81 100644
--- a/arch/x86/include/asm/fred.h
+++ b/arch/x86/include/asm/fred.h
@@ -110,7 +110,6 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { ret
static inline void cpu_init_fred_exceptions(void) { }
static inline void cpu_init_fred_rsps(void) { }
static inline void fred_complete_exception_setup(void) { }
-static inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }
static inline void fred_sync_rsp0(unsigned long rsp0) { }
static inline void fred_update_rsp0(void) { }
#endif /* CONFIG_X86_FRED */
diff --git a/arch/x86/include/asm/futex_robust.h b/arch/x86/include/asm/futex_robust.h
new file mode 100644
index 0000000..e879547
--- /dev/null
+++ b/arch/x86/include/asm/futex_robust.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_FUTEX_ROBUST_H
+#define _ASM_X86_FUTEX_ROBUST_H
+
+#include <asm/ptrace.h>
+
+static __always_inline void __user *x86_futex_robust_unlock_get_pop(struct pt_regs *regs)
+{
+ /*
+ * If ZF is set then the cmpxchg succeeded and the pending op pointer
+ * needs to be cleared.
+ */
+ return regs->flags & X86_EFLAGS_ZF ? (void __user *)regs->dx : NULL;
+}
+
+#define arch_futex_robust_unlock_get_pop(regs) \
+ x86_futex_robust_unlock_get_pop(regs)
+
+#endif /* _ASM_X86_FUTEX_ROBUST_H */
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 9314642a..dea60d6 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -4,51 +4,64 @@
#include <linux/threads.h>
+enum irq_stat_counts {
+ IRQ_COUNT_NMI,
+#ifdef CONFIG_X86_LOCAL_APIC
+ IRQ_COUNT_APIC_TIMER,
+ IRQ_COUNT_SPURIOUS,
+ IRQ_COUNT_APIC_PERF,
+ IRQ_COUNT_IRQ_WORK,
+ IRQ_COUNT_ICR_READ_RETRY,
+ IRQ_COUNT_X86_PLATFORM_IPI,
+#endif
+#ifdef CONFIG_SMP
+ IRQ_COUNT_RESCHEDULE,
+ IRQ_COUNT_CALL_FUNCTION,
+#endif
+ IRQ_COUNT_TLB,
+#ifdef CONFIG_X86_THERMAL_VECTOR
+ IRQ_COUNT_THERMAL_APIC,
+#endif
+#ifdef CONFIG_X86_MCE_THRESHOLD
+ IRQ_COUNT_THRESHOLD_APIC,
+#endif
+#ifdef CONFIG_X86_MCE_AMD
+ IRQ_COUNT_DEFERRED_ERROR,
+#endif
+#ifdef CONFIG_X86_MCE
+ IRQ_COUNT_MCE_EXCEPTION,
+ IRQ_COUNT_MCE_POLL,
+#endif
+#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
+ IRQ_COUNT_HYPERVISOR_CALLBACK,
+#endif
+#if IS_ENABLED(CONFIG_HYPERV)
+ IRQ_COUNT_HYPERV_REENLIGHTENMENT,
+ IRQ_COUNT_HYPERV_STIMER0,
+#endif
+#if IS_ENABLED(CONFIG_KVM)
+ IRQ_COUNT_POSTED_INTR,
+ IRQ_COUNT_POSTED_INTR_NESTED,
+ IRQ_COUNT_POSTED_INTR_WAKEUP,
+#endif
+#ifdef CONFIG_GUEST_PERF_EVENTS
+ IRQ_COUNT_PERF_GUEST_MEDIATED_PMI,
+#endif
+#ifdef CONFIG_X86_POSTED_MSI
+ IRQ_COUNT_POSTED_MSI_NOTIFICATION,
+#endif
+ IRQ_COUNT_PIC_APIC_ERROR,
+#ifdef CONFIG_X86_IO_APIC
+ IRQ_COUNT_IOAPIC_MISROUTED,
+#endif
+ IRQ_COUNT_MAX,
+};
+
typedef struct {
#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
u8 kvm_cpu_l1tf_flush_l1d;
#endif
- unsigned int __nmi_count; /* arch dependent */
-#ifdef CONFIG_X86_LOCAL_APIC
- unsigned int apic_timer_irqs; /* arch dependent */
- unsigned int irq_spurious_count;
- unsigned int icr_read_retry_count;
-#endif
-#if IS_ENABLED(CONFIG_KVM)
- unsigned int kvm_posted_intr_ipis;
- unsigned int kvm_posted_intr_wakeup_ipis;
- unsigned int kvm_posted_intr_nested_ipis;
-#endif
-#ifdef CONFIG_GUEST_PERF_EVENTS
- unsigned int perf_guest_mediated_pmis;
-#endif
- unsigned int x86_platform_ipis; /* arch dependent */
- unsigned int apic_perf_irqs;
- unsigned int apic_irq_work_irqs;
-#ifdef CONFIG_SMP
- unsigned int irq_resched_count;
- unsigned int irq_call_count;
-#endif
- unsigned int irq_tlb_count;
-#ifdef CONFIG_X86_THERMAL_VECTOR
- unsigned int irq_thermal_count;
-#endif
-#ifdef CONFIG_X86_MCE_THRESHOLD
- unsigned int irq_threshold_count;
-#endif
-#ifdef CONFIG_X86_MCE_AMD
- unsigned int irq_deferred_error_count;
-#endif
-#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
- unsigned int irq_hv_callback_count;
-#endif
-#if IS_ENABLED(CONFIG_HYPERV)
- unsigned int irq_hv_reenlightenment_count;
- unsigned int hyperv_stimer0_count;
-#endif
-#ifdef CONFIG_X86_POSTED_MSI
- unsigned int posted_msi_notification_count;
-#endif
+ unsigned int counts[IRQ_COUNT_MAX];
} ____cacheline_aligned irq_cpustat_t;
DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
@@ -58,15 +71,21 @@ DECLARE_PER_CPU_ALIGNED(struct pi_desc, posted_msi_pi_desc);
#endif
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) this_cpu_inc(irq_stat.member)
+#define inc_irq_stat(index) this_cpu_inc(irq_stat.counts[IRQ_COUNT_##index])
+void irq_stat_inc_and_enable(enum irq_stat_counts which);
+
+#ifdef CONFIG_X86_LOCAL_APIC
+#define inc_perf_irq_stat() inc_irq_stat(APIC_PERF)
+#else
+#define inc_perf_irq_stat() do { } while (0)
+#endif
extern void ack_bad_irq(unsigned int irq);
+#ifdef CONFIG_PROC_FS
extern u64 arch_irq_stat_cpu(unsigned int cpu);
#define arch_irq_stat_cpu arch_irq_stat_cpu
-
-extern u64 arch_irq_stat(void);
-#define arch_irq_stat arch_irq_stat
+#endif
DECLARE_PER_CPU_CACHE_HOT(u16, __softirq_pending);
#define local_softirq_pending_ref __softirq_pending
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index cbe19e6..47727d0 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -110,10 +110,6 @@ static inline void lock_vector_lock(void) {}
static inline void unlock_vector_lock(void) {}
#endif
-/* Statistics */
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
-
extern void elcr_set_level_irq(unsigned int irq);
extern char irq_entries_start[];
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 42bf6a5..20f5487 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -340,17 +340,14 @@ static __always_inline void __##func(struct pt_regs *regs)
__visible void noist_##func(struct pt_regs *regs)
/**
- * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
+ * DECLARE_IDTENTRY_VC - Declare a function for the VC entry point
* @vector: Vector number (ignored for C)
* @func: Function name of the entry point
*
- * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
- * safe_stack C handler.
+ * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE.
*/
#define DECLARE_IDTENTRY_VC(vector, func) \
- DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
- __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \
- __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code)
+ DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func);
/**
* DEFINE_IDTENTRY_IST - Emit code for IST entry points
@@ -391,26 +388,6 @@ static __always_inline void __##func(struct pt_regs *regs)
#define DEFINE_IDTENTRY_DF(func) \
DEFINE_IDTENTRY_RAW_ERRORCODE(func)
-/**
- * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
- * when raised from kernel mode
- * @func: Function name of the entry point
- *
- * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
- */
-#define DEFINE_IDTENTRY_VC_KERNEL(func) \
- DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
-
-/**
- * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
- * when raised from user mode
- * @func: Function name of the entry point
- *
- * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
- */
-#define DEFINE_IDTENTRY_VC_USER(func) \
- DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
-
#else /* CONFIG_X86_64 */
/**
diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h
index 950bfd0..66d964a 100644
--- a/arch/x86/include/asm/intel-family.h
+++ b/arch/x86/include/asm/intel-family.h
@@ -29,6 +29,7 @@
* _N,_P - other mobile parts
* _H - premium mobile parts
* _S - other client parts
+ * _R - ruggedized for harsh environment
*
* Historical OPTDIFFs:
*
@@ -151,6 +152,7 @@
#define INTEL_LUNARLAKE_M IFM(6, 0xBD) /* Lion Cove / Skymont */
#define INTEL_PANTHERLAKE_L IFM(6, 0xCC) /* Cougar Cove / Darkmont */
+#define INTEL_PANTHERLAKE_R IFM(6, 0xE5) /* Cougar Cove / Darkmont */
#define INTEL_WILDCATLAKE_L IFM(6, 0xD5)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c470e40..f14009f 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1504,6 +1504,7 @@ struct kvm_arch {
bool use_master_clock;
u64 master_kernel_ns;
u64 master_cycle_now;
+ struct ratelimit_state kvmclock_update_rs;
#ifdef CONFIG_KVM_HYPERV
struct kvm_hv hyperv;
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 0175d39a..e575b70 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -291,9 +291,6 @@ bool mce_is_memory_error(struct mce *m);
bool mce_is_correctable(struct mce *m);
bool mce_usable_address(struct mce *m);
-DECLARE_PER_CPU(unsigned, mce_exception_count);
-DECLARE_PER_CPU(unsigned, mce_poll_count);
-
typedef DECLARE_BITMAP(mce_banks_t, MAX_NR_BANKS);
DECLARE_PER_CPU(mce_banks_t, mce_poll_banks);
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 3c317d1..9cd136d 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -3,6 +3,7 @@
#define _ASM_X86_MICROCODE_H
#include <asm/msr.h>
+#include <asm/cpuid/api.h>
struct cpu_signature {
unsigned int sig;
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index a14a0f4..86554de 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -803,9 +803,10 @@
#define MSR_AMD64_LBR_SELECT 0xc000010e
/* Zen4 */
-#define MSR_ZEN4_BP_CFG 0xc001102e
+#define MSR_ZEN4_BP_CFG 0xc001102e
#define MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT 4
#define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5
+#define MSR_ZEN2_BP_CFG_BUG_FIX_BIT 33
/* Fam 19h MSRs */
#define MSR_F19H_UMC_PERF_CTL 0xc0010800
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 4099814..cef9a4c 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -40,12 +40,10 @@
#endif
#define __percpu_prefix
-#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg)
#else /* !CONFIG_CC_HAS_NAMED_AS: */
#define __percpu_prefix __force_percpu_prefix
-#define __percpu_seg_override
#endif /* CONFIG_CC_HAS_NAMED_AS */
@@ -82,7 +80,6 @@
#define __force_percpu_prefix
#define __percpu_prefix
-#define __percpu_seg_override
#define PER_CPU_VAR(var) (var)__percpu_rel
@@ -92,8 +89,6 @@
# define __my_cpu_type(var) typeof(var)
# define __my_cpu_ptr(ptr) (ptr)
# define __my_cpu_var(var) (var)
-
-# define __percpu_qual __percpu_seg_override
#else
# define __my_cpu_type(var) typeof(var) __percpu_seg_override
# define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
diff --git a/arch/x86/include/asm/percpu_types.h b/arch/x86/include/asm/percpu_types.h
new file mode 100644
index 0000000..0aa3e47
--- /dev/null
+++ b/arch/x86/include/asm/percpu_types.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_PERCPU_TYPES_H
+#define _ASM_X86_PERCPU_TYPES_H
+
+#if defined(CONFIG_SMP) && defined(CONFIG_CC_HAS_NAMED_AS)
+#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg)
+#else /* !CONFIG_CC_HAS_NAMED_AS: */
+#define __percpu_seg_override
+#endif
+
+#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL)
+#define __percpu_qual __percpu_seg_override
+#endif
+
+#include <asm-generic/percpu_types.h>
+
+#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 10b5355..87b1d4c 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -16,7 +16,7 @@ struct vm86;
#include <uapi/asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeatures.h>
-#include <asm/cpuid/api.h>
+#include <asm/cpuid/types.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>
#include <asm/percpu.h>
@@ -170,6 +170,7 @@ struct cpuinfo_x86 {
char x86_vendor_id[16];
char x86_model_id[64];
struct cpuinfo_topology topo;
+ struct cpuid_table cpuid;
/* in KB - valid for CPUS which support this call: */
unsigned int x86_cache_size;
int x86_cache_alignment; /* In bytes */
@@ -704,6 +705,11 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu)
return per_cpu(cpu_info.topo.l2c_id, cpu);
}
+static inline u32 per_cpu_core_id(unsigned int cpu)
+{
+ return per_cpu(cpu_info.topo.core_id, cpu);
+}
+
#ifdef CONFIG_CPU_SUP_AMD
/*
* Issue a DIV 0/1 insn to clear any division data from previous DIV
@@ -733,6 +739,7 @@ bool xen_set_default_idle(void);
#endif
void __noreturn stop_this_cpu(void *dummy);
+extern bool x86_hypervisor_present;
void microcode_check(struct cpuinfo_x86 *prev_info);
void store_cpu_caps(struct cpuinfo_x86 *info);
diff --git a/arch/x86/include/asm/purgatory.h b/arch/x86/include/asm/purgatory.h
index 2fee5e9..56a9e81 100644
--- a/arch/x86/include/asm/purgatory.h
+++ b/arch/x86/include/asm/purgatory.h
@@ -8,4 +8,4 @@
extern void purgatory(void);
#endif /* __ASSEMBLER__ */
-#endif /* _ASM_PURGATORY_H */
+#endif /* _ASM_X86_PURGATORY_H */
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 09e605c..594cfa1 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -661,7 +661,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages)
{
__snp_leak_pages(pfn, pages, true);
}
-void snp_prepare(void);
+int snp_prepare(void);
void snp_shutdown(void);
#else
static inline bool snp_probe_rmptable_info(void) { return false; }
@@ -679,7 +679,7 @@ static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp)
static inline void snp_leak_pages(u64 pfn, unsigned int npages) {}
static inline void kdump_sev_callback(void) { }
static inline void snp_fixup_e820_tables(void) {}
-static inline void snp_prepare(void) {}
+static inline int snp_prepare(void) { return -ENODEV; }
static inline void snp_shutdown(void) {}
#endif
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index f2d4921..4e73515 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -24,6 +24,10 @@ struct vdso_image {
long sym_int80_landing_pad;
long sym_vdso32_sigreturn_landing_pad;
long sym_vdso32_rt_sigreturn_landing_pad;
+ long sym___futex_list64_try_unlock_cs_start;
+ long sym___futex_list64_try_unlock_cs_end;
+ long sym___futex_list32_try_unlock_cs_start;
+ long sym___futex_list32_try_unlock_cs_end;
};
extern const struct vdso_image vdso64_image;
diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h
index eda233a..e26061d 100644
--- a/arch/x86/include/asm/vermagic.h
+++ b/arch/x86/include/asm/vermagic.h
@@ -5,8 +5,6 @@
#ifdef CONFIG_X86_64
/* X86_64 does not define MODULE_PROC_FAMILY */
-#elif defined CONFIG_M586
-#define MODULE_PROC_FAMILY "586 "
#elif defined CONFIG_M586TSC
#define MODULE_PROC_FAMILY "586TSC "
#elif defined CONFIG_M586MMX
@@ -31,10 +29,6 @@
#define MODULE_PROC_FAMILY "CRUSOE "
#elif defined CONFIG_MEFFICEON
#define MODULE_PROC_FAMILY "EFFICEON "
-#elif defined CONFIG_MWINCHIPC6
-#define MODULE_PROC_FAMILY "WINCHIPC6 "
-#elif defined CONFIG_MWINCHIP3D
-#define MODULE_PROC_FAMILY "WINCHIP3D "
#elif defined CONFIG_MCYRIXIII
#define MODULE_PROC_FAMILY "CYRIXIII "
#elif defined CONFIG_MVIAC3_2
diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h
index c2fc786..7c596ce 100644
--- a/arch/x86/include/asm/xen/hypervisor.h
+++ b/arch/x86/include/asm/xen/hypervisor.h
@@ -37,6 +37,7 @@ extern struct shared_info *HYPERVISOR_shared_info;
extern struct start_info *xen_start_info;
#include <asm/bug.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
#define XEN_SIGNATURE "XenVMMXenVMM"
diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c
index d7c8ef1..be4c5e9 100644
--- a/arch/x86/kernel/acpi/cppc.c
+++ b/arch/x86/kernel/acpi/cppc.c
@@ -88,19 +88,19 @@ static void amd_set_max_freq_ratio(void)
rc = cppc_get_perf_caps(0, &perf_caps);
if (rc) {
- pr_warn("Could not retrieve perf counters (%d)\n", rc);
+ pr_debug("Could not retrieve perf counters (%d)\n", rc);
return;
}
rc = amd_get_boost_ratio_numerator(0, &numerator);
if (rc) {
- pr_warn("Could not retrieve highest performance (%d)\n", rc);
+ pr_debug("Could not retrieve highest performance (%d)\n", rc);
return;
}
nominal_perf = perf_caps.nominal_perf;
if (!nominal_perf) {
- pr_warn("Could not retrieve nominal performance\n");
+ pr_debug("Could not retrieve nominal performance\n");
return;
}
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 6399049..aa1e199 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -64,6 +64,7 @@
#include <asm/tsc.h>
#include <asm/hypervisor.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/intel-family.h>
#include <asm/irq_regs.h>
#include <asm/cpu.h>
@@ -1045,7 +1046,7 @@ static void local_apic_timer_interrupt(void)
/*
* the NMI deadlock-detector uses this.
*/
- inc_irq_stat(apic_timer_irqs);
+ inc_irq_stat(APIC_TIMER);
evt->event_handler(evt);
}
@@ -2114,7 +2115,7 @@ static noinline void handle_spurious_interrupt(u8 vector)
trace_spurious_apic_entry(vector);
- inc_irq_stat(irq_spurious_count);
+ irq_stat_inc_and_enable(IRQ_COUNT_SPURIOUS);
/*
* If this is a spurious interrupt then do not acknowledge
@@ -2186,7 +2187,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
apic_write(APIC_ESR, 0);
v = apic_read(APIC_ESR);
apic_eoi();
- atomic_inc(&irq_err_count);
+ irq_stat_inc_and_enable(IRQ_COUNT_PIC_APIC_ERROR);
apic_pr_debug("APIC error on CPU%d: %02x", smp_processor_id(), v);
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 352ed55..7d7175d 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -1575,8 +1575,6 @@ static unsigned int startup_ioapic_irq(struct irq_data *data)
return was_pending;
}
-atomic_t irq_mis_count;
-
#ifdef CONFIG_GENERIC_PENDING_IRQ
static bool io_apic_level_ack_pending(struct mp_chip_data *data)
{
@@ -1713,7 +1711,7 @@ static void ioapic_ack_level(struct irq_data *irq_data)
* at the cpu.
*/
if (!(v & (1 << (i & 0x1f)))) {
- atomic_inc(&irq_mis_count);
+ irq_stat_inc_and_enable(IRQ_COUNT_IOAPIC_MISROUTED);
eoi_ioapic_pin(cfg->vector, irq_data->chip_data);
}
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index 98a57cb..c627bee 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -120,7 +120,7 @@ u32 apic_mem_wait_icr_idle_timeout(void)
for (cnt = 0; cnt < 1000; cnt++) {
if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
return 0;
- inc_irq_stat(icr_read_retry_count);
+ irq_stat_inc_and_enable(IRQ_COUNT_ICR_READ_RETRY);
udelay(100);
}
return APIC_ICR_BUSY;
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 2f8a58e..d2e8a84 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -19,6 +19,7 @@
obj-y := cacheinfo.o scattered.o
obj-y += topology_common.o topology_ext.o topology_amd.o
+obj-y += cpuid_parser.o
obj-y += common.o
obj-y += rdrand.o
obj-y += match.o
diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c
index 2c5b51a..dc119af 100644
--- a/arch/x86/kernel/cpu/acrn.c
+++ b/arch/x86/kernel/cpu/acrn.c
@@ -52,7 +52,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback)
* HYPERVISOR_CALLBACK_VECTOR.
*/
apic_eoi();
- inc_irq_stat(irq_hv_callback_count);
+ inc_irq_stat(HYPERVISOR_CALLBACK);
if (acrn_intr_handler)
acrn_intr_handler();
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 2d9ae6a..487ac14 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -16,6 +16,7 @@
#include <asm/cacheinfo.h>
#include <asm/cpu.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/spec-ctrl.h>
#include <asm/smp.h>
#include <asm/numa.h>
@@ -518,7 +519,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
break;
case 0x50 ... 0x5f:
case 0x80 ... 0xaf:
- case 0xc0 ... 0xcf:
+ case 0xc0 ... 0xef:
setup_force_cpu_cap(X86_FEATURE_ZEN6);
break;
default:
@@ -989,6 +990,9 @@ static void init_amd_zen2(struct cpuinfo_x86 *c)
/* Correct misconfigured CPUID on some clients. */
clear_cpu_cap(c, X86_FEATURE_INVLPGB);
+
+ if (!cpu_has(c, X86_FEATURE_HYPERVISOR))
+ msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN2_BP_CFG_BUG_FIX_BIT);
}
static void init_amd_zen3(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index 81695da9..681d2da 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -5,6 +5,7 @@
#include <asm/cpu.h>
#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
#include <asm/e820/api.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a4268c4..a3df21d 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1737,13 +1737,6 @@ static void __init cpu_parse_early_param(void)
int arglen;
#ifdef CONFIG_X86_32
- if (cmdline_find_option_bool(boot_command_line, "no387"))
-#ifdef CONFIG_MATH_EMULATION
- setup_clear_cpu_cap(X86_FEATURE_FPU);
-#else
- pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
-#endif
-
if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
setup_clear_cpu_cap(X86_FEATURE_FXSR);
#endif
@@ -1791,6 +1784,7 @@ static void __init cpu_parse_early_param(void)
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
{
memset(&c->x86_capability, 0, sizeof(c->x86_capability));
+ memset(&c->cpuid, 0, sizeof(c->cpuid));
c->extended_cpuid_level = 0;
if (!cpuid_feature())
@@ -1798,6 +1792,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
/* cyrix could have cpuid enabled via c_identify()*/
if (cpuid_feature()) {
+ cpuid_scan_cpu(c);
cpu_detect(c);
get_cpu_vendor(c);
intel_unlock_cpuid_leafs(c);
@@ -1970,8 +1965,8 @@ static void generic_identify(struct cpuinfo_x86 *c)
if (!cpuid_feature())
return;
+ cpuid_scan_cpu(c);
cpu_detect(c);
-
get_cpu_vendor(c);
intel_unlock_cpuid_leafs(c);
get_cpu_cap(c);
@@ -2023,6 +2018,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)
#endif
c->x86_cache_alignment = c->x86_clflush_size;
memset(&c->x86_capability, 0, sizeof(c->x86_capability));
+ memset(&c->cpuid, 0, sizeof(c->cpuid));
#ifdef CONFIG_X86_VMX_FEATURE_NAMES
memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
#endif
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 5c7a3a7..dca2d58 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -75,7 +75,6 @@ static inline struct amd_northbridge *amd_init_l3_cache(int index)
}
#endif
-unsigned int aperfmperf_get_khz(int cpu);
void cpu_select_mitigations(void);
extern void x86_spec_ctrl_setup_ap(void);
diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c
index 146f6f8..99801e8 100644
--- a/arch/x86/kernel/cpu/cpuid-deps.c
+++ b/arch/x86/kernel/cpu/cpuid-deps.c
@@ -92,6 +92,7 @@ static const struct cpuid_dep cpuid_deps[] = {
{ X86_FEATURE_FRED, X86_FEATURE_LKGS },
{ X86_FEATURE_SPEC_CTRL_SSBD, X86_FEATURE_SPEC_CTRL },
{ X86_FEATURE_LASS, X86_FEATURE_SMAP },
+ { X86_FEATURE_INVLPGB, X86_FEATURE_PCID },
{}
};
diff --git a/arch/x86/kernel/cpu/cpuid_parser.c b/arch/x86/kernel/cpu/cpuid_parser.c
new file mode 100644
index 0000000..898b0c4
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpuid_parser.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * CPUID parser; for populating the system's CPUID tables.
+ */
+
+#include <linux/kernel.h>
+
+#include <asm/cpuid/api.h>
+#include <asm/processor.h>
+
+#include "cpuid_parser.h"
+
+/* Clear a single CPUID table entry */
+static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output)
+{
+ struct cpuid_regs *regs = output->regs;
+
+ for (int i = 0; i < e->maxcnt; i++, regs++)
+ memset(regs, 0, sizeof(*regs));
+
+ memset(output->info, 0, sizeof(*output->info));
+}
+
+/*
+ * Leaf read functions:
+ */
+
+/*
+ * Default CPUID read function
+ * Satisfies the requirements stated at 'struct cpuid_parse_entry'->read().
+ */
+static void
+cpuid_read_generic(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output)
+{
+ struct cpuid_regs *regs = output->regs;
+
+ for (int i = 0; i < e->maxcnt; i++, regs++, output->info->nr_entries++)
+ cpuid_read_subleaf(e->leaf, e->subleaf + i, regs);
+}
+
+/*
+ * CPUID parser table:
+ */
+
+static const struct cpuid_parse_entry cpuid_parse_entries[] = {
+ CPUID_PARSE_ENTRIES
+};
+
+/*
+ * Leaf-independent parser code:
+ */
+
+static unsigned int cpuid_range_max_leaf(const struct cpuid_table *t, unsigned int range)
+{
+ const struct leaf_0x0_0 *l0 = __cpuid_table_subleaf(t, 0x0, 0);
+
+ switch (range) {
+ case CPUID_BASE_START: return l0 ? l0->max_std_leaf : 0;
+ default: return 0;
+ }
+}
+
+static void
+__cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+ unsigned int nr_entries, unsigned int start, unsigned int end, bool fill)
+{
+ const struct cpuid_parse_entry *entry = entries;
+ unsigned int range = CPUID_RANGE(start);
+
+ for (unsigned int i = 0; i < nr_entries; i++, entry++) {
+ struct cpuid_read_output output = {
+ .regs = cpuid_table_regs_p(t, entry->regs_offs),
+ .info = cpuid_table_info_p(t, entry->info_offs),
+ };
+
+ if (entry->leaf < start || entry->leaf > end)
+ continue;
+
+ cpuid_clear(entry, &output);
+
+ /*
+ * Read the range's anchor leaf unconditionally so that the cached
+ * maximum valid leaf value is available for the remaining entries.
+ */
+ if (fill && (entry->leaf == range || entry->leaf <= cpuid_range_max_leaf(t, range)))
+ entry->read(entry, &output);
+ }
+}
+
+/*
+ * Zero all cached CPUID entries within [@start-@end] range. This is needed when
+ * certain operations like MSR writes induce changes to the CPU's CPUID layout.
+ */
+static void
+__cpuid_zero_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+ unsigned int nr_entries, unsigned int start, unsigned int end)
+{
+ __cpuid_reset_table(t, entries, nr_entries, start, end, false);
+}
+
+static void
+__cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+ unsigned int nr_entries, unsigned int start, unsigned int end)
+{
+ __cpuid_reset_table(t, entries, nr_entries, start, end, true);
+}
+
+static void
+cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], unsigned int nr_entries)
+{
+ static const struct {
+ unsigned int start;
+ unsigned int end;
+ } ranges[] = {
+ { CPUID_BASE_START, CPUID_BASE_END },
+ };
+
+ for (unsigned int i = 0; i < ARRAY_SIZE(ranges); i++)
+ __cpuid_fill_table(t, entries, nr_entries, ranges[i].start, ranges[i].end);
+}
+
+static void __cpuid_scan_cpu_full(struct cpuinfo_x86 *c)
+{
+ unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries);
+ struct cpuid_table *table = &c->cpuid;
+
+ cpuid_fill_table(table, cpuid_parse_entries, nr_entries);
+}
+
+static void
+__cpuid_scan_cpu_partial(struct cpuinfo_x86 *c, unsigned int start_leaf, unsigned int end_leaf)
+{
+ unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries);
+ struct cpuid_table *table = &c->cpuid;
+
+ __cpuid_zero_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf);
+ __cpuid_fill_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf);
+}
+
+/*
+ * Call-site APIs:
+ */
+
+/**
+ * cpuid_scan_cpu() - Populate current CPU's CPUID table
+ * @c: CPU capability structure associated with the current CPU
+ *
+ * Populate the CPUID table embedded within @c with parsed CPUID data. All CPUID
+ * instructions are invoked locally, so this must be called on the CPU associated
+ * with @c.
+ */
+void cpuid_scan_cpu(struct cpuinfo_x86 *c)
+{
+ __cpuid_scan_cpu_full(c);
+}
+
+/**
+ * cpuid_refresh_range() - Rescan a CPUID table's leaf range
+ * @c: CPU capability structure associated with the current CPU
+ * @start: Start of leaf range to be re-scanned
+ * @end: End of leaf range
+ */
+void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end)
+{
+ if (WARN_ON_ONCE(start > end))
+ return;
+
+ if (WARN_ON_ONCE(CPUID_RANGE(start) != CPUID_RANGE(end)))
+ return;
+
+ __cpuid_scan_cpu_partial(c, start, end);
+}
+
+/**
+ * cpuid_refresh_leaf() - Rescan a CPUID table's leaf
+ * @c: CPU capability structure associated with the current CPU
+ * @leaf: Leaf to be re-scanned
+ */
+void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf)
+{
+ cpuid_refresh_range(c, leaf, leaf);
+}
diff --git a/arch/x86/kernel/cpu/cpuid_parser.h b/arch/x86/kernel/cpu/cpuid_parser.h
new file mode 100644
index 0000000..df62730
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpuid_parser.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ARCH_X86_CPUID_PARSER_H
+#define _ARCH_X86_CPUID_PARSER_H
+
+#include <asm/cpuid/types.h>
+
+/*
+ * Since accessing the CPUID leaves at 'struct cpuid_leaves' require compile time
+ * tokenization, split the CPUID parser into two stages: compile time macros for
+ * tokenizing the leaf/subleaf output offsets within the table, and generic runtime
+ * code to write to the relevant CPUID leaves using such offsets.
+ *
+ * The output of the compile time macros is cached by a compile time "parse entry"
+ * table (see 'struct cpuid_parse_entry'). The runtime parser code will utilize
+ * such offsets by passing them to the cpuid_table_*_p() functions.
+ */
+
+/*
+ * Compile time CPUID table offset calculations:
+ *
+ * @_leaf: CPUID leaf, in 0xN format
+ * @_subleaf: CPUID subleaf, in decimal format
+ */
+
+#define __cpuid_leaves_regs_offset(_leaf, _subleaf) \
+ offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf)
+
+#define __cpuid_leaves_info_offset(_leaf, _subleaf) \
+ offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf ## _ ## info)
+
+#define __cpuid_leaves_regs_maxcnt(_leaf, _subleaf) \
+ ARRAY_SIZE(((struct cpuid_leaves *)NULL)->leaf_ ## _leaf ## _ ## _subleaf)
+
+/*
+ * Translation of compile time offsets to generic runtime pointers:
+ */
+
+static inline struct cpuid_regs *
+cpuid_table_regs_p(const struct cpuid_table *t, unsigned long regs_offset)
+{
+ return (struct cpuid_regs *)((unsigned long)(&t->leaves) + regs_offset);
+}
+
+static inline struct leaf_parse_info *
+cpuid_table_info_p(const struct cpuid_table *t, unsigned long info_offset)
+{
+ return (struct leaf_parse_info *)((unsigned long)(&t->leaves) + info_offset);
+}
+
+/**
+ * struct cpuid_read_output - Output of a CPUID read operation
+ * @regs: Pointer to an array of CPUID outputs, where each array element covers the
+ * full EAX->EDX output range.
+ * @info: Pointer to query info; for saving the number of filled elements at @regs.
+ *
+ * A CPUID parser read function like cpuid_read_generic() or cpuid_read_0xN() uses this
+ * structure to save the CPUID query outputs. Actual storage for @regs and @info is
+ * provided by the read function caller, and is typically within the CPU's CPUID table.
+ *
+ * See struct cpuid_parse_entry.read().
+ */
+struct cpuid_read_output {
+ struct cpuid_regs *regs;
+ struct leaf_parse_info *info;
+};
+
+/**
+ * struct cpuid_parse_entry - CPUID parse table entry
+ * @leaf: Leaf number to be parsed
+ * @subleaf: Subleaf number to be parsed
+ * @regs_offs: Offset within 'struct cpuid_leaves' for saving the CPUID query output; to be
+ * passed to cpuid_table_regs_p().
+ * @info_offs: Offset within 'struct cpuid_leaves' for saving the CPUID query parse info; to be
+ * passed to cpuid_table_info_p().
+ * @maxcnt: Maximum number of output storage entries available for the CPUID query.
+ * @read: Read function for this entry. It must save the parsed CPUID output to the passed
+ * 'struct cpuid_read_output'->regs array of size >= @maxcnt. It must set
+ * 'struct cpuid_read_output'->info.nr_entries to the number of CPUID output entries
+ * parsed and filled. A generic implementation is provided at cpuid_read_generic().
+ */
+struct cpuid_parse_entry {
+ unsigned int leaf;
+ unsigned int subleaf;
+ unsigned int regs_offs;
+ unsigned int info_offs;
+ unsigned int maxcnt;
+ void (*read)(const struct cpuid_parse_entry *e, const struct cpuid_read_output *o);
+};
+
+#define __CPUID_PARSE_ENTRY(_leaf, _subleaf, _suffix, _reader_fn) \
+ { \
+ .leaf = _leaf, \
+ .subleaf = _subleaf, \
+ .regs_offs = __cpuid_leaves_regs_offset(_leaf, _suffix), \
+ .info_offs = __cpuid_leaves_info_offset(_leaf, _suffix), \
+ .maxcnt = __cpuid_leaves_regs_maxcnt(_leaf, _suffix), \
+ .read = cpuid_read_ ## _reader_fn, \
+ }
+
+/*
+ * CPUID_PARSE_ENTRY_N() is for parsing CPUID leaves with a subleaf range.
+ * Check <asm/cpuid/types.h> __CPUID_LEAF() vs. CPUID_LEAF_N().
+ */
+
+#define CPUID_PARSE_ENTRY(_leaf, _subleaf, _reader_fn) \
+ __CPUID_PARSE_ENTRY(_leaf, _subleaf, _subleaf, _reader_fn)
+
+#define CPUID_PARSE_ENTRY_N(_leaf, _reader_fn) \
+ __CPUID_PARSE_ENTRY(_leaf, __cpuid_leaf_first_subleaf(_leaf), n, _reader_fn)
+
+/*
+ * CPUID parser table:
+ */
+
+#define CPUID_PARSE_ENTRIES \
+ /* Leaf Subleaf Reader function */ \
+ CPUID_PARSE_ENTRY ( 0x0, 0, generic ), \
+ CPUID_PARSE_ENTRY ( 0x1, 0, generic ), \
+
+#endif /* _ARCH_X86_CPUID_PARSER_H */
diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
index 7f95a74..3e8891a 100644
--- a/arch/x86/kernel/cpu/hygon.c
+++ b/arch/x86/kernel/cpu/hygon.c
@@ -10,6 +10,7 @@
#include <asm/apic.h>
#include <asm/cpu.h>
+#include <asm/cpuid/api.h>
#include <asm/smp.h>
#include <asm/numa.h>
#include <asm/cacheinfo.h>
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index f28c0ef..abb3984 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -388,24 +388,15 @@ __setup("forcepae", forcepae_setup);
static void intel_workarounds(struct cpuinfo_x86 *c)
{
-#ifdef CONFIG_X86_F00F_BUG
/*
* All models of Pentium and Pentium with MMX technology CPUs
* have the F0 0F bug, which lets nonprivileged users lock up the
- * system. Announce that the fault handler will be checking for it.
+ * system. The fault handler always checks for it.
* The Quark is also family 5, but does not have the same bug.
*/
- clear_cpu_bug(c, X86_BUG_F00F);
- if (c->x86_vfm >= INTEL_FAM5_START && c->x86_vfm < INTEL_QUARK_X1000) {
- static int f00f_workaround_enabled;
-
+ if (IS_ENABLED(CONFIG_X86_F00F_BUG) &&
+ (c->x86_vfm >= INTEL_FAM5_START && c->x86_vfm < INTEL_QUARK_X1000))
set_cpu_bug(c, X86_BUG_F00F);
- if (!f00f_workaround_enabled) {
- pr_notice("Intel Pentium with F0 0F bug - workaround enabled.\n");
- f00f_workaround_enabled = 1;
- }
- }
-#endif
/*
* SEP CPUID bug: Pentium Pro reports SEP but doesn't have it until
diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c
index 6605a02..222fa9c 100644
--- a/arch/x86/kernel/cpu/mce/amd.c
+++ b/arch/x86/kernel/cpu/mce/amd.c
@@ -850,7 +850,7 @@ bool amd_mce_usable_address(struct mce *m)
DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error)
{
trace_deferred_error_apic_entry(DEFERRED_ERROR_VECTOR);
- inc_irq_stat(irq_deferred_error_count);
+ inc_irq_stat(DEFERRED_ERROR);
deferred_error_int_vector();
trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR);
apic_eoi();
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 8dd424a..9bba1e2 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -49,6 +49,7 @@
#include <asm/fred.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
#include <asm/traps.h>
#include <asm/tlbflush.h>
@@ -67,8 +68,6 @@ static DEFINE_MUTEX(mce_sysfs_mutex);
#define SPINUNIT 100 /* 100ns */
-DEFINE_PER_CPU(unsigned, mce_exception_count);
-
DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
@@ -90,7 +89,6 @@ struct mca_config mca_cfg __read_mostly = {
};
static DEFINE_PER_CPU(struct mce_hw_err, hw_errs_seen);
-static unsigned long mce_need_notify;
/*
* MCA banks polled by the period polling timer for corrected events.
@@ -152,8 +150,10 @@ EXPORT_PER_CPU_SYMBOL_GPL(injectm);
void mce_log(struct mce_hw_err *err)
{
- if (mce_gen_pool_add(err))
+ if (mce_gen_pool_add(err)) {
+ pr_info(HW_ERR "Machine check events logged\n");
irq_work_queue(&mce_irq_work);
+ }
}
EXPORT_SYMBOL_GPL(mce_log);
@@ -585,28 +585,6 @@ bool mce_is_correctable(struct mce *m)
}
EXPORT_SYMBOL_GPL(mce_is_correctable);
-/*
- * Notify the user(s) about new machine check events.
- * Can be called from interrupt context, but not from machine check/NMI
- * context.
- */
-static bool mce_notify_irq(void)
-{
- /* Not more than two messages every minute */
- static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
-
- if (test_and_clear_bit(0, &mce_need_notify)) {
- mce_work_trigger();
-
- if (__ratelimit(&ratelimit))
- pr_info(HW_ERR "Machine check events logged\n");
-
- return true;
- }
-
- return false;
-}
-
static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
void *data)
{
@@ -618,9 +596,7 @@ static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
/* Emit the trace record: */
trace_mce_record(err);
- set_bit(0, &mce_need_notify);
-
- mce_notify_irq();
+ mce_work_trigger();
return NOTIFY_DONE;
}
@@ -716,8 +692,6 @@ static noinstr void mce_read_aux(struct mce_hw_err *err, int i)
}
}
-DEFINE_PER_CPU(unsigned, mce_poll_count);
-
/*
* We have three scenarios for checking for Deferred errors:
*
@@ -820,7 +794,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce *m;
int i;
- this_cpu_inc(mce_poll_count);
+ inc_irq_stat(MCE_POLL);
mce_gather_info(&err, NULL);
m = &err.m;
@@ -1595,7 +1569,7 @@ noinstr void do_machine_check(struct pt_regs *regs)
*/
lmce = 1;
- this_cpu_inc(mce_exception_count);
+ inc_irq_stat(MCE_EXCEPTION);
mce_gather_info(&err, regs);
m = &err.m;
@@ -1804,7 +1778,7 @@ static void mce_timer_fn(struct timer_list *t)
* Alert userspace if needed. If we logged an MCE, reduce the polling
* interval, otherwise increase the polling interval.
*/
- if (mce_notify_irq())
+ if (!mce_gen_pool_empty())
iv = max(iv / 2, (unsigned long) HZ/100);
else
iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c
index d02c4f5..42c82c1 100644
--- a/arch/x86/kernel/cpu/mce/inject.c
+++ b/arch/x86/kernel/cpu/mce/inject.c
@@ -26,6 +26,7 @@
#include <asm/amd/nb.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/irq_vectors.h>
#include <asm/mce.h>
#include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c
index 0d13c9f..6c370d5 100644
--- a/arch/x86/kernel/cpu/mce/threshold.c
+++ b/arch/x86/kernel/cpu/mce/threshold.c
@@ -37,7 +37,7 @@ void (*mce_threshold_vector)(void) = default_threshold_interrupt;
DEFINE_IDTENTRY_SYSVEC(sysvec_threshold)
{
trace_threshold_apic_entry(THRESHOLD_APIC_VECTOR);
- inc_irq_stat(irq_threshold_count);
+ inc_irq_stat(THRESHOLD_APIC);
mce_threshold_vector();
trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR);
apic_eoi();
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index e533881..531dfb7 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -34,6 +34,7 @@
#include <asm/microcode.h>
#include <asm/processor.h>
+#include <asm/cpuid/api.h>
#include <asm/cmdline.h>
#include <asm/setup.h>
#include <asm/cpu.h>
@@ -233,11 +234,6 @@ static bool need_sha_check(u32 cur_rev)
{
u32 cutoff;
- if (!cur_rev) {
- cur_rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax);
- pr_info_once("No current revision, generating the lowest one: 0x%x\n", cur_rev);
- }
-
cutoff = get_cutoff_revision(cur_rev);
if (cutoff)
return cur_rev <= cutoff;
@@ -322,7 +318,7 @@ static u32 get_patch_level(void)
{
u32 rev, dummy __always_unused;
- if (IS_ENABLED(CONFIG_MICROCODE_DBG) && hypervisor_present) {
+ if (IS_ENABLED(CONFIG_MICROCODE_DBG) && x86_hypervisor_present) {
int cpu = smp_processor_id();
if (!microcode_rev[cpu]) {
@@ -338,6 +334,13 @@ static u32 get_patch_level(void)
}
native_rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
+ if (!rev) {
+ if (x86_family(bsp_cpuid_1_eax) < 0x17)
+ return rev;
+
+ rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax);
+ pr_info_once("No current revision, generating the lowest one: 0x%x\n", rev);
+ }
return rev;
}
@@ -714,7 +717,7 @@ static bool __apply_microcode_amd(struct microcode_amd *mc, u32 *cur_rev,
invlpg(p_addr_end);
}
- if (IS_ENABLED(CONFIG_MICROCODE_DBG) && hypervisor_present)
+ if (IS_ENABLED(CONFIG_MICROCODE_DBG) && x86_hypervisor_present)
microcode_rev[smp_processor_id()] = mc->hdr.patch_id;
/* verify patch application was successful */
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index 651202e..0dd0c72 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -34,6 +34,7 @@
#include <asm/apic.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/perf_event.h>
#include <asm/processor.h>
#include <asm/cmdline.h>
@@ -57,7 +58,7 @@ bool force_minrev = IS_ENABLED(CONFIG_MICROCODE_LATE_FORCE_MINREV);
u32 base_rev;
u32 microcode_rev[NR_CPUS] = {};
-bool hypervisor_present;
+bool __ro_after_init x86_hypervisor_present;
/*
* Synchronization.
@@ -118,15 +119,10 @@ bool __init microcode_loader_disabled(void)
/*
* Disable when:
*
- * 1) The CPU does not support CPUID.
- */
- if (!cpuid_feature()) {
- dis_ucode_ldr = true;
- return dis_ucode_ldr;
- }
-
- /*
- * 2) Bit 31 in CPUID[1]:ECX is clear
+ * 1) The CPU does not support CPUID, detected below in
+ * load_ucode_bsp().
+ *
+ * 2) Bit 31 in CPUID[1]:ECX is set
* The bit is reserved for hypervisor use. This is still not
* completely accurate as XEN PV guests don't see that CPUID bit
* set, but that's good enough as they don't land on the BSP
@@ -135,9 +131,7 @@ bool __init microcode_loader_disabled(void)
* 3) Certain AMD patch levels are not allowed to be
* overwritten.
*/
- hypervisor_present = native_cpuid_ecx(1) & BIT(31);
-
- if ((hypervisor_present && !IS_ENABLED(CONFIG_MICROCODE_DBG)) ||
+ if ((x86_hypervisor_present && !IS_ENABLED(CONFIG_MICROCODE_DBG)) ||
amd_check_current_patch_level())
dis_ucode_ldr = true;
@@ -179,6 +173,11 @@ void __init load_ucode_bsp(void)
early_parse_cmdline();
+ if (!cpuid_feature())
+ dis_ucode_ldr = true;
+ else
+ x86_hypervisor_present = native_cpuid_ecx(1) & BIT(31);
+
if (microcode_loader_disabled())
return;
diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
index 72c8809..af8b1d8 100644
--- a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
+++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
@@ -1,238 +1,241 @@
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000404 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002d0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x24 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa0000d1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003a2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000341 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x11f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 },
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Auto-generated by scripts/update-intel-ucode-defs.py */
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002e0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x43a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x43a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x25 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa000124 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003f0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xae, .steppings = 0x0002, .platform_mask = 0x97, .driver_data = 0x1000273 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000382 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x132 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x132 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x125 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002c0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002c0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 },
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 37ac4af..f4a444e 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -25,6 +25,7 @@
#include <linux/mm.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/setup.h>
@@ -138,6 +139,9 @@ u32 intel_get_platform_id(void)
{
unsigned int val[2];
+ if (x86_hypervisor_present)
+ return 0;
+
/*
* This can be called early. Use CPUID directly instead of
* relying on cpuinfo_x86 which may not be fully initialized.
diff --git a/arch/x86/kernel/cpu/microcode/internal.h b/arch/x86/kernel/cpu/microcode/internal.h
index 3b93c06..a10b547 100644
--- a/arch/x86/kernel/cpu/microcode/internal.h
+++ b/arch/x86/kernel/cpu/microcode/internal.h
@@ -48,7 +48,6 @@ extern struct early_load_data early_data;
extern struct ucode_cpu_info ucode_cpu_info[];
extern u32 microcode_rev[NR_CPUS];
extern u32 base_rev;
-extern bool hypervisor_present;
struct cpio_data find_microcode_in_initrd(const char *path);
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index b5b6a58..185d4f6 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -19,6 +19,7 @@
#include <linux/random.h>
#include <asm/processor.h>
#include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
#include <hyperv/hvhdk.h>
#include <asm/mshyperv.h>
#include <asm/desc.h>
@@ -154,7 +155,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback)
{
struct pt_regs *old_regs = set_irq_regs(regs);
- inc_irq_stat(irq_hv_callback_count);
+ inc_irq_stat(HYPERVISOR_CALLBACK);
if (mshv_handler)
mshv_handler();
@@ -193,7 +194,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0)
{
struct pt_regs *old_regs = set_irq_regs(regs);
- inc_irq_stat(hyperv_stimer0_count);
+ inc_irq_stat(HYPERV_STIMER0);
if (hv_stimer0_handler)
hv_stimer0_handler();
add_interrupt_randomness(HYPERV_STIMER0_VECTOR);
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 7667cf7..9c01d25 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -22,6 +22,7 @@
#include <linux/cpuhotplug.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/resctrl.h>
#include "internal.h"
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index 9bd87ba..c749ae2 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -21,6 +21,7 @@
#include <linux/resctrl.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include "internal.h"
@@ -377,7 +378,12 @@ static const struct x86_cpu_id snc_cpu_ids[] __initconst = {
static __init int snc_get_config(void)
{
- int ret = topology_num_nodes_per_package();
+ int ret;
+
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+ return 1;
+
+ ret = topology_num_nodes_per_package();
if (ret > 1 && !x86_match_cpu(snc_cpu_ids)) {
pr_warn("CoD enabled system? Resctrl not supported\n");
@@ -454,6 +460,7 @@ int __init rdt_get_l3_mon_config(struct rdt_resource *r)
(rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL) ||
rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL))) {
r->mon.mbm_cntr_assignable = true;
+ r->mon.mbm_cntr_configurable = true;
cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx);
r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1;
hw_res->mbm_cntr_assign_enabled = true;
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index 837d6a4..937129c 100644
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -6,6 +6,7 @@
#include <asm/memtype.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
#include "cpu.h"
diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c
index 4736197..9268289 100644
--- a/arch/x86/kernel/cpu/sgx/driver.c
+++ b/arch/x86/kernel/cpu/sgx/driver.c
@@ -6,7 +6,10 @@
#include <linux/mman.h>
#include <linux/security.h>
#include <linux/suspend.h>
+
+#include <asm/cpuid/api.h>
#include <asm/traps.h>
+
#include "driver.h"
#include "encl.h"
diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c
index 38b7fd2..4505f80 100644
--- a/arch/x86/kernel/cpu/sgx/main.c
+++ b/arch/x86/kernel/cpu/sgx/main.c
@@ -15,9 +15,12 @@
#include <linux/slab.h>
#include <linux/sysfs.h>
#include <linux/vmalloc.h>
+
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/sgx.h>
#include <asm/archrandom.h>
+
#include "driver.h"
#include "encl.h"
#include "encls.h"
diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c
index 6ac097e..da080d7 100644
--- a/arch/x86/kernel/cpu/topology_amd.c
+++ b/arch/x86/kernel/cpu/topology_amd.c
@@ -2,6 +2,7 @@
#include <linux/cpu.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/memtype.h>
#include <asm/msr.h>
#include <asm/processor.h>
@@ -79,7 +80,7 @@ static bool parse_8000_001e(struct topo_scan *tscan)
if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
return false;
- cpuid_leaf(0x8000001e, &leaf);
+ cpuid_read(0x8000001e, &leaf);
/*
* If leaf 0xb/0x26 is available, then the APIC ID and the domain
diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c
index d0d79d5..cf75134 100644
--- a/arch/x86/kernel/cpu/topology_common.c
+++ b/arch/x86/kernel/cpu/topology_common.c
@@ -6,6 +6,7 @@
#include <asm/intel-family.h>
#include <asm/apic.h>
#include <asm/processor.h>
+#include <asm/cpuid/api.h>
#include <asm/smp.h>
#include "cpu.h"
diff --git a/arch/x86/kernel/cpu/topology_ext.c b/arch/x86/kernel/cpu/topology_ext.c
index 467b032..60dfaa0 100644
--- a/arch/x86/kernel/cpu/topology_ext.c
+++ b/arch/x86/kernel/cpu/topology_ext.c
@@ -2,6 +2,7 @@
#include <linux/cpu.h>
#include <asm/apic.h>
+#include <asm/cpuid/api.h>
#include <asm/memtype.h>
#include <asm/processor.h>
@@ -70,7 +71,7 @@ static inline bool topo_subleaf(struct topo_scan *tscan, u32 leaf, u32 subleaf,
default: return false;
}
- cpuid_subleaf(leaf, subleaf, &sl);
+ cpuid_read_subleaf(leaf, subleaf, &sl);
if (!sl.num_processors || sl.type == INVALID_TYPE)
return false;
diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c
index 42c9398..1fdcd69 100644
--- a/arch/x86/kernel/cpu/transmeta.c
+++ b/arch/x86/kernel/cpu/transmeta.c
@@ -3,8 +3,11 @@
#include <linux/sched.h>
#include <linux/sched/clock.h>
#include <linux/mm.h>
+
#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
+
#include "cpu.h"
static void early_init_transmeta(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/umc.c b/arch/x86/kernel/cpu/umc.c
deleted file mode 100644
index 65a58a3..0000000
--- a/arch/x86/kernel/cpu/umc.c
+++ /dev/null
@@ -1,26 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/kernel.h>
-#include <asm/processor.h>
-#include "cpu.h"
-
-/*
- * UMC chips appear to be only either 386 or 486,
- * so no special init takes place.
- */
-
-static const struct cpu_dev umc_cpu_dev = {
- .c_vendor = "UMC",
- .c_ident = { "UMC UMC UMC" },
- .legacy_models = {
- { .family = 4, .model_names =
- {
- [1] = "U5D",
- [2] = "U5S",
- }
- },
- },
- .c_x86_vendor = X86_VENDOR_UMC,
-};
-
-cpu_dev_register(umc_cpu_dev);
-
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index eee0d1a..34b7357 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -33,6 +33,7 @@
#include <asm/div64.h>
#include <asm/x86_init.h>
#include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
#include <asm/timer.h>
#include <asm/apic.h>
#include <asm/vmware.h>
diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c
index 031379b..761aef5 100644
--- a/arch/x86/kernel/cpu/zhaoxin.c
+++ b/arch/x86/kernel/cpu/zhaoxin.c
@@ -4,6 +4,7 @@
#include <asm/cpu.h>
#include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include "cpu.h"
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index dae4362..b55fe9c 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -37,6 +37,7 @@
#include <linux/gfp.h>
#include <linux/completion.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
#include <asm/msr.h>
@@ -58,8 +59,7 @@ static void cpuid_smp_cpuid(void *cmd_block)
complete(&cmd->done);
}
-static ssize_t cpuid_read(struct file *file, char __user *buf,
- size_t count, loff_t *ppos)
+static ssize_t cpuid_read_f(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
char __user *tmp = buf;
struct cpuid_regs_done cmd;
@@ -119,7 +119,7 @@ static int cpuid_open(struct inode *inode, struct file *file)
static const struct file_operations cpuid_fops = {
.owner = THIS_MODULE,
.llseek = no_seek_end_llseek,
- .read = cpuid_read,
+ .read = cpuid_read_f,
.open = cpuid_open,
};
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 2a99927..eb72537 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -450,6 +450,10 @@ __init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entr
{
struct boot_e820_entry *entry = entries;
+ /* If there aren't any entries, we'll want to fall back to another source: */
+ if (!nr_entries)
+ return -ENOENT;
+
while (nr_entries) {
u64 start = entry->addr;
u64 size = entry->size;
@@ -458,7 +462,7 @@ __init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entr
/* Ignore the remaining entries on 64-bit overflow: */
if (start > end && likely(size))
- return -1;
+ return -EINVAL;
e820__range_add(start, size, type);
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index 6089838..584fb99 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -558,11 +558,6 @@ static inline void fpstate_init_fstate(struct fpstate *fpstate)
*/
void fpstate_init_user(struct fpstate *fpstate)
{
- if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
- fpstate_init_soft(&fpstate->regs.soft);
- return;
- }
-
xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures);
if (cpu_feature_enabled(X86_FEATURE_FXSR))
diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c
index ff988b9..0d33c21 100644
--- a/arch/x86/kernel/fpu/init.c
+++ b/arch/x86/kernel/fpu/init.c
@@ -36,12 +36,7 @@ static void fpu__init_cpu_generic(void)
write_cr0(cr0);
/* Flush out any pending x87 state: */
-#ifdef CONFIG_MATH_EMULATION
- if (!boot_cpu_has(X86_FEATURE_FPU))
- ;
- else
-#endif
- asm volatile ("fninit");
+ asm volatile ("fninit");
}
/*
@@ -86,13 +81,11 @@ static void __init fpu__init_system_early_generic(void)
setup_clear_cpu_cap(X86_FEATURE_FPU);
}
-#ifndef CONFIG_MATH_EMULATION
if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
for (;;)
asm volatile("hlt");
}
-#endif
}
/*
diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index c3ec251..20b638c 100644
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -27,14 +27,19 @@
static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
struct _fpx_sw_bytes *fx_sw)
{
+ int min_xstate_size = sizeof(struct fxregs_state) +
+ sizeof(struct xstate_header);
void __user *fpstate = fxbuf;
unsigned int magic2;
if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
return false;
- /* Check for the first magic field */
- if (fx_sw->magic1 != FP_XSTATE_MAGIC1)
+ /* Check for the first magic field and other error scenarios. */
+ if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
+ fx_sw->xstate_size < min_xstate_size ||
+ fx_sw->xstate_size > x86_task_fpu(current)->fpstate->user_size ||
+ fx_sw->xstate_size > fx_sw->extended_size)
goto setfx;
/*
@@ -43,7 +48,7 @@ static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
* fpstate layout with out copying the extended state information
* in the memory layout.
*/
- if (__get_user(magic2, (__u32 __user *)(fpstate + x86_task_fpu(current)->fpstate->user_size)))
+ if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
return false;
if (likely(magic2 == FP_XSTATE_MAGIC2))
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 0543b57..17d6edf 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -376,6 +376,13 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
}
/*
+ * Generated trampoline may contain rIP-relative addressing which
+ * displacement needs to be fixed.
+ */
+ text_poke_apply_relocation(trampoline, trampoline, size,
+ (void *)start_offset, size);
+
+ /*
* The address of the ftrace_ops that is used for this trampoline
* is stored at the end of the trampoline. This will be used to
* load the third parameter for the callback. Basically, that
diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c
index f67063d..f7a86b9 100644
--- a/arch/x86/kernel/i8259.c
+++ b/arch/x86/kernel/i8259.c
@@ -214,7 +214,7 @@ static void mask_and_ack_8259A(struct irq_data *data)
"spurious 8259A interrupt: IRQ%d.\n", irq);
spurious_irq_mask |= irqmask;
}
- atomic_inc(&irq_err_count);
+ irq_stat_inc_and_enable(IRQ_COUNT_PIC_APIC_ERROR);
/*
* Theoretically we do not have to handle this IRQ,
* but in Linux this does not cause problems and is
diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index 2604565..90a22e2 100644
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -268,6 +268,13 @@ void __init idt_setup_early_pf(void)
}
#endif
+#if IS_ENABLED(CONFIG_KVM_INTEL) && !defined(CONFIG_X86_64)
+void idt_entry_from_kvm(unsigned int vector)
+{
+ idt_do_interrupt_irqoff(gate_offset(idt_table + vector));
+}
+#endif
+
static void __init idt_map_in_cea(void)
{
/*
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index ec77be2..9d98c81 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -39,8 +39,6 @@ EXPORT_PER_CPU_SYMBOL(__softirq_pending);
DEFINE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr);
-atomic_t irq_err_count;
-
/*
* 'what should we do if we get a hw irq event on an illegal vector'.
* each architecture has to answer this themselves.
@@ -62,150 +60,131 @@ void ack_bad_irq(unsigned int irq)
apic_eoi();
}
-#define irq_stats(x) (&per_cpu(irq_stat, x))
+struct irq_stat_info {
+ unsigned int skip_vector;
+ const char *symbol;
+ const char *text;
+};
+
+#define DEFAULT_SUPPRESSED_VECTOR UINT_MAX
+
+#define ISS(idx, sym, txt) [IRQ_COUNT_##idx] = { .symbol = sym, .text = txt }
+
+#define ITS(idx, sym, txt) [IRQ_COUNT_##idx] = \
+ { .skip_vector = idx## _VECTOR, .symbol = sym, .text = txt }
+
+#define IDS(idx, sym, txt) [IRQ_COUNT_##idx] = \
+ { .skip_vector = DEFAULT_SUPPRESSED_VECTOR, .symbol = sym, .text = txt }
+
+static const struct irq_stat_info irq_stat_info[IRQ_COUNT_MAX] = {
+ ISS(NMI, "NMI", " Non-maskable interrupts\n"),
+#ifdef CONFIG_X86_LOCAL_APIC
+ ISS(APIC_TIMER, "LOC", " Local timer interrupts\n"),
+ IDS(SPURIOUS, "SPU", " Spurious interrupts\n"),
+ ISS(APIC_PERF, "PMI", " Performance monitoring interrupts\n"),
+ ISS(IRQ_WORK, "IWI", " IRQ work interrupts\n"),
+ IDS(ICR_READ_RETRY, "RTR", " APIC ICR read retries\n"),
+ ISS(X86_PLATFORM_IPI, "PLT", " Platform interrupts\n"),
+#endif
+#ifdef CONFIG_SMP
+ ISS(RESCHEDULE, "RES", " Rescheduling interrupts\n"),
+ ISS(CALL_FUNCTION, "CAL", " Function call interrupts\n"),
+#endif
+ ISS(TLB, "TLB", " TLB shootdowns\n"),
+#ifdef CONFIG_X86_THERMAL_VECTOR
+ ISS(THERMAL_APIC, "TRM", " Thermal event interrupt\n"),
+#endif
+#ifdef CONFIG_X86_MCE_THRESHOLD
+ ISS(THRESHOLD_APIC, "THR", " Threshold APIC interrupts\n"),
+#endif
+#ifdef CONFIG_X86_MCE_AMD
+ ISS(DEFERRED_ERROR, "DFR", " Deferred Error APIC interrupts\n"),
+#endif
+#ifdef CONFIG_X86_MCE
+ ISS(MCE_EXCEPTION, "MCE", " Machine check exceptions\n"),
+ ISS(MCE_POLL, "MCP", " Machine check polls\n"),
+#endif
+#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
+ ITS(HYPERVISOR_CALLBACK, "HYP", " Hypervisor callback interrupts\n"),
+#endif
+#if IS_ENABLED(CONFIG_HYPERV)
+ ITS(HYPERV_REENLIGHTENMENT, "HRE", " Hyper-V reenlightenment interrupts\n"),
+ ITS(HYPERV_STIMER0, "HVS", " Hyper-V stimer0 interrupts\n"),
+#endif
+#if IS_ENABLED(CONFIG_KVM)
+ ITS(POSTED_INTR, "PIN", " Posted-interrupt notification event\n"),
+ ITS(POSTED_INTR_NESTED, "NPI", " Nested posted-interrupt event\n"),
+ ITS(POSTED_INTR_WAKEUP, "PIW", " Posted-interrupt wakeup event\n"),
+#endif
+#ifdef CONFIG_GUEST_PERF_EVENTS
+ ISS(PERF_GUEST_MEDIATED_PMI, "VPMI", " Perf Guest Mediated PMI\n"),
+#endif
+#ifdef CONFIG_X86_POSTED_MSI
+ ISS(POSTED_MSI_NOTIFICATION, "PMN", " Posted MSI notification event\n"),
+#endif
+ IDS(PIC_APIC_ERROR, "ERR", " PIC/APIC error interrupts\n"),
+#ifdef CONFIG_X86_IO_APIC
+ IDS(IOAPIC_MISROUTED, "MIS", " Misrouted IO/APIC interrupts\n"),
+#endif
+};
+
+static DECLARE_BITMAP(irq_stat_count_show, IRQ_COUNT_MAX) __read_mostly;
+
+static int __init irq_init_stats(void)
+{
+ const struct irq_stat_info *info = irq_stat_info;
+
+ for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) {
+ if (!info->skip_vector || (info->skip_vector != DEFAULT_SUPPRESSED_VECTOR &&
+ test_bit(info->skip_vector, system_vectors)))
+ set_bit(i, irq_stat_count_show);
+ }
+
+#ifdef CONFIG_X86_LOCAL_APIC
+ if (!x86_platform_ipi_callback)
+ clear_bit(IRQ_COUNT_X86_PLATFORM_IPI, irq_stat_count_show);
+#endif
+
+#ifdef CONFIG_X86_POSTED_MSI
+ if (!posted_msi_enabled())
+ clear_bit(IRQ_COUNT_POSTED_MSI_NOTIFICATION, irq_stat_count_show);
+#endif
+
+#ifdef CONFIG_X86_MCE_AMD
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
+ boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
+ clear_bit(IRQ_COUNT_DEFERRED_ERROR, irq_stat_count_show);
+#endif
+ return 0;
+}
+late_initcall(irq_init_stats);
+
+/*
+ * Used for default disabled counters to increment the stats and to enable the
+ * entry for /proc/interrupts output.
+ */
+void irq_stat_inc_and_enable(enum irq_stat_counts which)
+{
+ this_cpu_inc(irq_stat.counts[which]);
+ set_bit(which, irq_stat_count_show);
+}
+
+#ifdef CONFIG_PROC_FS
/*
* /proc/interrupts printing for arch specific interrupts
*/
int arch_show_interrupts(struct seq_file *p, int prec)
{
- int j;
+ const struct irq_stat_info *info = irq_stat_info;
- seq_printf(p, "%*s: ", prec, "NMI");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
- seq_puts(p, " Non-maskable interrupts\n");
-#ifdef CONFIG_X86_LOCAL_APIC
- seq_printf(p, "%*s: ", prec, "LOC");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
- seq_puts(p, " Local timer interrupts\n");
+ for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) {
+ if (!test_bit(i, irq_stat_count_show))
+ continue;
- seq_printf(p, "%*s: ", prec, "SPU");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
- seq_puts(p, " Spurious interrupts\n");
- seq_printf(p, "%*s: ", prec, "PMI");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
- seq_puts(p, " Performance monitoring interrupts\n");
- seq_printf(p, "%*s: ", prec, "IWI");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
- seq_puts(p, " IRQ work interrupts\n");
- seq_printf(p, "%*s: ", prec, "RTR");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
- seq_puts(p, " APIC ICR read retries\n");
- if (x86_platform_ipi_callback) {
- seq_printf(p, "%*s: ", prec, "PLT");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
- seq_puts(p, " Platform interrupts\n");
+ seq_printf(p, "%*s:", prec, info->symbol);
+ irq_proc_emit_counts(p, &irq_stat.counts[i]);
+ seq_puts(p, info->text);
}
-#endif
-#ifdef CONFIG_SMP
- seq_printf(p, "%*s: ", prec, "RES");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
- seq_puts(p, " Rescheduling interrupts\n");
- seq_printf(p, "%*s: ", prec, "CAL");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
- seq_puts(p, " Function call interrupts\n");
- seq_printf(p, "%*s: ", prec, "TLB");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
- seq_puts(p, " TLB shootdowns\n");
-#endif
-#ifdef CONFIG_X86_THERMAL_VECTOR
- seq_printf(p, "%*s: ", prec, "TRM");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
- seq_puts(p, " Thermal event interrupts\n");
-#endif
-#ifdef CONFIG_X86_MCE_THRESHOLD
- seq_printf(p, "%*s: ", prec, "THR");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
- seq_puts(p, " Threshold APIC interrupts\n");
-#endif
-#ifdef CONFIG_X86_MCE_AMD
- seq_printf(p, "%*s: ", prec, "DFR");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
- seq_puts(p, " Deferred Error APIC interrupts\n");
-#endif
-#ifdef CONFIG_X86_MCE
- seq_printf(p, "%*s: ", prec, "MCE");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
- seq_puts(p, " Machine check exceptions\n");
- seq_printf(p, "%*s: ", prec, "MCP");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
- seq_puts(p, " Machine check polls\n");
-#endif
-#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
- if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
- seq_printf(p, "%*s: ", prec, "HYP");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->irq_hv_callback_count);
- seq_puts(p, " Hypervisor callback interrupts\n");
- }
-#endif
-#if IS_ENABLED(CONFIG_HYPERV)
- if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
- seq_printf(p, "%*s: ", prec, "HRE");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->irq_hv_reenlightenment_count);
- seq_puts(p, " Hyper-V reenlightenment interrupts\n");
- }
- if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
- seq_printf(p, "%*s: ", prec, "HVS");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->hyperv_stimer0_count);
- seq_puts(p, " Hyper-V stimer0 interrupts\n");
- }
-#endif
- seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
-#if defined(CONFIG_X86_IO_APIC)
- seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
-#endif
-#if IS_ENABLED(CONFIG_KVM)
- seq_printf(p, "%*s: ", prec, "PIN");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
- seq_puts(p, " Posted-interrupt notification event\n");
-
- seq_printf(p, "%*s: ", prec, "NPI");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->kvm_posted_intr_nested_ipis);
- seq_puts(p, " Nested posted-interrupt event\n");
-
- seq_printf(p, "%*s: ", prec, "PIW");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->kvm_posted_intr_wakeup_ipis);
- seq_puts(p, " Posted-interrupt wakeup event\n");
-#endif
-#ifdef CONFIG_GUEST_PERF_EVENTS
- seq_printf(p, "%*s: ", prec, "VPMI");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->perf_guest_mediated_pmis);
- seq_puts(p, " Perf Guest Mediated PMI\n");
-#endif
-#ifdef CONFIG_X86_POSTED_MSI
- seq_printf(p, "%*s: ", prec, "PMN");
- for_each_online_cpu(j)
- seq_printf(p, "%10u ",
- irq_stats(j)->posted_msi_notification_count);
- seq_puts(p, " Posted MSI notification event\n");
-#endif
return 0;
}
@@ -214,46 +193,14 @@ int arch_show_interrupts(struct seq_file *p, int prec)
*/
u64 arch_irq_stat_cpu(unsigned int cpu)
{
- u64 sum = irq_stats(cpu)->__nmi_count;
+ irq_cpustat_t *p = per_cpu_ptr(&irq_stat, cpu);
+ u64 sum = 0;
-#ifdef CONFIG_X86_LOCAL_APIC
- sum += irq_stats(cpu)->apic_timer_irqs;
- sum += irq_stats(cpu)->irq_spurious_count;
- sum += irq_stats(cpu)->apic_perf_irqs;
- sum += irq_stats(cpu)->apic_irq_work_irqs;
- sum += irq_stats(cpu)->icr_read_retry_count;
- if (x86_platform_ipi_callback)
- sum += irq_stats(cpu)->x86_platform_ipis;
-#endif
-#ifdef CONFIG_SMP
- sum += irq_stats(cpu)->irq_resched_count;
- sum += irq_stats(cpu)->irq_call_count;
-#endif
-#ifdef CONFIG_X86_THERMAL_VECTOR
- sum += irq_stats(cpu)->irq_thermal_count;
-#endif
-#ifdef CONFIG_X86_MCE_THRESHOLD
- sum += irq_stats(cpu)->irq_threshold_count;
-#endif
-#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
- sum += irq_stats(cpu)->irq_hv_callback_count;
-#endif
-#if IS_ENABLED(CONFIG_HYPERV)
- sum += irq_stats(cpu)->irq_hv_reenlightenment_count;
- sum += irq_stats(cpu)->hyperv_stimer0_count;
-#endif
-#ifdef CONFIG_X86_MCE
- sum += per_cpu(mce_exception_count, cpu);
- sum += per_cpu(mce_poll_count, cpu);
-#endif
+ for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++)
+ sum += p->counts[i];
return sum;
}
-
-u64 arch_irq_stat(void)
-{
- u64 sum = atomic_read(&irq_err_count);
- return sum;
-}
+#endif /* CONFIG_PROC_FS */
static __always_inline void handle_irq(struct irq_desc *desc,
struct pt_regs *regs)
@@ -338,7 +285,7 @@ DEFINE_IDTENTRY_IRQ(common_interrupt)
#ifdef CONFIG_X86_LOCAL_APIC
/* Function pointer for generic interrupt vector handling */
-void (*x86_platform_ipi_callback)(void) = NULL;
+void (*x86_platform_ipi_callback)(void) __ro_after_init = NULL;
/*
* Handler for X86_PLATFORM_IPI_VECTOR.
*/
@@ -348,7 +295,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
apic_eoi();
trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
- inc_irq_stat(x86_platform_ipis);
+ inc_irq_stat(X86_PLATFORM_IPI);
if (x86_platform_ipi_callback)
x86_platform_ipi_callback();
trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
@@ -363,7 +310,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
DEFINE_IDTENTRY_SYSVEC(sysvec_perf_guest_mediated_pmi_handler)
{
apic_eoi();
- inc_irq_stat(perf_guest_mediated_pmis);
+ inc_irq_stat(PERF_GUEST_MEDIATED_PMI);
perf_guest_handle_mediated_pmi();
}
#endif
@@ -389,7 +336,7 @@ EXPORT_SYMBOL_FOR_KVM(kvm_set_posted_intr_wakeup_handler);
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
{
apic_eoi();
- inc_irq_stat(kvm_posted_intr_ipis);
+ inc_irq_stat(POSTED_INTR);
}
/*
@@ -398,7 +345,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
{
apic_eoi();
- inc_irq_stat(kvm_posted_intr_wakeup_ipis);
+ inc_irq_stat(POSTED_INTR_WAKEUP);
kvm_posted_intr_wakeup_handler();
}
@@ -408,7 +355,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
{
apic_eoi();
- inc_irq_stat(kvm_posted_intr_nested_ipis);
+ inc_irq_stat(POSTED_INTR_NESTED);
}
#endif
@@ -482,7 +429,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification)
/* Mark the handler active for intel_ack_posted_msi_irq() */
__this_cpu_write(posted_msi_handler_active, true);
- inc_irq_stat(posted_msi_notification_count);
+ inc_irq_stat(POSTED_MSI_NOTIFICATION);
irq_enter();
/*
@@ -577,7 +524,7 @@ static void smp_thermal_vector(void)
DEFINE_IDTENTRY_SYSVEC(sysvec_thermal)
{
trace_thermal_apic_entry(THERMAL_APIC_VECTOR);
- inc_irq_stat(irq_thermal_count);
+ inc_irq_stat(THERMAL_APIC);
smp_thermal_vector();
trace_thermal_apic_exit(THERMAL_APIC_VECTOR);
apic_eoi();
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c
index b0a24de..308c624 100644
--- a/arch/x86/kernel/irq_work.c
+++ b/arch/x86/kernel/irq_work.c
@@ -18,7 +18,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work)
{
apic_eoi();
trace_irq_work_entry(IRQ_WORK_VECTOR);
- inc_irq_stat(apic_irq_work_irqs);
+ inc_irq_stat(IRQ_WORK);
irq_work_run();
trace_irq_work_exit(IRQ_WORK_VECTOR);
}
diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c
index 9e9a591..f58ce92 100644
--- a/arch/x86/kernel/jailhouse.c
+++ b/arch/x86/kernel/jailhouse.c
@@ -17,6 +17,7 @@
#include <asm/io_apic.h>
#include <asm/acpi.h>
#include <asm/cpu.h>
+#include <asm/cpuid/api.h>
#include <asm/hypervisor.h>
#include <asm/i8259.h>
#include <asm/irqdomain.h>
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 29226d1..dcef84d 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -41,6 +41,7 @@
#include <asm/hypervisor.h>
#include <asm/mtrr.h>
#include <asm/tlb.h>
+#include <asm/cpuid/api.h>
#include <asm/cpuidle_haltpoll.h>
#include <asm/msr.h>
#include <asm/ptrace.h>
@@ -304,7 +305,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt)
apic_eoi();
- inc_irq_stat(irq_hv_callback_count);
+ inc_irq_stat(HYPERVISOR_CALLBACK);
if (__this_cpu_read(async_pf_enabled)) {
token = __this_cpu_read(apf_reason.token);
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index b5991d5..cb3d0ca 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -87,6 +87,27 @@ static u64 kvm_clock_get_cycles(struct clocksource *cs)
return kvm_clock_read();
}
+static u64 kvm_clock_get_cycles_snapshot(struct clocksource *cs,
+ struct clocksource_hw_snapshot *chs)
+{
+ struct pvclock_vcpu_time_info *src;
+ unsigned version;
+ u64 ret, tsc;
+
+ preempt_disable_notrace();
+ src = this_cpu_pvti();
+ do {
+ version = pvclock_read_begin(src);
+ tsc = rdtsc_ordered();
+ ret = __pvclock_read_cycles(src, tsc);
+ } while (pvclock_read_retry(src, version));
+ preempt_enable_notrace();
+
+ chs->hw_cycles = tsc;
+ chs->hw_csid = CSID_X86_TSC;
+ return ret;
+}
+
static noinstr u64 kvm_sched_clock_read(void)
{
return pvclock_clocksource_read_nowd(this_cpu_pvti()) - kvm_sched_clock_offset;
@@ -156,13 +177,14 @@ static int kvm_cs_enable(struct clocksource *cs)
}
static struct clocksource kvm_clock = {
- .name = "kvm-clock",
- .read = kvm_clock_get_cycles,
- .rating = 400,
- .mask = CLOCKSOURCE_MASK(64),
- .flags = CLOCK_SOURCE_IS_CONTINUOUS,
- .id = CSID_X86_KVM_CLK,
- .enable = kvm_cs_enable,
+ .name = "kvm-clock",
+ .read = kvm_clock_get_cycles,
+ .read_snapshot = kvm_clock_get_cycles_snapshot,
+ .rating = 400,
+ .mask = CLOCKSOURCE_MASK(64),
+ .flags = CLOCK_SOURCE_IS_CONTINUOUS,
+ .id = CSID_X86_KVM_CLK,
+ .enable = kvm_cs_enable,
};
static void kvm_register_clock(char *txt)
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3d239ed..3c9f60d 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -576,7 +576,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi)
irq_state = irqentry_nmi_enter(regs);
- inc_irq_stat(__nmi_count);
+ inc_irq_stat(NMI);
if (IS_ENABLED(CONFIG_NMI_CHECK_CPU) && ignore_nmis) {
WRITE_ONCE(nsp->idt_ignored, nsp->idt_ignored + 1);
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx)
{
exc_nmi(regs);
}
-EXPORT_SYMBOL_FOR_KVM(asm_exc_nmi_kvm_vmx);
#endif
#ifdef CONFIG_NMI_CHECK_CPU
@@ -725,7 +724,7 @@ DEFINE_FREDENTRY_NMI(exc_nmi)
irq_state = irqentry_nmi_enter(regs);
- inc_irq_stat(__nmi_count);
+ inc_irq_stat(NMI);
default_do_nmi(regs);
irqentry_nmi_exit(regs, irq_state);
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 792fa96..44f29fc 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -24,6 +24,7 @@
#include <asm/time.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
+#include <asm/cpuid/api.h>
#include <asm/delay.h>
#include <asm/fixmap.h>
#include <asm/apic.h>
diff --git a/arch/x86/kernel/pmem.c b/arch/x86/kernel/pmem.c
index 23154d2..04fb221 100644
--- a/arch/x86/kernel/pmem.c
+++ b/arch/x86/kernel/pmem.c
@@ -27,6 +27,8 @@ static __init int register_e820_pmem(void)
* simply here to trigger the module to load on demand.
*/
pdev = platform_device_alloc("e820_pmem", -1);
+ if (!pdev)
+ return -ENOMEM;
rc = platform_device_add(pdev);
if (rc)
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 4c718f8..a554f19 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -969,7 +969,7 @@ void amd_e400_c1e_apic_setup(void)
void __init arch_post_acpi_subsys_init(void)
{
- u32 lo, hi;
+ u64 val;
if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
return;
@@ -979,8 +979,8 @@ void __init arch_post_acpi_subsys_init(void)
* the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
* MSR_K8_INT_PENDING_MSG.
*/
- rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
- if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
+ rdmsrq(MSR_K8_INT_PENDING_MSG, val);
+ if (!(val & K8_INTP_C1E_ACTIVE_MASK))
return;
boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);
diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 4ffba68..eaeb774 100644
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -136,6 +136,14 @@
* %r13 original CR4 when relocate_kernel() was invoked
*/
+ /*
+ * Set return address to 0 if not preserving context. The purgatory
+ * shipped in kexec-tools will unconditionally look for the return
+ * address on the stack and set a kexec_jump_back_entry= command
+ * line option if it's non-zero. There's no other way that it can
+ * tell a preserve-context (kjump) kexec from a normal one.
+ */
+ pushq $0
/* store the start address on the stack */
pushq %rdx
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index cbf95fe..985103c 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -250,7 +250,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
{
apic_eoi();
trace_reschedule_entry(RESCHEDULE_VECTOR);
- inc_irq_stat(irq_resched_count);
+ inc_irq_stat(RESCHEDULE);
scheduler_ipi();
trace_reschedule_exit(RESCHEDULE_VECTOR);
}
@@ -259,7 +259,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
{
apic_eoi();
trace_call_function_entry(CALL_FUNCTION_VECTOR);
- inc_irq_stat(irq_call_count);
+ inc_irq_stat(CALL_FUNCTION);
generic_smp_call_function_interrupt();
trace_call_function_exit(CALL_FUNCTION_VECTOR);
}
@@ -268,7 +268,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single)
{
apic_eoi();
trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR);
- inc_irq_stat(irq_call_count);
+ inc_irq_stat(CALL_FUNCTION);
generic_smp_call_function_single_interrupt();
trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR);
}
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 294a8ea..cb999fe 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -424,6 +424,21 @@ static const struct x86_cpu_id intel_cod_cpu[] = {
{}
};
+/*
+ * Allows splitting the LLC by matching 'core_id % split_llc'.
+ *
+ * This is mostly a debug hack to emulate systems with multiple LLCs per node
+ * on systems that do not naturally have this.
+ */
+static unsigned int split_llc = 0;
+
+static int __init split_llc_setup(char *str)
+{
+ get_option(&str, &split_llc);
+ return 0;
+}
+early_param("split_llc", split_llc_setup);
+
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
@@ -438,6 +453,11 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))
return false;
+ if (split_llc &&
+ (per_cpu_core_id(cpu1) % split_llc) !=
+ (per_cpu_core_id(cpu2) % split_llc))
+ return false;
+
/*
* Allow the SNC topology without warning. Return of false
* means 'c' does not share the LLC of 'o'. This will be
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 0ca3912..30aa836 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -1475,13 +1475,6 @@ DEFINE_IDTENTRY(exc_coprocessor_error)
DEFINE_IDTENTRY(exc_simd_coprocessor_error)
{
- if (IS_ENABLED(CONFIG_X86_INVD_BUG)) {
- /* AMD 486 bug: INVD in CPL 0 raises #XF instead of #GP */
- if (!static_cpu_has(X86_FEATURE_XMM)) {
- __exc_general_protection(regs, 0);
- return;
- }
- }
math_error(regs, X86_TRAP_XF);
}
@@ -1550,20 +1543,6 @@ DEFINE_IDTENTRY(exc_device_not_available)
if (handle_xfd_event(regs))
return;
-#ifdef CONFIG_MATH_EMULATION
- if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) {
- struct math_emu_info info = { };
-
- cond_local_irq_enable(regs);
-
- info.regs = regs;
- math_emulate(&info);
-
- cond_local_irq_disable(regs);
- return;
- }
-#endif
-
/* This should not happen. */
if (WARN(cr0 & X86_CR0_TS, "CR0.TS was set")) {
/* Try to fix it up and carry on. */
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index c8c6cc0..8013dcc 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -4481,7 +4481,7 @@ static const struct opcode opcode_map_0f_38[256] = {
X16(N), X16(N),
/* 0x20 - 0x2f */
X8(N),
- X2(N), GP(SrcReg | DstMem | ModRM | Mov | Aligned, &pfx_0f_e7_0f_38_2a), N, N, N, N, N,
+ X2(N), GP(SrcMem | DstReg | ModRM | Mov | Aligned, &pfx_0f_e7_0f_38_2a), N, N, N, N, N,
/* 0x30 - 0x7f */
X16(N), X16(N), X16(N), X16(N), X16(N),
/* 0x80 - 0xef */
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 9b140bb..4438eca 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2040,7 +2040,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
* flush). Translate the address here so the memory can be uniformly
* read with kvm_read_guest().
*/
- if (!hc->fast && is_guest_mode(vcpu)) {
+ if (!hc->fast && mmu_is_nested(vcpu)) {
hc->ingpa = translate_nested_gpa(vcpu, hc->ingpa, 0, NULL);
if (unlikely(hc->ingpa == INVALID_GPA))
return HV_STATUS_INVALID_HYPERCALL_INPUT;
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index e3ec4d8..4078e62 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -667,13 +667,15 @@ bool __kvm_apic_update_irr(unsigned long *pir, void *regs, int *max_irr)
u32 *__pir = (void *)pir_vals;
u32 i, vec;
u32 irr_val, prev_irr_val;
- int max_updated_irr;
+ int max_new_irr;
- max_updated_irr = -1;
- *max_irr = -1;
-
- if (!pi_harvest_pir(pir, pir_vals))
+ if (!pi_harvest_pir(pir, pir_vals)) {
+ *max_irr = apic_find_highest_vector(regs + APIC_IRR);
return false;
+ }
+
+ max_new_irr = -1;
+ *max_irr = -1;
for (i = vec = 0; i <= 7; i++, vec += 32) {
u32 *p_irr = (u32 *)(regs + APIC_IRR + i * 0x10);
@@ -688,25 +690,25 @@ bool __kvm_apic_update_irr(unsigned long *pir, void *regs, int *max_irr)
!try_cmpxchg(p_irr, &prev_irr_val, irr_val));
if (prev_irr_val != irr_val)
- max_updated_irr = __fls(irr_val ^ prev_irr_val) + vec;
+ max_new_irr = __fls(irr_val ^ prev_irr_val) + vec;
}
if (irr_val)
*max_irr = __fls(irr_val) + vec;
}
- return ((max_updated_irr != -1) &&
- (max_updated_irr == *max_irr));
+ return max_new_irr != -1 && max_new_irr == *max_irr;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_apic_update_irr);
bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, unsigned long *pir, int *max_irr)
{
struct kvm_lapic *apic = vcpu->arch.apic;
- bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr);
+ bool max_irr_is_from_pir;
- if (unlikely(!apic->apicv_active && irr_updated))
+ max_irr_is_from_pir = __kvm_apic_update_irr(pir, apic->regs, max_irr);
+ if (unlikely(!apic->apicv_active && max_irr_is_from_pir))
apic->irr_pending = true;
- return irr_updated;
+ return max_irr_is_from_pir;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_apic_update_irr);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 24fbc9e..91843e92 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -52,6 +52,7 @@
#include <asm/page.h>
#include <asm/memtype.h>
#include <asm/cmpxchg.h>
+#include <asm/cpuid/api.h>
#include <asm/io.h>
#include <asm/set_memory.h>
#include <asm/spec-ctrl.h>
@@ -182,6 +183,8 @@ static struct kmem_cache *pte_list_desc_cache;
struct kmem_cache *mmu_page_header_cache;
static void mmu_spte_set(u64 *sptep, u64 spte);
+static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
+ u64 *spte, struct list_head *invalid_list);
struct kvm_mmu_role_regs {
const unsigned long cr0;
@@ -1287,19 +1290,6 @@ static void drop_spte(struct kvm *kvm, u64 *sptep)
rmap_remove(kvm, sptep);
}
-static void drop_large_spte(struct kvm *kvm, u64 *sptep, bool flush)
-{
- struct kvm_mmu_page *sp;
-
- sp = sptep_to_sp(sptep);
- WARN_ON_ONCE(sp->role.level == PG_LEVEL_4K);
-
- drop_spte(kvm, sptep);
-
- if (flush)
- kvm_flush_remote_tlbs_sptep(kvm, sptep);
-}
-
/*
* Write-protect on the specified @sptep, @pt_protect indicates whether
* spte write-protection is caused by protecting shadow page table.
@@ -2466,7 +2456,8 @@ static struct kvm_mmu_page *kvm_mmu_get_child_sp(struct kvm_vcpu *vcpu,
{
union kvm_mmu_page_role role;
- if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep))
+ if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep) &&
+ spte_to_child_sp(*sptep) && spte_to_child_sp(*sptep)->gfn == gfn)
return ERR_PTR(-EEXIST);
role = kvm_mmu_child_role(sptep, direct, access);
@@ -2536,6 +2527,23 @@ static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
__shadow_walk_next(iterator, *iterator->sptep);
}
+/*
+ * Note: while normally KVM uses a "bool flush" return value to let
+ * the caller batch flushes, __link_shadow_page() flushes immediately
+ * before populating the parent PTE with the new shadow page. The
+ * typical callers, direct_map() and FNAME(fetch)(), are not going
+ * to zap more than one huge SPTE anyway.
+ *
+ * The only exception, where @flush can be false, is when a huge SPTE
+ * is replaced with a shadow page SPTE with a fully populated page table,
+ * which can happen from shadow_mmu_split_huge_page(). In this case,
+ * no memory is unmapped across the change to the page tables and no
+ * immediate flush is needed for correctness.
+ *
+ * Even in that case, calls to kvm_mmu_commit_zap_page() are not
+ * batched. Doing so would require adding an invalid_list argument
+ * all the way down to __walk_slot_rmaps().
+ */
static void __link_shadow_page(struct kvm *kvm,
struct kvm_mmu_memory_cache *cache, u64 *sptep,
struct kvm_mmu_page *sp, bool flush)
@@ -2544,13 +2552,18 @@ static void __link_shadow_page(struct kvm *kvm,
BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
- /*
- * If an SPTE is present already, it must be a leaf and therefore
- * a large one. Drop it, and flush the TLB if needed, before
- * installing sp.
- */
- if (is_shadow_present_pte(*sptep))
- drop_large_spte(kvm, sptep, flush);
+ if (is_shadow_present_pte(*sptep)) {
+ struct kvm_mmu_page *parent_sp;
+ LIST_HEAD(invalid_list);
+
+ parent_sp = sptep_to_sp(sptep);
+ WARN_ON_ONCE(parent_sp->role.level == PG_LEVEL_4K);
+
+ if (mmu_page_zap_pte(kvm, parent_sp, sptep, &invalid_list))
+ kvm_mmu_commit_zap_page(kvm, &invalid_list);
+ else if (flush)
+ kvm_flush_remote_tlbs_sptep(kvm, sptep);
+ }
spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c
index 85a0473..4e753386 100644
--- a/arch/x86/kvm/mmu/spte.c
+++ b/arch/x86/kvm/mmu/spte.c
@@ -15,6 +15,7 @@
#include "x86.h"
#include "spte.h"
+#include <asm/cpuid/api.h>
#include <asm/e820/api.h>
#include <asm/memtype.h>
#include <asm/vmx.h>
diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h
index 657f5f7..2ad2578 100644
--- a/arch/x86/kvm/reverse_cpuid.h
+++ b/arch/x86/kvm/reverse_cpuid.h
@@ -3,8 +3,10 @@
#define ARCH_X86_KVM_REVERSE_CPUID_H
#include <uapi/asm/kvm.h>
+
#include <asm/cpufeature.h>
#include <asm/cpufeatures.h>
+#include <asm/cpuid/types.h>
/*
* Define a KVM-only feature flag.
diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c
index adf2118..cdd5a6d 100644
--- a/arch/x86/kvm/svm/avic.c
+++ b/arch/x86/kvm/svm/avic.c
@@ -207,6 +207,35 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
/*
+ * Flush the TLB when enabling (x2)AVIC and when transitioning between
+ * xAVIC and x2AVIC, as the CPU may have inserted a TLB entry for the
+ * "wrong" mapping.
+ *
+ * KVM uses a per-VM "scratch" page to back the APIC memslot, because
+ * KVM also uses per-VM page tables *and* maintains the page table (NPT
+ * or shadow page) mappings for said memslot even if one or more vCPUs
+ * have their local APIC hardware-disabled or are in x2APIC mode, i.e.
+ * even if one or more vCPUs' APIC MMIO BAR is effectively disabled.
+ *
+ * If xAVIC is fully enabled, hardware ignores the physical address in
+ * KVM's page tables, i.e. in the leaf SPTE for the APIC memslot, and
+ * instead redirects the access to the AVIC backing page, i.e. to the
+ * vCPU's virtual APIC page. If xAVIC is not enabled (APIC is either
+ * hardware-disabled or in x2APIC mode), then guest accesses will use
+ * the page table mapping verbatim, i.e. will access the per-VM scratch
+ * page, as normal memory.
+ *
+ * In both cases, the CPU is allowed to cache TLB entries for the APIC
+ * base GPA. So, KVM needs to flush the TLB when enabling xAVIC, as
+ * accesses need to be redirected to the virtual APIC page, but the TLB
+ * may contain entries pointing at the scratch page. KVM also needs to
+ * flush the TLB when enabling x2AVIC, as accesses need to go to the
+ * scratch page, but the TLB may contain entries tagged as xAVIC, i.e.
+ * entries pointing to the vCPU's virtual APIC page.
+ */
+ kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
+
+ /*
* Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR
* accesses, while interrupt injection to a running vCPU can be
* achieved using AVIC doorbell. KVM disables the APIC access page
@@ -219,12 +248,6 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
/* Disabling MSR intercept for x2APIC registers */
avic_set_x2apic_msr_interception(svm, false);
} else {
- /*
- * Flush the TLB, the guest may have inserted a non-APIC
- * mapping into the TLB while AVIC was disabled.
- */
- kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
-
/* Enabling MSR intercept for x2APIC registers */
avic_set_x2apic_msr_interception(svm, true);
}
@@ -1300,12 +1323,14 @@ bool __init avic_hardware_setup(void)
}
/*
- * Disable IPI virtualization for AMD Family 17h CPUs (Zen1 and Zen2)
- * due to erratum 1235, which results in missed VM-Exits on the sender
- * and thus missed wake events for blocking vCPUs due to the CPU
- * failing to see a software update to clear IsRunning.
+ * Disable IPI virtualization for AMD Family 17h (Zen1 and Zen2) and
+ * Hygon Family 18h (derived from AMD Zen1) CPUs due to erratum 1235,
+ * which results in missed VM-Exits on the sender and thus missed wake
+ * events for blocking vCPUs due to the CPU failing to see a software
+ * update to clear IsRunning.
*/
- enable_ipiv = enable_ipiv && boot_cpu_data.x86 != 0x17;
+ if (boot_cpu_data.x86 == 0x17 || boot_cpu_data.x86 == 0x18)
+ enable_ipiv = false;
amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 961804d..b340dc9 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -160,6 +160,16 @@ void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm)
if (!intercept_smi)
vmcb_clr_intercept(&vmcb02->control, INTERCEPT_SMI);
+ /*
+ * Intercept PAUSE if and only if L1 wants to. KVM intercepts PAUSE so
+ * that a vCPU that may be spinning waiting for a lock can be scheduled
+ * out in favor of the vCPU that holds said lock. KVM doesn't support
+ * yielding across L2 vCPUs, as KVM has limited visilibity into which
+ * L2 vCPUs are in the same L2 VM, i.e. may be contending for locks.
+ */
+ if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE))
+ vmcb_clr_intercept(&vmcb02->control, INTERCEPT_PAUSE);
+
if (nested_vmcb_needs_vls_intercept(svm)) {
/*
* If the virtual VMLOAD/VMSAVE is not enabled for the L2,
@@ -819,7 +829,6 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
struct vmcb *vmcb01 = svm->vmcb01.ptr;
struct kvm_vcpu *vcpu = &svm->vcpu;
- u32 pause_count12, pause_thresh12;
nested_svm_transition_tlb_flush(vcpu);
@@ -947,31 +956,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
vmcb02->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE;
if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER))
- pause_count12 = vmcb12_ctrl->pause_filter_count;
+ vmcb02->control.pause_filter_count = vmcb12_ctrl->pause_filter_count;
else
- pause_count12 = 0;
+ vmcb02->control.pause_filter_count = 0;
if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD))
- pause_thresh12 = vmcb12_ctrl->pause_filter_thresh;
+ vmcb02->control.pause_filter_thresh = vmcb12_ctrl->pause_filter_thresh;
else
- pause_thresh12 = 0;
- if (kvm_pause_in_guest(svm->vcpu.kvm)) {
- /* use guest values since host doesn't intercept PAUSE */
- vmcb02->control.pause_filter_count = pause_count12;
- vmcb02->control.pause_filter_thresh = pause_thresh12;
-
- } else {
- /* start from host values otherwise */
- vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count;
- vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh;
-
- /* ... but ensure filtering is disabled if so requested. */
- if (vmcb12_is_intercept(vmcb12_ctrl, INTERCEPT_PAUSE)) {
- if (!pause_count12)
- vmcb02->control.pause_filter_count = 0;
- if (!pause_thresh12)
- vmcb02->control.pause_filter_thresh = 0;
- }
- }
+ vmcb02->control.pause_filter_thresh = 0;
/*
* Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to
@@ -1298,12 +1289,6 @@ void nested_svm_vmexit(struct vcpu_svm *svm)
/* in case we halted in L2 */
kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
- if (!kvm_pause_in_guest(vcpu->kvm)) {
- vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count;
- vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
-
- }
-
/*
* Invalidate last_bus_lock_rip unless KVM is still waiting for the
* guest to make forward progress before re-enabling bus lock detection.
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index c2126b3..f5c3667 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -23,6 +23,7 @@
#include <asm/pkru.h>
#include <asm/trapnr.h>
+#include <asm/cpuid/api.h>
#include <asm/fpu/xcr.h>
#include <asm/fpu/xstate.h>
#include <asm/debugreg.h>
@@ -3662,23 +3663,26 @@ int pre_sev_run(struct vcpu_svm *svm, int cpu)
}
#define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE)
-static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
+static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 min_len)
{
struct vmcb_control_area *control = &svm->vmcb->control;
u64 ghcb_scratch_beg, ghcb_scratch_end;
u64 scratch_gpa_beg, scratch_gpa_end;
void *scratch_va;
+ if (WARN_ON_ONCE(!min_len))
+ goto e_scratch;
+
scratch_gpa_beg = svm->sev_es.sw_scratch;
if (!scratch_gpa_beg) {
pr_err("vmgexit: scratch gpa not provided\n");
goto e_scratch;
}
- scratch_gpa_end = scratch_gpa_beg + len;
+ scratch_gpa_end = scratch_gpa_beg + min_len;
if (scratch_gpa_end < scratch_gpa_beg) {
pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n",
- len, scratch_gpa_beg);
+ min_len, scratch_gpa_beg);
goto e_scratch;
}
@@ -3702,21 +3706,27 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
scratch_va = (void *)svm->sev_es.ghcb;
scratch_va += (scratch_gpa_beg - control->ghcb_gpa);
+
+ svm->sev_es.ghcb_sa_len = ghcb_scratch_end - scratch_gpa_beg;
} else {
+ /* GHCB v2 requires the scratch area to be within the GHCB. */
+ if (to_kvm_sev_info(svm->vcpu.kvm)->ghcb_version >= 2)
+ goto e_scratch;
+
/*
* The guest memory must be read into a kernel buffer, so
* limit the size
*/
- if (len > GHCB_SCRATCH_AREA_LIMIT) {
+ if (min_len > GHCB_SCRATCH_AREA_LIMIT) {
pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n",
- len, GHCB_SCRATCH_AREA_LIMIT);
+ min_len, GHCB_SCRATCH_AREA_LIMIT);
goto e_scratch;
}
- scratch_va = kvzalloc(len, GFP_KERNEL_ACCOUNT);
+ scratch_va = kvzalloc(min_len, GFP_KERNEL_ACCOUNT);
if (!scratch_va)
return -ENOMEM;
- if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) {
+ if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, min_len)) {
/* Unable to copy scratch area from guest */
pr_err("vmgexit: kvm_read_guest for scratch area failed\n");
@@ -3732,11 +3742,10 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
*/
svm->sev_es.ghcb_sa_sync = sync;
svm->sev_es.ghcb_sa_free = true;
+ svm->sev_es.ghcb_sa_len = min_len;
}
svm->sev_es.ghcb_sa = scratch_va;
- svm->sev_es.ghcb_sa_len = len;
-
return 0;
e_scratch:
@@ -3833,7 +3842,7 @@ struct psc_buffer {
struct psc_entry entries[];
} __packed;
-static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc);
+static int snp_begin_psc(struct vcpu_svm *svm);
static void snp_complete_psc(struct vcpu_svm *svm, u64 psc_ret)
{
@@ -3864,9 +3873,9 @@ static void __snp_complete_one_psc(struct vcpu_svm *svm)
*/
for (idx = svm->sev_es.psc_idx; svm->sev_es.psc_inflight;
svm->sev_es.psc_inflight--, idx++) {
- struct psc_entry *entry = &entries[idx];
+ struct psc_entry entry = READ_ONCE(entries[idx]);
- entry->cur_page = entry->pagesize ? 512 : 1;
+ entries[idx].cur_page = entry.pagesize ? 512 : 1;
}
hdr->cur_entry = idx;
@@ -3875,7 +3884,6 @@ static void __snp_complete_one_psc(struct vcpu_svm *svm)
static int snp_complete_one_psc(struct kvm_vcpu *vcpu)
{
struct vcpu_svm *svm = to_svm(vcpu);
- struct psc_buffer *psc = svm->sev_es.ghcb_sa;
if (vcpu->run->hypercall.ret) {
snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
@@ -3885,16 +3893,18 @@ static int snp_complete_one_psc(struct kvm_vcpu *vcpu)
__snp_complete_one_psc(svm);
/* Handle the next range (if any). */
- return snp_begin_psc(svm, psc);
+ return snp_begin_psc(svm);
}
-static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc)
+static int snp_begin_psc(struct vcpu_svm *svm)
{
+ struct vcpu_sev_es_state *sev_es = &svm->sev_es;
+ struct psc_buffer *psc = sev_es->ghcb_sa;
struct psc_entry *entries = psc->entries;
struct kvm_vcpu *vcpu = &svm->vcpu;
struct psc_hdr *hdr = &psc->hdr;
struct psc_entry entry_start;
- u16 idx, idx_start, idx_end;
+ u16 idx, idx_start, idx_end, max_nr_entries;
int npages;
bool huge;
u64 gfn;
@@ -3904,6 +3914,19 @@ static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc)
return 1;
}
+ /*
+ * GHCB v2 requires the scratch area to reside within the GHCB itself,
+ * and PSC requests are only supported for GHCB v2+. Thus it should be
+ * impossible to exceed the max PSC entry count (which is derived from
+ * the size of the shared GHCB buffer).
+ */
+ max_nr_entries = (sev_es->ghcb_sa_len - sizeof(struct psc_hdr)) /
+ sizeof(struct psc_entry);
+ if (WARN_ON_ONCE(max_nr_entries > VMGEXIT_PSC_MAX_COUNT)) {
+ snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC);
+ return 1;
+ }
+
next_range:
/* There should be no other PSCs in-flight at this point. */
if (WARN_ON_ONCE(svm->sev_es.psc_inflight)) {
@@ -3916,17 +3939,17 @@ static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc)
* validation, so take care to only use validated copies of values used
* for things like array indexing.
*/
- idx_start = hdr->cur_entry;
- idx_end = hdr->end_entry;
+ idx_start = READ_ONCE(hdr->cur_entry);
+ idx_end = READ_ONCE(hdr->end_entry);
- if (idx_end >= VMGEXIT_PSC_MAX_COUNT) {
+ if (idx_end >= max_nr_entries) {
snp_complete_psc(svm, VMGEXIT_PSC_ERROR_INVALID_HDR);
return 1;
}
/* Find the start of the next range which needs processing. */
for (idx = idx_start; idx <= idx_end; idx++, hdr->cur_entry++) {
- entry_start = entries[idx];
+ entry_start = READ_ONCE(entries[idx]);
gfn = entry_start.gfn;
huge = entry_start.pagesize;
@@ -3970,7 +3993,7 @@ static int snp_begin_psc(struct vcpu_svm *svm, struct psc_buffer *psc)
* KVM_HC_MAP_GPA_RANGE exit.
*/
while (++idx <= idx_end) {
- struct psc_entry entry = entries[idx];
+ struct psc_entry entry = READ_ONCE(entries[idx]);
if (entry.operation != entry_start.operation ||
entry.gfn != entry_start.gfn + npages ||
@@ -4493,13 +4516,22 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
case SVM_VMGEXIT_MMIO_READ:
case SVM_VMGEXIT_MMIO_WRITE: {
bool is_write = control->exit_code == SVM_VMGEXIT_MMIO_WRITE;
+ u64 len = control->exit_info_2;
- ret = setup_vmgexit_scratch(svm, !is_write, control->exit_info_2);
+ if (!len)
+ return 1;
+
+ if (to_kvm_sev_info(vcpu->kvm)->ghcb_version >= 2 && len > 8) {
+ svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
+ return 1;
+ }
+
+ ret = setup_vmgexit_scratch(svm, !is_write, len);
if (ret)
break;
- ret = kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1,
- control->exit_info_2, svm->sev_es.ghcb_sa);
+ ret = kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1, len,
+ svm->sev_es.ghcb_sa);
break;
}
case SVM_VMGEXIT_NMI_COMPLETE:
@@ -4546,11 +4578,11 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
vcpu->run->system_event.data[0] = control->ghcb_gpa;
break;
case SVM_VMGEXIT_PSC:
- ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
+ ret = setup_vmgexit_scratch(svm, true, sizeof(struct psc_hdr));
if (ret)
break;
- ret = snp_begin_psc(svm, svm->sev_es.ghcb_sa);
+ ret = snp_begin_psc(svm);
break;
case SVM_VMGEXIT_AP_CREATION:
ret = sev_snp_ap_creation(svm);
@@ -4572,6 +4604,11 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
control->exit_info_1, control->exit_info_2);
ret = -EINVAL;
break;
+ case SVM_EXIT_IOIO:
+ if (!((control->exit_info_1 & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT))
+ return 1;
+
+ fallthrough;
default:
ret = svm_invoke_exit_handler(vcpu, control->exit_code);
}
@@ -4592,6 +4629,9 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in)
if (unlikely(check_mul_overflow(count, size, &bytes)))
return -EINVAL;
+ if (!bytes)
+ return 1;
+
r = setup_vmgexit_scratch(svm, in, bytes);
if (r)
return r;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index e7fdd7a..d38a21b 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -41,6 +41,7 @@
#include <asm/irq_remapping.h>
#include <asm/spec-ctrl.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/traps.h>
#include <asm/reboot.h>
#include <asm/fpu/api.h>
@@ -913,7 +914,15 @@ static void grow_ple_window(struct kvm_vcpu *vcpu)
struct vmcb_control_area *control = &svm->vmcb->control;
int old = control->pause_filter_count;
- if (kvm_pause_in_guest(vcpu->kvm))
+ /* Adjusting pause_filter_count makes no sense if PLE is disabled. */
+ WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm));
+
+ /*
+ * While running L2, KVM should intercept PAUSE if and only if L1 wants
+ * to intercept PAUSE, and L1's intercept should take priority, i.e.
+ * KVM should never handle a PAUSE intercept from L2.
+ */
+ if (WARN_ON_ONCE(is_guest_mode(vcpu)))
return;
control->pause_filter_count = __grow_ple_window(old,
@@ -934,7 +943,10 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu)
struct vmcb_control_area *control = &svm->vmcb->control;
int old = control->pause_filter_count;
- if (kvm_pause_in_guest(vcpu->kvm))
+ /* Adjusting pause_filter_count makes no sense if PLE is disabled. */
+ WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm));
+
+ if (is_guest_mode(vcpu))
return;
control->pause_filter_count =
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index e7fdbe9..0db25bb 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -154,7 +154,7 @@ TRACE_EVENT(kvm_xen_hypercall,
__entry->a2 = a2;
__entry->a3 = a3;
__entry->a4 = a4;
- __entry->a4 = a5;
+ __entry->a5 = a5;
),
TP_printk("cpl %d nr 0x%lx a0 0x%lx a1 0x%lx a2 0x%lx a3 0x%lx a4 0x%lx a5 %lx",
diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h
index 56cacc0..3156827 100644
--- a/arch/x86/kvm/vmx/capabilities.h
+++ b/arch/x86/kvm/vmx/capabilities.h
@@ -14,6 +14,7 @@ extern bool __read_mostly flexpriority_enabled;
extern bool __read_mostly enable_ept;
extern bool __read_mostly enable_unrestricted_guest;
extern bool __read_mostly enable_ept_ad_bits;
+extern bool __read_mostly enable_cet;
extern bool __read_mostly enable_pml;
extern int __read_mostly pt_mode;
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 27eb76e..74e0b01 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -15,6 +15,7 @@
#include <linux/perf_event.h>
#include <asm/msr.h>
#include <asm/perf_event.h>
+#include <asm/cpuid/api.h>
#include "x86.h"
#include "cpuid.h"
#include "lapic.h"
diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c
index df1d0cf..29a1f8e 100644
--- a/arch/x86/kvm/vmx/sgx.c
+++ b/arch/x86/kvm/vmx/sgx.c
@@ -2,6 +2,7 @@
/* Copyright(c) 2021 Intel Corporation. */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/sgx.h>
diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S
index 8a481da..ff1f254 100644
--- a/arch/x86/kvm/vmx/vmenter.S
+++ b/arch/x86/kvm/vmx/vmenter.S
@@ -31,38 +31,6 @@
#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
#endif
-.macro VMX_DO_EVENT_IRQOFF call_insn call_target
- /*
- * Unconditionally create a stack frame, getting the correct RSP on the
- * stack (for x86-64) would take two instructions anyways, and RBP can
- * be used to restore RSP to make objtool happy (see below).
- */
- push %_ASM_BP
- mov %_ASM_SP, %_ASM_BP
-
-#ifdef CONFIG_X86_64
- /*
- * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
- * creating the synthetic interrupt stack frame for the IRQ/NMI.
- */
- and $-16, %rsp
- push $__KERNEL_DS
- push %rbp
-#endif
- pushf
- push $__KERNEL_CS
- \call_insn \call_target
-
- /*
- * "Restore" RSP from RBP, even though IRET has already unwound RSP to
- * the correct value. objtool doesn't know the callee will IRET and,
- * without the explicit restore, thinks the stack is getting walloped.
- * Using an unwind hint is problematic due to x86-64's dynamic alignment.
- */
- leave
- RET
-.endm
-
.section .noinstr.text, "ax"
/**
@@ -320,10 +288,6 @@
SYM_FUNC_END(__vmx_vcpu_run)
-SYM_FUNC_START(vmx_do_nmi_irqoff)
- VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
-SYM_FUNC_END(vmx_do_nmi_irqoff)
-
#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
/**
@@ -375,13 +339,3 @@
RET
SYM_FUNC_END(vmread_error_trampoline)
#endif
-
-.section .text, "ax"
-
-#ifndef CONFIG_X86_FRED
-
-SYM_FUNC_START(vmx_do_interrupt_irqoff)
- VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
-SYM_FUNC_END(vmx_do_interrupt_irqoff)
-
-#endif
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index a29896a..ede773c 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -33,6 +33,7 @@
#include <asm/asm.h>
#include <asm/cpu.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/debugreg.h>
#include <asm/desc.h>
#include <asm/fpu/api.h>
@@ -108,6 +109,9 @@ module_param_named(unrestricted_guest,
bool __read_mostly enable_ept_ad_bits = 1;
module_param_named(eptad, enable_ept_ad_bits, bool, 0444);
+bool __read_mostly enable_cet = 1;
+module_param_named(cet, enable_cet, bool, 0444);
+
static bool __read_mostly emulate_invalid_guest_state = true;
module_param(emulate_invalid_guest_state, bool, 0444);
@@ -4476,7 +4480,7 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
* SSP is reloaded from IA32_PL3_SSP. Check SDM Vol.2A/B Chapter
* 3 and 4 for details.
*/
- if (cpu_has_load_cet_ctrl()) {
+ if (enable_cet) {
vmcs_writel(HOST_S_CET, kvm_host.s_cet);
vmcs_writel(HOST_SSP, 0);
vmcs_writel(HOST_INTR_SSP_TABLE, 0);
@@ -4532,6 +4536,10 @@ static u32 vmx_get_initial_vmentry_ctrl(void)
if (vmx_pt_mode_is_system())
vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP |
VM_ENTRY_LOAD_IA32_RTIT_CTL);
+
+ if (!enable_cet)
+ vmentry_ctrl &= ~VM_ENTRY_LOAD_CET_STATE;
+
/*
* IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically.
*/
@@ -4546,6 +4554,9 @@ static u32 vmx_get_initial_vmexit_ctrl(void)
{
u32 vmexit_ctrl = vmcs_config.vmexit_ctrl;
+ if (!enable_cet)
+ vmexit_ctrl &= ~VM_EXIT_LOAD_CET_STATE;
+
/*
* Not used by KVM and never set in vmcs01 or vmcs02, but emulated for
* nested virtualization and thus allowed to be set in vmcs12.
@@ -7029,8 +7040,8 @@ static void vmx_set_rvi(int vector)
int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
{
struct vcpu_vt *vt = to_vt(vcpu);
+ bool max_irr_is_from_pir;
int max_irr;
- bool got_posted_interrupt;
if (KVM_BUG_ON(!enable_apicv, vcpu->kvm))
return -EIO;
@@ -7042,17 +7053,22 @@ int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
* But on x86 this is just a compiler barrier anyway.
*/
smp_mb__after_atomic();
- got_posted_interrupt =
- kvm_apic_update_irr(vcpu, vt->pi_desc.pir, &max_irr);
+ max_irr_is_from_pir = kvm_apic_update_irr(vcpu, vt->pi_desc.pir,
+ &max_irr);
} else {
max_irr = kvm_lapic_find_highest_irr(vcpu);
- got_posted_interrupt = false;
+ max_irr_is_from_pir = false;
}
/*
- * Newly recognized interrupts are injected via either virtual interrupt
- * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is
- * disabled in two cases:
+ * If APICv is enabled and L2 is not active, then update the Requesting
+ * Virtual Interrupt (RVI) portion of vmcs01.GUEST_INTR_STATUS with the
+ * highest priority IRR to deliver the IRQ via Virtual Interrupt
+ * Delivery. Note, this is required even if the highest priority IRQ
+ * was already pending in the IRR, as RVI isn't updated in lockstep with
+ * the IRR (unlike apic->irr_pending).
+ *
+ * For the cases where Virtual Interrupt Delivery can't be used:
*
* 1) If L2 is running and the vCPU has a new pending interrupt. If L1
* wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a
@@ -7063,10 +7079,29 @@ int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
* 2) If APICv is disabled for this vCPU, assigned devices may still
* attempt to post interrupts. The posted interrupt vector will cause
* a VM-Exit and the subsequent entry will call sync_pir_to_irr.
+ *
+ * In both cases, set KVM_REQ_EVENT if and only if the highest priority
+ * pending IRQ came from the PIR, as setting KVM_REQ_EVENT if any IRQ
+ * is pending may put the vCPU into an infinite loop, e.g. if the IRQ
+ * is blocked, then it will stay pending until an IRQ window is opened.
+ *
+ * Note! It's possible that one or more IRQs were moved from the PIR
+ * to the IRR _without_ max_irr_is_from_pir being true! I.e. if there
+ * was a higher priority IRQ already pending in the IRR. Not setting
+ * KVM_REQ_EVENT in this case is intentional and safe. If APICv is
+ * inactive, or L2 is running with exit-on-interrupt off (in vmcs12),
+ * i.e. without nested virtual interrupt delivery, then there's no need
+ * to request an IRQ window as the lower priority IRQ only needs to be
+ * delivered when the higher priority IRQ is dismissed from the ISR,
+ * i.e. on the next EOI, and EOIs are always intercepted if APICv is
+ * disabled or if L2 is running without nested VID. If L2 is running
+ * exit-on-interrupt on (in vmcs12), then the higher priority IRQ will
+ * trigger a nested VM-Exit, at which point KVM will re-evaluate L1's
+ * pending IRQs.
*/
if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu))
vmx_set_rvi(max_irr);
- else if (got_posted_interrupt)
+ else if (max_irr_is_from_pir)
kvm_make_request(KVM_REQ_EVENT, vcpu);
return max_irr;
@@ -7083,9 +7118,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
}
-void vmx_do_interrupt_irqoff(unsigned long entry);
-void vmx_do_nmi_irqoff(void);
-
static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
{
/*
@@ -7127,17 +7159,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
"unexpected VM-Exit interrupt info: 0x%x", intr_info))
return;
- /*
- * Invoke the kernel's IRQ handler for the vector. Use the FRED path
- * when it's available even if FRED isn't fully enabled, e.g. even if
- * FRED isn't supported in hardware, in order to avoid the indirect
- * CALL in the non-FRED path.
- */
kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
- if (IS_ENABLED(CONFIG_X86_FRED))
- fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
- else
- vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));
+ x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
kvm_after_interrupt(vcpu);
vcpu->arch.at_instruction_boundary = true;
@@ -7447,10 +7470,7 @@ noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
- if (cpu_feature_enabled(X86_FEATURE_FRED))
- fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
- else
- vmx_do_nmi_irqoff();
+ x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
kvm_after_interrupt(vcpu);
}
@@ -8131,7 +8151,7 @@ static __init void vmx_set_cpu_caps(void)
* VMX_BASIC[bit56] == 0, inject #CP at VMX entry with error code
* fails, so disable CET in this case too.
*/
- if (!cpu_has_load_cet_ctrl() || !enable_unrestricted_guest ||
+ if (!enable_cet || !enable_unrestricted_guest ||
!cpu_has_vmx_basic_no_hw_errcode_cc()) {
kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
kvm_cpu_cap_clear(X86_FEATURE_IBT);
@@ -8606,6 +8626,9 @@ __init int vmx_hardware_setup(void)
!cpu_has_vmx_invept_global())
enable_ept = 0;
+ if (!cpu_has_load_cet_ctrl())
+ enable_cet = 0;
+
/* NX support is required for shadow paging. */
if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) {
pr_err_ratelimited("NX (Execute Disable) not supported\n");
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0a1b63c..0550359 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4876,7 +4876,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
r = tdp_enabled;
break;
case KVM_CAP_X86_APIC_BUS_CYCLES_NS:
- r = APIC_BUS_CYCLE_NS_DEFAULT;
+ r = kvm ? kvm->arch.apic_bus_cycle_ns : APIC_BUS_CYCLE_NS_DEFAULT;
break;
case KVM_CAP_EXIT_HYPERCALL:
r = KVM_EXIT_HYPERCALL_VALID_MASK;
@@ -5227,8 +5227,13 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
* On a host with synchronized TSC, there is no need to update
* kvmclock on vcpu->cpu migration
*/
- if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
- kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+ if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) {
+ if (__ratelimit(&vcpu->kvm->arch.kvmclock_update_rs))
+ kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+ else
+ kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
+ }
+
if (vcpu->cpu != cpu)
kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
vcpu->cpu = cpu;
@@ -13366,6 +13371,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
raw_spin_lock_init(&kvm->arch.tsc_write_lock);
mutex_init(&kvm->arch.apic_map_lock);
seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
+ ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10);
+ ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE);
kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
@@ -14323,7 +14330,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
* the RAP (Return Address Predicator).
*/
if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
- kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS);
+ kvm_register_mark_dirty(vcpu, VCPU_EXREG_ERAPS);
kvm_invalidate_pcid(vcpu, operand.pcid);
return kvm_skip_emulated_instruction(vcpu);
diff --git a/arch/x86/math-emu/Makefile b/arch/x86/math-emu/Makefile
deleted file mode 100644
index 02211fc..0000000
--- a/arch/x86/math-emu/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for wm-FPU-emu
-#
-
-#DEBUG = -DDEBUGGING
-DEBUG =
-PARANOID = -DPARANOID
-ccflags-y += $(PARANOID) $(DEBUG) -fno-builtin $(MATH_EMULATION)
-asflags-y += $(PARANOID)
-
-# From 'C' language sources:
-C_OBJS =fpu_entry.o errors.o \
- fpu_arith.o fpu_aux.o fpu_etc.o fpu_tags.o fpu_trig.o \
- load_store.o get_address.o \
- poly_atan.o poly_l2.o poly_2xm1.o poly_sin.o poly_tan.o \
- reg_add_sub.o reg_compare.o reg_constant.o reg_convert.o \
- reg_ld_str.o reg_divide.o reg_mul.o
-
-# From 80x86 assembler sources:
-A_OBJS =reg_u_add.o reg_u_div.o reg_u_mul.o reg_u_sub.o \
- div_small.o reg_norm.o reg_round.o \
- wm_shrx.o wm_sqrt.o \
- div_Xsig.o polynom_Xsig.o round_Xsig.o \
- shr_Xsig.o mul_Xsig.o
-
-obj-y =$(C_OBJS) $(A_OBJS)
-
-proto:
- cproto -e -DMAKING_PROTO *.c >fpu_proto.h
diff --git a/arch/x86/math-emu/README b/arch/x86/math-emu/README
deleted file mode 100644
index e6235491d..0000000
--- a/arch/x86/math-emu/README
+++ /dev/null
@@ -1,427 +0,0 @@
- +---------------------------------------------------------------------------+
- | wm-FPU-emu an FPU emulator for 80386 and 80486SX microprocessors. |
- | |
- | Copyright (C) 1992,1993,1994,1995,1996,1997,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@melbpc.org.au |
- | |
- | This program is free software; you can redistribute it and/or modify |
- | it under the terms of the GNU General Public License version 2 as |
- | published by the Free Software Foundation. |
- | |
- | This program is distributed in the hope that it will be useful, |
- | but WITHOUT ANY WARRANTY; without even the implied warranty of |
- | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
- | GNU General Public License for more details. |
- | |
- | You should have received a copy of the GNU General Public License |
- | along with this program; if not, write to the Free Software |
- | Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
- | |
- +---------------------------------------------------------------------------+
-
-
-
-wm-FPU-emu is an FPU emulator for Linux. It is derived from wm-emu387
-which was my 80387 emulator for early versions of djgpp (gcc under
-msdos); wm-emu387 was in turn based upon emu387 which was written by
-DJ Delorie for djgpp. The interface to the Linux kernel is based upon
-the original Linux math emulator by Linus Torvalds.
-
-My target FPU for wm-FPU-emu is that described in the Intel486
-Programmer's Reference Manual (1992 edition). Unfortunately, numerous
-facets of the functioning of the FPU are not well covered in the
-Reference Manual. The information in the manual has been supplemented
-with measurements on real 80486's. Unfortunately, it is simply not
-possible to be sure that all of the peculiarities of the 80486 have
-been discovered, so there is always likely to be obscure differences
-in the detailed behaviour of the emulator and a real 80486.
-
-wm-FPU-emu does not implement all of the behaviour of the 80486 FPU,
-but is very close. See "Limitations" later in this file for a list of
-some differences.
-
-Please report bugs, etc to me at:
- billm@melbpc.org.au
-or b.metzenthen@medoto.unimelb.edu.au
-
-For more information on the emulator and on floating point topics, see
-my web pages, currently at http://www.suburbia.net/~billm/
-
-
---Bill Metzenthen
- December 1999
-
-
------------------------ Internals of wm-FPU-emu -----------------------
-
-Numeric algorithms:
-(1) Add, subtract, and multiply. Nothing remarkable in these.
-(2) Divide has been tuned to get reasonable performance. The algorithm
- is not the obvious one which most people seem to use, but is designed
- to take advantage of the characteristics of the 80386. I expect that
- it has been invented many times before I discovered it, but I have not
- seen it. It is based upon one of those ideas which one carries around
- for years without ever bothering to check it out.
-(3) The sqrt function has been tuned to get good performance. It is based
- upon Newton's classic method. Performance was improved by capitalizing
- upon the properties of Newton's method, and the code is once again
- structured taking account of the 80386 characteristics.
-(4) The trig, log, and exp functions are based in each case upon quasi-
- "optimal" polynomial approximations. My definition of "optimal" was
- based upon getting good accuracy with reasonable speed.
-(5) The argument reducing code for the trig function effectively uses
- a value of pi which is accurate to more than 128 bits. As a consequence,
- the reduced argument is accurate to more than 64 bits for arguments up
- to a few pi, and accurate to more than 64 bits for most arguments,
- even for arguments approaching 2^63. This is far superior to an
- 80486, which uses a value of pi which is accurate to 66 bits.
-
-The code of the emulator is complicated slightly by the need to
-account for a limited form of re-entrancy. Normally, the emulator will
-emulate each FPU instruction to completion without interruption.
-However, it may happen that when the emulator is accessing the user
-memory space, swapping may be needed. In this case the emulator may be
-temporarily suspended while disk i/o takes place. During this time
-another process may use the emulator, thereby perhaps changing static
-variables. The code which accesses user memory is confined to five
-files:
- fpu_entry.c
- reg_ld_str.c
- load_store.c
- get_address.c
- errors.c
-As from version 1.12 of the emulator, no static variables are used
-(apart from those in the kernel's per-process tables). The emulator is
-therefore now fully re-entrant, rather than having just the restricted
-form of re-entrancy which is required by the Linux kernel.
-
------------------------ Limitations of wm-FPU-emu -----------------------
-
-There are a number of differences between the current wm-FPU-emu
-(version 2.01) and the 80486 FPU (apart from bugs). The differences
-are fewer than those which applied to the 1.xx series of the emulator.
-Some of the more important differences are listed below:
-
-The Roundup flag does not have much meaning for the transcendental
-functions and its 80486 value with these functions is likely to differ
-from its emulator value.
-
-In a few rare cases the Underflow flag obtained with the emulator will
-be different from that obtained with an 80486. This occurs when the
-following conditions apply simultaneously:
-(a) the operands have a higher precision than the current setting of the
- precision control (PC) flags.
-(b) the underflow exception is masked.
-(c) the magnitude of the exact result (before rounding) is less than 2^-16382.
-(d) the magnitude of the final result (after rounding) is exactly 2^-16382.
-(e) the magnitude of the exact result would be exactly 2^-16382 if the
- operands were rounded to the current precision before the arithmetic
- operation was performed.
-If all of these apply, the emulator will set the Underflow flag but a real
-80486 will not.
-
-NOTE: Certain formats of Extended Real are UNSUPPORTED. They are
-unsupported by the 80486. They are the Pseudo-NaNs, Pseudoinfinities,
-and Unnormals. None of these will be generated by an 80486 or by the
-emulator. Do not use them. The emulator treats them differently in
-detail from the way an 80486 does.
-
-Self modifying code can cause the emulator to fail. An example of such
-code is:
- movl %esp,[%ebx]
- fld1
-The FPU instruction may be (usually will be) loaded into the pre-fetch
-queue of the CPU before the mov instruction is executed. If the
-destination of the 'movl' overlaps the FPU instruction then the bytes
-in the prefetch queue and memory will be inconsistent when the FPU
-instruction is executed. The emulator will be invoked but will not be
-able to find the instruction which caused the device-not-present
-exception. For this case, the emulator cannot emulate the behaviour of
-an 80486DX.
-
-Handling of the address size override prefix byte (0x67) has not been
-extensively tested yet. A major problem exists because using it in
-vm86 mode can cause a general protection fault. Address offsets
-greater than 0xffff appear to be illegal in vm86 mode but are quite
-acceptable (and work) in real mode. A small test program developed to
-check the addressing, and which runs successfully in real mode,
-crashes dosemu under Linux and also brings Windows down with a general
-protection fault message when run under the MS-DOS prompt of Windows
-3.1. (The program simply reads data from a valid address).
-
-The emulator supports 16-bit protected mode, with one difference from
-an 80486DX. A 80486DX will allow some floating point instructions to
-write a few bytes below the lowest address of the stack. The emulator
-will not allow this in 16-bit protected mode: no instructions are
-allowed to write outside the bounds set by the protection.
-
------------------------ Performance of wm-FPU-emu -----------------------
-
-Speed.
------
-
-The speed of floating point computation with the emulator will depend
-upon instruction mix. Relative performance is best for the instructions
-which require most computation. The simple instructions are adversely
-affected by the FPU instruction trap overhead.
-
-
-Timing: Some simple timing tests have been made on the emulator functions.
-The times include load/store instructions. All times are in microseconds
-measured on a 33MHz 386 with 64k cache. The Turbo C tests were under
-ms-dos, the next two columns are for emulators running with the djgpp
-ms-dos extender. The final column is for wm-FPU-emu in Linux 0.97,
-using libm4.0 (hard).
-
-function Turbo C djgpp 1.06 WM-emu387 wm-FPU-emu
-
- + 60.5 154.8 76.5 139.4
- - 61.1-65.5 157.3-160.8 76.2-79.5 142.9-144.7
- * 71.0 190.8 79.6 146.6
- / 61.2-75.0 261.4-266.9 75.3-91.6 142.2-158.1
-
- sin() 310.8 4692.0 319.0 398.5
- cos() 284.4 4855.2 308.0 388.7
- tan() 495.0 8807.1 394.9 504.7
- atan() 328.9 4866.4 601.1 419.5-491.9
-
- sqrt() 128.7 crashed 145.2 227.0
- log() 413.1-419.1 5103.4-5354.21 254.7-282.2 409.4-437.1
- exp() 479.1 6619.2 469.1 850.8
-
-
-The performance under Linux is improved by the use of look-ahead code.
-The following results show the improvement which is obtained under
-Linux due to the look-ahead code. Also given are the times for the
-original Linux emulator with the 4.1 'soft' lib.
-
- [ Linus' note: I changed look-ahead to be the default under linux, as
- there was no reason not to use it after I had edited it to be
- disabled during tracing ]
-
- wm-FPU-emu w original w
- look-ahead 'soft' lib
- + 106.4 190.2
- - 108.6-111.6 192.4-216.2
- * 113.4 193.1
- / 108.8-124.4 700.1-706.2
-
- sin() 390.5 2642.0
- cos() 381.5 2767.4
- tan() 496.5 3153.3
- atan() 367.2-435.5 2439.4-3396.8
-
- sqrt() 195.1 4732.5
- log() 358.0-387.5 3359.2-3390.3
- exp() 619.3 4046.4
-
-
-These figures are now somewhat out-of-date. The emulator has become
-progressively slower for most functions as more of the 80486 features
-have been implemented.
-
-
------------------------ Accuracy of wm-FPU-emu -----------------------
-
-
-The accuracy of the emulator is in almost all cases equal to or better
-than that of an Intel 80486 FPU.
-
-The results of the basic arithmetic functions (+,-,*,/), and fsqrt
-match those of an 80486 FPU. They are the best possible; the error for
-these never exceeds 1/2 an lsb. The fprem and fprem1 instructions
-return exact results; they have no error.
-
-
-The following table compares the emulator accuracy for the sqrt(),
-trig and log functions against the Turbo C "emulator". For this table,
-each function was tested at about 400 points. Ideal worst-case results
-would be 64 bits. The reduced Turbo C accuracy of cos() and tan() for
-arguments greater than pi/4 can be thought of as being related to the
-precision of the argument x; e.g. an argument of pi/2-(1e-10) which is
-accurate to 64 bits can result in a relative accuracy in cos() of
-about 64 + log2(cos(x)) = 31 bits.
-
-
-Function Tested x range Worst result Turbo C
- (relative bits)
-
-sqrt(x) 1 .. 2 64.1 63.2
-atan(x) 1e-10 .. 200 64.2 62.8
-cos(x) 0 .. pi/2-(1e-10) 64.4 (x <= pi/4) 62.4
- 64.1 (x = pi/2-(1e-10)) 31.9
-sin(x) 1e-10 .. pi/2 64.0 62.8
-tan(x) 1e-10 .. pi/2-(1e-10) 64.0 (x <= pi/4) 62.1
- 64.1 (x = pi/2-(1e-10)) 31.9
-exp(x) 0 .. 1 63.1 ** 62.9
-log(x) 1+1e-6 .. 2 63.8 ** 62.1
-
-** The accuracy for exp() and log() is low because the FPU (emulator)
-does not compute them directly; two operations are required.
-
-
-The emulator passes the "paranoia" tests (compiled with gcc 2.3.3 or
-later) for 'float' variables (24 bit precision numbers) when precision
-control is set to 24, 53 or 64 bits, and for 'double' variables (53
-bit precision numbers) when precision control is set to 53 bits (a
-properly performing FPU cannot pass the 'paranoia' tests for 'double'
-variables when precision control is set to 64 bits).
-
-The code for reducing the argument for the trig functions (fsin, fcos,
-fptan and fsincos) has been improved and now effectively uses a value
-for pi which is accurate to more than 128 bits precision. As a
-consequence, the accuracy of these functions for large arguments has
-been dramatically improved (and is now very much better than an 80486
-FPU). There is also now no degradation of accuracy for fcos and fptan
-for operands close to pi/2. Measured results are (note that the
-definition of accuracy has changed slightly from that used for the
-above table):
-
-Function Tested x range Worst result
- (absolute bits)
-
-cos(x) 0 .. 9.22e+18 62.0
-sin(x) 1e-16 .. 9.22e+18 62.1
-tan(x) 1e-16 .. 9.22e+18 61.8
-
-It is possible with some effort to find very large arguments which
-give much degraded precision. For example, the integer number
- 8227740058411162616.0
-is within about 10e-7 of a multiple of pi. To find the tan (for
-example) of this number to 64 bits precision it would be necessary to
-have a value of pi which had about 150 bits precision. The FPU
-emulator computes the result to about 42.6 bits precision (the correct
-result is about -9.739715e-8). On the other hand, an 80486 FPU returns
-0.01059, which in relative terms is hopelessly inaccurate.
-
-For arguments close to critical angles (which occur at multiples of
-pi/2) the emulator is more accurate than an 80486 FPU. For very large
-arguments, the emulator is far more accurate.
-
-
-Prior to version 1.20 of the emulator, the accuracy of the results for
-the transcendental functions (in their principal range) was not as
-good as the results from an 80486 FPU. From version 1.20, the accuracy
-has been considerably improved and these functions now give measured
-worst-case results which are better than the worst-case results given
-by an 80486 FPU.
-
-The following table gives the measured results for the emulator. The
-number of randomly selected arguments in each case is about half a
-million. The group of three columns gives the frequency of the given
-accuracy in number of times per million, thus the second of these
-columns shows that an accuracy of between 63.80 and 63.89 bits was
-found at a rate of 133 times per one million measurements for fsin.
-The results show that the fsin, fcos and fptan instructions return
-results which are in error (i.e. less accurate than the best possible
-result (which is 64 bits)) for about one per cent of all arguments
-between -pi/2 and +pi/2. The other instructions have a lower
-frequency of results which are in error. The last two columns give
-the worst accuracy which was found (in bits) and the approximate value
-of the argument which produced it.
-
- frequency (per M)
- ------------------- ---------------
-instr arg range # tests 63.7 63.8 63.9 worst at arg
- bits bits bits bits
------ ------------ ------- ---- ---- ----- ----- --------
-fsin (0,pi/2) 547756 0 133 10673 63.89 0.451317
-fcos (0,pi/2) 547563 0 126 10532 63.85 0.700801
-fptan (0,pi/2) 536274 11 267 10059 63.74 0.784876
-fpatan 4 quadrants 517087 0 8 1855 63.88 0.435121 (4q)
-fyl2x (0,20) 541861 0 0 1323 63.94 1.40923 (x)
-fyl2xp1 (-.293,.414) 520256 0 0 5678 63.93 0.408542 (x)
-f2xm1 (-1,1) 538847 4 481 6488 63.79 0.167709
-
-
-Tests performed on an 80486 FPU showed results of lower accuracy. The
-following table gives the results which were obtained with an AMD
-486DX2/66 (other tests indicate that an Intel 486DX produces
-identical results). The tests were basically the same as those used
-to measure the emulator (the values, being random, were in general not
-the same). The total number of tests for each instruction are given
-at the end of the table, in case each about 100k tests were performed.
-Another line of figures at the end of the table shows that most of the
-instructions return results which are in error for more than 10
-percent of the arguments tested.
-
-The numbers in the body of the table give the approx number of times a
-result of the given accuracy in bits (given in the left-most column)
-was obtained per one million arguments. For three of the instructions,
-two columns of results are given: * The second column for f2xm1 gives
-the number cases where the results of the first column were for a
-positive argument, this shows that this instruction gives better
-results for positive arguments than it does for negative. * In the
-cases of fcos and fptan, the first column gives the results when all
-cases where arguments greater than 1.5 were removed from the results
-given in the second column. Unlike the emulator, an 80486 FPU returns
-results of relatively poor accuracy for these instructions when the
-argument approaches pi/2. The table does not show those cases when the
-accuracy of the results were less than 62 bits, which occurs quite
-often for fsin and fptan when the argument approaches pi/2. This poor
-accuracy is discussed above in relation to the Turbo C "emulator", and
-the accuracy of the value of pi.
-
-
-bits f2xm1 f2xm1 fpatan fcos fcos fyl2x fyl2xp1 fsin fptan fptan
-62.0 0 0 0 0 437 0 0 0 0 925
-62.1 0 0 10 0 894 0 0 0 0 1023
-62.2 14 0 0 0 1033 0 0 0 0 945
-62.3 57 0 0 0 1202 0 0 0 0 1023
-62.4 385 0 0 10 1292 0 23 0 0 1178
-62.5 1140 0 0 119 1649 0 39 0 0 1149
-62.6 2037 0 0 189 1620 0 16 0 0 1169
-62.7 5086 14 0 646 2315 10 101 35 39 1402
-62.8 8818 86 0 984 3050 59 287 131 224 2036
-62.9 11340 1355 0 2126 4153 79 605 357 321 1948
-63.0 15557 4750 0 3319 5376 246 1281 862 808 2688
-63.1 20016 8288 0 4620 6628 511 2569 1723 1510 3302
-63.2 24945 11127 10 6588 8098 1120 4470 2968 2990 4724
-63.3 25686 12382 69 8774 10682 1906 6775 4482 5474 7236
-63.4 29219 14722 79 11109 12311 3094 9414 7259 8912 10587
-63.5 30458 14936 393 13802 15014 5874 12666 9609 13762 15262
-63.6 32439 16448 1277 17945 19028 10226 15537 14657 19158 20346
-63.7 35031 16805 4067 23003 23947 18910 20116 21333 25001 26209
-63.8 33251 15820 7673 24781 25675 24617 25354 24440 29433 30329
-63.9 33293 16833 18529 28318 29233 31267 31470 27748 29676 30601
-
-Per cent with error:
- 30.9 3.2 18.5 9.8 13.1 11.6 17.4
-Total arguments tested:
- 70194 70099 101784 100641 100641 101799 128853 114893 102675 102675
-
-
-------------------------- Contributors -------------------------------
-
-A number of people have contributed to the development of the
-emulator, often by just reporting bugs, sometimes with suggested
-fixes, and a few kind people have provided me with access in one way
-or another to an 80486 machine. Contributors include (to those people
-who I may have forgotten, please forgive me):
-
-Linus Torvalds
-Tommy.Thorn@daimi.aau.dk
-Andrew.Tridgell@anu.edu.au
-Nick Holloway, alfie@dcs.warwick.ac.uk
-Hermano Moura, moura@dcs.gla.ac.uk
-Jon Jagger, J.Jagger@scp.ac.uk
-Lennart Benschop
-Brian Gallew, geek+@CMU.EDU
-Thomas Staniszewski, ts3v+@andrew.cmu.edu
-Martin Howell, mph@plasma.apana.org.au
-M Saggaf, alsaggaf@athena.mit.edu
-Peter Barker, PETER@socpsy.sci.fau.edu
-tom@vlsivie.tuwien.ac.at
-Dan Russel, russed@rpi.edu
-Daniel Carosone, danielce@ee.mu.oz.au
-cae@jpmorgan.com
-Hamish Coleman, t933093@minyos.xx.rmit.oz.au
-Bruce Evans, bde@kralizec.zeta.org.au
-Timo Korvola, Timo.Korvola@hut.fi
-Rick Lyons, rick@razorback.brisnet.org.au
-Rick, jrs@world.std.com
-
-...and numerous others who responded to my request for help with
-a real 80486.
-
diff --git a/arch/x86/math-emu/control_w.h b/arch/x86/math-emu/control_w.h
deleted file mode 100644
index 93cbc89..0000000
--- a/arch/x86/math-emu/control_w.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | control_w.h |
- | |
- | Copyright (C) 1992,1993 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@vaxc.cc.monash.edu.au |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _CONTROLW_H_
-#define _CONTROLW_H_
-
-#ifdef __ASSEMBLER__
-#define _Const_(x) $##x
-#else
-#define _Const_(x) x
-#endif
-
-#define CW_RC _Const_(0x0C00) /* rounding control */
-#define CW_PC _Const_(0x0300) /* precision control */
-
-#define CW_Precision Const_(0x0020) /* loss of precision mask */
-#define CW_Underflow Const_(0x0010) /* underflow mask */
-#define CW_Overflow Const_(0x0008) /* overflow mask */
-#define CW_ZeroDiv Const_(0x0004) /* divide by zero mask */
-#define CW_Denormal Const_(0x0002) /* denormalized operand mask */
-#define CW_Invalid Const_(0x0001) /* invalid operation mask */
-
-#define CW_Exceptions _Const_(0x003f) /* all masks */
-
-#define RC_RND _Const_(0x0000)
-#define RC_DOWN _Const_(0x0400)
-#define RC_UP _Const_(0x0800)
-#define RC_CHOP _Const_(0x0C00)
-
-/* p 15-5: Precision control bits affect only the following:
- ADD, SUB(R), MUL, DIV(R), and SQRT */
-#define PR_24_BITS _Const_(0x000)
-#define PR_53_BITS _Const_(0x200)
-#define PR_64_BITS _Const_(0x300)
-#define PR_RESERVED_BITS _Const_(0x100)
-/* FULL_PRECISION simulates all exceptions masked */
-#define FULL_PRECISION (PR_64_BITS | RC_RND | 0x3f)
-
-#endif /* _CONTROLW_H_ */
diff --git a/arch/x86/math-emu/div_Xsig.S b/arch/x86/math-emu/div_Xsig.S
deleted file mode 100644
index 8c270ab..0000000
--- a/arch/x86/math-emu/div_Xsig.S
+++ /dev/null
@@ -1,367 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "div_Xsig.S"
-/*---------------------------------------------------------------------------+
- | div_Xsig.S |
- | |
- | Division subroutine for 96 bit quantities |
- | |
- | Copyright (C) 1994,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Divide the 96 bit quantity pointed to by a, by that pointed to by b, and |
- | put the 96 bit result at the location d. |
- | |
- | The result may not be accurate to 96 bits. It is intended for use where |
- | a result better than 64 bits is required. The result should usually be |
- | good to at least 94 bits. |
- | The returned result is actually divided by one half. This is done to |
- | prevent overflow. |
- | |
- | .aaaaaaaaaaaaaa / .bbbbbbbbbbbbb -> .dddddddddddd |
- | |
- | void div_Xsig(Xsig *a, Xsig *b, Xsig *dest) |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-
-#define XsigLL(x) (x)
-#define XsigL(x) 4(x)
-#define XsigH(x) 8(x)
-
-
-#ifndef NON_REENTRANT_FPU
-/*
- Local storage on the stack:
- Accumulator: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- */
-#define FPU_accum_3 -4(%ebp)
-#define FPU_accum_2 -8(%ebp)
-#define FPU_accum_1 -12(%ebp)
-#define FPU_accum_0 -16(%ebp)
-#define FPU_result_3 -20(%ebp)
-#define FPU_result_2 -24(%ebp)
-#define FPU_result_1 -28(%ebp)
-
-#else
-.data
-/*
- Local storage in a static area:
- Accumulator: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- */
- .align 4,0
-FPU_accum_3:
- .long 0
-FPU_accum_2:
- .long 0
-FPU_accum_1:
- .long 0
-FPU_accum_0:
- .long 0
-FPU_result_3:
- .long 0
-FPU_result_2:
- .long 0
-FPU_result_1:
- .long 0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-SYM_FUNC_START(div_Xsig)
- pushl %ebp
- movl %esp,%ebp
-#ifndef NON_REENTRANT_FPU
- subl $28,%esp
-#endif /* NON_REENTRANT_FPU */
-
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%esi /* pointer to num */
- movl PARAM2,%ebx /* pointer to denom */
-
-#ifdef PARANOID
- testl $0x80000000, XsigH(%ebx) /* Divisor */
- je L_bugged
-#endif /* PARANOID */
-
-
-/*---------------------------------------------------------------------------+
- | Divide: Return arg1/arg2 to arg3. |
- | |
- | The maximum returned value is (ignoring exponents) |
- | .ffffffff ffffffff |
- | ------------------ = 1.ffffffff fffffffe |
- | .80000000 00000000 |
- | and the minimum is |
- | .80000000 00000000 |
- | ------------------ = .80000000 00000001 (rounded) |
- | .ffffffff ffffffff |
- | |
- +---------------------------------------------------------------------------*/
-
- /* Save extended dividend in local register */
-
- /* Divide by 2 to prevent overflow */
- clc
- movl XsigH(%esi),%eax
- rcrl %eax
- movl %eax,FPU_accum_3
- movl XsigL(%esi),%eax
- rcrl %eax
- movl %eax,FPU_accum_2
- movl XsigLL(%esi),%eax
- rcrl %eax
- movl %eax,FPU_accum_1
- movl $0,%eax
- rcrl %eax
- movl %eax,FPU_accum_0
-
- movl FPU_accum_2,%eax /* Get the current num */
- movl FPU_accum_3,%edx
-
-/*----------------------------------------------------------------------*/
-/* Initialization done.
- Do the first 32 bits. */
-
- /* We will divide by a number which is too large */
- movl XsigH(%ebx),%ecx
- addl $1,%ecx
- jnc LFirst_div_not_1
-
- /* here we need to divide by 100000000h,
- i.e., no division at all.. */
- mov %edx,%eax
- jmp LFirst_div_done
-
-LFirst_div_not_1:
- divl %ecx /* Divide the numerator by the augmented
- denom ms dw */
-
-LFirst_div_done:
- movl %eax,FPU_result_3 /* Put the result in the answer */
-
- mull XsigH(%ebx) /* mul by the ms dw of the denom */
-
- subl %eax,FPU_accum_2 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_3
-
- movl FPU_result_3,%eax /* Get the result back */
- mull XsigL(%ebx) /* now mul the ls dw of the denom */
-
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
- sbbl $0,FPU_accum_3
- je LDo_2nd_32_bits /* Must check for non-zero result here */
-
-#ifdef PARANOID
- jb L_bugged_1
-#endif /* PARANOID */
-
- /* need to subtract another once of the denom */
- incl FPU_result_3 /* Correct the answer */
-
- movl XsigL(%ebx),%eax
- movl XsigH(%ebx),%edx
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
-
-#ifdef PARANOID
- sbbl $0,FPU_accum_3
- jne L_bugged_1 /* Must check for non-zero result here */
-#endif /* PARANOID */
-
-/*----------------------------------------------------------------------*/
-/* Half of the main problem is done, there is just a reduced numerator
- to handle now.
- Work with the second 32 bits, FPU_accum_0 not used from now on */
-LDo_2nd_32_bits:
- movl FPU_accum_2,%edx /* get the reduced num */
- movl FPU_accum_1,%eax
-
- /* need to check for possible subsequent overflow */
- cmpl XsigH(%ebx),%edx
- jb LDo_2nd_div
- ja LPrevent_2nd_overflow
-
- cmpl XsigL(%ebx),%eax
- jb LDo_2nd_div
-
-LPrevent_2nd_overflow:
-/* The numerator is greater or equal, would cause overflow */
- /* prevent overflow */
- subl XsigL(%ebx),%eax
- sbbl XsigH(%ebx),%edx
- movl %edx,FPU_accum_2
- movl %eax,FPU_accum_1
-
- incl FPU_result_3 /* Reflect the subtraction in the answer */
-
-#ifdef PARANOID
- je L_bugged_2 /* Can't bump the result to 1.0 */
-#endif /* PARANOID */
-
-LDo_2nd_div:
- cmpl $0,%ecx /* augmented denom msw */
- jnz LSecond_div_not_1
-
- /* %ecx == 0, we are dividing by 1.0 */
- mov %edx,%eax
- jmp LSecond_div_done
-
-LSecond_div_not_1:
- divl %ecx /* Divide the numerator by the denom ms dw */
-
-LSecond_div_done:
- movl %eax,FPU_result_2 /* Put the result in the answer */
-
- mull XsigH(%ebx) /* mul by the ms dw of the denom */
-
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
-#endif /* PARANOID */
-
- movl FPU_result_2,%eax /* Get the result back */
- mull XsigL(%ebx) /* now mul the ls dw of the denom */
-
- subl %eax,FPU_accum_0 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_1 /* Subtract from the num local reg */
- sbbl $0,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
-#endif /* PARANOID */
-
- jz LDo_3rd_32_bits
-
-#ifdef PARANOID
- cmpl $1,FPU_accum_2
- jne L_bugged_2
-#endif /* PARANOID */
-
- /* need to subtract another once of the denom */
- movl XsigL(%ebx),%eax
- movl XsigH(%ebx),%edx
- subl %eax,FPU_accum_0 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_1
- sbbl $0,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
- jne L_bugged_2
-#endif /* PARANOID */
-
- addl $1,FPU_result_2 /* Correct the answer */
- adcl $0,FPU_result_3
-
-#ifdef PARANOID
- jc L_bugged_2 /* Must check for non-zero result here */
-#endif /* PARANOID */
-
-/*----------------------------------------------------------------------*/
-/* The division is essentially finished here, we just need to perform
- tidying operations.
- Deal with the 3rd 32 bits */
-LDo_3rd_32_bits:
- /* We use an approximation for the third 32 bits.
- To take account of the 3rd 32 bits of the divisor
- (call them del), we subtract del * (a/b) */
-
- movl FPU_result_3,%eax /* a/b */
- mull XsigLL(%ebx) /* del */
-
- subl %edx,FPU_accum_1
-
- /* A borrow indicates that the result is negative */
- jnb LTest_over
-
- movl XsigH(%ebx),%edx
- addl %edx,FPU_accum_1
-
- subl $1,FPU_result_2 /* Adjust the answer */
- sbbl $0,FPU_result_3
-
- /* The above addition might not have been enough, check again. */
- movl FPU_accum_1,%edx /* get the reduced num */
- cmpl XsigH(%ebx),%edx /* denom */
- jb LDo_3rd_div
-
- movl XsigH(%ebx),%edx
- addl %edx,FPU_accum_1
-
- subl $1,FPU_result_2 /* Adjust the answer */
- sbbl $0,FPU_result_3
- jmp LDo_3rd_div
-
-LTest_over:
- movl FPU_accum_1,%edx /* get the reduced num */
-
- /* need to check for possible subsequent overflow */
- cmpl XsigH(%ebx),%edx /* denom */
- jb LDo_3rd_div
-
- /* prevent overflow */
- subl XsigH(%ebx),%edx
- movl %edx,FPU_accum_1
-
- addl $1,FPU_result_2 /* Reflect the subtraction in the answer */
- adcl $0,FPU_result_3
-
-LDo_3rd_div:
- movl FPU_accum_0,%eax
- movl FPU_accum_1,%edx
- divl XsigH(%ebx)
-
- movl %eax,FPU_result_1 /* Rough estimate of third word */
-
- movl PARAM3,%esi /* pointer to answer */
-
- movl FPU_result_1,%eax
- movl %eax,XsigLL(%esi)
- movl FPU_result_2,%eax
- movl %eax,XsigL(%esi)
- movl FPU_result_3,%eax
- movl %eax,XsigH(%esi)
-
-L_exit:
- popl %ebx
- popl %edi
- popl %esi
-
- leave
- RET
-
-
-#ifdef PARANOID
-/* The logic is wrong if we got here */
-L_bugged:
- pushl EX_INTERNAL|0x240
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_bugged_1:
- pushl EX_INTERNAL|0x241
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_bugged_2:
- pushl EX_INTERNAL|0x242
- call EXCEPTION
- pop %ebx
- jmp L_exit
-#endif /* PARANOID */
-SYM_FUNC_END(div_Xsig)
diff --git a/arch/x86/math-emu/div_small.S b/arch/x86/math-emu/div_small.S
deleted file mode 100644
index 637439b..0000000
--- a/arch/x86/math-emu/div_small.S
+++ /dev/null
@@ -1,48 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "div_small.S"
-/*---------------------------------------------------------------------------+
- | div_small.S |
- | |
- | Divide a 64 bit integer by a 32 bit integer & return remainder. |
- | |
- | Copyright (C) 1992,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | unsigned long FPU_div_small(unsigned long long *x, unsigned long y) |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(FPU_div_small)
- pushl %ebp
- movl %esp,%ebp
-
- pushl %esi
-
- movl PARAM1,%esi /* pointer to num */
- movl PARAM2,%ecx /* The denominator */
-
- movl 4(%esi),%eax /* Get the current num msw */
- xorl %edx,%edx
- divl %ecx
-
- movl %eax,4(%esi)
-
- movl (%esi),%eax /* Get the num lsw */
- divl %ecx
-
- movl %eax,(%esi)
-
- movl %edx,%eax /* Return the remainder in eax */
-
- popl %esi
-
- leave
- RET
-SYM_FUNC_END(FPU_div_small)
diff --git a/arch/x86/math-emu/errors.c b/arch/x86/math-emu/errors.c
deleted file mode 100644
index ec071cb..0000000
--- a/arch/x86/math-emu/errors.c
+++ /dev/null
@@ -1,686 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | errors.c |
- | |
- | The error handling functions for wm-FPU-emu |
- | |
- | Copyright (C) 1992,1993,1994,1996 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note: |
- | The file contains code which accesses user memory. |
- | Emulator static data may change when user memory is accessed, due to |
- | other processes using the emulator while swapping is in progress. |
- +---------------------------------------------------------------------------*/
-
-#include <linux/signal.h>
-
-#include <linux/uaccess.h>
-
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "exception.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "reg_constant.h"
-#include "version.h"
-
-/* */
-#undef PRINT_MESSAGES
-/* */
-
-#if 0
-void Un_impl(void)
-{
- u_char byte1, FPU_modrm;
- unsigned long address = FPU_ORIG_EIP;
-
- RE_ENTRANT_CHECK_OFF;
- /* No need to check access_ok(), we have previously fetched these bytes. */
- printk("Unimplemented FPU Opcode at eip=%p : ", (void __user *)address);
- if (FPU_CS == __USER_CS) {
- while (1) {
- FPU_get_user(byte1, (u_char __user *) address);
- if ((byte1 & 0xf8) == 0xd8)
- break;
- printk("[%02x]", byte1);
- address++;
- }
- printk("%02x ", byte1);
- FPU_get_user(FPU_modrm, 1 + (u_char __user *) address);
-
- if (FPU_modrm >= 0300)
- printk("%02x (%02x+%d)\n", FPU_modrm, FPU_modrm & 0xf8,
- FPU_modrm & 7);
- else
- printk("/%d\n", (FPU_modrm >> 3) & 7);
- } else {
- printk("cs selector = %04x\n", FPU_CS);
- }
-
- RE_ENTRANT_CHECK_ON;
-
- EXCEPTION(EX_Invalid);
-
-}
-#endif /* 0 */
-
-/*
- Called for opcodes which are illegal and which are known to result in a
- SIGILL with a real 80486.
- */
-void FPU_illegal(void)
-{
- math_abort(FPU_info, SIGILL);
-}
-
-void FPU_printall(void)
-{
- int i;
- static const char *tag_desc[] = { "Valid", "Zero", "ERROR", "Empty",
- "DeNorm", "Inf", "NaN"
- };
- u_char byte1, FPU_modrm;
- unsigned long address = FPU_ORIG_EIP;
-
- RE_ENTRANT_CHECK_OFF;
- /* No need to check access_ok(), we have previously fetched these bytes. */
- printk("At %p:", (void *)address);
- if (FPU_CS == __USER_CS) {
-#define MAX_PRINTED_BYTES 20
- for (i = 0; i < MAX_PRINTED_BYTES; i++) {
- FPU_get_user(byte1, (u_char __user *) address);
- if ((byte1 & 0xf8) == 0xd8) {
- printk(" %02x", byte1);
- break;
- }
- printk(" [%02x]", byte1);
- address++;
- }
- if (i == MAX_PRINTED_BYTES)
- printk(" [more..]\n");
- else {
- FPU_get_user(FPU_modrm, 1 + (u_char __user *) address);
-
- if (FPU_modrm >= 0300)
- printk(" %02x (%02x+%d)\n", FPU_modrm,
- FPU_modrm & 0xf8, FPU_modrm & 7);
- else
- printk(" /%d, mod=%d rm=%d\n",
- (FPU_modrm >> 3) & 7,
- (FPU_modrm >> 6) & 3, FPU_modrm & 7);
- }
- } else {
- printk("%04x\n", FPU_CS);
- }
-
- partial_status = status_word();
-
-#ifdef DEBUGGING
- if (partial_status & SW_Backward)
- printk("SW: backward compatibility\n");
- if (partial_status & SW_C3)
- printk("SW: condition bit 3\n");
- if (partial_status & SW_C2)
- printk("SW: condition bit 2\n");
- if (partial_status & SW_C1)
- printk("SW: condition bit 1\n");
- if (partial_status & SW_C0)
- printk("SW: condition bit 0\n");
- if (partial_status & SW_Summary)
- printk("SW: exception summary\n");
- if (partial_status & SW_Stack_Fault)
- printk("SW: stack fault\n");
- if (partial_status & SW_Precision)
- printk("SW: loss of precision\n");
- if (partial_status & SW_Underflow)
- printk("SW: underflow\n");
- if (partial_status & SW_Overflow)
- printk("SW: overflow\n");
- if (partial_status & SW_Zero_Div)
- printk("SW: divide by zero\n");
- if (partial_status & SW_Denorm_Op)
- printk("SW: denormalized operand\n");
- if (partial_status & SW_Invalid)
- printk("SW: invalid operation\n");
-#endif /* DEBUGGING */
-
- printk(" SW: b=%d st=%d es=%d sf=%d cc=%d%d%d%d ef=%d%d%d%d%d%d\n", partial_status & 0x8000 ? 1 : 0, /* busy */
- (partial_status & 0x3800) >> 11, /* stack top pointer */
- partial_status & 0x80 ? 1 : 0, /* Error summary status */
- partial_status & 0x40 ? 1 : 0, /* Stack flag */
- partial_status & SW_C3 ? 1 : 0, partial_status & SW_C2 ? 1 : 0, /* cc */
- partial_status & SW_C1 ? 1 : 0, partial_status & SW_C0 ? 1 : 0, /* cc */
- partial_status & SW_Precision ? 1 : 0,
- partial_status & SW_Underflow ? 1 : 0,
- partial_status & SW_Overflow ? 1 : 0,
- partial_status & SW_Zero_Div ? 1 : 0,
- partial_status & SW_Denorm_Op ? 1 : 0,
- partial_status & SW_Invalid ? 1 : 0);
-
- printk(" CW: ic=%d rc=%d%d pc=%d%d iem=%d ef=%d%d%d%d%d%d\n",
- control_word & 0x1000 ? 1 : 0,
- (control_word & 0x800) >> 11, (control_word & 0x400) >> 10,
- (control_word & 0x200) >> 9, (control_word & 0x100) >> 8,
- control_word & 0x80 ? 1 : 0,
- control_word & SW_Precision ? 1 : 0,
- control_word & SW_Underflow ? 1 : 0,
- control_word & SW_Overflow ? 1 : 0,
- control_word & SW_Zero_Div ? 1 : 0,
- control_word & SW_Denorm_Op ? 1 : 0,
- control_word & SW_Invalid ? 1 : 0);
-
- for (i = 0; i < 8; i++) {
- FPU_REG *r = &st(i);
- u_char tagi = FPU_gettagi(i);
-
- switch (tagi) {
- case TAG_Empty:
- continue;
- case TAG_Zero:
- case TAG_Special:
- /* Update tagi for the printk below */
- tagi = FPU_Special(r);
- fallthrough;
- case TAG_Valid:
- printk("st(%d) %c .%04lx %04lx %04lx %04lx e%+-6d ", i,
- getsign(r) ? '-' : '+',
- (long)(r->sigh >> 16),
- (long)(r->sigh & 0xFFFF),
- (long)(r->sigl >> 16),
- (long)(r->sigl & 0xFFFF),
- exponent(r) - EXP_BIAS + 1);
- break;
- default:
- printk("Whoops! Error in errors.c: tag%d is %d ", i,
- tagi);
- continue;
- }
- printk("%s\n", tag_desc[(int)(unsigned)tagi]);
- }
-
- RE_ENTRANT_CHECK_ON;
-
-}
-
-static struct {
- int type;
- const char *name;
-} exception_names[] = {
- {
- EX_StackOver, "stack overflow"}, {
- EX_StackUnder, "stack underflow"}, {
- EX_Precision, "loss of precision"}, {
- EX_Underflow, "underflow"}, {
- EX_Overflow, "overflow"}, {
- EX_ZeroDiv, "divide by zero"}, {
- EX_Denormal, "denormalized operand"}, {
- EX_Invalid, "invalid operation"}, {
- EX_INTERNAL, "INTERNAL BUG in " FPU_VERSION}, {
- 0, NULL}
-};
-
-/*
- EX_INTERNAL is always given with a code which indicates where the
- error was detected.
-
- Internal error types:
- 0x14 in fpu_etc.c
- 0x1nn in a *.c file:
- 0x101 in reg_add_sub.c
- 0x102 in reg_mul.c
- 0x104 in poly_atan.c
- 0x105 in reg_mul.c
- 0x107 in fpu_trig.c
- 0x108 in reg_compare.c
- 0x109 in reg_compare.c
- 0x110 in reg_add_sub.c
- 0x111 in fpe_entry.c
- 0x112 in fpu_trig.c
- 0x113 in errors.c
- 0x115 in fpu_trig.c
- 0x116 in fpu_trig.c
- 0x117 in fpu_trig.c
- 0x118 in fpu_trig.c
- 0x119 in fpu_trig.c
- 0x120 in poly_atan.c
- 0x121 in reg_compare.c
- 0x122 in reg_compare.c
- 0x123 in reg_compare.c
- 0x125 in fpu_trig.c
- 0x126 in fpu_entry.c
- 0x127 in poly_2xm1.c
- 0x128 in fpu_entry.c
- 0x129 in fpu_entry.c
- 0x130 in get_address.c
- 0x131 in get_address.c
- 0x132 in get_address.c
- 0x133 in get_address.c
- 0x140 in load_store.c
- 0x141 in load_store.c
- 0x150 in poly_sin.c
- 0x151 in poly_sin.c
- 0x160 in reg_ld_str.c
- 0x161 in reg_ld_str.c
- 0x162 in reg_ld_str.c
- 0x163 in reg_ld_str.c
- 0x164 in reg_ld_str.c
- 0x170 in fpu_tags.c
- 0x171 in fpu_tags.c
- 0x172 in fpu_tags.c
- 0x180 in reg_convert.c
- 0x2nn in an *.S file:
- 0x201 in reg_u_add.S
- 0x202 in reg_u_div.S
- 0x203 in reg_u_div.S
- 0x204 in reg_u_div.S
- 0x205 in reg_u_mul.S
- 0x206 in reg_u_sub.S
- 0x207 in wm_sqrt.S
- 0x208 in reg_div.S
- 0x209 in reg_u_sub.S
- 0x210 in reg_u_sub.S
- 0x211 in reg_u_sub.S
- 0x212 in reg_u_sub.S
- 0x213 in wm_sqrt.S
- 0x214 in wm_sqrt.S
- 0x215 in wm_sqrt.S
- 0x220 in reg_norm.S
- 0x221 in reg_norm.S
- 0x230 in reg_round.S
- 0x231 in reg_round.S
- 0x232 in reg_round.S
- 0x233 in reg_round.S
- 0x234 in reg_round.S
- 0x235 in reg_round.S
- 0x236 in reg_round.S
- 0x240 in div_Xsig.S
- 0x241 in div_Xsig.S
- 0x242 in div_Xsig.S
- */
-
-asmlinkage __visible void FPU_exception(int n)
-{
- int i, int_type;
-
- int_type = 0; /* Needed only to stop compiler warnings */
- if (n & EX_INTERNAL) {
- int_type = n - EX_INTERNAL;
- n = EX_INTERNAL;
- /* Set lots of exception bits! */
- partial_status |= (SW_Exc_Mask | SW_Summary | SW_Backward);
- } else {
- /* Extract only the bits which we use to set the status word */
- n &= (SW_Exc_Mask);
- /* Set the corresponding exception bit */
- partial_status |= n;
- /* Set summary bits iff exception isn't masked */
- if (partial_status & ~control_word & CW_Exceptions)
- partial_status |= (SW_Summary | SW_Backward);
- if (n & (SW_Stack_Fault | EX_Precision)) {
- if (!(n & SW_C1))
- /* This bit distinguishes over- from underflow for a stack fault,
- and roundup from round-down for precision loss. */
- partial_status &= ~SW_C1;
- }
- }
-
- RE_ENTRANT_CHECK_OFF;
- if ((~control_word & n & CW_Exceptions) || (n == EX_INTERNAL)) {
- /* Get a name string for error reporting */
- for (i = 0; exception_names[i].type; i++)
- if ((exception_names[i].type & n) ==
- exception_names[i].type)
- break;
-
- if (exception_names[i].type) {
-#ifdef PRINT_MESSAGES
- printk("FP Exception: %s!\n", exception_names[i].name);
-#endif /* PRINT_MESSAGES */
- } else
- printk("FPU emulator: Unknown Exception: 0x%04x!\n", n);
-
- if (n == EX_INTERNAL) {
- printk("FPU emulator: Internal error type 0x%04x\n",
- int_type);
- FPU_printall();
- }
-#ifdef PRINT_MESSAGES
- else
- FPU_printall();
-#endif /* PRINT_MESSAGES */
-
- /*
- * The 80486 generates an interrupt on the next non-control FPU
- * instruction. So we need some means of flagging it.
- * We use the ES (Error Summary) bit for this.
- */
- }
- RE_ENTRANT_CHECK_ON;
-
-#ifdef __DEBUG__
- math_abort(FPU_info, SIGFPE);
-#endif /* __DEBUG__ */
-
-}
-
-/* Real operation attempted on a NaN. */
-/* Returns < 0 if the exception is unmasked */
-int real_1op_NaN(FPU_REG *a)
-{
- int signalling, isNaN;
-
- isNaN = (exponent(a) == EXP_OVER) && (a->sigh & 0x80000000);
-
- /* The default result for the case of two "equal" NaNs (signs may
- differ) is chosen to reproduce 80486 behaviour */
- signalling = isNaN && !(a->sigh & 0x40000000);
-
- if (!signalling) {
- if (!isNaN) { /* pseudo-NaN, or other unsupported? */
- if (control_word & CW_Invalid) {
- /* Masked response */
- reg_copy(&CONST_QNaN, a);
- }
- EXCEPTION(EX_Invalid);
- return (!(control_word & CW_Invalid) ? FPU_Exception :
- 0) | TAG_Special;
- }
- return TAG_Special;
- }
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- if (!(a->sigh & 0x80000000)) { /* pseudo-NaN ? */
- reg_copy(&CONST_QNaN, a);
- }
- /* ensure a Quiet NaN */
- a->sigh |= 0x40000000;
- }
-
- EXCEPTION(EX_Invalid);
-
- return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special;
-}
-
-/* Real operation attempted on two operands, one a NaN. */
-/* Returns < 0 if the exception is unmasked */
-int real_2op_NaN(FPU_REG const *b, u_char tagb,
- int deststnr, FPU_REG const *defaultNaN)
-{
- FPU_REG *dest = &st(deststnr);
- FPU_REG const *a = dest;
- u_char taga = FPU_gettagi(deststnr);
- FPU_REG const *x;
- int signalling, unsupported;
-
- if (taga == TAG_Special)
- taga = FPU_Special(a);
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
-
- /* TW_NaN is also used for unsupported data types. */
- unsupported = ((taga == TW_NaN)
- && !((exponent(a) == EXP_OVER)
- && (a->sigh & 0x80000000)))
- || ((tagb == TW_NaN)
- && !((exponent(b) == EXP_OVER) && (b->sigh & 0x80000000)));
- if (unsupported) {
- if (control_word & CW_Invalid) {
- /* Masked response */
- FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr);
- }
- EXCEPTION(EX_Invalid);
- return (!(control_word & CW_Invalid) ? FPU_Exception : 0) |
- TAG_Special;
- }
-
- if (taga == TW_NaN) {
- x = a;
- if (tagb == TW_NaN) {
- signalling = !(a->sigh & b->sigh & 0x40000000);
- if (significand(b) > significand(a))
- x = b;
- else if (significand(b) == significand(a)) {
- /* The default result for the case of two "equal" NaNs (signs may
- differ) is chosen to reproduce 80486 behaviour */
- x = defaultNaN;
- }
- } else {
- /* return the quiet version of the NaN in a */
- signalling = !(a->sigh & 0x40000000);
- }
- } else
-#ifdef PARANOID
- if (tagb == TW_NaN)
-#endif /* PARANOID */
- {
- signalling = !(b->sigh & 0x40000000);
- x = b;
- }
-#ifdef PARANOID
- else {
- signalling = 0;
- EXCEPTION(EX_INTERNAL | 0x113);
- x = &CONST_QNaN;
- }
-#endif /* PARANOID */
-
- if ((!signalling) || (control_word & CW_Invalid)) {
- if (!x)
- x = b;
-
- if (!(x->sigh & 0x80000000)) /* pseudo-NaN ? */
- x = &CONST_QNaN;
-
- FPU_copy_to_regi(x, TAG_Special, deststnr);
-
- if (!signalling)
- return TAG_Special;
-
- /* ensure a Quiet NaN */
- dest->sigh |= 0x40000000;
- }
-
- EXCEPTION(EX_Invalid);
-
- return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special;
-}
-
-/* Invalid arith operation on Valid registers */
-/* Returns < 0 if the exception is unmasked */
-asmlinkage __visible int arith_invalid(int deststnr)
-{
-
- EXCEPTION(EX_Invalid);
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr);
- }
-
- return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Valid;
-
-}
-
-/* Divide a finite number by zero */
-asmlinkage __visible int FPU_divide_by_zero(int deststnr, u_char sign)
-{
- FPU_REG *dest = &st(deststnr);
- int tag = TAG_Valid;
-
- if (control_word & CW_ZeroDiv) {
- /* The masked response */
- FPU_copy_to_regi(&CONST_INF, TAG_Special, deststnr);
- setsign(dest, sign);
- tag = TAG_Special;
- }
-
- EXCEPTION(EX_ZeroDiv);
-
- return (!(control_word & CW_ZeroDiv) ? FPU_Exception : 0) | tag;
-
-}
-
-/* This may be called often, so keep it lean */
-int set_precision_flag(int flags)
-{
- if (control_word & CW_Precision) {
- partial_status &= ~(SW_C1 & flags);
- partial_status |= flags; /* The masked response */
- return 0;
- } else {
- EXCEPTION(flags);
- return 1;
- }
-}
-
-/* This may be called often, so keep it lean */
-asmlinkage __visible void set_precision_flag_up(void)
-{
- if (control_word & CW_Precision)
- partial_status |= (SW_Precision | SW_C1); /* The masked response */
- else
- EXCEPTION(EX_Precision | SW_C1);
-}
-
-/* This may be called often, so keep it lean */
-asmlinkage __visible void set_precision_flag_down(void)
-{
- if (control_word & CW_Precision) { /* The masked response */
- partial_status &= ~SW_C1;
- partial_status |= SW_Precision;
- } else
- EXCEPTION(EX_Precision);
-}
-
-asmlinkage __visible int denormal_operand(void)
-{
- if (control_word & CW_Denormal) { /* The masked response */
- partial_status |= SW_Denorm_Op;
- return TAG_Special;
- } else {
- EXCEPTION(EX_Denormal);
- return TAG_Special | FPU_Exception;
- }
-}
-
-asmlinkage __visible int arith_overflow(FPU_REG *dest)
-{
- int tag = TAG_Valid;
-
- if (control_word & CW_Overflow) {
- /* The masked response */
-/* ###### The response here depends upon the rounding mode */
- reg_copy(&CONST_INF, dest);
- tag = TAG_Special;
- } else {
- /* Subtract the magic number from the exponent */
- addexponent(dest, (-3 * (1 << 13)));
- }
-
- EXCEPTION(EX_Overflow);
- if (control_word & CW_Overflow) {
- /* The overflow exception is masked. */
- /* By definition, precision is lost.
- The roundup bit (C1) is also set because we have
- "rounded" upwards to Infinity. */
- EXCEPTION(EX_Precision | SW_C1);
- return tag;
- }
-
- return tag;
-
-}
-
-asmlinkage __visible int arith_underflow(FPU_REG *dest)
-{
- int tag = TAG_Valid;
-
- if (control_word & CW_Underflow) {
- /* The masked response */
- if (exponent16(dest) <= EXP_UNDER - 63) {
- reg_copy(&CONST_Z, dest);
- partial_status &= ~SW_C1; /* Round down. */
- tag = TAG_Zero;
- } else {
- stdexp(dest);
- }
- } else {
- /* Add the magic number to the exponent. */
- addexponent(dest, (3 * (1 << 13)) + EXTENDED_Ebias);
- }
-
- EXCEPTION(EX_Underflow);
- if (control_word & CW_Underflow) {
- /* The underflow exception is masked. */
- EXCEPTION(EX_Precision);
- return tag;
- }
-
- return tag;
-
-}
-
-void FPU_stack_overflow(void)
-{
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- top--;
- FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
- }
-
- EXCEPTION(EX_StackOver);
-
- return;
-
-}
-
-void FPU_stack_underflow(void)
-{
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
- }
-
- EXCEPTION(EX_StackUnder);
-
- return;
-
-}
-
-void FPU_stack_underflow_i(int i)
-{
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i);
- }
-
- EXCEPTION(EX_StackUnder);
-
- return;
-
-}
-
-void FPU_stack_underflow_pop(int i)
-{
-
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i);
- FPU_pop();
- }
-
- EXCEPTION(EX_StackUnder);
-
- return;
-
-}
diff --git a/arch/x86/math-emu/exception.h b/arch/x86/math-emu/exception.h
deleted file mode 100644
index 59961d3..0000000
--- a/arch/x86/math-emu/exception.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | exception.h |
- | |
- | Copyright (C) 1992 W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@vaxc.cc.monash.edu.au |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _EXCEPTION_H_
-#define _EXCEPTION_H_
-
-#ifdef __ASSEMBLER__
-#define Const_(x) $##x
-#else
-#define Const_(x) x
-#endif
-
-#ifndef SW_C1
-#include "fpu_emu.h"
-#endif /* SW_C1 */
-
-#define FPU_BUSY Const_(0x8000) /* FPU busy bit (8087 compatibility) */
-#define EX_ErrorSummary Const_(0x0080) /* Error summary status */
-/* Special exceptions: */
-#define EX_INTERNAL Const_(0x8000) /* Internal error in wm-FPU-emu */
-#define EX_StackOver Const_(0x0041|SW_C1) /* stack overflow */
-#define EX_StackUnder Const_(0x0041) /* stack underflow */
-/* Exception flags: */
-#define EX_Precision Const_(0x0020) /* loss of precision */
-#define EX_Underflow Const_(0x0010) /* underflow */
-#define EX_Overflow Const_(0x0008) /* overflow */
-#define EX_ZeroDiv Const_(0x0004) /* divide by zero */
-#define EX_Denormal Const_(0x0002) /* denormalized operand */
-#define EX_Invalid Const_(0x0001) /* invalid operation */
-
-#define PRECISION_LOST_UP Const_((EX_Precision | SW_C1))
-#define PRECISION_LOST_DOWN Const_(EX_Precision)
-
-#ifndef __ASSEMBLER__
-
-#ifdef DEBUG
-#define EXCEPTION(x) { printk("exception in %s at line %d\n", \
- __FILE__, __LINE__); FPU_exception(x); }
-#else
-#define EXCEPTION(x) FPU_exception(x)
-#endif
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _EXCEPTION_H_ */
diff --git a/arch/x86/math-emu/fpu_arith.c b/arch/x86/math-emu/fpu_arith.c
deleted file mode 100644
index 09006dc..0000000
--- a/arch/x86/math-emu/fpu_arith.c
+++ /dev/null
@@ -1,153 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_arith.c |
- | |
- | Code to implement the FPU register/register arithmetic instructions |
- | |
- | Copyright (C) 1992,1993,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "status_w.h"
-
-void fadd__(void)
-{
- /* fadd st,st(i) */
- int i = FPU_rm;
- clear_C1();
- FPU_add(&st(i), FPU_gettagi(i), 0, control_word);
-}
-
-void fmul__(void)
-{
- /* fmul st,st(i) */
- int i = FPU_rm;
- clear_C1();
- FPU_mul(&st(i), FPU_gettagi(i), 0, control_word);
-}
-
-void fsub__(void)
-{
- /* fsub st,st(i) */
- clear_C1();
- FPU_sub(0, FPU_rm, control_word);
-}
-
-void fsubr_(void)
-{
- /* fsubr st,st(i) */
- clear_C1();
- FPU_sub(REV, FPU_rm, control_word);
-}
-
-void fdiv__(void)
-{
- /* fdiv st,st(i) */
- clear_C1();
- FPU_div(0, FPU_rm, control_word);
-}
-
-void fdivr_(void)
-{
- /* fdivr st,st(i) */
- clear_C1();
- FPU_div(REV, FPU_rm, control_word);
-}
-
-void fadd_i(void)
-{
- /* fadd st(i),st */
- int i = FPU_rm;
- clear_C1();
- FPU_add(&st(i), FPU_gettagi(i), i, control_word);
-}
-
-void fmul_i(void)
-{
- /* fmul st(i),st */
- clear_C1();
- FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word);
-}
-
-void fsubri(void)
-{
- /* fsubr st(i),st */
- clear_C1();
- FPU_sub(DEST_RM, FPU_rm, control_word);
-}
-
-void fsub_i(void)
-{
- /* fsub st(i),st */
- clear_C1();
- FPU_sub(REV | DEST_RM, FPU_rm, control_word);
-}
-
-void fdivri(void)
-{
- /* fdivr st(i),st */
- clear_C1();
- FPU_div(DEST_RM, FPU_rm, control_word);
-}
-
-void fdiv_i(void)
-{
- /* fdiv st(i),st */
- clear_C1();
- FPU_div(REV | DEST_RM, FPU_rm, control_word);
-}
-
-void faddp_(void)
-{
- /* faddp st(i),st */
- int i = FPU_rm;
- clear_C1();
- if (FPU_add(&st(i), FPU_gettagi(i), i, control_word) >= 0)
- FPU_pop();
-}
-
-void fmulp_(void)
-{
- /* fmulp st(i),st */
- clear_C1();
- if (FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word) >= 0)
- FPU_pop();
-}
-
-void fsubrp(void)
-{
- /* fsubrp st(i),st */
- clear_C1();
- if (FPU_sub(DEST_RM, FPU_rm, control_word) >= 0)
- FPU_pop();
-}
-
-void fsubp_(void)
-{
- /* fsubp st(i),st */
- clear_C1();
- if (FPU_sub(REV | DEST_RM, FPU_rm, control_word) >= 0)
- FPU_pop();
-}
-
-void fdivrp(void)
-{
- /* fdivrp st(i),st */
- clear_C1();
- if (FPU_div(DEST_RM, FPU_rm, control_word) >= 0)
- FPU_pop();
-}
-
-void fdivp_(void)
-{
- /* fdivp st(i),st */
- clear_C1();
- if (FPU_div(REV | DEST_RM, FPU_rm, control_word) >= 0)
- FPU_pop();
-}
diff --git a/arch/x86/math-emu/fpu_asm.h b/arch/x86/math-emu/fpu_asm.h
deleted file mode 100644
index a83353d..0000000
--- a/arch/x86/math-emu/fpu_asm.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | fpu_asm.h |
- | |
- | Copyright (C) 1992,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_ASM_H_
-#define _FPU_ASM_H_
-
-#include <linux/linkage.h>
-
-#define EXCEPTION FPU_exception
-
-#define PARAM1 8(%ebp)
-#define PARAM2 12(%ebp)
-#define PARAM3 16(%ebp)
-#define PARAM4 20(%ebp)
-#define PARAM5 24(%ebp)
-#define PARAM6 28(%ebp)
-#define PARAM7 32(%ebp)
-
-#define SIGL_OFFSET 0
-#define EXP(x) 8(x)
-#define SIG(x) SIGL_OFFSET##(x)
-#define SIGL(x) SIGL_OFFSET##(x)
-#define SIGH(x) 4(x)
-
-#endif /* _FPU_ASM_H_ */
diff --git a/arch/x86/math-emu/fpu_aux.c b/arch/x86/math-emu/fpu_aux.c
deleted file mode 100644
index 5f253ae..0000000
--- a/arch/x86/math-emu/fpu_aux.c
+++ /dev/null
@@ -1,267 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_aux.c |
- | |
- | Code to implement some of the FPU auxiliary instructions. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-
-static void fnop(void)
-{
-}
-
-static void fclex(void)
-{
- partial_status &=
- ~(SW_Backward | SW_Summary | SW_Stack_Fault | SW_Precision |
- SW_Underflow | SW_Overflow | SW_Zero_Div | SW_Denorm_Op |
- SW_Invalid);
- no_ip_update = 1;
-}
-
-/* Needs to be externally visible */
-void fpstate_init_soft(struct swregs_state *soft)
-{
- struct address *oaddr, *iaddr;
- memset(soft, 0, sizeof(*soft));
- soft->cwd = 0x037f;
- soft->swd = 0;
- soft->ftop = 0; /* We don't keep top in the status word internally. */
- soft->twd = 0xffff;
- /* The behaviour is different from that detailed in
- Section 15.1.6 of the Intel manual */
- oaddr = (struct address *)&soft->foo;
- oaddr->offset = 0;
- oaddr->selector = 0;
- iaddr = (struct address *)&soft->fip;
- iaddr->offset = 0;
- iaddr->selector = 0;
- iaddr->opcode = 0;
- soft->no_update = 1;
-}
-
-void finit(void)
-{
- fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft);
-}
-
-/*
- * These are nops on the i387..
- */
-#define feni fnop
-#define fdisi fnop
-#define fsetpm fnop
-
-static FUNC const finit_table[] = {
- feni, fdisi, fclex, finit,
- fsetpm, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void finit_(void)
-{
- (finit_table[FPU_rm]) ();
-}
-
-static void fstsw_ax(void)
-{
- *(short *)&FPU_EAX = status_word();
- no_ip_update = 1;
-}
-
-static FUNC const fstsw_table[] = {
- fstsw_ax, FPU_illegal, FPU_illegal, FPU_illegal,
- FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void fstsw_(void)
-{
- (fstsw_table[FPU_rm]) ();
-}
-
-static FUNC const fp_nop_table[] = {
- fnop, FPU_illegal, FPU_illegal, FPU_illegal,
- FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void fp_nop(void)
-{
- (fp_nop_table[FPU_rm]) ();
-}
-
-void fld_i_(void)
-{
- FPU_REG *st_new_ptr;
- int i;
- u_char tag;
-
- if (STACK_OVERFLOW) {
- FPU_stack_overflow();
- return;
- }
-
- /* fld st(i) */
- i = FPU_rm;
- if (NOT_EMPTY(i)) {
- reg_copy(&st(i), st_new_ptr);
- tag = FPU_gettagi(i);
- push();
- FPU_settag0(tag);
- } else {
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_stack_underflow();
- } else
- EXCEPTION(EX_StackUnder);
- }
-
-}
-
-void fxch_i(void)
-{
- /* fxch st(i) */
- FPU_REG t;
- int i = FPU_rm;
- FPU_REG *st0_ptr = &st(0), *sti_ptr = &st(i);
- long tag_word = fpu_tag_word;
- int regnr = top & 7, regnri = ((regnr + i) & 7);
- u_char st0_tag = (tag_word >> (regnr * 2)) & 3;
- u_char sti_tag = (tag_word >> (regnri * 2)) & 3;
-
- if (st0_tag == TAG_Empty) {
- if (sti_tag == TAG_Empty) {
- FPU_stack_underflow();
- FPU_stack_underflow_i(i);
- return;
- }
- if (control_word & CW_Invalid) {
- /* Masked response */
- FPU_copy_to_reg0(sti_ptr, sti_tag);
- }
- FPU_stack_underflow_i(i);
- return;
- }
- if (sti_tag == TAG_Empty) {
- if (control_word & CW_Invalid) {
- /* Masked response */
- FPU_copy_to_regi(st0_ptr, st0_tag, i);
- }
- FPU_stack_underflow();
- return;
- }
- clear_C1();
-
- reg_copy(st0_ptr, &t);
- reg_copy(sti_ptr, st0_ptr);
- reg_copy(&t, sti_ptr);
-
- tag_word &= ~(3 << (regnr * 2)) & ~(3 << (regnri * 2));
- tag_word |= (sti_tag << (regnr * 2)) | (st0_tag << (regnri * 2));
- fpu_tag_word = tag_word;
-}
-
-static void fcmovCC(void)
-{
- /* fcmovCC st(i) */
- int i = FPU_rm;
- FPU_REG *st0_ptr = &st(0);
- FPU_REG *sti_ptr = &st(i);
- long tag_word = fpu_tag_word;
- int regnr = top & 7;
- int regnri = (top + i) & 7;
- u_char sti_tag = (tag_word >> (regnri * 2)) & 3;
-
- if (sti_tag == TAG_Empty) {
- FPU_stack_underflow();
- clear_C1();
- return;
- }
- reg_copy(sti_ptr, st0_ptr);
- tag_word &= ~(3 << (regnr * 2));
- tag_word |= (sti_tag << (regnr * 2));
- fpu_tag_word = tag_word;
-}
-
-void fcmovb(void)
-{
- if (FPU_EFLAGS & X86_EFLAGS_CF)
- fcmovCC();
-}
-
-void fcmove(void)
-{
- if (FPU_EFLAGS & X86_EFLAGS_ZF)
- fcmovCC();
-}
-
-void fcmovbe(void)
-{
- if (FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF))
- fcmovCC();
-}
-
-void fcmovu(void)
-{
- if (FPU_EFLAGS & X86_EFLAGS_PF)
- fcmovCC();
-}
-
-void fcmovnb(void)
-{
- if (!(FPU_EFLAGS & X86_EFLAGS_CF))
- fcmovCC();
-}
-
-void fcmovne(void)
-{
- if (!(FPU_EFLAGS & X86_EFLAGS_ZF))
- fcmovCC();
-}
-
-void fcmovnbe(void)
-{
- if (!(FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF)))
- fcmovCC();
-}
-
-void fcmovnu(void)
-{
- if (!(FPU_EFLAGS & X86_EFLAGS_PF))
- fcmovCC();
-}
-
-void ffree_(void)
-{
- /* ffree st(i) */
- FPU_settagi(FPU_rm, TAG_Empty);
-}
-
-void ffreep(void)
-{
- /* ffree st(i) + pop - unofficial code */
- FPU_settagi(FPU_rm, TAG_Empty);
- FPU_pop();
-}
-
-void fst_i_(void)
-{
- /* fst st(i) */
- FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm);
-}
-
-void fstp_i(void)
-{
- /* fstp st(i) */
- FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm);
- FPU_pop();
-}
diff --git a/arch/x86/math-emu/fpu_emu.h b/arch/x86/math-emu/fpu_emu.h
deleted file mode 100644
index def569c..0000000
--- a/arch/x86/math-emu/fpu_emu.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | fpu_emu.h |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_EMU_H_
-#define _FPU_EMU_H_
-
-/*
- * Define PECULIAR_486 to get a closer approximation to 80486 behaviour,
- * rather than behaviour which appears to be cleaner.
- * This is a matter of opinion: for all I know, the 80486 may simply
- * be complying with the IEEE spec. Maybe one day I'll get to see the
- * spec...
- */
-#define PECULIAR_486
-
-#ifdef __ASSEMBLER__
-#include "fpu_asm.h"
-#define Const(x) $##x
-#else
-#define Const(x) x
-#endif
-
-#define EXP_BIAS Const(0)
-#define EXP_OVER Const(0x4000) /* smallest invalid large exponent */
-#define EXP_UNDER Const(-0x3fff) /* largest invalid small exponent */
-#define EXP_WAY_UNDER Const(-0x6000) /* Below the smallest denormal, but
- still a 16 bit nr. */
-#define EXP_Infinity EXP_OVER
-#define EXP_NaN EXP_OVER
-
-#define EXTENDED_Ebias Const(0x3fff)
-#define EXTENDED_Emin (-0x3ffe) /* smallest valid exponent */
-
-#define SIGN_POS Const(0)
-#define SIGN_NEG Const(0x80)
-
-#define SIGN_Positive Const(0)
-#define SIGN_Negative Const(0x8000)
-
-/* Keep the order TAG_Valid, TAG_Zero, TW_Denormal */
-/* The following fold to 2 (Special) in the Tag Word */
-#define TW_Denormal Const(4) /* De-normal */
-#define TW_Infinity Const(5) /* + or - infinity */
-#define TW_NaN Const(6) /* Not a Number */
-#define TW_Unsupported Const(7) /* Not supported by an 80486 */
-
-#define TAG_Valid Const(0) /* valid */
-#define TAG_Zero Const(1) /* zero */
-#define TAG_Special Const(2) /* De-normal, + or - infinity,
- or Not a Number */
-#define TAG_Empty Const(3) /* empty */
-#define TAG_Error Const(0x80) /* probably need to abort */
-
-#define LOADED_DATA Const(10101) /* Special st() number to identify
- loaded data (not on stack). */
-
-/* A few flags (must be >= 0x10). */
-#define REV 0x10
-#define DEST_RM 0x20
-#define LOADED 0x40
-
-#define FPU_Exception Const(0x80000000) /* Added to tag returns. */
-
-#ifndef __ASSEMBLER__
-
-#include "fpu_system.h"
-
-#include <uapi/asm/sigcontext.h> /* for struct _fpstate */
-#include <asm/math_emu.h>
-#include <linux/linkage.h>
-
-/*
-#define RE_ENTRANT_CHECKING
- */
-
-#ifdef RE_ENTRANT_CHECKING
-extern u_char emulating;
-# define RE_ENTRANT_CHECK_OFF emulating = 0
-# define RE_ENTRANT_CHECK_ON emulating = 1
-#else
-# define RE_ENTRANT_CHECK_OFF
-# define RE_ENTRANT_CHECK_ON
-#endif /* RE_ENTRANT_CHECKING */
-
-#define FWAIT_OPCODE 0x9b
-#define OP_SIZE_PREFIX 0x66
-#define ADDR_SIZE_PREFIX 0x67
-#define PREFIX_CS 0x2e
-#define PREFIX_DS 0x3e
-#define PREFIX_ES 0x26
-#define PREFIX_SS 0x36
-#define PREFIX_FS 0x64
-#define PREFIX_GS 0x65
-#define PREFIX_REPE 0xf3
-#define PREFIX_REPNE 0xf2
-#define PREFIX_LOCK 0xf0
-#define PREFIX_CS_ 1
-#define PREFIX_DS_ 2
-#define PREFIX_ES_ 3
-#define PREFIX_FS_ 4
-#define PREFIX_GS_ 5
-#define PREFIX_SS_ 6
-#define PREFIX_DEFAULT 7
-
-struct address {
- unsigned int offset;
- unsigned int selector:16;
- unsigned int opcode:11;
- unsigned int empty:5;
-};
-struct fpu__reg {
- unsigned sigl;
- unsigned sigh;
- short exp;
-};
-
-typedef void (*FUNC) (void);
-typedef struct fpu__reg FPU_REG;
-typedef void (*FUNC_ST0) (FPU_REG *st0_ptr, u_char st0_tag);
-typedef struct {
- u_char address_size, operand_size, segment;
-} overrides;
-/* This structure is 32 bits: */
-typedef struct {
- overrides override;
- u_char default_mode;
-} fpu_addr_modes;
-/* PROTECTED has a restricted meaning in the emulator; it is used
- to signal that the emulator needs to do special things to ensure
- that protection is respected in a segmented model. */
-#define PROTECTED 4
-#define SIXTEEN 1 /* We rely upon this being 1 (true) */
-#define VM86 SIXTEEN
-#define PM16 (SIXTEEN | PROTECTED)
-#define SEG32 PROTECTED
-extern u_char const data_sizes_16[32];
-
-#define register_base ((u_char *) registers )
-#define fpu_register(x) ( * ((FPU_REG *)( register_base + 10 * (x & 7) )) )
-#define st(x) ( * ((FPU_REG *)( register_base + 10 * ((top+x) & 7) )) )
-
-#define STACK_OVERFLOW (FPU_stackoverflow(&st_new_ptr))
-#define NOT_EMPTY(i) (!FPU_empty_i(i))
-
-#define NOT_EMPTY_ST0 (st0_tag ^ TAG_Empty)
-
-#define poppop() { FPU_pop(); FPU_pop(); }
-
-/* push() does not affect the tags */
-#define push() { top--; }
-
-#define signbyte(a) (((u_char *)(a))[9])
-#define getsign(a) (signbyte(a) & 0x80)
-#define setsign(a,b) { if ((b) != 0) signbyte(a) |= 0x80; else signbyte(a) &= 0x7f; }
-#define copysign(a,b) { if (getsign(a)) signbyte(b) |= 0x80; \
- else signbyte(b) &= 0x7f; }
-#define changesign(a) { signbyte(a) ^= 0x80; }
-#define setpositive(a) { signbyte(a) &= 0x7f; }
-#define setnegative(a) { signbyte(a) |= 0x80; }
-#define signpositive(a) ( (signbyte(a) & 0x80) == 0 )
-#define signnegative(a) (signbyte(a) & 0x80)
-
-static inline void reg_copy(FPU_REG const *x, FPU_REG *y)
-{
- *(short *)&(y->exp) = *(const short *)&(x->exp);
- *(long long *)&(y->sigl) = *(const long long *)&(x->sigl);
-}
-
-#define exponent(x) (((*(short *)&((x)->exp)) & 0x7fff) - EXTENDED_Ebias)
-#define setexponentpos(x,y) { (*(short *)&((x)->exp)) = \
- ((y) + EXTENDED_Ebias) & 0x7fff; }
-#define exponent16(x) (*(short *)&((x)->exp))
-#define setexponent16(x,y) { (*(short *)&((x)->exp)) = (u16)(y); }
-#define addexponent(x,y) { (*(short *)&((x)->exp)) += (y); }
-#define stdexp(x) { (*(short *)&((x)->exp)) += EXTENDED_Ebias; }
-
-#define isdenormal(ptr) (exponent(ptr) == EXP_BIAS+EXP_UNDER)
-
-#define significand(x) ( ((unsigned long long *)&((x)->sigl))[0] )
-
-/*----- Prototypes for functions written in assembler -----*/
-/* extern void reg_move(FPU_REG *a, FPU_REG *b); */
-
-asmlinkage int FPU_normalize(FPU_REG *x);
-asmlinkage int FPU_normalize_nuo(FPU_REG *x);
-asmlinkage int FPU_u_sub(FPU_REG const *arg1, FPU_REG const *arg2,
- FPU_REG * answ, unsigned int control_w, u_char sign,
- int expa, int expb);
-asmlinkage int FPU_u_mul(FPU_REG const *arg1, FPU_REG const *arg2,
- FPU_REG * answ, unsigned int control_w, u_char sign,
- int expon);
-asmlinkage int FPU_u_div(FPU_REG const *arg1, FPU_REG const *arg2,
- FPU_REG * answ, unsigned int control_w, u_char sign);
-asmlinkage int FPU_u_add(FPU_REG const *arg1, FPU_REG const *arg2,
- FPU_REG * answ, unsigned int control_w, u_char sign,
- int expa, int expb);
-asmlinkage int wm_sqrt(FPU_REG *n, int dummy1, int dummy2,
- unsigned int control_w, u_char sign);
-asmlinkage unsigned FPU_shrx(void *l, unsigned x);
-asmlinkage unsigned FPU_shrxs(void *v, unsigned x);
-asmlinkage unsigned long FPU_div_small(unsigned long long *x, unsigned long y);
-asmlinkage int FPU_round(FPU_REG *arg, unsigned int extent, int dummy,
- unsigned int control_w, u_char sign);
-
-#ifndef MAKING_PROTO
-#include "fpu_proto.h"
-#endif
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _FPU_EMU_H_ */
diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c
deleted file mode 100644
index 5034df6..0000000
--- a/arch/x86/math-emu/fpu_entry.c
+++ /dev/null
@@ -1,718 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_entry.c |
- | |
- | The entry functions for wm-FPU-emu |
- | |
- | Copyright (C) 1992,1993,1994,1996,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | See the files "README" and "COPYING" for further copyright and warranty |
- | information. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note: |
- | The file contains code which accesses user memory. |
- | Emulator static data may change when user memory is accessed, due to |
- | other processes using the emulator while swapping is in progress. |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | math_emulate(), restore_i387_soft() and save_i387_soft() are the only |
- | entry points for wm-FPU-emu. |
- +---------------------------------------------------------------------------*/
-
-#include <linux/signal.h>
-#include <linux/regset.h>
-
-#include <linux/uaccess.h>
-#include <asm/traps.h>
-#include <asm/user.h>
-#include <asm/fpu/api.h>
-#include <asm/fpu/regset.h>
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "exception.h"
-#include "control_w.h"
-#include "status_w.h"
-
-#define __BAD__ FPU_illegal /* Illegal on an 80486, causes SIGILL */
-
-/* fcmovCC and f(u)comi(p) are enabled if CPUID(1).EDX(15) "cmov" is set */
-
-/* WARNING: "u" entries are not documented by Intel in their 80486 manual
- and may not work on FPU clones or later Intel FPUs.
- Changes to support them provided by Linus Torvalds. */
-
-static FUNC const st_instr_table[64] = {
-/* Opcode: d8 d9 da db */
-/* dc dd de df */
-/* c0..7 */ fadd__, fld_i_, fcmovb, fcmovnb,
-/* c0..7 */ fadd_i, ffree_, faddp_, ffreep,/*u*/
-/* c8..f */ fmul__, fxch_i, fcmove, fcmovne,
-/* c8..f */ fmul_i, fxch_i,/*u*/ fmulp_, fxch_i,/*u*/
-/* d0..7 */ fcom_st, fp_nop, fcmovbe, fcmovnbe,
-/* d0..7 */ fcom_st,/*u*/ fst_i_, fcompst,/*u*/ fstp_i,/*u*/
-/* d8..f */ fcompst, fstp_i,/*u*/ fcmovu, fcmovnu,
-/* d8..f */ fcompst,/*u*/ fstp_i, fcompp, fstp_i,/*u*/
-/* e0..7 */ fsub__, FPU_etc, __BAD__, finit_,
-/* e0..7 */ fsubri, fucom_, fsubrp, fstsw_,
-/* e8..f */ fsubr_, fconst, fucompp, fucomi_,
-/* e8..f */ fsub_i, fucomp, fsubp_, fucomip,
-/* f0..7 */ fdiv__, FPU_triga, __BAD__, fcomi_,
-/* f0..7 */ fdivri, __BAD__, fdivrp, fcomip,
-/* f8..f */ fdivr_, FPU_trigb, __BAD__, __BAD__,
-/* f8..f */ fdiv_i, __BAD__, fdivp_, __BAD__,
-};
-
-#define _NONE_ 0 /* Take no special action */
-#define _REG0_ 1 /* Need to check for not empty st(0) */
-#define _REGI_ 2 /* Need to check for not empty st(0) and st(rm) */
-#define _REGi_ 0 /* Uses st(rm) */
-#define _PUSH_ 3 /* Need to check for space to push onto stack */
-#define _null_ 4 /* Function illegal or not implemented */
-#define _REGIi 5 /* Uses st(0) and st(rm), result to st(rm) */
-#define _REGIp 6 /* Uses st(0) and st(rm), result to st(rm) then pop */
-#define _REGIc 0 /* Compare st(0) and st(rm) */
-#define _REGIn 0 /* Uses st(0) and st(rm), but handle checks later */
-
-static u_char const type_table[64] = {
-/* Opcode: d8 d9 da db dc dd de df */
-/* c0..7 */ _REGI_, _NONE_, _REGIn, _REGIn, _REGIi, _REGi_, _REGIp, _REGi_,
-/* c8..f */ _REGI_, _REGIn, _REGIn, _REGIn, _REGIi, _REGI_, _REGIp, _REGI_,
-/* d0..7 */ _REGIc, _NONE_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_,
-/* d8..f */ _REGIc, _REG0_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_,
-/* e0..7 */ _REGI_, _NONE_, _null_, _NONE_, _REGIi, _REGIc, _REGIp, _NONE_,
-/* e8..f */ _REGI_, _NONE_, _REGIc, _REGIc, _REGIi, _REGIc, _REGIp, _REGIc,
-/* f0..7 */ _REGI_, _NONE_, _null_, _REGIc, _REGIi, _null_, _REGIp, _REGIc,
-/* f8..f */ _REGI_, _NONE_, _null_, _null_, _REGIi, _null_, _REGIp, _null_,
-};
-
-#ifdef RE_ENTRANT_CHECKING
-u_char emulating = 0;
-#endif /* RE_ENTRANT_CHECKING */
-
-static int valid_prefix(u_char *Byte, u_char __user ** fpu_eip,
- overrides * override);
-
-void math_emulate(struct math_emu_info *info)
-{
- u_char FPU_modrm, byte1;
- unsigned short code;
- fpu_addr_modes addr_modes;
- int unmasked;
- FPU_REG loaded_data;
- FPU_REG *st0_ptr;
- u_char loaded_tag, st0_tag;
- void __user *data_address;
- struct address data_sel_off;
- struct address entry_sel_off;
- unsigned long code_base = 0;
- unsigned long code_limit = 0; /* Initialized to stop compiler warnings */
- struct desc_struct code_descriptor;
-
-#ifdef RE_ENTRANT_CHECKING
- if (emulating) {
- printk("ERROR: wm-FPU-emu is not RE-ENTRANT!\n");
- }
- RE_ENTRANT_CHECK_ON;
-#endif /* RE_ENTRANT_CHECKING */
-
- FPU_info = info;
-
- FPU_ORIG_EIP = FPU_EIP;
-
- if ((FPU_EFLAGS & 0x00020000) != 0) {
- /* Virtual 8086 mode */
- addr_modes.default_mode = VM86;
- FPU_EIP += code_base = FPU_CS << 4;
- code_limit = code_base + 0xffff; /* Assumes code_base <= 0xffff0000 */
- } else if (FPU_CS == __USER_CS && FPU_DS == __USER_DS) {
- addr_modes.default_mode = 0;
- } else if (FPU_CS == __KERNEL_CS) {
- printk("math_emulate: %04x:%08lx\n", FPU_CS, FPU_EIP);
- panic("Math emulation needed in kernel");
- } else {
-
- if ((FPU_CS & 4) != 4) { /* Must be in the LDT */
- /* Can only handle segmented addressing via the LDT
- for now, and it must be 16 bit */
- printk("FPU emulator: Unsupported addressing mode\n");
- math_abort(FPU_info, SIGILL);
- }
-
- code_descriptor = FPU_get_ldt_descriptor(FPU_CS);
- if (code_descriptor.d) {
- /* The above test may be wrong, the book is not clear */
- /* Segmented 32 bit protected mode */
- addr_modes.default_mode = SEG32;
- } else {
- /* 16 bit protected mode */
- addr_modes.default_mode = PM16;
- }
- FPU_EIP += code_base = seg_get_base(&code_descriptor);
- code_limit = seg_get_limit(&code_descriptor) + 1;
- code_limit *= seg_get_granularity(&code_descriptor);
- code_limit += code_base - 1;
- if (code_limit < code_base)
- code_limit = 0xffffffff;
- }
-
- FPU_lookahead = !(FPU_EFLAGS & X86_EFLAGS_TF);
-
- if (!valid_prefix(&byte1, (u_char __user **) & FPU_EIP,
- &addr_modes.override)) {
- RE_ENTRANT_CHECK_OFF;
- printk
- ("FPU emulator: Unknown prefix byte 0x%02x, probably due to\n"
- "FPU emulator: self-modifying code! (emulation impossible)\n",
- byte1);
- RE_ENTRANT_CHECK_ON;
- EXCEPTION(EX_INTERNAL | 0x126);
- math_abort(FPU_info, SIGILL);
- }
-
- do_another_FPU_instruction:
-
- no_ip_update = 0;
-
- FPU_EIP++; /* We have fetched the prefix and first code bytes. */
-
- if (addr_modes.default_mode) {
- /* This checks for the minimum instruction bytes.
- We also need to check any extra (address mode) code access. */
- if (FPU_EIP > code_limit)
- math_abort(FPU_info, SIGSEGV);
- }
-
- if ((byte1 & 0xf8) != 0xd8) {
- if (byte1 == FWAIT_OPCODE) {
- if (partial_status & SW_Summary)
- goto do_the_FPU_interrupt;
- else
- goto FPU_fwait_done;
- }
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x128);
- math_abort(FPU_info, SIGILL);
-#endif /* PARANOID */
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(FPU_modrm, (u_char __user *) FPU_EIP);
- RE_ENTRANT_CHECK_ON;
- FPU_EIP++;
-
- if (partial_status & SW_Summary) {
- /* Ignore the error for now if the current instruction is a no-wait
- control instruction */
- /* The 80486 manual contradicts itself on this topic,
- but a real 80486 uses the following instructions:
- fninit, fnstenv, fnsave, fnstsw, fnstenv, fnclex.
- */
- code = (FPU_modrm << 8) | byte1;
- if (!((((code & 0xf803) == 0xe003) || /* fnclex, fninit, fnstsw */
- (((code & 0x3003) == 0x3001) && /* fnsave, fnstcw, fnstenv,
- fnstsw */
- ((code & 0xc000) != 0xc000))))) {
- /*
- * We need to simulate the action of the kernel to FPU
- * interrupts here.
- */
- do_the_FPU_interrupt:
-
- FPU_EIP = FPU_ORIG_EIP; /* Point to current FPU instruction. */
-
- RE_ENTRANT_CHECK_OFF;
- current->thread.trap_nr = X86_TRAP_MF;
- current->thread.error_code = 0;
- send_sig(SIGFPE, current, 1);
- return;
- }
- }
-
- entry_sel_off.offset = FPU_ORIG_EIP;
- entry_sel_off.selector = FPU_CS;
- entry_sel_off.opcode = (byte1 << 8) | FPU_modrm;
- entry_sel_off.empty = 0;
-
- FPU_rm = FPU_modrm & 7;
-
- if (FPU_modrm < 0300) {
- /* All of these instructions use the mod/rm byte to get a data address */
-
- if ((addr_modes.default_mode & SIXTEEN)
- ^ (addr_modes.override.address_size == ADDR_SIZE_PREFIX))
- data_address =
- FPU_get_address_16(FPU_modrm, &FPU_EIP,
- &data_sel_off, addr_modes);
- else
- data_address =
- FPU_get_address(FPU_modrm, &FPU_EIP, &data_sel_off,
- addr_modes);
-
- if (addr_modes.default_mode) {
- if (FPU_EIP - 1 > code_limit)
- math_abort(FPU_info, SIGSEGV);
- }
-
- if (!(byte1 & 1)) {
- unsigned short status1 = partial_status;
-
- st0_ptr = &st(0);
- st0_tag = FPU_gettag0();
-
- /* Stack underflow has priority */
- if (NOT_EMPTY_ST0) {
- if (addr_modes.default_mode & PROTECTED) {
- /* This table works for 16 and 32 bit protected mode */
- if (access_limit <
- data_sizes_16[(byte1 >> 1) & 3])
- math_abort(FPU_info, SIGSEGV);
- }
-
- unmasked = 0; /* Do this here to stop compiler warnings. */
- switch ((byte1 >> 1) & 3) {
- case 0:
- unmasked =
- FPU_load_single((float __user *)
- data_address,
- &loaded_data);
- loaded_tag = unmasked & 0xff;
- unmasked &= ~0xff;
- break;
- case 1:
- loaded_tag =
- FPU_load_int32((long __user *)
- data_address,
- &loaded_data);
- break;
- case 2:
- unmasked =
- FPU_load_double((double __user *)
- data_address,
- &loaded_data);
- loaded_tag = unmasked & 0xff;
- unmasked &= ~0xff;
- break;
- case 3:
- default: /* Used here to suppress gcc warnings. */
- loaded_tag =
- FPU_load_int16((short __user *)
- data_address,
- &loaded_data);
- break;
- }
-
- /* No more access to user memory, it is safe
- to use static data now */
-
- /* NaN operands have the next priority. */
- /* We have to delay looking at st(0) until after
- loading the data, because that data might contain an SNaN */
- if (((st0_tag == TAG_Special) && isNaN(st0_ptr))
- || ((loaded_tag == TAG_Special)
- && isNaN(&loaded_data))) {
- /* Restore the status word; we might have loaded a
- denormal. */
- partial_status = status1;
- if ((FPU_modrm & 0x30) == 0x10) {
- /* fcom or fcomp */
- EXCEPTION(EX_Invalid);
- setcc(SW_C3 | SW_C2 | SW_C0);
- if ((FPU_modrm & 0x08)
- && (control_word &
- CW_Invalid))
- FPU_pop(); /* fcomp, masked, so we pop. */
- } else {
- if (loaded_tag == TAG_Special)
- loaded_tag =
- FPU_Special
- (&loaded_data);
-#ifdef PECULIAR_486
- /* This is not really needed, but gives behaviour
- identical to an 80486 */
- if ((FPU_modrm & 0x28) == 0x20)
- /* fdiv or fsub */
- real_2op_NaN
- (&loaded_data,
- loaded_tag, 0,
- &loaded_data);
- else
-#endif /* PECULIAR_486 */
- /* fadd, fdivr, fmul, or fsubr */
- real_2op_NaN
- (&loaded_data,
- loaded_tag, 0,
- st0_ptr);
- }
- goto reg_mem_instr_done;
- }
-
- if (unmasked && !((FPU_modrm & 0x30) == 0x10)) {
- /* Is not a comparison instruction. */
- if ((FPU_modrm & 0x38) == 0x38) {
- /* fdivr */
- if ((st0_tag == TAG_Zero) &&
- ((loaded_tag == TAG_Valid)
- || (loaded_tag ==
- TAG_Special
- &&
- isdenormal
- (&loaded_data)))) {
- if (FPU_divide_by_zero
- (0,
- getsign
- (&loaded_data))
- < 0) {
- /* We use the fact here that the unmasked
- exception in the loaded data was for a
- denormal operand */
- /* Restore the state of the denormal op bit */
- partial_status
- &=
- ~SW_Denorm_Op;
- partial_status
- |=
- status1 &
- SW_Denorm_Op;
- } else
- setsign(st0_ptr,
- getsign
- (&loaded_data));
- }
- }
- goto reg_mem_instr_done;
- }
-
- switch ((FPU_modrm >> 3) & 7) {
- case 0: /* fadd */
- clear_C1();
- FPU_add(&loaded_data, loaded_tag, 0,
- control_word);
- break;
- case 1: /* fmul */
- clear_C1();
- FPU_mul(&loaded_data, loaded_tag, 0,
- control_word);
- break;
- case 2: /* fcom */
- FPU_compare_st_data(&loaded_data,
- loaded_tag);
- break;
- case 3: /* fcomp */
- if (!FPU_compare_st_data
- (&loaded_data, loaded_tag)
- && !unmasked)
- FPU_pop();
- break;
- case 4: /* fsub */
- clear_C1();
- FPU_sub(LOADED | loaded_tag,
- (int)&loaded_data,
- control_word);
- break;
- case 5: /* fsubr */
- clear_C1();
- FPU_sub(REV | LOADED | loaded_tag,
- (int)&loaded_data,
- control_word);
- break;
- case 6: /* fdiv */
- clear_C1();
- FPU_div(LOADED | loaded_tag,
- (int)&loaded_data,
- control_word);
- break;
- case 7: /* fdivr */
- clear_C1();
- if (st0_tag == TAG_Zero)
- partial_status = status1; /* Undo any denorm tag,
- zero-divide has priority. */
- FPU_div(REV | LOADED | loaded_tag,
- (int)&loaded_data,
- control_word);
- break;
- }
- } else {
- if ((FPU_modrm & 0x30) == 0x10) {
- /* The instruction is fcom or fcomp */
- EXCEPTION(EX_StackUnder);
- setcc(SW_C3 | SW_C2 | SW_C0);
- if ((FPU_modrm & 0x08)
- && (control_word & CW_Invalid))
- FPU_pop(); /* fcomp */
- } else
- FPU_stack_underflow();
- }
- reg_mem_instr_done:
- operand_address = data_sel_off;
- } else {
- if (!(no_ip_update =
- FPU_load_store(((FPU_modrm & 0x38) | (byte1 & 6))
- >> 1, addr_modes, data_address))) {
- operand_address = data_sel_off;
- }
- }
-
- } else {
- /* None of these instructions access user memory */
- u_char instr_index = (FPU_modrm & 0x38) | (byte1 & 7);
-
-#ifdef PECULIAR_486
- /* This is supposed to be undefined, but a real 80486 seems
- to do this: */
- operand_address.offset = 0;
- operand_address.selector = FPU_DS;
-#endif /* PECULIAR_486 */
-
- st0_ptr = &st(0);
- st0_tag = FPU_gettag0();
- switch (type_table[(int)instr_index]) {
- case _NONE_: /* also _REGIc: _REGIn */
- break;
- case _REG0_:
- if (!NOT_EMPTY_ST0) {
- FPU_stack_underflow();
- goto FPU_instruction_done;
- }
- break;
- case _REGIi:
- if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
- FPU_stack_underflow_i(FPU_rm);
- goto FPU_instruction_done;
- }
- break;
- case _REGIp:
- if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
- FPU_stack_underflow_pop(FPU_rm);
- goto FPU_instruction_done;
- }
- break;
- case _REGI_:
- if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
- FPU_stack_underflow();
- goto FPU_instruction_done;
- }
- break;
- case _PUSH_: /* Only used by the fld st(i) instruction */
- break;
- case _null_:
- FPU_illegal();
- goto FPU_instruction_done;
- default:
- EXCEPTION(EX_INTERNAL | 0x111);
- goto FPU_instruction_done;
- }
- (*st_instr_table[(int)instr_index]) ();
-
- FPU_instruction_done:
- ;
- }
-
- if (!no_ip_update)
- instruction_address = entry_sel_off;
-
- FPU_fwait_done:
-
-#ifdef DEBUG
- RE_ENTRANT_CHECK_OFF;
- FPU_printall();
- RE_ENTRANT_CHECK_ON;
-#endif /* DEBUG */
-
- if (FPU_lookahead && !need_resched()) {
- FPU_ORIG_EIP = FPU_EIP - code_base;
- if (valid_prefix(&byte1, (u_char __user **) & FPU_EIP,
- &addr_modes.override))
- goto do_another_FPU_instruction;
- }
-
- if (addr_modes.default_mode)
- FPU_EIP -= code_base;
-
- RE_ENTRANT_CHECK_OFF;
-}
-
-/* Support for prefix bytes is not yet complete. To properly handle
- all prefix bytes, further changes are needed in the emulator code
- which accesses user address space. Access to separate segments is
- important for msdos emulation. */
-static int valid_prefix(u_char *Byte, u_char __user **fpu_eip,
- overrides * override)
-{
- u_char byte;
- u_char __user *ip = *fpu_eip;
-
- *override = (overrides) {
- 0, 0, PREFIX_DEFAULT}; /* defaults */
-
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(byte, ip);
- RE_ENTRANT_CHECK_ON;
-
- while (1) {
- switch (byte) {
- case ADDR_SIZE_PREFIX:
- override->address_size = ADDR_SIZE_PREFIX;
- goto do_next_byte;
-
- case OP_SIZE_PREFIX:
- override->operand_size = OP_SIZE_PREFIX;
- goto do_next_byte;
-
- case PREFIX_CS:
- override->segment = PREFIX_CS_;
- goto do_next_byte;
- case PREFIX_ES:
- override->segment = PREFIX_ES_;
- goto do_next_byte;
- case PREFIX_SS:
- override->segment = PREFIX_SS_;
- goto do_next_byte;
- case PREFIX_FS:
- override->segment = PREFIX_FS_;
- goto do_next_byte;
- case PREFIX_GS:
- override->segment = PREFIX_GS_;
- goto do_next_byte;
- case PREFIX_DS:
- override->segment = PREFIX_DS_;
- goto do_next_byte;
-
-/* lock is not a valid prefix for FPU instructions,
- let the cpu handle it to generate a SIGILL. */
-/* case PREFIX_LOCK: */
-
- /* rep.. prefixes have no meaning for FPU instructions */
- case PREFIX_REPE:
- case PREFIX_REPNE:
-
- do_next_byte:
- ip++;
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(byte, ip);
- RE_ENTRANT_CHECK_ON;
- break;
- case FWAIT_OPCODE:
- *Byte = byte;
- return 1;
- default:
- if ((byte & 0xf8) == 0xd8) {
- *Byte = byte;
- *fpu_eip = ip;
- return 1;
- } else {
- /* Not a valid sequence of prefix bytes followed by
- an FPU instruction. */
- *Byte = byte; /* Needed for error message. */
- return 0;
- }
- }
- }
-}
-
-void math_abort(struct math_emu_info *info, unsigned int signal)
-{
- FPU_EIP = FPU_ORIG_EIP;
- current->thread.trap_nr = X86_TRAP_MF;
- current->thread.error_code = 0;
- send_sig(signal, current, 1);
- RE_ENTRANT_CHECK_OFF;
- __asm__("movl %0,%%esp ; ret": :"g"(((long)info) - 4));
-#ifdef PARANOID
- printk("ERROR: wm-FPU-emu math_abort failed!\n");
-#endif /* PARANOID */
-}
-
-#define S387 ((struct swregs_state *)s387)
-#define sstatus_word() \
- ((S387->swd & ~SW_Top & 0xffff) | ((S387->ftop << SW_Top_Shift) & SW_Top))
-
-int fpregs_soft_set(struct task_struct *target,
- const struct user_regset *regset,
- unsigned int pos, unsigned int count,
- const void *kbuf, const void __user *ubuf)
-{
- struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft;
- void *space = s387->st_space;
- int ret;
- int offset, other, i, tags, regnr, tag, newtop;
-
- RE_ENTRANT_CHECK_OFF;
- ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, s387, 0,
- offsetof(struct swregs_state, st_space));
- RE_ENTRANT_CHECK_ON;
-
- if (ret)
- return ret;
-
- S387->ftop = (S387->swd >> SW_Top_Shift) & 7;
- offset = (S387->ftop & 7) * 10;
- other = 80 - offset;
-
- RE_ENTRANT_CHECK_OFF;
-
- /* Copy all registers in stack order. */
- ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- space + offset, 0, other);
- if (!ret && offset)
- ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- space, 0, offset);
-
- RE_ENTRANT_CHECK_ON;
-
- /* The tags may need to be corrected now. */
- tags = S387->twd;
- newtop = S387->ftop;
- for (i = 0; i < 8; i++) {
- regnr = (i + newtop) & 7;
- if (((tags >> ((regnr & 7) * 2)) & 3) != TAG_Empty) {
- /* The loaded data over-rides all other cases. */
- tag =
- FPU_tagof((FPU_REG *) ((u_char *) S387->st_space +
- 10 * regnr));
- tags &= ~(3 << (regnr * 2));
- tags |= (tag & 3) << (regnr * 2);
- }
- }
- S387->twd = tags;
-
- return ret;
-}
-
-int fpregs_soft_get(struct task_struct *target,
- const struct user_regset *regset,
- struct membuf to)
-{
- struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft;
- const void *space = s387->st_space;
- int offset = (S387->ftop & 7) * 10, other = 80 - offset;
-
- RE_ENTRANT_CHECK_OFF;
-
-#ifdef PECULIAR_486
- S387->cwd &= ~0xe080;
- /* An 80486 sets nearly all of the reserved bits to 1. */
- S387->cwd |= 0xffff0040;
- S387->swd = sstatus_word() | 0xffff0000;
- S387->twd |= 0xffff0000;
- S387->fcs &= ~0xf8000000;
- S387->fos |= 0xffff0000;
-#endif /* PECULIAR_486 */
-
- membuf_write(&to, s387, offsetof(struct swregs_state, st_space));
- membuf_write(&to, space + offset, other);
- membuf_write(&to, space, offset);
-
- RE_ENTRANT_CHECK_ON;
-
- return 0;
-}
diff --git a/arch/x86/math-emu/fpu_etc.c b/arch/x86/math-emu/fpu_etc.c
deleted file mode 100644
index 39423ec..0000000
--- a/arch/x86/math-emu/fpu_etc.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_etc.c |
- | |
- | Implement a few FPU instructions. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "reg_constant.h"
-
-static void fchs(FPU_REG *st0_ptr, u_char st0tag)
-{
- if (st0tag ^ TAG_Empty) {
- signbyte(st0_ptr) ^= SIGN_NEG;
- clear_C1();
- } else
- FPU_stack_underflow();
-}
-
-static void fabs(FPU_REG *st0_ptr, u_char st0tag)
-{
- if (st0tag ^ TAG_Empty) {
- setpositive(st0_ptr);
- clear_C1();
- } else
- FPU_stack_underflow();
-}
-
-static void ftst_(FPU_REG *st0_ptr, u_char st0tag)
-{
- switch (st0tag) {
- case TAG_Zero:
- setcc(SW_C3);
- break;
- case TAG_Valid:
- if (getsign(st0_ptr) == SIGN_POS)
- setcc(0);
- else
- setcc(SW_C0);
- break;
- case TAG_Special:
- switch (FPU_Special(st0_ptr)) {
- case TW_Denormal:
- if (getsign(st0_ptr) == SIGN_POS)
- setcc(0);
- else
- setcc(SW_C0);
- if (denormal_operand() < 0) {
-#ifdef PECULIAR_486
- /* This is weird! */
- if (getsign(st0_ptr) == SIGN_POS)
- setcc(SW_C3);
-#endif /* PECULIAR_486 */
- return;
- }
- break;
- case TW_NaN:
- setcc(SW_C0 | SW_C2 | SW_C3); /* Operand is not comparable */
- EXCEPTION(EX_Invalid);
- break;
- case TW_Infinity:
- if (getsign(st0_ptr) == SIGN_POS)
- setcc(0);
- else
- setcc(SW_C0);
- break;
- default:
- setcc(SW_C0 | SW_C2 | SW_C3); /* Operand is not comparable */
- EXCEPTION(EX_INTERNAL | 0x14);
- break;
- }
- break;
- case TAG_Empty:
- setcc(SW_C0 | SW_C2 | SW_C3);
- EXCEPTION(EX_StackUnder);
- break;
- }
-}
-
-static void fxam(FPU_REG *st0_ptr, u_char st0tag)
-{
- int c = 0;
- switch (st0tag) {
- case TAG_Empty:
- c = SW_C3 | SW_C0;
- break;
- case TAG_Zero:
- c = SW_C3;
- break;
- case TAG_Valid:
- c = SW_C2;
- break;
- case TAG_Special:
- switch (FPU_Special(st0_ptr)) {
- case TW_Denormal:
- c = SW_C2 | SW_C3; /* Denormal */
- break;
- case TW_NaN:
- /* We also use NaN for unsupported types. */
- if ((st0_ptr->sigh & 0x80000000)
- && (exponent(st0_ptr) == EXP_OVER))
- c = SW_C0;
- break;
- case TW_Infinity:
- c = SW_C2 | SW_C0;
- break;
- }
- }
- if (getsign(st0_ptr) == SIGN_NEG)
- c |= SW_C1;
- setcc(c);
-}
-
-static void FPU_ST0_illegal(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_illegal();
-}
-
-static FUNC_ST0 const fp_etc_table[] = {
- fchs, fabs, FPU_ST0_illegal, FPU_ST0_illegal,
- ftst_, fxam, FPU_ST0_illegal, FPU_ST0_illegal,
-};
-
-void FPU_etc(void)
-{
- (fp_etc_table[FPU_rm]) (&st(0), FPU_gettag0());
-}
diff --git a/arch/x86/math-emu/fpu_proto.h b/arch/x86/math-emu/fpu_proto.h
deleted file mode 100644
index 94c4023..0000000
--- a/arch/x86/math-emu/fpu_proto.h
+++ /dev/null
@@ -1,157 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _FPU_PROTO_H
-#define _FPU_PROTO_H
-
-/* errors.c */
-extern void FPU_illegal(void);
-extern void FPU_printall(void);
-asmlinkage void FPU_exception(int n);
-extern int real_1op_NaN(FPU_REG *a);
-extern int real_2op_NaN(FPU_REG const *b, u_char tagb, int deststnr,
- FPU_REG const *defaultNaN);
-asmlinkage int arith_invalid(int deststnr);
-asmlinkage int FPU_divide_by_zero(int deststnr, u_char sign);
-extern int set_precision_flag(int flags);
-asmlinkage void set_precision_flag_up(void);
-asmlinkage void set_precision_flag_down(void);
-asmlinkage int denormal_operand(void);
-asmlinkage int arith_overflow(FPU_REG *dest);
-asmlinkage int arith_underflow(FPU_REG *dest);
-extern void FPU_stack_overflow(void);
-extern void FPU_stack_underflow(void);
-extern void FPU_stack_underflow_i(int i);
-extern void FPU_stack_underflow_pop(int i);
-/* fpu_arith.c */
-extern void fadd__(void);
-extern void fmul__(void);
-extern void fsub__(void);
-extern void fsubr_(void);
-extern void fdiv__(void);
-extern void fdivr_(void);
-extern void fadd_i(void);
-extern void fmul_i(void);
-extern void fsubri(void);
-extern void fsub_i(void);
-extern void fdivri(void);
-extern void fdiv_i(void);
-extern void faddp_(void);
-extern void fmulp_(void);
-extern void fsubrp(void);
-extern void fsubp_(void);
-extern void fdivrp(void);
-extern void fdivp_(void);
-/* fpu_aux.c */
-extern void finit(void);
-extern void finit_(void);
-extern void fstsw_(void);
-extern void fp_nop(void);
-extern void fld_i_(void);
-extern void fxch_i(void);
-extern void fcmovb(void);
-extern void fcmove(void);
-extern void fcmovbe(void);
-extern void fcmovu(void);
-extern void fcmovnb(void);
-extern void fcmovne(void);
-extern void fcmovnbe(void);
-extern void fcmovnu(void);
-extern void ffree_(void);
-extern void ffreep(void);
-extern void fst_i_(void);
-extern void fstp_i(void);
-/* fpu_entry.c */
-extern void math_emulate(struct math_emu_info *info);
-extern void math_abort(struct math_emu_info *info, unsigned int signal);
-/* fpu_etc.c */
-extern void FPU_etc(void);
-/* fpu_tags.c */
-extern int FPU_gettag0(void);
-extern int FPU_gettagi(int stnr);
-extern int FPU_gettag(int regnr);
-extern void FPU_settag0(int tag);
-extern void FPU_settagi(int stnr, int tag);
-extern void FPU_settag(int regnr, int tag);
-extern int FPU_Special(FPU_REG const *ptr);
-extern int isNaN(FPU_REG const *ptr);
-extern void FPU_pop(void);
-extern int FPU_empty_i(int stnr);
-extern int FPU_stackoverflow(FPU_REG ** st_new_ptr);
-extern void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr);
-extern void FPU_copy_to_reg1(FPU_REG const *r, u_char tag);
-extern void FPU_copy_to_reg0(FPU_REG const *r, u_char tag);
-/* fpu_trig.c */
-extern void FPU_triga(void);
-extern void FPU_trigb(void);
-/* get_address.c */
-extern void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip,
- struct address *addr,
- fpu_addr_modes addr_modes);
-extern void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip,
- struct address *addr,
- fpu_addr_modes addr_modes);
-/* load_store.c */
-extern int FPU_load_store(u_char type, fpu_addr_modes addr_modes,
- void __user * data_address);
-/* poly_2xm1.c */
-extern int poly_2xm1(u_char sign, FPU_REG * arg, FPU_REG *result);
-/* poly_atan.c */
-extern void poly_atan(FPU_REG * st0_ptr, u_char st0_tag, FPU_REG *st1_ptr,
- u_char st1_tag);
-/* poly_l2.c */
-extern void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign);
-extern int poly_l2p1(u_char s0, u_char s1, FPU_REG *r0, FPU_REG *r1,
- FPU_REG * d);
-/* poly_sin.c */
-extern void poly_sine(FPU_REG *st0_ptr);
-extern void poly_cos(FPU_REG *st0_ptr);
-/* poly_tan.c */
-extern void poly_tan(FPU_REG *st0_ptr);
-/* reg_add_sub.c */
-extern int FPU_add(FPU_REG const *b, u_char tagb, int destrnr, int control_w);
-extern int FPU_sub(int flags, int rm, int control_w);
-/* reg_compare.c */
-extern int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag);
-extern void fcom_st(void);
-extern void fcompst(void);
-extern void fcompp(void);
-extern void fucom_(void);
-extern void fucomp(void);
-extern void fucompp(void);
-extern void fcomi_(void);
-extern void fcomip(void);
-extern void fucomi_(void);
-extern void fucomip(void);
-/* reg_constant.c */
-extern void fconst(void);
-/* reg_ld_str.c */
-extern int FPU_load_extended(long double __user *s, int stnr);
-extern int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data);
-extern int FPU_load_single(float __user *single, FPU_REG *loaded_data);
-extern int FPU_load_int64(long long __user *_s);
-extern int FPU_load_int32(long __user *_s, FPU_REG *loaded_data);
-extern int FPU_load_int16(short __user *_s, FPU_REG *loaded_data);
-extern int FPU_load_bcd(u_char __user *s);
-extern int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag,
- long double __user * d);
-extern int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag,
- double __user * dfloat);
-extern int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag,
- float __user * single);
-extern int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag,
- long long __user * d);
-extern int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d);
-extern int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d);
-extern int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d);
-extern int FPU_round_to_int(FPU_REG *r, u_char tag);
-extern u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s);
-extern void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address);
-extern u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d);
-extern void fsave(fpu_addr_modes addr_modes, u_char __user *data_address);
-extern int FPU_tagof(FPU_REG *ptr);
-/* reg_mul.c */
-extern int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w);
-
-extern int FPU_div(int flags, int regrm, int control_w);
-/* reg_convert.c */
-extern int FPU_to_exp16(FPU_REG const *a, FPU_REG *x);
-#endif /* _FPU_PROTO_H */
diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h
deleted file mode 100644
index 5e238e9..0000000
--- a/arch/x86/math-emu/fpu_system.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | fpu_system.h |
- | |
- | Copyright (C) 1992,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_SYSTEM_H
-#define _FPU_SYSTEM_H
-
-/* system dependent definitions */
-
-#include <linux/sched.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-
-#include <asm/desc.h>
-#include <asm/mmu_context.h>
-
-static inline struct desc_struct FPU_get_ldt_descriptor(unsigned seg)
-{
- static struct desc_struct zero_desc;
- struct desc_struct ret = zero_desc;
-
-#ifdef CONFIG_MODIFY_LDT_SYSCALL
- seg >>= 3;
- mutex_lock(¤t->mm->context.lock);
- if (current->mm->context.ldt && seg < current->mm->context.ldt->nr_entries)
- ret = current->mm->context.ldt->entries[seg];
- mutex_unlock(¤t->mm->context.lock);
-#endif
- return ret;
-}
-
-#define SEG_TYPE_WRITABLE (1U << 1)
-#define SEG_TYPE_EXPANDS_DOWN (1U << 2)
-#define SEG_TYPE_EXECUTE (1U << 3)
-#define SEG_TYPE_EXPAND_MASK (SEG_TYPE_EXPANDS_DOWN | SEG_TYPE_EXECUTE)
-#define SEG_TYPE_EXECUTE_MASK (SEG_TYPE_WRITABLE | SEG_TYPE_EXECUTE)
-
-static inline unsigned long seg_get_base(struct desc_struct *d)
-{
- unsigned long base = (unsigned long)d->base2 << 24;
-
- return base | ((unsigned long)d->base1 << 16) | d->base0;
-}
-
-static inline unsigned long seg_get_limit(struct desc_struct *d)
-{
- return ((unsigned long)d->limit1 << 16) | d->limit0;
-}
-
-static inline unsigned long seg_get_granularity(struct desc_struct *d)
-{
- return d->g ? 4096 : 1;
-}
-
-static inline bool seg_expands_down(struct desc_struct *d)
-{
- return (d->type & SEG_TYPE_EXPAND_MASK) == SEG_TYPE_EXPANDS_DOWN;
-}
-
-static inline bool seg_execute_only(struct desc_struct *d)
-{
- return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_EXECUTE;
-}
-
-static inline bool seg_writable(struct desc_struct *d)
-{
- return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_WRITABLE;
-}
-
-#define I387 (&x86_task_fpu(current)->fpstate->regs)
-#define FPU_info (I387->soft.info)
-
-#define FPU_CS (*(unsigned short *) &(FPU_info->regs->cs))
-#define FPU_SS (*(unsigned short *) &(FPU_info->regs->ss))
-#define FPU_DS (*(unsigned short *) &(FPU_info->regs->ds))
-#define FPU_EAX (FPU_info->regs->ax)
-#define FPU_EFLAGS (FPU_info->regs->flags)
-#define FPU_EIP (FPU_info->regs->ip)
-#define FPU_ORIG_EIP (FPU_info->___orig_eip)
-
-#define FPU_lookahead (I387->soft.lookahead)
-
-/* nz if ip_offset and cs_selector are not to be set for the current
- instruction. */
-#define no_ip_update (*(u_char *)&(I387->soft.no_update))
-#define FPU_rm (*(u_char *)&(I387->soft.rm))
-
-/* Number of bytes of data which can be legally accessed by the current
- instruction. This only needs to hold a number <= 108, so a byte will do. */
-#define access_limit (*(u_char *)&(I387->soft.alimit))
-
-#define partial_status (I387->soft.swd)
-#define control_word (I387->soft.cwd)
-#define fpu_tag_word (I387->soft.twd)
-#define registers (I387->soft.st_space)
-#define top (I387->soft.ftop)
-
-#define instruction_address (*(struct address *)&I387->soft.fip)
-#define operand_address (*(struct address *)&I387->soft.foo)
-
-#define FPU_access_ok(y,z) if ( !access_ok(y,z) ) \
- math_abort(FPU_info,SIGSEGV)
-#define FPU_abort math_abort(FPU_info, SIGSEGV)
-#define FPU_copy_from_user(to, from, n) \
- do { if (copy_from_user(to, from, n)) FPU_abort; } while (0)
-
-#undef FPU_IGNORE_CODE_SEGV
-#ifdef FPU_IGNORE_CODE_SEGV
-/* access_ok() is very expensive, and causes the emulator to run
- about 20% slower if applied to the code. Anyway, errors due to bad
- code addresses should be much rarer than errors due to bad data
- addresses. */
-#define FPU_code_access_ok(z)
-#else
-/* A simpler test than access_ok() can probably be done for
- FPU_code_access_ok() because the only possible error is to step
- past the upper boundary of a legal code area. */
-#define FPU_code_access_ok(z) FPU_access_ok((void __user *)FPU_EIP,z)
-#endif
-
-#define FPU_get_user(x,y) do { if (get_user((x),(y))) FPU_abort; } while (0)
-#define FPU_put_user(x,y) do { if (put_user((x),(y))) FPU_abort; } while (0)
-
-#endif
diff --git a/arch/x86/math-emu/fpu_tags.c b/arch/x86/math-emu/fpu_tags.c
deleted file mode 100644
index bff95d4..0000000
--- a/arch/x86/math-emu/fpu_tags.c
+++ /dev/null
@@ -1,116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_tags.c |
- | |
- | Set FPU register tags. |
- | |
- | Copyright (C) 1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "exception.h"
-
-void FPU_pop(void)
-{
- fpu_tag_word |= 3 << ((top & 7) * 2);
- top++;
-}
-
-int FPU_gettag0(void)
-{
- return (fpu_tag_word >> ((top & 7) * 2)) & 3;
-}
-
-int FPU_gettagi(int stnr)
-{
- return (fpu_tag_word >> (((top + stnr) & 7) * 2)) & 3;
-}
-
-int FPU_gettag(int regnr)
-{
- return (fpu_tag_word >> ((regnr & 7) * 2)) & 3;
-}
-
-void FPU_settag0(int tag)
-{
- int regnr = top;
- regnr &= 7;
- fpu_tag_word &= ~(3 << (regnr * 2));
- fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-void FPU_settagi(int stnr, int tag)
-{
- int regnr = stnr + top;
- regnr &= 7;
- fpu_tag_word &= ~(3 << (regnr * 2));
- fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-void FPU_settag(int regnr, int tag)
-{
- regnr &= 7;
- fpu_tag_word &= ~(3 << (regnr * 2));
- fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-int FPU_Special(FPU_REG const *ptr)
-{
- int exp = exponent(ptr);
-
- if (exp == EXP_BIAS + EXP_UNDER)
- return TW_Denormal;
- else if (exp != EXP_BIAS + EXP_OVER)
- return TW_NaN;
- else if ((ptr->sigh == 0x80000000) && (ptr->sigl == 0))
- return TW_Infinity;
- return TW_NaN;
-}
-
-int isNaN(FPU_REG const *ptr)
-{
- return ((exponent(ptr) == EXP_BIAS + EXP_OVER)
- && !((ptr->sigh == 0x80000000) && (ptr->sigl == 0)));
-}
-
-int FPU_empty_i(int stnr)
-{
- int regnr = (top + stnr) & 7;
-
- return ((fpu_tag_word >> (regnr * 2)) & 3) == TAG_Empty;
-}
-
-int FPU_stackoverflow(FPU_REG ** st_new_ptr)
-{
- *st_new_ptr = &st(-1);
-
- return ((fpu_tag_word >> (((top - 1) & 7) * 2)) & 3) != TAG_Empty;
-}
-
-void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr)
-{
- reg_copy(r, &st(stnr));
- FPU_settagi(stnr, tag);
-}
-
-void FPU_copy_to_reg1(FPU_REG const *r, u_char tag)
-{
- reg_copy(r, &st(1));
- FPU_settagi(1, tag);
-}
-
-void FPU_copy_to_reg0(FPU_REG const *r, u_char tag)
-{
- int regnr = top;
- regnr &= 7;
-
- reg_copy(r, &st(0));
-
- fpu_tag_word &= ~(3 << (regnr * 2));
- fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
diff --git a/arch/x86/math-emu/fpu_trig.c b/arch/x86/math-emu/fpu_trig.c
deleted file mode 100644
index 85daf98..0000000
--- a/arch/x86/math-emu/fpu_trig.c
+++ /dev/null
@@ -1,1649 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | fpu_trig.c |
- | |
- | Implementation of the FPU "transcendental" functions. |
- | |
- | Copyright (C) 1992,1993,1994,1997,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@melbpc.org.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "reg_constant.h"
-
-static void rem_kernel(unsigned long long st0, unsigned long long *y,
- unsigned long long st1, unsigned long long q, int n);
-
-#define BETTER_THAN_486
-
-#define FCOS 4
-
-/* Used only by fptan, fsin, fcos, and fsincos. */
-/* This routine produces very accurate results, similar to
- using a value of pi with more than 128 bits precision. */
-/* Limited measurements show no results worse than 64 bit precision
- except for the results for arguments close to 2^63, where the
- precision of the result sometimes degrades to about 63.9 bits */
-static int trig_arg(FPU_REG *st0_ptr, int even)
-{
- FPU_REG tmp;
- u_char tmptag;
- unsigned long long q;
- int old_cw = control_word, saved_status = partial_status;
- int tag, st0_tag = TAG_Valid;
-
- if (exponent(st0_ptr) >= 63) {
- partial_status |= SW_C2; /* Reduction incomplete. */
- return -1;
- }
-
- control_word &= ~CW_RC;
- control_word |= RC_CHOP;
-
- setpositive(st0_ptr);
- tag = FPU_u_div(st0_ptr, &CONST_PI2, &tmp, PR_64_BITS | RC_CHOP | 0x3f,
- SIGN_POS);
-
- FPU_round_to_int(&tmp, tag); /* Fortunately, this can't overflow
- to 2^64 */
- q = significand(&tmp);
- if (q) {
- rem_kernel(significand(st0_ptr),
- &significand(&tmp),
- significand(&CONST_PI2),
- q, exponent(st0_ptr) - exponent(&CONST_PI2));
- setexponent16(&tmp, exponent(&CONST_PI2));
- st0_tag = FPU_normalize(&tmp);
- FPU_copy_to_reg0(&tmp, st0_tag);
- }
-
- if ((even && !(q & 1)) || (!even && (q & 1))) {
- st0_tag =
- FPU_sub(REV | LOADED | TAG_Valid, (int)&CONST_PI2,
- FULL_PRECISION);
-
-#ifdef BETTER_THAN_486
- /* So far, the results are exact but based upon a 64 bit
- precision approximation to pi/2. The technique used
- now is equivalent to using an approximation to pi/2 which
- is accurate to about 128 bits. */
- if ((exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64)
- || (q > 1)) {
- /* This code gives the effect of having pi/2 to better than
- 128 bits precision. */
-
- significand(&tmp) = q + 1;
- setexponent16(&tmp, 63);
- FPU_normalize(&tmp);
- tmptag =
- FPU_u_mul(&CONST_PI2extra, &tmp, &tmp,
- FULL_PRECISION, SIGN_POS,
- exponent(&CONST_PI2extra) +
- exponent(&tmp));
- setsign(&tmp, getsign(&CONST_PI2extra));
- st0_tag = FPU_add(&tmp, tmptag, 0, FULL_PRECISION);
- if (signnegative(st0_ptr)) {
- /* CONST_PI2extra is negative, so the result of the addition
- can be negative. This means that the argument is actually
- in a different quadrant. The correction is always < pi/2,
- so it can't overflow into yet another quadrant. */
- setpositive(st0_ptr);
- q++;
- }
- }
-#endif /* BETTER_THAN_486 */
- }
-#ifdef BETTER_THAN_486
- else {
- /* So far, the results are exact but based upon a 64 bit
- precision approximation to pi/2. The technique used
- now is equivalent to using an approximation to pi/2 which
- is accurate to about 128 bits. */
- if (((q > 0)
- && (exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64))
- || (q > 1)) {
- /* This code gives the effect of having p/2 to better than
- 128 bits precision. */
-
- significand(&tmp) = q;
- setexponent16(&tmp, 63);
- FPU_normalize(&tmp); /* This must return TAG_Valid */
- tmptag =
- FPU_u_mul(&CONST_PI2extra, &tmp, &tmp,
- FULL_PRECISION, SIGN_POS,
- exponent(&CONST_PI2extra) +
- exponent(&tmp));
- setsign(&tmp, getsign(&CONST_PI2extra));
- st0_tag = FPU_sub(LOADED | (tmptag & 0x0f), (int)&tmp,
- FULL_PRECISION);
- if ((exponent(st0_ptr) == exponent(&CONST_PI2)) &&
- ((st0_ptr->sigh > CONST_PI2.sigh)
- || ((st0_ptr->sigh == CONST_PI2.sigh)
- && (st0_ptr->sigl > CONST_PI2.sigl)))) {
- /* CONST_PI2extra is negative, so the result of the
- subtraction can be larger than pi/2. This means
- that the argument is actually in a different quadrant.
- The correction is always < pi/2, so it can't overflow
- into yet another quadrant. */
- st0_tag =
- FPU_sub(REV | LOADED | TAG_Valid,
- (int)&CONST_PI2, FULL_PRECISION);
- q++;
- }
- }
- }
-#endif /* BETTER_THAN_486 */
-
- FPU_settag0(st0_tag);
- control_word = old_cw;
- partial_status = saved_status & ~SW_C2; /* Reduction complete. */
-
- return (q & 3) | even;
-}
-
-/* Convert a long to register */
-static void convert_l2reg(long const *arg, int deststnr)
-{
- int tag;
- long num = *arg;
- u_char sign;
- FPU_REG *dest = &st(deststnr);
-
- if (num == 0) {
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
- return;
- }
-
- if (num > 0) {
- sign = SIGN_POS;
- } else {
- num = -num;
- sign = SIGN_NEG;
- }
-
- dest->sigh = num;
- dest->sigl = 0;
- setexponent16(dest, 31);
- tag = FPU_normalize(dest);
- FPU_settagi(deststnr, tag);
- setsign(dest, sign);
- return;
-}
-
-static void single_arg_error(FPU_REG *st0_ptr, u_char st0_tag)
-{
- if (st0_tag == TAG_Empty)
- FPU_stack_underflow(); /* Puts a QNaN in st(0) */
- else if (st0_tag == TW_NaN)
- real_1op_NaN(st0_ptr); /* return with a NaN in st(0) */
-#ifdef PARANOID
- else
- EXCEPTION(EX_INTERNAL | 0x0112);
-#endif /* PARANOID */
-}
-
-static void single_arg_2_error(FPU_REG *st0_ptr, u_char st0_tag)
-{
- int isNaN;
-
- switch (st0_tag) {
- case TW_NaN:
- isNaN = (exponent(st0_ptr) == EXP_OVER)
- && (st0_ptr->sigh & 0x80000000);
- if (isNaN && !(st0_ptr->sigh & 0x40000000)) { /* Signaling ? */
- EXCEPTION(EX_Invalid);
- if (control_word & CW_Invalid) {
- /* The masked response */
- /* Convert to a QNaN */
- st0_ptr->sigh |= 0x40000000;
- push();
- FPU_copy_to_reg0(st0_ptr, TAG_Special);
- }
- } else if (isNaN) {
- /* A QNaN */
- push();
- FPU_copy_to_reg0(st0_ptr, TAG_Special);
- } else {
- /* pseudoNaN or other unsupported */
- EXCEPTION(EX_Invalid);
- if (control_word & CW_Invalid) {
- /* The masked response */
- FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
- push();
- FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
- }
- }
- break; /* return with a NaN in st(0) */
-#ifdef PARANOID
- default:
- EXCEPTION(EX_INTERNAL | 0x0112);
-#endif /* PARANOID */
- }
-}
-
-/*---------------------------------------------------------------------------*/
-
-static void f2xm1(FPU_REG *st0_ptr, u_char tag)
-{
- FPU_REG a;
-
- clear_C1();
-
- if (tag == TAG_Valid) {
- /* For an 80486 FPU, the result is undefined if the arg is >= 1.0 */
- if (exponent(st0_ptr) < 0) {
- denormal_arg:
-
- FPU_to_exp16(st0_ptr, &a);
-
- /* poly_2xm1(x) requires 0 < st(0) < 1. */
- poly_2xm1(getsign(st0_ptr), &a, st0_ptr);
- }
- set_precision_flag_up(); /* 80486 appears to always do this */
- return;
- }
-
- if (tag == TAG_Zero)
- return;
-
- if (tag == TAG_Special)
- tag = FPU_Special(st0_ptr);
-
- switch (tag) {
- case TW_Denormal:
- if (denormal_operand() < 0)
- return;
- goto denormal_arg;
- case TW_Infinity:
- if (signnegative(st0_ptr)) {
- /* -infinity gives -1 (p16-10) */
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
- setnegative(st0_ptr);
- }
- return;
- default:
- single_arg_error(st0_ptr, tag);
- }
-}
-
-static void fptan(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st_new_ptr;
- int q;
- u_char arg_sign = getsign(st0_ptr);
-
- /* Stack underflow has higher priority */
- if (st0_tag == TAG_Empty) {
- FPU_stack_underflow(); /* Puts a QNaN in st(0) */
- if (control_word & CW_Invalid) {
- st_new_ptr = &st(-1);
- push();
- FPU_stack_underflow(); /* Puts a QNaN in the new st(0) */
- }
- return;
- }
-
- if (STACK_OVERFLOW) {
- FPU_stack_overflow();
- return;
- }
-
- if (st0_tag == TAG_Valid) {
- if (exponent(st0_ptr) > -40) {
- if ((q = trig_arg(st0_ptr, 0)) == -1) {
- /* Operand is out of range */
- return;
- }
-
- poly_tan(st0_ptr);
- setsign(st0_ptr, (q & 1) ^ (arg_sign != 0));
- set_precision_flag_up(); /* We do not really know if up or down */
- } else {
- /* For a small arg, the result == the argument */
- /* Underflow may happen */
-
- denormal_arg:
-
- FPU_to_exp16(st0_ptr, st0_ptr);
-
- st0_tag =
- FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign);
- FPU_settag0(st0_tag);
- }
- push();
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
- return;
- }
-
- if (st0_tag == TAG_Zero) {
- push();
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
- setcc(0);
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
-
- if (st0_tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return;
-
- goto denormal_arg;
- }
-
- if (st0_tag == TW_Infinity) {
- /* The 80486 treats infinity as an invalid operand */
- if (arith_invalid(0) >= 0) {
- st_new_ptr = &st(-1);
- push();
- arith_invalid(0);
- }
- return;
- }
-
- single_arg_2_error(st0_ptr, st0_tag);
-}
-
-static void fxtract(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st_new_ptr;
- u_char sign;
- register FPU_REG *st1_ptr = st0_ptr; /* anticipate */
-
- if (STACK_OVERFLOW) {
- FPU_stack_overflow();
- return;
- }
-
- clear_C1();
-
- if (st0_tag == TAG_Valid) {
- long e;
-
- push();
- sign = getsign(st1_ptr);
- reg_copy(st1_ptr, st_new_ptr);
- setexponent16(st_new_ptr, exponent(st_new_ptr));
-
- denormal_arg:
-
- e = exponent16(st_new_ptr);
- convert_l2reg(&e, 1);
- setexponentpos(st_new_ptr, 0);
- setsign(st_new_ptr, sign);
- FPU_settag0(TAG_Valid); /* Needed if arg was a denormal */
- return;
- } else if (st0_tag == TAG_Zero) {
- sign = getsign(st0_ptr);
-
- if (FPU_divide_by_zero(0, SIGN_NEG) < 0)
- return;
-
- push();
- FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
- setsign(st_new_ptr, sign);
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
-
- if (st0_tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return;
-
- push();
- sign = getsign(st1_ptr);
- FPU_to_exp16(st1_ptr, st_new_ptr);
- goto denormal_arg;
- } else if (st0_tag == TW_Infinity) {
- sign = getsign(st0_ptr);
- setpositive(st0_ptr);
- push();
- FPU_copy_to_reg0(&CONST_INF, TAG_Special);
- setsign(st_new_ptr, sign);
- return;
- } else if (st0_tag == TW_NaN) {
- if (real_1op_NaN(st0_ptr) < 0)
- return;
-
- push();
- FPU_copy_to_reg0(st0_ptr, TAG_Special);
- return;
- } else if (st0_tag == TAG_Empty) {
- /* Is this the correct behaviour? */
- if (control_word & EX_Invalid) {
- FPU_stack_underflow();
- push();
- FPU_stack_underflow();
- } else
- EXCEPTION(EX_StackUnder);
- }
-#ifdef PARANOID
- else
- EXCEPTION(EX_INTERNAL | 0x119);
-#endif /* PARANOID */
-}
-
-static void fdecstp(FPU_REG *st0_ptr, u_char st0_tag)
-{
- clear_C1();
- top--;
-}
-
-static void fincstp(FPU_REG *st0_ptr, u_char st0_tag)
-{
- clear_C1();
- top++;
-}
-
-static void fsqrt_(FPU_REG *st0_ptr, u_char st0_tag)
-{
- int expon;
-
- clear_C1();
-
- if (st0_tag == TAG_Valid) {
- u_char tag;
-
- if (signnegative(st0_ptr)) {
- arith_invalid(0); /* sqrt(negative) is invalid */
- return;
- }
-
- /* make st(0) in [1.0 .. 4.0) */
- expon = exponent(st0_ptr);
-
- denormal_arg:
-
- setexponent16(st0_ptr, (expon & 1));
-
- /* Do the computation, the sign of the result will be positive. */
- tag = wm_sqrt(st0_ptr, 0, 0, control_word, SIGN_POS);
- addexponent(st0_ptr, expon >> 1);
- FPU_settag0(tag);
- return;
- }
-
- if (st0_tag == TAG_Zero)
- return;
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
-
- if (st0_tag == TW_Infinity) {
- if (signnegative(st0_ptr))
- arith_invalid(0); /* sqrt(-Infinity) is invalid */
- return;
- } else if (st0_tag == TW_Denormal) {
- if (signnegative(st0_ptr)) {
- arith_invalid(0); /* sqrt(negative) is invalid */
- return;
- }
-
- if (denormal_operand() < 0)
- return;
-
- FPU_to_exp16(st0_ptr, st0_ptr);
-
- expon = exponent16(st0_ptr);
-
- goto denormal_arg;
- }
-
- single_arg_error(st0_ptr, st0_tag);
-
-}
-
-static void frndint_(FPU_REG *st0_ptr, u_char st0_tag)
-{
- int flags, tag;
-
- if (st0_tag == TAG_Valid) {
- u_char sign;
-
- denormal_arg:
-
- sign = getsign(st0_ptr);
-
- if (exponent(st0_ptr) > 63)
- return;
-
- if (st0_tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return;
- }
-
- /* Fortunately, this can't overflow to 2^64 */
- if ((flags = FPU_round_to_int(st0_ptr, st0_tag)))
- set_precision_flag(flags);
-
- setexponent16(st0_ptr, 63);
- tag = FPU_normalize(st0_ptr);
- setsign(st0_ptr, sign);
- FPU_settag0(tag);
- return;
- }
-
- if (st0_tag == TAG_Zero)
- return;
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
-
- if (st0_tag == TW_Denormal)
- goto denormal_arg;
- else if (st0_tag == TW_Infinity)
- return;
- else
- single_arg_error(st0_ptr, st0_tag);
-}
-
-static int f_sin(FPU_REG *st0_ptr, u_char tag)
-{
- u_char arg_sign = getsign(st0_ptr);
-
- if (tag == TAG_Valid) {
- int q;
-
- if (exponent(st0_ptr) > -40) {
- if ((q = trig_arg(st0_ptr, 0)) == -1) {
- /* Operand is out of range */
- return 1;
- }
-
- poly_sine(st0_ptr);
-
- if (q & 2)
- changesign(st0_ptr);
-
- setsign(st0_ptr, getsign(st0_ptr) ^ arg_sign);
-
- /* We do not really know if up or down */
- set_precision_flag_up();
- return 0;
- } else {
- /* For a small arg, the result == the argument */
- set_precision_flag_up(); /* Must be up. */
- return 0;
- }
- }
-
- if (tag == TAG_Zero) {
- setcc(0);
- return 0;
- }
-
- if (tag == TAG_Special)
- tag = FPU_Special(st0_ptr);
-
- if (tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return 1;
-
- /* For a small arg, the result == the argument */
- /* Underflow may happen */
- FPU_to_exp16(st0_ptr, st0_ptr);
-
- tag = FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign);
-
- FPU_settag0(tag);
-
- return 0;
- } else if (tag == TW_Infinity) {
- /* The 80486 treats infinity as an invalid operand */
- arith_invalid(0);
- return 1;
- } else {
- single_arg_error(st0_ptr, tag);
- return 1;
- }
-}
-
-static void fsin(FPU_REG *st0_ptr, u_char tag)
-{
- f_sin(st0_ptr, tag);
-}
-
-static int f_cos(FPU_REG *st0_ptr, u_char tag)
-{
- u_char st0_sign;
-
- st0_sign = getsign(st0_ptr);
-
- if (tag == TAG_Valid) {
- int q;
-
- if (exponent(st0_ptr) > -40) {
- if ((exponent(st0_ptr) < 0)
- || ((exponent(st0_ptr) == 0)
- && (significand(st0_ptr) <=
- 0xc90fdaa22168c234LL))) {
- poly_cos(st0_ptr);
-
- /* We do not really know if up or down */
- set_precision_flag_down();
-
- return 0;
- } else if ((q = trig_arg(st0_ptr, FCOS)) != -1) {
- poly_sine(st0_ptr);
-
- if ((q + 1) & 2)
- changesign(st0_ptr);
-
- /* We do not really know if up or down */
- set_precision_flag_down();
-
- return 0;
- } else {
- /* Operand is out of range */
- return 1;
- }
- } else {
- denormal_arg:
-
- setcc(0);
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-#ifdef PECULIAR_486
- set_precision_flag_down(); /* 80486 appears to do this. */
-#else
- set_precision_flag_up(); /* Must be up. */
-#endif /* PECULIAR_486 */
- return 0;
- }
- } else if (tag == TAG_Zero) {
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
- setcc(0);
- return 0;
- }
-
- if (tag == TAG_Special)
- tag = FPU_Special(st0_ptr);
-
- if (tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return 1;
-
- goto denormal_arg;
- } else if (tag == TW_Infinity) {
- /* The 80486 treats infinity as an invalid operand */
- arith_invalid(0);
- return 1;
- } else {
- single_arg_error(st0_ptr, tag); /* requires st0_ptr == &st(0) */
- return 1;
- }
-}
-
-static void fcos(FPU_REG *st0_ptr, u_char st0_tag)
-{
- f_cos(st0_ptr, st0_tag);
-}
-
-static void fsincos(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st_new_ptr;
- FPU_REG arg;
- u_char tag;
-
- /* Stack underflow has higher priority */
- if (st0_tag == TAG_Empty) {
- FPU_stack_underflow(); /* Puts a QNaN in st(0) */
- if (control_word & CW_Invalid) {
- st_new_ptr = &st(-1);
- push();
- FPU_stack_underflow(); /* Puts a QNaN in the new st(0) */
- }
- return;
- }
-
- if (STACK_OVERFLOW) {
- FPU_stack_overflow();
- return;
- }
-
- if (st0_tag == TAG_Special)
- tag = FPU_Special(st0_ptr);
- else
- tag = st0_tag;
-
- if (tag == TW_NaN) {
- single_arg_2_error(st0_ptr, TW_NaN);
- return;
- } else if (tag == TW_Infinity) {
- /* The 80486 treats infinity as an invalid operand */
- if (arith_invalid(0) >= 0) {
- /* Masked response */
- push();
- arith_invalid(0);
- }
- return;
- }
-
- reg_copy(st0_ptr, &arg);
- if (!f_sin(st0_ptr, st0_tag)) {
- push();
- FPU_copy_to_reg0(&arg, st0_tag);
- f_cos(&st(0), st0_tag);
- } else {
- /* An error, so restore st(0) */
- FPU_copy_to_reg0(&arg, st0_tag);
- }
-}
-
-/*---------------------------------------------------------------------------*/
-/* The following all require two arguments: st(0) and st(1) */
-
-/* A lean, mean kernel for the fprem instructions. This relies upon
- the division and rounding to an integer in do_fprem giving an
- exact result. Because of this, rem_kernel() needs to deal only with
- the least significant 64 bits, the more significant bits of the
- result must be zero.
- */
-static void rem_kernel(unsigned long long st0, unsigned long long *y,
- unsigned long long st1, unsigned long long q, int n)
-{
- int dummy;
- unsigned long long x;
-
- x = st0 << n;
-
- /* Do the required multiplication and subtraction in the one operation */
-
- /* lsw x -= lsw st1 * lsw q */
- asm volatile ("mull %4; subl %%eax,%0; sbbl %%edx,%1":"=m"
- (((unsigned *)&x)[0]), "=m"(((unsigned *)&x)[1]),
- "=a"(dummy)
- :"2"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[0])
- :"%dx");
- /* msw x -= msw st1 * lsw q */
- asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]),
- "=a"(dummy)
- :"1"(((unsigned *)&st1)[1]), "m"(((unsigned *)&q)[0])
- :"%dx");
- /* msw x -= lsw st1 * msw q */
- asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]),
- "=a"(dummy)
- :"1"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[1])
- :"%dx");
-
- *y = x;
-}
-
-/* Remainder of st(0) / st(1) */
-/* This routine produces exact results, i.e. there is never any
- rounding or truncation, etc of the result. */
-static void do_fprem(FPU_REG *st0_ptr, u_char st0_tag, int round)
-{
- FPU_REG *st1_ptr = &st(1);
- u_char st1_tag = FPU_gettagi(1);
-
- if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
- FPU_REG tmp, st0, st1;
- u_char st0_sign, st1_sign;
- u_char tmptag;
- int tag;
- int old_cw;
- int expdif;
- long long q;
- unsigned short saved_status;
- int cc;
-
- fprem_valid:
- /* Convert registers for internal use. */
- st0_sign = FPU_to_exp16(st0_ptr, &st0);
- st1_sign = FPU_to_exp16(st1_ptr, &st1);
- expdif = exponent16(&st0) - exponent16(&st1);
-
- old_cw = control_word;
- cc = 0;
-
- /* We want the status following the denorm tests, but don't want
- the status changed by the arithmetic operations. */
- saved_status = partial_status;
- control_word &= ~CW_RC;
- control_word |= RC_CHOP;
-
- if (expdif < 64) {
- /* This should be the most common case */
-
- if (expdif > -2) {
- u_char sign = st0_sign ^ st1_sign;
- tag = FPU_u_div(&st0, &st1, &tmp,
- PR_64_BITS | RC_CHOP | 0x3f,
- sign);
- setsign(&tmp, sign);
-
- if (exponent(&tmp) >= 0) {
- FPU_round_to_int(&tmp, tag); /* Fortunately, this can't
- overflow to 2^64 */
- q = significand(&tmp);
-
- rem_kernel(significand(&st0),
- &significand(&tmp),
- significand(&st1),
- q, expdif);
-
- setexponent16(&tmp, exponent16(&st1));
- } else {
- reg_copy(&st0, &tmp);
- q = 0;
- }
-
- if ((round == RC_RND)
- && (tmp.sigh & 0xc0000000)) {
- /* We may need to subtract st(1) once more,
- to get a result <= 1/2 of st(1). */
- unsigned long long x;
- expdif =
- exponent16(&st1) - exponent16(&tmp);
- if (expdif <= 1) {
- if (expdif == 0)
- x = significand(&st1) -
- significand(&tmp);
- else /* expdif is 1 */
- x = (significand(&st1)
- << 1) -
- significand(&tmp);
- if ((x < significand(&tmp)) ||
- /* or equi-distant (from 0 & st(1)) and q is odd */
- ((x == significand(&tmp))
- && (q & 1))) {
- st0_sign = !st0_sign;
- significand(&tmp) = x;
- q++;
- }
- }
- }
-
- if (q & 4)
- cc |= SW_C0;
- if (q & 2)
- cc |= SW_C3;
- if (q & 1)
- cc |= SW_C1;
- } else {
- control_word = old_cw;
- setcc(0);
- return;
- }
- } else {
- /* There is a large exponent difference ( >= 64 ) */
- /* To make much sense, the code in this section should
- be done at high precision. */
- int exp_1, N;
- u_char sign;
-
- /* prevent overflow here */
- /* N is 'a number between 32 and 63' (p26-113) */
- reg_copy(&st0, &tmp);
- tmptag = st0_tag;
- N = (expdif & 0x0000001f) + 32; /* This choice gives results
- identical to an AMD 486 */
- setexponent16(&tmp, N);
- exp_1 = exponent16(&st1);
- setexponent16(&st1, 0);
- expdif -= N;
-
- sign = getsign(&tmp) ^ st1_sign;
- tag =
- FPU_u_div(&tmp, &st1, &tmp,
- PR_64_BITS | RC_CHOP | 0x3f, sign);
- setsign(&tmp, sign);
-
- FPU_round_to_int(&tmp, tag); /* Fortunately, this can't
- overflow to 2^64 */
-
- rem_kernel(significand(&st0),
- &significand(&tmp),
- significand(&st1),
- significand(&tmp), exponent(&tmp)
- );
- setexponent16(&tmp, exp_1 + expdif);
-
- /* It is possible for the operation to be complete here.
- What does the IEEE standard say? The Intel 80486 manual
- implies that the operation will never be completed at this
- point, and the behaviour of a real 80486 confirms this.
- */
- if (!(tmp.sigh | tmp.sigl)) {
- /* The result is zero */
- control_word = old_cw;
- partial_status = saved_status;
- FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
- setsign(&st0, st0_sign);
-#ifdef PECULIAR_486
- setcc(SW_C2);
-#else
- setcc(0);
-#endif /* PECULIAR_486 */
- return;
- }
- cc = SW_C2;
- }
-
- control_word = old_cw;
- partial_status = saved_status;
- tag = FPU_normalize_nuo(&tmp);
- reg_copy(&tmp, st0_ptr);
-
- /* The only condition to be looked for is underflow,
- and it can occur here only if underflow is unmasked. */
- if ((exponent16(&tmp) <= EXP_UNDER) && (tag != TAG_Zero)
- && !(control_word & CW_Underflow)) {
- setcc(cc);
- tag = arith_underflow(st0_ptr);
- setsign(st0_ptr, st0_sign);
- FPU_settag0(tag);
- return;
- } else if ((exponent16(&tmp) > EXP_UNDER) || (tag == TAG_Zero)) {
- stdexp(st0_ptr);
- setsign(st0_ptr, st0_sign);
- } else {
- tag =
- FPU_round(st0_ptr, 0, 0, FULL_PRECISION, st0_sign);
- }
- FPU_settag0(tag);
- setcc(cc);
-
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
- if (st1_tag == TAG_Special)
- st1_tag = FPU_Special(st1_ptr);
-
- if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
- || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
- || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
- if (denormal_operand() < 0)
- return;
- goto fprem_valid;
- } else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
- FPU_stack_underflow();
- return;
- } else if (st0_tag == TAG_Zero) {
- if (st1_tag == TAG_Valid) {
- setcc(0);
- return;
- } else if (st1_tag == TW_Denormal) {
- if (denormal_operand() < 0)
- return;
- setcc(0);
- return;
- } else if (st1_tag == TAG_Zero) {
- arith_invalid(0);
- return;
- } /* fprem(?,0) always invalid */
- else if (st1_tag == TW_Infinity) {
- setcc(0);
- return;
- }
- } else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
- if (st1_tag == TAG_Zero) {
- arith_invalid(0); /* fprem(Valid,Zero) is invalid */
- return;
- } else if (st1_tag != TW_NaN) {
- if (((st0_tag == TW_Denormal)
- || (st1_tag == TW_Denormal))
- && (denormal_operand() < 0))
- return;
-
- if (st1_tag == TW_Infinity) {
- /* fprem(Valid,Infinity) is o.k. */
- setcc(0);
- return;
- }
- }
- } else if (st0_tag == TW_Infinity) {
- if (st1_tag != TW_NaN) {
- arith_invalid(0); /* fprem(Infinity,?) is invalid */
- return;
- }
- }
-
- /* One of the registers must contain a NaN if we got here. */
-
-#ifdef PARANOID
- if ((st0_tag != TW_NaN) && (st1_tag != TW_NaN))
- EXCEPTION(EX_INTERNAL | 0x118);
-#endif /* PARANOID */
-
- real_2op_NaN(st1_ptr, st1_tag, 0, st1_ptr);
-
-}
-
-/* ST(1) <- ST(1) * log ST; pop ST */
-static void fyl2x(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st1_ptr = &st(1), exponent;
- u_char st1_tag = FPU_gettagi(1);
- u_char sign;
- int e, tag;
-
- clear_C1();
-
- if ((st0_tag == TAG_Valid) && (st1_tag == TAG_Valid)) {
- both_valid:
- /* Both regs are Valid or Denormal */
- if (signpositive(st0_ptr)) {
- if (st0_tag == TW_Denormal)
- FPU_to_exp16(st0_ptr, st0_ptr);
- else
- /* Convert st(0) for internal use. */
- setexponent16(st0_ptr, exponent(st0_ptr));
-
- if ((st0_ptr->sigh == 0x80000000)
- && (st0_ptr->sigl == 0)) {
- /* Special case. The result can be precise. */
- u_char esign;
- e = exponent16(st0_ptr);
- if (e >= 0) {
- exponent.sigh = e;
- esign = SIGN_POS;
- } else {
- exponent.sigh = -e;
- esign = SIGN_NEG;
- }
- exponent.sigl = 0;
- setexponent16(&exponent, 31);
- tag = FPU_normalize_nuo(&exponent);
- stdexp(&exponent);
- setsign(&exponent, esign);
- tag =
- FPU_mul(&exponent, tag, 1, FULL_PRECISION);
- if (tag >= 0)
- FPU_settagi(1, tag);
- } else {
- /* The usual case */
- sign = getsign(st1_ptr);
- if (st1_tag == TW_Denormal)
- FPU_to_exp16(st1_ptr, st1_ptr);
- else
- /* Convert st(1) for internal use. */
- setexponent16(st1_ptr,
- exponent(st1_ptr));
- poly_l2(st0_ptr, st1_ptr, sign);
- }
- } else {
- /* negative */
- if (arith_invalid(1) < 0)
- return;
- }
-
- FPU_pop();
-
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
- if (st1_tag == TAG_Special)
- st1_tag = FPU_Special(st1_ptr);
-
- if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
- FPU_stack_underflow_pop(1);
- return;
- } else if ((st0_tag <= TW_Denormal) && (st1_tag <= TW_Denormal)) {
- if (st0_tag == TAG_Zero) {
- if (st1_tag == TAG_Zero) {
- /* Both args zero is invalid */
- if (arith_invalid(1) < 0)
- return;
- } else {
- u_char sign;
- sign = getsign(st1_ptr) ^ SIGN_NEG;
- if (FPU_divide_by_zero(1, sign) < 0)
- return;
-
- setsign(st1_ptr, sign);
- }
- } else if (st1_tag == TAG_Zero) {
- /* st(1) contains zero, st(0) valid <> 0 */
- /* Zero is the valid answer */
- sign = getsign(st1_ptr);
-
- if (signnegative(st0_ptr)) {
- /* log(negative) */
- if (arith_invalid(1) < 0)
- return;
- } else if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
- else {
- if (exponent(st0_ptr) < 0)
- sign ^= SIGN_NEG;
-
- FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
- setsign(st1_ptr, sign);
- }
- } else {
- /* One or both operands are denormals. */
- if (denormal_operand() < 0)
- return;
- goto both_valid;
- }
- } else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) {
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
- return;
- }
- /* One or both arg must be an infinity */
- else if (st0_tag == TW_Infinity) {
- if ((signnegative(st0_ptr)) || (st1_tag == TAG_Zero)) {
- /* log(-infinity) or 0*log(infinity) */
- if (arith_invalid(1) < 0)
- return;
- } else {
- u_char sign = getsign(st1_ptr);
-
- if ((st1_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- FPU_copy_to_reg1(&CONST_INF, TAG_Special);
- setsign(st1_ptr, sign);
- }
- }
- /* st(1) must be infinity here */
- else if (((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal))
- && (signpositive(st0_ptr))) {
- if (exponent(st0_ptr) >= 0) {
- if ((exponent(st0_ptr) == 0) &&
- (st0_ptr->sigh == 0x80000000) &&
- (st0_ptr->sigl == 0)) {
- /* st(0) holds 1.0 */
- /* infinity*log(1) */
- if (arith_invalid(1) < 0)
- return;
- }
- /* else st(0) is positive and > 1.0 */
- } else {
- /* st(0) is positive and < 1.0 */
-
- if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- changesign(st1_ptr);
- }
- } else {
- /* st(0) must be zero or negative */
- if (st0_tag == TAG_Zero) {
- /* This should be invalid, but a real 80486 is happy with it. */
-
-#ifndef PECULIAR_486
- sign = getsign(st1_ptr);
- if (FPU_divide_by_zero(1, sign) < 0)
- return;
-#endif /* PECULIAR_486 */
-
- changesign(st1_ptr);
- } else if (arith_invalid(1) < 0) /* log(negative) */
- return;
- }
-
- FPU_pop();
-}
-
-static void fpatan(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st1_ptr = &st(1);
- u_char st1_tag = FPU_gettagi(1);
- int tag;
-
- clear_C1();
- if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
- valid_atan:
-
- poly_atan(st0_ptr, st0_tag, st1_ptr, st1_tag);
-
- FPU_pop();
-
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
- if (st1_tag == TAG_Special)
- st1_tag = FPU_Special(st1_ptr);
-
- if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
- || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
- || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
- if (denormal_operand() < 0)
- return;
-
- goto valid_atan;
- } else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
- FPU_stack_underflow_pop(1);
- return;
- } else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) {
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) >= 0)
- FPU_pop();
- return;
- } else if ((st0_tag == TW_Infinity) || (st1_tag == TW_Infinity)) {
- u_char sign = getsign(st1_ptr);
- if (st0_tag == TW_Infinity) {
- if (st1_tag == TW_Infinity) {
- if (signpositive(st0_ptr)) {
- FPU_copy_to_reg1(&CONST_PI4, TAG_Valid);
- } else {
- setpositive(st1_ptr);
- tag =
- FPU_u_add(&CONST_PI4, &CONST_PI2,
- st1_ptr, FULL_PRECISION,
- SIGN_POS,
- exponent(&CONST_PI4),
- exponent(&CONST_PI2));
- if (tag >= 0)
- FPU_settagi(1, tag);
- }
- } else {
- if ((st1_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- if (signpositive(st0_ptr)) {
- FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
- setsign(st1_ptr, sign); /* An 80486 preserves the sign */
- FPU_pop();
- return;
- } else {
- FPU_copy_to_reg1(&CONST_PI, TAG_Valid);
- }
- }
- } else {
- /* st(1) is infinity, st(0) not infinity */
- if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- FPU_copy_to_reg1(&CONST_PI2, TAG_Valid);
- }
- setsign(st1_ptr, sign);
- } else if (st1_tag == TAG_Zero) {
- /* st(0) must be valid or zero */
- u_char sign = getsign(st1_ptr);
-
- if ((st0_tag == TW_Denormal) && (denormal_operand() < 0))
- return;
-
- if (signpositive(st0_ptr)) {
- /* An 80486 preserves the sign */
- FPU_pop();
- return;
- }
-
- FPU_copy_to_reg1(&CONST_PI, TAG_Valid);
- setsign(st1_ptr, sign);
- } else if (st0_tag == TAG_Zero) {
- /* st(1) must be TAG_Valid here */
- u_char sign = getsign(st1_ptr);
-
- if ((st1_tag == TW_Denormal) && (denormal_operand() < 0))
- return;
-
- FPU_copy_to_reg1(&CONST_PI2, TAG_Valid);
- setsign(st1_ptr, sign);
- }
-#ifdef PARANOID
- else
- EXCEPTION(EX_INTERNAL | 0x125);
-#endif /* PARANOID */
-
- FPU_pop();
- set_precision_flag_up(); /* We do not really know if up or down */
-}
-
-static void fprem(FPU_REG *st0_ptr, u_char st0_tag)
-{
- do_fprem(st0_ptr, st0_tag, RC_CHOP);
-}
-
-static void fprem1(FPU_REG *st0_ptr, u_char st0_tag)
-{
- do_fprem(st0_ptr, st0_tag, RC_RND);
-}
-
-static void fyl2xp1(FPU_REG *st0_ptr, u_char st0_tag)
-{
- u_char sign, sign1;
- FPU_REG *st1_ptr = &st(1), a, b;
- u_char st1_tag = FPU_gettagi(1);
-
- clear_C1();
- if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
- valid_yl2xp1:
-
- sign = getsign(st0_ptr);
- sign1 = getsign(st1_ptr);
-
- FPU_to_exp16(st0_ptr, &a);
- FPU_to_exp16(st1_ptr, &b);
-
- if (poly_l2p1(sign, sign1, &a, &b, st1_ptr))
- return;
-
- FPU_pop();
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
- if (st1_tag == TAG_Special)
- st1_tag = FPU_Special(st1_ptr);
-
- if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
- || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
- || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
- if (denormal_operand() < 0)
- return;
-
- goto valid_yl2xp1;
- } else if ((st0_tag == TAG_Empty) | (st1_tag == TAG_Empty)) {
- FPU_stack_underflow_pop(1);
- return;
- } else if (st0_tag == TAG_Zero) {
- switch (st1_tag) {
- case TW_Denormal:
- if (denormal_operand() < 0)
- return;
- fallthrough;
- case TAG_Zero:
- case TAG_Valid:
- setsign(st0_ptr, getsign(st0_ptr) ^ getsign(st1_ptr));
- FPU_copy_to_reg1(st0_ptr, st0_tag);
- break;
-
- case TW_Infinity:
- /* Infinity*log(1) */
- if (arith_invalid(1) < 0)
- return;
- break;
-
- case TW_NaN:
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
- return;
- break;
-
- default:
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x116);
- return;
-#endif /* PARANOID */
- break;
- }
- } else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
- switch (st1_tag) {
- case TAG_Zero:
- if (signnegative(st0_ptr)) {
- if (exponent(st0_ptr) >= 0) {
- /* st(0) holds <= -1.0 */
-#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */
- changesign(st1_ptr);
-#else
- if (arith_invalid(1) < 0)
- return;
-#endif /* PECULIAR_486 */
- } else if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
- else
- changesign(st1_ptr);
- } else if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
- break;
-
- case TW_Infinity:
- if (signnegative(st0_ptr)) {
- if ((exponent(st0_ptr) >= 0) &&
- !((st0_ptr->sigh == 0x80000000) &&
- (st0_ptr->sigl == 0))) {
- /* st(0) holds < -1.0 */
-#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */
- changesign(st1_ptr);
-#else
- if (arith_invalid(1) < 0)
- return;
-#endif /* PECULIAR_486 */
- } else if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
- else
- changesign(st1_ptr);
- } else if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
- break;
-
- case TW_NaN:
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
- return;
- }
-
- } else if (st0_tag == TW_NaN) {
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
- return;
- } else if (st0_tag == TW_Infinity) {
- if (st1_tag == TW_NaN) {
- if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
- return;
- } else if (signnegative(st0_ptr)) {
-#ifndef PECULIAR_486
- /* This should have higher priority than denormals, but... */
- if (arith_invalid(1) < 0) /* log(-infinity) */
- return;
-#endif /* PECULIAR_486 */
- if ((st1_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-#ifdef PECULIAR_486
- /* Denormal operands actually get higher priority */
- if (arith_invalid(1) < 0) /* log(-infinity) */
- return;
-#endif /* PECULIAR_486 */
- } else if (st1_tag == TAG_Zero) {
- /* log(infinity) */
- if (arith_invalid(1) < 0)
- return;
- }
-
- /* st(1) must be valid here. */
-
- else if ((st1_tag == TW_Denormal) && (denormal_operand() < 0))
- return;
-
- /* The Manual says that log(Infinity) is invalid, but a real
- 80486 sensibly says that it is o.k. */
- else {
- u_char sign = getsign(st1_ptr);
- FPU_copy_to_reg1(&CONST_INF, TAG_Special);
- setsign(st1_ptr, sign);
- }
- }
-#ifdef PARANOID
- else {
- EXCEPTION(EX_INTERNAL | 0x117);
- return;
- }
-#endif /* PARANOID */
-
- FPU_pop();
- return;
-
-}
-
-static void fscale(FPU_REG *st0_ptr, u_char st0_tag)
-{
- FPU_REG *st1_ptr = &st(1);
- u_char st1_tag = FPU_gettagi(1);
- int old_cw = control_word;
- u_char sign = getsign(st0_ptr);
-
- clear_C1();
- if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
- long scale;
- FPU_REG tmp;
-
- /* Convert register for internal use. */
- setexponent16(st0_ptr, exponent(st0_ptr));
-
- valid_scale:
-
- if (exponent(st1_ptr) > 30) {
- /* 2^31 is far too large, would require 2^(2^30) or 2^(-2^30) */
-
- if (signpositive(st1_ptr)) {
- EXCEPTION(EX_Overflow);
- FPU_copy_to_reg0(&CONST_INF, TAG_Special);
- } else {
- EXCEPTION(EX_Underflow);
- FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
- }
- setsign(st0_ptr, sign);
- return;
- }
-
- control_word &= ~CW_RC;
- control_word |= RC_CHOP;
- reg_copy(st1_ptr, &tmp);
- FPU_round_to_int(&tmp, st1_tag); /* This can never overflow here */
- control_word = old_cw;
- scale = signnegative(st1_ptr) ? -tmp.sigl : tmp.sigl;
- scale += exponent16(st0_ptr);
-
- setexponent16(st0_ptr, scale);
-
- /* Use FPU_round() to properly detect under/overflow etc */
- FPU_round(st0_ptr, 0, 0, control_word, sign);
-
- return;
- }
-
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
- if (st1_tag == TAG_Special)
- st1_tag = FPU_Special(st1_ptr);
-
- if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
- switch (st1_tag) {
- case TAG_Valid:
- /* st(0) must be a denormal */
- if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- FPU_to_exp16(st0_ptr, st0_ptr); /* Will not be left on stack */
- goto valid_scale;
-
- case TAG_Zero:
- if (st0_tag == TW_Denormal)
- denormal_operand();
- return;
-
- case TW_Denormal:
- denormal_operand();
- return;
-
- case TW_Infinity:
- if ((st0_tag == TW_Denormal)
- && (denormal_operand() < 0))
- return;
-
- if (signpositive(st1_ptr))
- FPU_copy_to_reg0(&CONST_INF, TAG_Special);
- else
- FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
- setsign(st0_ptr, sign);
- return;
-
- case TW_NaN:
- real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
- return;
- }
- } else if (st0_tag == TAG_Zero) {
- switch (st1_tag) {
- case TAG_Valid:
- case TAG_Zero:
- return;
-
- case TW_Denormal:
- denormal_operand();
- return;
-
- case TW_Infinity:
- if (signpositive(st1_ptr))
- arith_invalid(0); /* Zero scaled by +Infinity */
- return;
-
- case TW_NaN:
- real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
- return;
- }
- } else if (st0_tag == TW_Infinity) {
- switch (st1_tag) {
- case TAG_Valid:
- case TAG_Zero:
- return;
-
- case TW_Denormal:
- denormal_operand();
- return;
-
- case TW_Infinity:
- if (signnegative(st1_ptr))
- arith_invalid(0); /* Infinity scaled by -Infinity */
- return;
-
- case TW_NaN:
- real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
- return;
- }
- } else if (st0_tag == TW_NaN) {
- if (st1_tag != TAG_Empty) {
- real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
- return;
- }
- }
-#ifdef PARANOID
- if (!((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty))) {
- EXCEPTION(EX_INTERNAL | 0x115);
- return;
- }
-#endif
-
- /* At least one of st(0), st(1) must be empty */
- FPU_stack_underflow();
-
-}
-
-/*---------------------------------------------------------------------------*/
-
-static FUNC_ST0 const trig_table_a[] = {
- f2xm1, fyl2x, fptan, fpatan,
- fxtract, fprem1, fdecstp, fincstp,
-};
-
-void FPU_triga(void)
-{
- (trig_table_a[FPU_rm]) (&st(0), FPU_gettag0());
-}
-
-static FUNC_ST0 const trig_table_b[] = {
- fprem, fyl2xp1, fsqrt_, fsincos, frndint_, fscale, fsin, fcos
-};
-
-void FPU_trigb(void)
-{
- (trig_table_b[FPU_rm]) (&st(0), FPU_gettag0());
-}
diff --git a/arch/x86/math-emu/get_address.c b/arch/x86/math-emu/get_address.c
deleted file mode 100644
index 4a9fd90..0000000
--- a/arch/x86/math-emu/get_address.c
+++ /dev/null
@@ -1,401 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | get_address.c |
- | |
- | Get the effective address from an FPU instruction. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note: |
- | The file contains code which accesses user memory. |
- | Emulator static data may change when user memory is accessed, due to |
- | other processes using the emulator while swapping is in progress. |
- +---------------------------------------------------------------------------*/
-
-#include <linux/stddef.h>
-
-#include <linux/uaccess.h>
-#include <asm/vm86.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-
-#define FPU_WRITE_BIT 0x10
-
-static int reg_offset[] = {
- offsetof(struct pt_regs, ax),
- offsetof(struct pt_regs, cx),
- offsetof(struct pt_regs, dx),
- offsetof(struct pt_regs, bx),
- offsetof(struct pt_regs, sp),
- offsetof(struct pt_regs, bp),
- offsetof(struct pt_regs, si),
- offsetof(struct pt_regs, di)
-};
-
-#define REG_(x) (*(long *)(reg_offset[(x)] + (u_char *)FPU_info->regs))
-
-static int reg_offset_vm86[] = {
- offsetof(struct pt_regs, cs),
- offsetof(struct kernel_vm86_regs, ds),
- offsetof(struct kernel_vm86_regs, es),
- offsetof(struct kernel_vm86_regs, fs),
- offsetof(struct kernel_vm86_regs, gs),
- offsetof(struct pt_regs, ss),
- offsetof(struct kernel_vm86_regs, ds)
-};
-
-#define VM86_REG_(x) (*(unsigned short *) \
- (reg_offset_vm86[((unsigned)x)] + (u_char *)FPU_info->regs))
-
-static int reg_offset_pm[] = {
- offsetof(struct pt_regs, cs),
- offsetof(struct pt_regs, ds),
- offsetof(struct pt_regs, es),
- offsetof(struct pt_regs, fs),
- offsetof(struct pt_regs, ds), /* dummy, not saved on stack */
- offsetof(struct pt_regs, ss),
- offsetof(struct pt_regs, ds)
-};
-
-#define PM_REG_(x) (*(unsigned short *) \
- (reg_offset_pm[((unsigned)x)] + (u_char *)FPU_info->regs))
-
-/* Decode the SIB byte. This function assumes mod != 0 */
-static int sib(int mod, unsigned long *fpu_eip)
-{
- u_char ss, index, base;
- long offset;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(base, (u_char __user *) (*fpu_eip)); /* The SIB byte */
- RE_ENTRANT_CHECK_ON;
- (*fpu_eip)++;
- ss = base >> 6;
- index = (base >> 3) & 7;
- base &= 7;
-
- if ((mod == 0) && (base == 5))
- offset = 0; /* No base register */
- else
- offset = REG_(base);
-
- if (index == 4) {
- /* No index register */
- /* A non-zero ss is illegal */
- if (ss)
- EXCEPTION(EX_Invalid);
- } else {
- offset += (REG_(index)) << ss;
- }
-
- if (mod == 1) {
- /* 8 bit signed displacement */
- long displacement;
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(displacement, (signed char __user *)(*fpu_eip));
- offset += displacement;
- RE_ENTRANT_CHECK_ON;
- (*fpu_eip)++;
- } else if (mod == 2 || base == 5) { /* The second condition also has mod==0 */
- /* 32 bit displacement */
- long displacement;
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(4);
- FPU_get_user(displacement, (long __user *)(*fpu_eip));
- offset += displacement;
- RE_ENTRANT_CHECK_ON;
- (*fpu_eip) += 4;
- }
-
- return offset;
-}
-
-static unsigned long vm86_segment(u_char segment, struct address *addr)
-{
- segment--;
-#ifdef PARANOID
- if (segment > PREFIX_SS_) {
- EXCEPTION(EX_INTERNAL | 0x130);
- math_abort(FPU_info, SIGSEGV);
- }
-#endif /* PARANOID */
- addr->selector = VM86_REG_(segment);
- return (unsigned long)VM86_REG_(segment) << 4;
-}
-
-/* This should work for 16 and 32 bit protected mode. */
-static long pm_address(u_char FPU_modrm, u_char segment,
- struct address *addr, long offset)
-{
- struct desc_struct descriptor;
- unsigned long base_address, limit, address, seg_top;
-
- segment--;
-
-#ifdef PARANOID
- /* segment is unsigned, so this also detects if segment was 0: */
- if (segment > PREFIX_SS_) {
- EXCEPTION(EX_INTERNAL | 0x132);
- math_abort(FPU_info, SIGSEGV);
- }
-#endif /* PARANOID */
-
- switch (segment) {
- case PREFIX_GS_ - 1:
- /* user gs handling can be lazy, use special accessors */
- savesegment(gs, addr->selector);
- break;
- default:
- addr->selector = PM_REG_(segment);
- }
-
- descriptor = FPU_get_ldt_descriptor(addr->selector);
- base_address = seg_get_base(&descriptor);
- address = base_address + offset;
- limit = seg_get_limit(&descriptor) + 1;
- limit *= seg_get_granularity(&descriptor);
- limit += base_address - 1;
- if (limit < base_address)
- limit = 0xffffffff;
-
- if (seg_expands_down(&descriptor)) {
- if (descriptor.g) {
- seg_top = 0xffffffff;
- } else {
- seg_top = base_address + (1 << 20);
- if (seg_top < base_address)
- seg_top = 0xffffffff;
- }
- access_limit =
- (address <= limit) || (address >= seg_top) ? 0 :
- ((seg_top - address) >= 255 ? 255 : seg_top - address);
- } else {
- access_limit =
- (address > limit) || (address < base_address) ? 0 :
- ((limit - address) >= 254 ? 255 : limit - address + 1);
- }
- if (seg_execute_only(&descriptor) ||
- (!seg_writable(&descriptor) && (FPU_modrm & FPU_WRITE_BIT))) {
- access_limit = 0;
- }
- return address;
-}
-
-/*
- MOD R/M byte: MOD == 3 has a special use for the FPU
- SIB byte used iff R/M = 100b
-
- 7 6 5 4 3 2 1 0
- ..... ......... .........
- MOD OPCODE(2) R/M
-
- SIB byte
-
- 7 6 5 4 3 2 1 0
- ..... ......... .........
- SS INDEX BASE
-
-*/
-
-void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip,
- struct address *addr, fpu_addr_modes addr_modes)
-{
- u_char mod;
- unsigned rm = FPU_modrm & 7;
- long *cpu_reg_ptr;
- int address = 0; /* Initialized just to stop compiler warnings. */
-
- /* Memory accessed via the cs selector is write protected
- in `non-segmented' 32 bit protected mode. */
- if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT)
- && (addr_modes.override.segment == PREFIX_CS_)) {
- math_abort(FPU_info, SIGSEGV);
- }
-
- addr->selector = FPU_DS; /* Default, for 32 bit non-segmented mode. */
-
- mod = (FPU_modrm >> 6) & 3;
-
- if (rm == 4 && mod != 3) {
- address = sib(mod, fpu_eip);
- } else {
- cpu_reg_ptr = ®_(rm);
- switch (mod) {
- case 0:
- if (rm == 5) {
- /* Special case: disp32 */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(4);
- FPU_get_user(address,
- (unsigned long __user
- *)(*fpu_eip));
- (*fpu_eip) += 4;
- RE_ENTRANT_CHECK_ON;
- addr->offset = address;
- return (void __user *)address;
- } else {
- address = *cpu_reg_ptr; /* Just return the contents
- of the cpu register */
- addr->offset = address;
- return (void __user *)address;
- }
- case 1:
- /* 8 bit signed displacement */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(address, (signed char __user *)(*fpu_eip));
- RE_ENTRANT_CHECK_ON;
- (*fpu_eip)++;
- break;
- case 2:
- /* 32 bit displacement */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(4);
- FPU_get_user(address, (long __user *)(*fpu_eip));
- (*fpu_eip) += 4;
- RE_ENTRANT_CHECK_ON;
- break;
- case 3:
- /* Not legal for the FPU */
- EXCEPTION(EX_Invalid);
- }
- address += *cpu_reg_ptr;
- }
-
- addr->offset = address;
-
- switch (addr_modes.default_mode) {
- case 0:
- break;
- case VM86:
- address += vm86_segment(addr_modes.override.segment, addr);
- break;
- case PM16:
- case SEG32:
- address = pm_address(FPU_modrm, addr_modes.override.segment,
- addr, address);
- break;
- default:
- EXCEPTION(EX_INTERNAL | 0x133);
- }
-
- return (void __user *)address;
-}
-
-void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip,
- struct address *addr, fpu_addr_modes addr_modes)
-{
- u_char mod;
- unsigned rm = FPU_modrm & 7;
- int address = 0; /* Default used for mod == 0 */
-
- /* Memory accessed via the cs selector is write protected
- in `non-segmented' 32 bit protected mode. */
- if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT)
- && (addr_modes.override.segment == PREFIX_CS_)) {
- math_abort(FPU_info, SIGSEGV);
- }
-
- addr->selector = FPU_DS; /* Default, for 32 bit non-segmented mode. */
-
- mod = (FPU_modrm >> 6) & 3;
-
- switch (mod) {
- case 0:
- if (rm == 6) {
- /* Special case: disp16 */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(2);
- FPU_get_user(address,
- (unsigned short __user *)(*fpu_eip));
- (*fpu_eip) += 2;
- RE_ENTRANT_CHECK_ON;
- goto add_segment;
- }
- break;
- case 1:
- /* 8 bit signed displacement */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(1);
- FPU_get_user(address, (signed char __user *)(*fpu_eip));
- RE_ENTRANT_CHECK_ON;
- (*fpu_eip)++;
- break;
- case 2:
- /* 16 bit displacement */
- RE_ENTRANT_CHECK_OFF;
- FPU_code_access_ok(2);
- FPU_get_user(address, (unsigned short __user *)(*fpu_eip));
- (*fpu_eip) += 2;
- RE_ENTRANT_CHECK_ON;
- break;
- case 3:
- /* Not legal for the FPU */
- EXCEPTION(EX_Invalid);
- break;
- }
- switch (rm) {
- case 0:
- address += FPU_info->regs->bx + FPU_info->regs->si;
- break;
- case 1:
- address += FPU_info->regs->bx + FPU_info->regs->di;
- break;
- case 2:
- address += FPU_info->regs->bp + FPU_info->regs->si;
- if (addr_modes.override.segment == PREFIX_DEFAULT)
- addr_modes.override.segment = PREFIX_SS_;
- break;
- case 3:
- address += FPU_info->regs->bp + FPU_info->regs->di;
- if (addr_modes.override.segment == PREFIX_DEFAULT)
- addr_modes.override.segment = PREFIX_SS_;
- break;
- case 4:
- address += FPU_info->regs->si;
- break;
- case 5:
- address += FPU_info->regs->di;
- break;
- case 6:
- address += FPU_info->regs->bp;
- if (addr_modes.override.segment == PREFIX_DEFAULT)
- addr_modes.override.segment = PREFIX_SS_;
- break;
- case 7:
- address += FPU_info->regs->bx;
- break;
- }
-
- add_segment:
- address &= 0xffff;
-
- addr->offset = address;
-
- switch (addr_modes.default_mode) {
- case 0:
- break;
- case VM86:
- address += vm86_segment(addr_modes.override.segment, addr);
- break;
- case PM16:
- case SEG32:
- address = pm_address(FPU_modrm, addr_modes.override.segment,
- addr, address);
- break;
- default:
- EXCEPTION(EX_INTERNAL | 0x131);
- }
-
- return (void __user *)address;
-}
diff --git a/arch/x86/math-emu/load_store.c b/arch/x86/math-emu/load_store.c
deleted file mode 100644
index 4092df7..0000000
--- a/arch/x86/math-emu/load_store.c
+++ /dev/null
@@ -1,322 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | load_store.c |
- | |
- | This file contains most of the code to interpret the FPU instructions |
- | which load and store from user memory. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note: |
- | The file contains code which accesses user memory. |
- | Emulator static data may change when user memory is accessed, due to |
- | other processes using the emulator while swapping is in progress. |
- +---------------------------------------------------------------------------*/
-
-#include <linux/uaccess.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-
-#define _NONE_ 0 /* st0_ptr etc not needed */
-#define _REG0_ 1 /* Will be storing st(0) */
-#define _PUSH_ 3 /* Need to check for space to push onto stack */
-#define _null_ 4 /* Function illegal or not implemented */
-
-#define pop_0() { FPU_settag0(TAG_Empty); top++; }
-
-/* index is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */
-static u_char const type_table[32] = {
- _PUSH_, _PUSH_, _PUSH_, _PUSH_, /* /0: d9:fld f32, db:fild m32, dd:fld f64, df:fild m16 */
- _null_, _REG0_, _REG0_, _REG0_, /* /1: d9:undef, db,dd,df:fisttp m32/64/16 */
- _REG0_, _REG0_, _REG0_, _REG0_, /* /2: d9:fst f32, db:fist m32, dd:fst f64, df:fist m16 */
- _REG0_, _REG0_, _REG0_, _REG0_, /* /3: d9:fstp f32, db:fistp m32, dd:fstp f64, df:fistp m16 */
- _NONE_, _null_, _NONE_, _PUSH_,
- _NONE_, _PUSH_, _null_, _PUSH_,
- _NONE_, _null_, _NONE_, _REG0_,
- _NONE_, _REG0_, _NONE_, _REG0_
-};
-
-u_char const data_sizes_16[32] = {
- 4, 4, 8, 2,
- 0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */
- 4, 4, 8, 2,
- 4, 4, 8, 2,
- 14, 0, 94, 10, 2, 10, 0, 8,
- 14, 0, 94, 10, 2, 10, 2, 8
-};
-
-static u_char const data_sizes_32[32] = {
- 4, 4, 8, 2,
- 0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */
- 4, 4, 8, 2,
- 4, 4, 8, 2,
- 28, 0, 108, 10, 2, 10, 0, 8,
- 28, 0, 108, 10, 2, 10, 2, 8
-};
-
-int FPU_load_store(u_char type, fpu_addr_modes addr_modes,
- void __user * data_address)
-{
- FPU_REG loaded_data;
- FPU_REG *st0_ptr;
- u_char st0_tag = TAG_Empty; /* This is just to stop a gcc warning. */
- u_char loaded_tag;
- int sv_cw;
-
- st0_ptr = NULL; /* Initialized just to stop compiler warnings. */
-
- if (addr_modes.default_mode & PROTECTED) {
- if (addr_modes.default_mode == SEG32) {
- if (access_limit < data_sizes_32[type])
- math_abort(FPU_info, SIGSEGV);
- } else if (addr_modes.default_mode == PM16) {
- if (access_limit < data_sizes_16[type])
- math_abort(FPU_info, SIGSEGV);
- }
-#ifdef PARANOID
- else
- EXCEPTION(EX_INTERNAL | 0x140);
-#endif /* PARANOID */
- }
-
- switch (type_table[type]) {
- case _NONE_:
- break;
- case _REG0_:
- st0_ptr = &st(0); /* Some of these instructions pop after
- storing */
- st0_tag = FPU_gettag0();
- break;
- case _PUSH_:
- {
- if (FPU_gettagi(-1) != TAG_Empty) {
- FPU_stack_overflow();
- return 0;
- }
- top--;
- st0_ptr = &st(0);
- }
- break;
- case _null_:
- FPU_illegal();
- return 0;
-#ifdef PARANOID
- default:
- EXCEPTION(EX_INTERNAL | 0x141);
- return 0;
-#endif /* PARANOID */
- }
-
- switch (type) {
- /* type is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */
- case 000: /* fld m32real (d9 /0) */
- clear_C1();
- loaded_tag =
- FPU_load_single((float __user *)data_address, &loaded_data);
- if ((loaded_tag == TAG_Special)
- && isNaN(&loaded_data)
- && (real_1op_NaN(&loaded_data) < 0)) {
- top++;
- break;
- }
- FPU_copy_to_reg0(&loaded_data, loaded_tag);
- break;
- case 001: /* fild m32int (db /0) */
- clear_C1();
- loaded_tag =
- FPU_load_int32((long __user *)data_address, &loaded_data);
- FPU_copy_to_reg0(&loaded_data, loaded_tag);
- break;
- case 002: /* fld m64real (dd /0) */
- clear_C1();
- loaded_tag =
- FPU_load_double((double __user *)data_address,
- &loaded_data);
- if ((loaded_tag == TAG_Special)
- && isNaN(&loaded_data)
- && (real_1op_NaN(&loaded_data) < 0)) {
- top++;
- break;
- }
- FPU_copy_to_reg0(&loaded_data, loaded_tag);
- break;
- case 003: /* fild m16int (df /0) */
- clear_C1();
- loaded_tag =
- FPU_load_int16((short __user *)data_address, &loaded_data);
- FPU_copy_to_reg0(&loaded_data, loaded_tag);
- break;
- /* case 004: undefined (d9 /1) */
- /* fisttp are enabled if CPUID(1).ECX(0) "sse3" is set */
- case 005: /* fisttp m32int (db /1) */
- clear_C1();
- sv_cw = control_word;
- control_word |= RC_CHOP;
- if (FPU_store_int32
- (st0_ptr, st0_tag, (long __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- control_word = sv_cw;
- break;
- case 006: /* fisttp m64int (dd /1) */
- clear_C1();
- sv_cw = control_word;
- control_word |= RC_CHOP;
- if (FPU_store_int64
- (st0_ptr, st0_tag, (long long __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- control_word = sv_cw;
- break;
- case 007: /* fisttp m16int (df /1) */
- clear_C1();
- sv_cw = control_word;
- control_word |= RC_CHOP;
- if (FPU_store_int16
- (st0_ptr, st0_tag, (short __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- control_word = sv_cw;
- break;
- case 010: /* fst m32real */
- clear_C1();
- FPU_store_single(st0_ptr, st0_tag,
- (float __user *)data_address);
- break;
- case 011: /* fist m32int */
- clear_C1();
- FPU_store_int32(st0_ptr, st0_tag, (long __user *)data_address);
- break;
- case 012: /* fst m64real */
- clear_C1();
- FPU_store_double(st0_ptr, st0_tag,
- (double __user *)data_address);
- break;
- case 013: /* fist m16int */
- clear_C1();
- FPU_store_int16(st0_ptr, st0_tag, (short __user *)data_address);
- break;
- case 014: /* fstp m32real */
- clear_C1();
- if (FPU_store_single
- (st0_ptr, st0_tag, (float __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 015: /* fistp m32int */
- clear_C1();
- if (FPU_store_int32
- (st0_ptr, st0_tag, (long __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 016: /* fstp m64real */
- clear_C1();
- if (FPU_store_double
- (st0_ptr, st0_tag, (double __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 017: /* fistp m16int */
- clear_C1();
- if (FPU_store_int16
- (st0_ptr, st0_tag, (short __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 020: /* fldenv m14/28byte */
- fldenv(addr_modes, (u_char __user *) data_address);
- /* Ensure that the values just loaded are not changed by
- fix-up operations. */
- return 1;
- case 022: /* frstor m94/108byte */
- FPU_frstor(addr_modes, (u_char __user *) data_address);
- /* Ensure that the values just loaded are not changed by
- fix-up operations. */
- return 1;
- case 023: /* fbld m80dec */
- clear_C1();
- loaded_tag = FPU_load_bcd((u_char __user *) data_address);
- FPU_settag0(loaded_tag);
- break;
- case 024: /* fldcw */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(data_address, 2);
- FPU_get_user(control_word,
- (unsigned short __user *)data_address);
- RE_ENTRANT_CHECK_ON;
- if (partial_status & ~control_word & CW_Exceptions)
- partial_status |= (SW_Summary | SW_Backward);
- else
- partial_status &= ~(SW_Summary | SW_Backward);
-#ifdef PECULIAR_486
- control_word |= 0x40; /* An 80486 appears to always set this bit */
-#endif /* PECULIAR_486 */
- return 1;
- case 025: /* fld m80real */
- clear_C1();
- loaded_tag =
- FPU_load_extended((long double __user *)data_address, 0);
- FPU_settag0(loaded_tag);
- break;
- case 027: /* fild m64int */
- clear_C1();
- loaded_tag = FPU_load_int64((long long __user *)data_address);
- if (loaded_tag == TAG_Error)
- return 0;
- FPU_settag0(loaded_tag);
- break;
- case 030: /* fstenv m14/28byte */
- fstenv(addr_modes, (u_char __user *) data_address);
- return 1;
- case 032: /* fsave */
- fsave(addr_modes, (u_char __user *) data_address);
- return 1;
- case 033: /* fbstp m80dec */
- clear_C1();
- if (FPU_store_bcd
- (st0_ptr, st0_tag, (u_char __user *) data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 034: /* fstcw m16int */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(data_address, 2);
- FPU_put_user(control_word,
- (unsigned short __user *)data_address);
- RE_ENTRANT_CHECK_ON;
- return 1;
- case 035: /* fstp m80real */
- clear_C1();
- if (FPU_store_extended
- (st0_ptr, st0_tag, (long double __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- case 036: /* fstsw m2byte */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(data_address, 2);
- FPU_put_user(status_word(),
- (unsigned short __user *)data_address);
- RE_ENTRANT_CHECK_ON;
- return 1;
- case 037: /* fistp m64int */
- clear_C1();
- if (FPU_store_int64
- (st0_ptr, st0_tag, (long long __user *)data_address))
- pop_0(); /* pop only if the number was actually stored
- (see the 80486 manual p16-28) */
- break;
- }
- return 0;
-}
diff --git a/arch/x86/math-emu/mul_Xsig.S b/arch/x86/math-emu/mul_Xsig.S
deleted file mode 100644
index 54a031b..0000000
--- a/arch/x86/math-emu/mul_Xsig.S
+++ /dev/null
@@ -1,179 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | mul_Xsig.S |
- | |
- | Multiply a 12 byte fixed point number by another fixed point number. |
- | |
- | Copyright (C) 1992,1994,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Call from C as: |
- | void mul32_Xsig(Xsig *x, unsigned b) |
- | |
- | void mul64_Xsig(Xsig *x, unsigned long long *b) |
- | |
- | void mul_Xsig_Xsig(Xsig *x, unsigned *b) |
- | |
- | The result is neither rounded nor normalized, and the ls bit or so may |
- | be wrong. |
- | |
- +---------------------------------------------------------------------------*/
- .file "mul_Xsig.S"
-
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(mul32_Xsig)
- pushl %ebp
- movl %esp,%ebp
- subl $16,%esp
- pushl %esi
-
- movl PARAM1,%esi
- movl PARAM2,%ecx
-
- xor %eax,%eax
- movl %eax,-4(%ebp)
- movl %eax,-8(%ebp)
-
- movl (%esi),%eax /* lsl of Xsig */
- mull %ecx /* msl of b */
- movl %edx,-12(%ebp)
-
- movl 4(%esi),%eax /* midl of Xsig */
- mull %ecx /* msl of b */
- addl %eax,-12(%ebp)
- adcl %edx,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull %ecx /* msl of b */
- addl %eax,-8(%ebp)
- adcl %edx,-4(%ebp)
-
- movl -12(%ebp),%eax
- movl %eax,(%esi)
- movl -8(%ebp),%eax
- movl %eax,4(%esi)
- movl -4(%ebp),%eax
- movl %eax,8(%esi)
-
- popl %esi
- leave
- RET
-SYM_FUNC_END(mul32_Xsig)
-
-
-SYM_FUNC_START(mul64_Xsig)
- pushl %ebp
- movl %esp,%ebp
- subl $16,%esp
- pushl %esi
-
- movl PARAM1,%esi
- movl PARAM2,%ecx
-
- xor %eax,%eax
- movl %eax,-4(%ebp)
- movl %eax,-8(%ebp)
-
- movl (%esi),%eax /* lsl of Xsig */
- mull 4(%ecx) /* msl of b */
- movl %edx,-12(%ebp)
-
- movl 4(%esi),%eax /* midl of Xsig */
- mull (%ecx) /* lsl of b */
- addl %edx,-12(%ebp)
- adcl $0,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 4(%esi),%eax /* midl of Xsig */
- mull 4(%ecx) /* msl of b */
- addl %eax,-12(%ebp)
- adcl %edx,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull (%ecx) /* lsl of b */
- addl %eax,-12(%ebp)
- adcl %edx,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull 4(%ecx) /* msl of b */
- addl %eax,-8(%ebp)
- adcl %edx,-4(%ebp)
-
- movl -12(%ebp),%eax
- movl %eax,(%esi)
- movl -8(%ebp),%eax
- movl %eax,4(%esi)
- movl -4(%ebp),%eax
- movl %eax,8(%esi)
-
- popl %esi
- leave
- RET
-SYM_FUNC_END(mul64_Xsig)
-
-
-
-SYM_FUNC_START(mul_Xsig_Xsig)
- pushl %ebp
- movl %esp,%ebp
- subl $16,%esp
- pushl %esi
-
- movl PARAM1,%esi
- movl PARAM2,%ecx
-
- xor %eax,%eax
- movl %eax,-4(%ebp)
- movl %eax,-8(%ebp)
-
- movl (%esi),%eax /* lsl of Xsig */
- mull 8(%ecx) /* msl of b */
- movl %edx,-12(%ebp)
-
- movl 4(%esi),%eax /* midl of Xsig */
- mull 4(%ecx) /* midl of b */
- addl %edx,-12(%ebp)
- adcl $0,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull (%ecx) /* lsl of b */
- addl %edx,-12(%ebp)
- adcl $0,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 4(%esi),%eax /* midl of Xsig */
- mull 8(%ecx) /* msl of b */
- addl %eax,-12(%ebp)
- adcl %edx,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull 4(%ecx) /* midl of b */
- addl %eax,-12(%ebp)
- adcl %edx,-8(%ebp)
- adcl $0,-4(%ebp)
-
- movl 8(%esi),%eax /* msl of Xsig */
- mull 8(%ecx) /* msl of b */
- addl %eax,-8(%ebp)
- adcl %edx,-4(%ebp)
-
- movl -12(%ebp),%edx
- movl %edx,(%esi)
- movl -8(%ebp),%edx
- movl %edx,4(%esi)
- movl -4(%ebp),%edx
- movl %edx,8(%esi)
-
- popl %esi
- leave
- RET
-SYM_FUNC_END(mul_Xsig_Xsig)
diff --git a/arch/x86/math-emu/poly.h b/arch/x86/math-emu/poly.h
deleted file mode 100644
index 654bfe4..0000000
--- a/arch/x86/math-emu/poly.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | poly.h |
- | |
- | Header file for the FPU-emu poly*.c source files. |
- | |
- | Copyright (C) 1994,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@melbpc.org.au |
- | |
- | Declarations and definitions for functions operating on Xsig (12-byte |
- | extended-significand) quantities. |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _POLY_H
-#define _POLY_H
-
-/* This 12-byte structure is used to improve the accuracy of computation
- of transcendental functions.
- Intended to be used to get results better than 8-byte computation
- allows. 9-byte would probably be sufficient.
- */
-typedef struct {
- unsigned long lsw;
- unsigned long midw;
- unsigned long msw;
-} Xsig;
-
-asmlinkage void mul64(unsigned long long const *a, unsigned long long const *b,
- unsigned long long *result);
-asmlinkage void polynomial_Xsig(Xsig *, const unsigned long long *x,
- const unsigned long long terms[], const int n);
-
-asmlinkage void mul32_Xsig(Xsig *, const unsigned long mult);
-asmlinkage void mul64_Xsig(Xsig *, const unsigned long long *mult);
-asmlinkage void mul_Xsig_Xsig(Xsig *dest, const Xsig *mult);
-
-asmlinkage void shr_Xsig(Xsig *, const int n);
-asmlinkage int round_Xsig(Xsig *);
-asmlinkage int norm_Xsig(Xsig *);
-asmlinkage void div_Xsig(Xsig *x1, const Xsig *x2, Xsig *dest);
-
-/* Macro to extract the most significant 32 bits from a long long */
-#define LL_MSW(x) (((unsigned long *)&x)[1])
-
-/* Macro to initialize an Xsig struct */
-#define MK_XSIG(a,b,c) { c, b, a }
-
-/* Macro to access the 8 ms bytes of an Xsig as a long long */
-#define XSIG_LL(x) (*(unsigned long long *)&x.midw)
-
-/*
- Need to run gcc with optimizations on to get these to
- actually be in-line.
- */
-
-/* Multiply two fixed-point 32 bit numbers, producing a 32 bit result.
- The answer is the ms word of the product. */
-/* Some versions of gcc make it difficult to stop eax from being clobbered.
- Merely specifying that it is used doesn't work...
- */
-static inline unsigned long mul_32_32(const unsigned long arg1,
- const unsigned long arg2)
-{
- int retval;
- asm volatile ("mull %2; movl %%edx,%%eax":"=a" (retval)
- :"0"(arg1), "g"(arg2)
- :"dx");
- return retval;
-}
-
-/* Add the 12 byte Xsig x2 to Xsig dest, with no checks for overflow. */
-static inline void add_Xsig_Xsig(Xsig *dest, const Xsig *x2)
-{
- asm volatile ("movl %1,%%edi; movl %2,%%esi;\n"
- "movl (%%esi),%%eax; addl %%eax,(%%edi);\n"
- "movl 4(%%esi),%%eax; adcl %%eax,4(%%edi);\n"
- "movl 8(%%esi),%%eax; adcl %%eax,8(%%edi);\n":"=g"
- (*dest):"g"(dest), "g"(x2)
- :"ax", "si", "di");
-}
-
-/* Add the 12 byte Xsig x2 to Xsig dest, adjust exp if overflow occurs. */
-/* Note: the constraints in the asm statement didn't always work properly
- with gcc 2.5.8. Changing from using edi to using ecx got around the
- problem, but keep fingers crossed! */
-static inline void add_two_Xsig(Xsig *dest, const Xsig *x2, long int *exp)
-{
- asm volatile ("movl %2,%%ecx; movl %3,%%esi;\n"
- "movl (%%esi),%%eax; addl %%eax,(%%ecx);\n"
- "movl 4(%%esi),%%eax; adcl %%eax,4(%%ecx);\n"
- "movl 8(%%esi),%%eax; adcl %%eax,8(%%ecx);\n"
- "jnc 0f;\n"
- "rcrl 8(%%ecx); rcrl 4(%%ecx); rcrl (%%ecx)\n"
- "movl %4,%%ecx; incl (%%ecx)\n"
- "movl $1,%%eax; jmp 1f;\n"
- "0: xorl %%eax,%%eax;\n" "1:\n":"=g" (*exp), "=g"(*dest)
- :"g"(dest), "g"(x2), "g"(exp)
- :"cx", "si", "ax");
-}
-
-/* Negate (subtract from 1.0) the 12 byte Xsig */
-/* This is faster in a loop on my 386 than using the "neg" instruction. */
-static inline void negate_Xsig(Xsig *x)
-{
- asm volatile ("movl %1,%%esi;\n"
- "xorl %%ecx,%%ecx;\n"
- "movl %%ecx,%%eax; subl (%%esi),%%eax; movl %%eax,(%%esi);\n"
- "movl %%ecx,%%eax; sbbl 4(%%esi),%%eax; movl %%eax,4(%%esi);\n"
- "movl %%ecx,%%eax; sbbl 8(%%esi),%%eax; movl %%eax,8(%%esi);\n":"=g"
- (*x):"g"(x):"si", "ax", "cx");
-}
-
-#endif /* _POLY_H */
diff --git a/arch/x86/math-emu/poly_2xm1.c b/arch/x86/math-emu/poly_2xm1.c
deleted file mode 100644
index aa33006b..0000000
--- a/arch/x86/math-emu/poly_2xm1.c
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | poly_2xm1.c |
- | |
- | Function to compute 2^x-1 by a polynomial approximation. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define HIPOWER 11
-static const unsigned long long lterms[HIPOWER] = {
- 0x0000000000000000LL, /* This term done separately as 12 bytes */
- 0xf5fdeffc162c7543LL,
- 0x1c6b08d704a0bfa6LL,
- 0x0276556df749cc21LL,
- 0x002bb0ffcf14f6b8LL,
- 0x0002861225ef751cLL,
- 0x00001ffcbfcd5422LL,
- 0x00000162c005d5f1LL,
- 0x0000000da96ccb1bLL,
- 0x0000000078d1b897LL,
- 0x000000000422b029LL
-};
-
-static const Xsig hiterm = MK_XSIG(0xb17217f7, 0xd1cf79ab, 0xc8a39194);
-
-/* Four slices: 0.0 : 0.25 : 0.50 : 0.75 : 1.0,
- These numbers are 2^(1/4), 2^(1/2), and 2^(3/4)
- */
-static const Xsig shiftterm0 = MK_XSIG(0, 0, 0);
-static const Xsig shiftterm1 = MK_XSIG(0x9837f051, 0x8db8a96f, 0x46ad2318);
-static const Xsig shiftterm2 = MK_XSIG(0xb504f333, 0xf9de6484, 0x597d89b3);
-static const Xsig shiftterm3 = MK_XSIG(0xd744fcca, 0xd69d6af4, 0x39a68bb9);
-
-static const Xsig *shiftterm[] = { &shiftterm0, &shiftterm1,
- &shiftterm2, &shiftterm3
-};
-
-/*--- poly_2xm1() -----------------------------------------------------------+
- | Requires st(0) which is TAG_Valid and < 1. |
- +---------------------------------------------------------------------------*/
-int poly_2xm1(u_char sign, FPU_REG *arg, FPU_REG *result)
-{
- long int exponent, shift;
- unsigned long long Xll;
- Xsig accumulator, Denom, argSignif;
- u_char tag;
-
- exponent = exponent16(arg);
-
-#ifdef PARANOID
- if (exponent >= 0) { /* Don't want a |number| >= 1.0 */
- /* Number negative, too large, or not Valid. */
- EXCEPTION(EX_INTERNAL | 0x127);
- return 1;
- }
-#endif /* PARANOID */
-
- argSignif.lsw = 0;
- XSIG_LL(argSignif) = Xll = significand(arg);
-
- if (exponent == -1) {
- shift = (argSignif.msw & 0x40000000) ? 3 : 2;
- /* subtract 0.5 or 0.75 */
- exponent -= 2;
- XSIG_LL(argSignif) <<= 2;
- Xll <<= 2;
- } else if (exponent == -2) {
- shift = 1;
- /* subtract 0.25 */
- exponent--;
- XSIG_LL(argSignif) <<= 1;
- Xll <<= 1;
- } else
- shift = 0;
-
- if (exponent < -2) {
- /* Shift the argument right by the required places. */
- if (FPU_shrx(&Xll, -2 - exponent) >= 0x80000000U)
- Xll++; /* round up */
- }
-
- accumulator.lsw = accumulator.midw = accumulator.msw = 0;
- polynomial_Xsig(&accumulator, &Xll, lterms, HIPOWER - 1);
- mul_Xsig_Xsig(&accumulator, &argSignif);
- shr_Xsig(&accumulator, 3);
-
- mul_Xsig_Xsig(&argSignif, &hiterm); /* The leading term */
- add_two_Xsig(&accumulator, &argSignif, &exponent);
-
- if (shift) {
- /* The argument is large, use the identity:
- f(x+a) = f(a) * (f(x) + 1) - 1;
- */
- shr_Xsig(&accumulator, -exponent);
- accumulator.msw |= 0x80000000; /* add 1.0 */
- mul_Xsig_Xsig(&accumulator, shiftterm[shift]);
- accumulator.msw &= 0x3fffffff; /* subtract 1.0 */
- exponent = 1;
- }
-
- if (sign != SIGN_POS) {
- /* The argument is negative, use the identity:
- f(-x) = -f(x) / (1 + f(x))
- */
- Denom.lsw = accumulator.lsw;
- XSIG_LL(Denom) = XSIG_LL(accumulator);
- if (exponent < 0)
- shr_Xsig(&Denom, -exponent);
- else if (exponent > 0) {
- /* exponent must be 1 here */
- XSIG_LL(Denom) <<= 1;
- if (Denom.lsw & 0x80000000)
- XSIG_LL(Denom) |= 1;
- (Denom.lsw) <<= 1;
- }
- Denom.msw |= 0x80000000; /* add 1.0 */
- div_Xsig(&accumulator, &Denom, &accumulator);
- }
-
- /* Convert to 64 bit signed-compatible */
- exponent += round_Xsig(&accumulator);
-
- result = &st(0);
- significand(result) = XSIG_LL(accumulator);
- setexponent16(result, exponent);
-
- tag = FPU_round(result, 1, 0, FULL_PRECISION, sign);
-
- setsign(result, sign);
- FPU_settag0(tag);
-
- return 0;
-
-}
diff --git a/arch/x86/math-emu/poly_atan.c b/arch/x86/math-emu/poly_atan.c
deleted file mode 100644
index 7e7412c..0000000
--- a/arch/x86/math-emu/poly_atan.c
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | poly_atan.c |
- | |
- | Compute the arctan of a FPU_REG, using a polynomial approximation. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define HIPOWERon 6 /* odd poly, negative terms */
-static const unsigned long long oddnegterms[HIPOWERon] = {
- 0x0000000000000000LL, /* Dummy (not for - 1.0) */
- 0x015328437f756467LL,
- 0x0005dda27b73dec6LL,
- 0x0000226bf2bfb91aLL,
- 0x000000ccc439c5f7LL,
- 0x0000000355438407LL
-};
-
-#define HIPOWERop 6 /* odd poly, positive terms */
-static const unsigned long long oddplterms[HIPOWERop] = {
-/* 0xaaaaaaaaaaaaaaabLL, transferred to fixedpterm[] */
- 0x0db55a71875c9ac2LL,
- 0x0029fce2d67880b0LL,
- 0x0000dfd3908b4596LL,
- 0x00000550fd61dab4LL,
- 0x0000001c9422b3f9LL,
- 0x000000003e3301e1LL
-};
-
-static const unsigned long long denomterm = 0xebd9b842c5c53a0eLL;
-
-static const Xsig fixedpterm = MK_XSIG(0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa);
-
-static const Xsig pi_signif = MK_XSIG(0xc90fdaa2, 0x2168c234, 0xc4c6628b);
-
-/*--- poly_atan() -----------------------------------------------------------+
- | |
- +---------------------------------------------------------------------------*/
-void poly_atan(FPU_REG *st0_ptr, u_char st0_tag,
- FPU_REG *st1_ptr, u_char st1_tag)
-{
- u_char transformed, inverted, sign1, sign2;
- int exponent;
- long int dummy_exp;
- Xsig accumulator, Numer, Denom, accumulatore, argSignif, argSq, argSqSq;
- u_char tag;
-
- sign1 = getsign(st0_ptr);
- sign2 = getsign(st1_ptr);
- if (st0_tag == TAG_Valid) {
- exponent = exponent(st0_ptr);
- } else {
- /* This gives non-compatible stack contents... */
- FPU_to_exp16(st0_ptr, st0_ptr);
- exponent = exponent16(st0_ptr);
- }
- if (st1_tag == TAG_Valid) {
- exponent -= exponent(st1_ptr);
- } else {
- /* This gives non-compatible stack contents... */
- FPU_to_exp16(st1_ptr, st1_ptr);
- exponent -= exponent16(st1_ptr);
- }
-
- if ((exponent < 0) || ((exponent == 0) &&
- ((st0_ptr->sigh < st1_ptr->sigh) ||
- ((st0_ptr->sigh == st1_ptr->sigh) &&
- (st0_ptr->sigl < st1_ptr->sigl))))) {
- inverted = 1;
- Numer.lsw = Denom.lsw = 0;
- XSIG_LL(Numer) = significand(st0_ptr);
- XSIG_LL(Denom) = significand(st1_ptr);
- } else {
- inverted = 0;
- exponent = -exponent;
- Numer.lsw = Denom.lsw = 0;
- XSIG_LL(Numer) = significand(st1_ptr);
- XSIG_LL(Denom) = significand(st0_ptr);
- }
- div_Xsig(&Numer, &Denom, &argSignif);
- exponent += norm_Xsig(&argSignif);
-
- if ((exponent >= -1)
- || ((exponent == -2) && (argSignif.msw > 0xd413ccd0))) {
- /* The argument is greater than sqrt(2)-1 (=0.414213562...) */
- /* Convert the argument by an identity for atan */
- transformed = 1;
-
- if (exponent >= 0) {
-#ifdef PARANOID
- if (!((exponent == 0) &&
- (argSignif.lsw == 0) && (argSignif.midw == 0) &&
- (argSignif.msw == 0x80000000))) {
- EXCEPTION(EX_INTERNAL | 0x104); /* There must be a logic error */
- return;
- }
-#endif /* PARANOID */
- argSignif.msw = 0; /* Make the transformed arg -> 0.0 */
- } else {
- Numer.lsw = Denom.lsw = argSignif.lsw;
- XSIG_LL(Numer) = XSIG_LL(Denom) = XSIG_LL(argSignif);
-
- if (exponent < -1)
- shr_Xsig(&Numer, -1 - exponent);
- negate_Xsig(&Numer);
-
- shr_Xsig(&Denom, -exponent);
- Denom.msw |= 0x80000000;
-
- div_Xsig(&Numer, &Denom, &argSignif);
-
- exponent = -1 + norm_Xsig(&argSignif);
- }
- } else {
- transformed = 0;
- }
-
- argSq.lsw = argSignif.lsw;
- argSq.midw = argSignif.midw;
- argSq.msw = argSignif.msw;
- mul_Xsig_Xsig(&argSq, &argSq);
-
- argSqSq.lsw = argSq.lsw;
- argSqSq.midw = argSq.midw;
- argSqSq.msw = argSq.msw;
- mul_Xsig_Xsig(&argSqSq, &argSqSq);
-
- accumulatore.lsw = argSq.lsw;
- XSIG_LL(accumulatore) = XSIG_LL(argSq);
-
- shr_Xsig(&argSq, 2 * (-1 - exponent - 1));
- shr_Xsig(&argSqSq, 4 * (-1 - exponent - 1));
-
- /* Now have argSq etc with binary point at the left
- .1xxxxxxxx */
-
- /* Do the basic fixed point polynomial evaluation */
- accumulator.msw = accumulator.midw = accumulator.lsw = 0;
- polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq),
- oddplterms, HIPOWERop - 1);
- mul64_Xsig(&accumulator, &XSIG_LL(argSq));
- negate_Xsig(&accumulator);
- polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq), oddnegterms,
- HIPOWERon - 1);
- negate_Xsig(&accumulator);
- add_two_Xsig(&accumulator, &fixedpterm, &dummy_exp);
-
- mul64_Xsig(&accumulatore, &denomterm);
- shr_Xsig(&accumulatore, 1 + 2 * (-1 - exponent));
- accumulatore.msw |= 0x80000000;
-
- div_Xsig(&accumulator, &accumulatore, &accumulator);
-
- mul_Xsig_Xsig(&accumulator, &argSignif);
- mul_Xsig_Xsig(&accumulator, &argSq);
-
- shr_Xsig(&accumulator, 3);
- negate_Xsig(&accumulator);
- add_Xsig_Xsig(&accumulator, &argSignif);
-
- if (transformed) {
- /* compute pi/4 - accumulator */
- shr_Xsig(&accumulator, -1 - exponent);
- negate_Xsig(&accumulator);
- add_Xsig_Xsig(&accumulator, &pi_signif);
- exponent = -1;
- }
-
- if (inverted) {
- /* compute pi/2 - accumulator */
- shr_Xsig(&accumulator, -exponent);
- negate_Xsig(&accumulator);
- add_Xsig_Xsig(&accumulator, &pi_signif);
- exponent = 0;
- }
-
- if (sign1) {
- /* compute pi - accumulator */
- shr_Xsig(&accumulator, 1 - exponent);
- negate_Xsig(&accumulator);
- add_Xsig_Xsig(&accumulator, &pi_signif);
- exponent = 1;
- }
-
- exponent += round_Xsig(&accumulator);
-
- significand(st1_ptr) = XSIG_LL(accumulator);
- setexponent16(st1_ptr, exponent);
-
- tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign2);
- FPU_settagi(1, tag);
-
- set_precision_flag_up(); /* We do not really know if up or down,
- use this as the default. */
-
-}
diff --git a/arch/x86/math-emu/poly_l2.c b/arch/x86/math-emu/poly_l2.c
deleted file mode 100644
index 98b6949..0000000
--- a/arch/x86/math-emu/poly_l2.c
+++ /dev/null
@@ -1,245 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | poly_l2.c |
- | |
- | Compute the base 2 log of a FPU_REG, using a polynomial approximation. |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-static void log2_kernel(FPU_REG const *arg, u_char argsign,
- Xsig * accum_result, long int *expon);
-
-/*--- poly_l2() -------------------------------------------------------------+
- | Base 2 logarithm by a polynomial approximation. |
- +---------------------------------------------------------------------------*/
-void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign)
-{
- long int exponent, expon, expon_expon;
- Xsig accumulator, expon_accum, yaccum;
- u_char sign, argsign;
- FPU_REG x;
- int tag;
-
- exponent = exponent16(st0_ptr);
-
- /* From st0_ptr, make a number > sqrt(2)/2 and < sqrt(2) */
- if (st0_ptr->sigh > (unsigned)0xb504f334) {
- /* Treat as sqrt(2)/2 < st0_ptr < 1 */
- significand(&x) = -significand(st0_ptr);
- setexponent16(&x, -1);
- exponent++;
- argsign = SIGN_NEG;
- } else {
- /* Treat as 1 <= st0_ptr < sqrt(2) */
- x.sigh = st0_ptr->sigh - 0x80000000;
- x.sigl = st0_ptr->sigl;
- setexponent16(&x, 0);
- argsign = SIGN_POS;
- }
- tag = FPU_normalize_nuo(&x);
-
- if (tag == TAG_Zero) {
- expon = 0;
- accumulator.msw = accumulator.midw = accumulator.lsw = 0;
- } else {
- log2_kernel(&x, argsign, &accumulator, &expon);
- }
-
- if (exponent < 0) {
- sign = SIGN_NEG;
- exponent = -exponent;
- } else
- sign = SIGN_POS;
- expon_accum.msw = exponent;
- expon_accum.midw = expon_accum.lsw = 0;
- if (exponent) {
- expon_expon = 31 + norm_Xsig(&expon_accum);
- shr_Xsig(&accumulator, expon_expon - expon);
-
- if (sign ^ argsign)
- negate_Xsig(&accumulator);
- add_Xsig_Xsig(&accumulator, &expon_accum);
- } else {
- expon_expon = expon;
- sign = argsign;
- }
-
- yaccum.lsw = 0;
- XSIG_LL(yaccum) = significand(st1_ptr);
- mul_Xsig_Xsig(&accumulator, &yaccum);
-
- expon_expon += round_Xsig(&accumulator);
-
- if (accumulator.msw == 0) {
- FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
- return;
- }
-
- significand(st1_ptr) = XSIG_LL(accumulator);
- setexponent16(st1_ptr, expon_expon + exponent16(st1_ptr) + 1);
-
- tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign ^ st1_sign);
- FPU_settagi(1, tag);
-
- set_precision_flag_up(); /* 80486 appears to always do this */
-
- return;
-
-}
-
-/*--- poly_l2p1() -----------------------------------------------------------+
- | Base 2 logarithm by a polynomial approximation. |
- | log2(x+1) |
- +---------------------------------------------------------------------------*/
-int poly_l2p1(u_char sign0, u_char sign1,
- FPU_REG * st0_ptr, FPU_REG * st1_ptr, FPU_REG * dest)
-{
- u_char tag;
- long int exponent;
- Xsig accumulator, yaccum;
-
- if (exponent16(st0_ptr) < 0) {
- log2_kernel(st0_ptr, sign0, &accumulator, &exponent);
-
- yaccum.lsw = 0;
- XSIG_LL(yaccum) = significand(st1_ptr);
- mul_Xsig_Xsig(&accumulator, &yaccum);
-
- exponent += round_Xsig(&accumulator);
-
- exponent += exponent16(st1_ptr) + 1;
- if (exponent < EXP_WAY_UNDER)
- exponent = EXP_WAY_UNDER;
-
- significand(dest) = XSIG_LL(accumulator);
- setexponent16(dest, exponent);
-
- tag = FPU_round(dest, 1, 0, FULL_PRECISION, sign0 ^ sign1);
- FPU_settagi(1, tag);
-
- if (tag == TAG_Valid)
- set_precision_flag_up(); /* 80486 appears to always do this */
- } else {
- /* The magnitude of st0_ptr is far too large. */
-
- if (sign0 != SIGN_POS) {
- /* Trying to get the log of a negative number. */
-#ifdef PECULIAR_486 /* Stupid 80486 doesn't worry about log(negative). */
- changesign(st1_ptr);
-#else
- if (arith_invalid(1) < 0)
- return 1;
-#endif /* PECULIAR_486 */
- }
-
- /* 80486 appears to do this */
- if (sign0 == SIGN_NEG)
- set_precision_flag_down();
- else
- set_precision_flag_up();
- }
-
- if (exponent(dest) <= EXP_UNDER)
- EXCEPTION(EX_Underflow);
-
- return 0;
-
-}
-
-#undef HIPOWER
-#define HIPOWER 10
-static const unsigned long long logterms[HIPOWER] = {
- 0x2a8eca5705fc2ef0LL,
- 0xf6384ee1d01febceLL,
- 0x093bb62877cdf642LL,
- 0x006985d8a9ec439bLL,
- 0x0005212c4f55a9c8LL,
- 0x00004326a16927f0LL,
- 0x0000038d1d80a0e7LL,
- 0x0000003141cc80c6LL,
- 0x00000002b1668c9fLL,
- 0x000000002c7a46aaLL
-};
-
-static const unsigned long leadterm = 0xb8000000;
-
-/*--- log2_kernel() ---------------------------------------------------------+
- | Base 2 logarithm by a polynomial approximation. |
- | log2(x+1) |
- +---------------------------------------------------------------------------*/
-static void log2_kernel(FPU_REG const *arg, u_char argsign, Xsig *accum_result,
- long int *expon)
-{
- long int exponent, adj;
- unsigned long long Xsq;
- Xsig accumulator, Numer, Denom, argSignif, arg_signif;
-
- exponent = exponent16(arg);
- Numer.lsw = Denom.lsw = 0;
- XSIG_LL(Numer) = XSIG_LL(Denom) = significand(arg);
- if (argsign == SIGN_POS) {
- shr_Xsig(&Denom, 2 - (1 + exponent));
- Denom.msw |= 0x80000000;
- div_Xsig(&Numer, &Denom, &argSignif);
- } else {
- shr_Xsig(&Denom, 1 - (1 + exponent));
- negate_Xsig(&Denom);
- if (Denom.msw & 0x80000000) {
- div_Xsig(&Numer, &Denom, &argSignif);
- exponent++;
- } else {
- /* Denom must be 1.0 */
- argSignif.lsw = Numer.lsw;
- argSignif.midw = Numer.midw;
- argSignif.msw = Numer.msw;
- }
- }
-
-#ifndef PECULIAR_486
- /* Should check here that |local_arg| is within the valid range */
- if (exponent >= -2) {
- if ((exponent > -2) || (argSignif.msw > (unsigned)0xafb0ccc0)) {
- /* The argument is too large */
- }
- }
-#endif /* PECULIAR_486 */
-
- arg_signif.lsw = argSignif.lsw;
- XSIG_LL(arg_signif) = XSIG_LL(argSignif);
- adj = norm_Xsig(&argSignif);
- accumulator.lsw = argSignif.lsw;
- XSIG_LL(accumulator) = XSIG_LL(argSignif);
- mul_Xsig_Xsig(&accumulator, &accumulator);
- shr_Xsig(&accumulator, 2 * (-1 - (1 + exponent + adj)));
- Xsq = XSIG_LL(accumulator);
- if (accumulator.lsw & 0x80000000)
- Xsq++;
-
- accumulator.msw = accumulator.midw = accumulator.lsw = 0;
- /* Do the basic fixed point polynomial evaluation */
- polynomial_Xsig(&accumulator, &Xsq, logterms, HIPOWER - 1);
-
- mul_Xsig_Xsig(&accumulator, &argSignif);
- shr_Xsig(&accumulator, 6 - adj);
-
- mul32_Xsig(&arg_signif, leadterm);
- add_two_Xsig(&accumulator, &arg_signif, &exponent);
-
- *expon = exponent + 1;
- accum_result->lsw = accumulator.lsw;
- accum_result->midw = accumulator.midw;
- accum_result->msw = accumulator.msw;
-
-}
diff --git a/arch/x86/math-emu/poly_sin.c b/arch/x86/math-emu/poly_sin.c
deleted file mode 100644
index c192fba..0000000
--- a/arch/x86/math-emu/poly_sin.c
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | poly_sin.c |
- | |
- | Computation of an approximation of the sin function and the cosine |
- | function by a polynomial. |
- | |
- | Copyright (C) 1992,1993,1994,1997,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@melbpc.org.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define N_COEFF_P 4
-#define N_COEFF_N 4
-
-static const unsigned long long pos_terms_l[N_COEFF_P] = {
- 0xaaaaaaaaaaaaaaabLL,
- 0x00d00d00d00cf906LL,
- 0x000006b99159a8bbLL,
- 0x000000000d7392e6LL
-};
-
-static const unsigned long long neg_terms_l[N_COEFF_N] = {
- 0x2222222222222167LL,
- 0x0002e3bc74aab624LL,
- 0x0000000b09229062LL,
- 0x00000000000c7973LL
-};
-
-#define N_COEFF_PH 4
-#define N_COEFF_NH 4
-static const unsigned long long pos_terms_h[N_COEFF_PH] = {
- 0x0000000000000000LL,
- 0x05b05b05b05b0406LL,
- 0x000049f93edd91a9LL,
- 0x00000000c9c9ed62LL
-};
-
-static const unsigned long long neg_terms_h[N_COEFF_NH] = {
- 0xaaaaaaaaaaaaaa98LL,
- 0x001a01a01a019064LL,
- 0x0000008f76c68a77LL,
- 0x0000000000d58f5eLL
-};
-
-/*--- poly_sine() -----------------------------------------------------------+
- | |
- +---------------------------------------------------------------------------*/
-void poly_sine(FPU_REG *st0_ptr)
-{
- int exponent, echange;
- Xsig accumulator, argSqrd, argTo4;
- unsigned long fix_up, adj;
- unsigned long long fixed_arg;
- FPU_REG result;
-
- exponent = exponent(st0_ptr);
-
- accumulator.lsw = accumulator.midw = accumulator.msw = 0;
-
- /* Split into two ranges, for arguments below and above 1.0 */
- /* The boundary between upper and lower is approx 0.88309101259 */
- if ((exponent < -1)
- || ((exponent == -1) && (st0_ptr->sigh <= 0xe21240aa))) {
- /* The argument is <= 0.88309101259 */
-
- argSqrd.msw = st0_ptr->sigh;
- argSqrd.midw = st0_ptr->sigl;
- argSqrd.lsw = 0;
- mul64_Xsig(&argSqrd, &significand(st0_ptr));
- shr_Xsig(&argSqrd, 2 * (-1 - exponent));
- argTo4.msw = argSqrd.msw;
- argTo4.midw = argSqrd.midw;
- argTo4.lsw = argSqrd.lsw;
- mul_Xsig_Xsig(&argTo4, &argTo4);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l,
- N_COEFF_N - 1);
- mul_Xsig_Xsig(&accumulator, &argSqrd);
- negate_Xsig(&accumulator);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l,
- N_COEFF_P - 1);
-
- shr_Xsig(&accumulator, 2); /* Divide by four */
- accumulator.msw |= 0x80000000; /* Add 1.0 */
-
- mul64_Xsig(&accumulator, &significand(st0_ptr));
- mul64_Xsig(&accumulator, &significand(st0_ptr));
- mul64_Xsig(&accumulator, &significand(st0_ptr));
-
- /* Divide by four, FPU_REG compatible, etc */
- exponent = 3 * exponent;
-
- /* The minimum exponent difference is 3 */
- shr_Xsig(&accumulator, exponent(st0_ptr) - exponent);
-
- negate_Xsig(&accumulator);
- XSIG_LL(accumulator) += significand(st0_ptr);
-
- echange = round_Xsig(&accumulator);
-
- setexponentpos(&result, exponent(st0_ptr) + echange);
- } else {
- /* The argument is > 0.88309101259 */
- /* We use sin(st(0)) = cos(pi/2-st(0)) */
-
- fixed_arg = significand(st0_ptr);
-
- if (exponent == 0) {
- /* The argument is >= 1.0 */
-
- /* Put the binary point at the left. */
- fixed_arg <<= 1;
- }
- /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
- fixed_arg = 0x921fb54442d18469LL - fixed_arg;
- /* There is a special case which arises due to rounding, to fix here. */
- if (fixed_arg == 0xffffffffffffffffLL)
- fixed_arg = 0;
-
- XSIG_LL(argSqrd) = fixed_arg;
- argSqrd.lsw = 0;
- mul64_Xsig(&argSqrd, &fixed_arg);
-
- XSIG_LL(argTo4) = XSIG_LL(argSqrd);
- argTo4.lsw = argSqrd.lsw;
- mul_Xsig_Xsig(&argTo4, &argTo4);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h,
- N_COEFF_NH - 1);
- mul_Xsig_Xsig(&accumulator, &argSqrd);
- negate_Xsig(&accumulator);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h,
- N_COEFF_PH - 1);
- negate_Xsig(&accumulator);
-
- mul64_Xsig(&accumulator, &fixed_arg);
- mul64_Xsig(&accumulator, &fixed_arg);
-
- shr_Xsig(&accumulator, 3);
- negate_Xsig(&accumulator);
-
- add_Xsig_Xsig(&accumulator, &argSqrd);
-
- shr_Xsig(&accumulator, 1);
-
- accumulator.lsw |= 1; /* A zero accumulator here would cause problems */
- negate_Xsig(&accumulator);
-
- /* The basic computation is complete. Now fix the answer to
- compensate for the error due to the approximation used for
- pi/2
- */
-
- /* This has an exponent of -65 */
- fix_up = 0x898cc517;
- /* The fix-up needs to be improved for larger args */
- if (argSqrd.msw & 0xffc00000) {
- /* Get about 32 bit precision in these: */
- fix_up -= mul_32_32(0x898cc517, argSqrd.msw) / 6;
- }
- fix_up = mul_32_32(fix_up, LL_MSW(fixed_arg));
-
- adj = accumulator.lsw; /* temp save */
- accumulator.lsw -= fix_up;
- if (accumulator.lsw > adj)
- XSIG_LL(accumulator)--;
-
- echange = round_Xsig(&accumulator);
-
- setexponentpos(&result, echange - 1);
- }
-
- significand(&result) = XSIG_LL(accumulator);
- setsign(&result, getsign(st0_ptr));
- FPU_copy_to_reg0(&result, TAG_Valid);
-
-#ifdef PARANOID
- if ((exponent(&result) >= 0)
- && (significand(&result) > 0x8000000000000000LL)) {
- EXCEPTION(EX_INTERNAL | 0x150);
- }
-#endif /* PARANOID */
-
-}
-
-/*--- poly_cos() ------------------------------------------------------------+
- | |
- +---------------------------------------------------------------------------*/
-void poly_cos(FPU_REG *st0_ptr)
-{
- FPU_REG result;
- long int exponent, exp2, echange;
- Xsig accumulator, argSqrd, fix_up, argTo4;
- unsigned long long fixed_arg;
-
-#ifdef PARANOID
- if ((exponent(st0_ptr) > 0)
- || ((exponent(st0_ptr) == 0)
- && (significand(st0_ptr) > 0xc90fdaa22168c234LL))) {
- EXCEPTION(EX_Invalid);
- FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
- return;
- }
-#endif /* PARANOID */
-
- exponent = exponent(st0_ptr);
-
- accumulator.lsw = accumulator.midw = accumulator.msw = 0;
-
- if ((exponent < -1)
- || ((exponent == -1) && (st0_ptr->sigh <= 0xb00d6f54))) {
- /* arg is < 0.687705 */
-
- argSqrd.msw = st0_ptr->sigh;
- argSqrd.midw = st0_ptr->sigl;
- argSqrd.lsw = 0;
- mul64_Xsig(&argSqrd, &significand(st0_ptr));
-
- if (exponent < -1) {
- /* shift the argument right by the required places */
- shr_Xsig(&argSqrd, 2 * (-1 - exponent));
- }
-
- argTo4.msw = argSqrd.msw;
- argTo4.midw = argSqrd.midw;
- argTo4.lsw = argSqrd.lsw;
- mul_Xsig_Xsig(&argTo4, &argTo4);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h,
- N_COEFF_NH - 1);
- mul_Xsig_Xsig(&accumulator, &argSqrd);
- negate_Xsig(&accumulator);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h,
- N_COEFF_PH - 1);
- negate_Xsig(&accumulator);
-
- mul64_Xsig(&accumulator, &significand(st0_ptr));
- mul64_Xsig(&accumulator, &significand(st0_ptr));
- shr_Xsig(&accumulator, -2 * (1 + exponent));
-
- shr_Xsig(&accumulator, 3);
- negate_Xsig(&accumulator);
-
- add_Xsig_Xsig(&accumulator, &argSqrd);
-
- shr_Xsig(&accumulator, 1);
-
- /* It doesn't matter if accumulator is all zero here, the
- following code will work ok */
- negate_Xsig(&accumulator);
-
- if (accumulator.lsw & 0x80000000)
- XSIG_LL(accumulator)++;
- if (accumulator.msw == 0) {
- /* The result is 1.0 */
- FPU_copy_to_reg0(&CONST_1, TAG_Valid);
- return;
- } else {
- significand(&result) = XSIG_LL(accumulator);
-
- /* will be a valid positive nr with expon = -1 */
- setexponentpos(&result, -1);
- }
- } else {
- fixed_arg = significand(st0_ptr);
-
- if (exponent == 0) {
- /* The argument is >= 1.0 */
-
- /* Put the binary point at the left. */
- fixed_arg <<= 1;
- }
- /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
- fixed_arg = 0x921fb54442d18469LL - fixed_arg;
- /* There is a special case which arises due to rounding, to fix here. */
- if (fixed_arg == 0xffffffffffffffffLL)
- fixed_arg = 0;
-
- exponent = -1;
- exp2 = -1;
-
- /* A shift is needed here only for a narrow range of arguments,
- i.e. for fixed_arg approx 2^-32, but we pick up more... */
- if (!(LL_MSW(fixed_arg) & 0xffff0000)) {
- fixed_arg <<= 16;
- exponent -= 16;
- exp2 -= 16;
- }
-
- XSIG_LL(argSqrd) = fixed_arg;
- argSqrd.lsw = 0;
- mul64_Xsig(&argSqrd, &fixed_arg);
-
- if (exponent < -1) {
- /* shift the argument right by the required places */
- shr_Xsig(&argSqrd, 2 * (-1 - exponent));
- }
-
- argTo4.msw = argSqrd.msw;
- argTo4.midw = argSqrd.midw;
- argTo4.lsw = argSqrd.lsw;
- mul_Xsig_Xsig(&argTo4, &argTo4);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l,
- N_COEFF_N - 1);
- mul_Xsig_Xsig(&accumulator, &argSqrd);
- negate_Xsig(&accumulator);
-
- polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l,
- N_COEFF_P - 1);
-
- shr_Xsig(&accumulator, 2); /* Divide by four */
- accumulator.msw |= 0x80000000; /* Add 1.0 */
-
- mul64_Xsig(&accumulator, &fixed_arg);
- mul64_Xsig(&accumulator, &fixed_arg);
- mul64_Xsig(&accumulator, &fixed_arg);
-
- /* Divide by four, FPU_REG compatible, etc */
- exponent = 3 * exponent;
-
- /* The minimum exponent difference is 3 */
- shr_Xsig(&accumulator, exp2 - exponent);
-
- negate_Xsig(&accumulator);
- XSIG_LL(accumulator) += fixed_arg;
-
- /* The basic computation is complete. Now fix the answer to
- compensate for the error due to the approximation used for
- pi/2
- */
-
- /* This has an exponent of -65 */
- XSIG_LL(fix_up) = 0x898cc51701b839a2ll;
- fix_up.lsw = 0;
-
- /* The fix-up needs to be improved for larger args */
- if (argSqrd.msw & 0xffc00000) {
- /* Get about 32 bit precision in these: */
- fix_up.msw -= mul_32_32(0x898cc517, argSqrd.msw) / 2;
- fix_up.msw += mul_32_32(0x898cc517, argTo4.msw) / 24;
- }
-
- exp2 += norm_Xsig(&accumulator);
- shr_Xsig(&accumulator, 1); /* Prevent overflow */
- exp2++;
- shr_Xsig(&fix_up, 65 + exp2);
-
- add_Xsig_Xsig(&accumulator, &fix_up);
-
- echange = round_Xsig(&accumulator);
-
- setexponentpos(&result, exp2 + echange);
- significand(&result) = XSIG_LL(accumulator);
- }
-
- FPU_copy_to_reg0(&result, TAG_Valid);
-
-#ifdef PARANOID
- if ((exponent(&result) >= 0)
- && (significand(&result) > 0x8000000000000000LL)) {
- EXCEPTION(EX_INTERNAL | 0x151);
- }
-#endif /* PARANOID */
-
-}
diff --git a/arch/x86/math-emu/poly_tan.c b/arch/x86/math-emu/poly_tan.c
deleted file mode 100644
index 1f5b1d7..0000000
--- a/arch/x86/math-emu/poly_tan.c
+++ /dev/null
@@ -1,213 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | poly_tan.c |
- | |
- | Compute the tan of a FPU_REG, using a polynomial approximation. |
- | |
- | Copyright (C) 1992,1993,1994,1997,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@melbpc.org.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define HiPOWERop 3 /* odd poly, positive terms */
-static const unsigned long long oddplterm[HiPOWERop] = {
- 0x0000000000000000LL,
- 0x0051a1cf08fca228LL,
- 0x0000000071284ff7LL
-};
-
-#define HiPOWERon 2 /* odd poly, negative terms */
-static const unsigned long long oddnegterm[HiPOWERon] = {
- 0x1291a9a184244e80LL,
- 0x0000583245819c21LL
-};
-
-#define HiPOWERep 2 /* even poly, positive terms */
-static const unsigned long long evenplterm[HiPOWERep] = {
- 0x0e848884b539e888LL,
- 0x00003c7f18b887daLL
-};
-
-#define HiPOWERen 2 /* even poly, negative terms */
-static const unsigned long long evennegterm[HiPOWERen] = {
- 0xf1f0200fd51569ccLL,
- 0x003afb46105c4432LL
-};
-
-static const unsigned long long twothirds = 0xaaaaaaaaaaaaaaabLL;
-
-/*--- poly_tan() ------------------------------------------------------------+
- | |
- +---------------------------------------------------------------------------*/
-void poly_tan(FPU_REG *st0_ptr)
-{
- long int exponent;
- int invert;
- Xsig argSq, argSqSq, accumulatoro, accumulatore, accum,
- argSignif, fix_up;
- unsigned long adj;
-
- exponent = exponent(st0_ptr);
-
-#ifdef PARANOID
- if (signnegative(st0_ptr)) { /* Can't hack a number < 0.0 */
- arith_invalid(0);
- return;
- } /* Need a positive number */
-#endif /* PARANOID */
-
- /* Split the problem into two domains, smaller and larger than pi/4 */
- if ((exponent == 0)
- || ((exponent == -1) && (st0_ptr->sigh > 0xc90fdaa2))) {
- /* The argument is greater than (approx) pi/4 */
- invert = 1;
- accum.lsw = 0;
- XSIG_LL(accum) = significand(st0_ptr);
-
- if (exponent == 0) {
- /* The argument is >= 1.0 */
- /* Put the binary point at the left. */
- XSIG_LL(accum) <<= 1;
- }
- /* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
- XSIG_LL(accum) = 0x921fb54442d18469LL - XSIG_LL(accum);
- /* This is a special case which arises due to rounding. */
- if (XSIG_LL(accum) == 0xffffffffffffffffLL) {
- FPU_settag0(TAG_Valid);
- significand(st0_ptr) = 0x8a51e04daabda360LL;
- setexponent16(st0_ptr,
- (0x41 + EXTENDED_Ebias) | SIGN_Negative);
- return;
- }
-
- argSignif.lsw = accum.lsw;
- XSIG_LL(argSignif) = XSIG_LL(accum);
- exponent = -1 + norm_Xsig(&argSignif);
- } else {
- invert = 0;
- argSignif.lsw = 0;
- XSIG_LL(accum) = XSIG_LL(argSignif) = significand(st0_ptr);
-
- if (exponent < -1) {
- /* shift the argument right by the required places */
- if (FPU_shrx(&XSIG_LL(accum), -1 - exponent) >=
- 0x80000000U)
- XSIG_LL(accum)++; /* round up */
- }
- }
-
- XSIG_LL(argSq) = XSIG_LL(accum);
- argSq.lsw = accum.lsw;
- mul_Xsig_Xsig(&argSq, &argSq);
- XSIG_LL(argSqSq) = XSIG_LL(argSq);
- argSqSq.lsw = argSq.lsw;
- mul_Xsig_Xsig(&argSqSq, &argSqSq);
-
- /* Compute the negative terms for the numerator polynomial */
- accumulatoro.msw = accumulatoro.midw = accumulatoro.lsw = 0;
- polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddnegterm,
- HiPOWERon - 1);
- mul_Xsig_Xsig(&accumulatoro, &argSq);
- negate_Xsig(&accumulatoro);
- /* Add the positive terms */
- polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddplterm,
- HiPOWERop - 1);
-
- /* Compute the positive terms for the denominator polynomial */
- accumulatore.msw = accumulatore.midw = accumulatore.lsw = 0;
- polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evenplterm,
- HiPOWERep - 1);
- mul_Xsig_Xsig(&accumulatore, &argSq);
- negate_Xsig(&accumulatore);
- /* Add the negative terms */
- polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evennegterm,
- HiPOWERen - 1);
- /* Multiply by arg^2 */
- mul64_Xsig(&accumulatore, &XSIG_LL(argSignif));
- mul64_Xsig(&accumulatore, &XSIG_LL(argSignif));
- /* de-normalize and divide by 2 */
- shr_Xsig(&accumulatore, -2 * (1 + exponent) + 1);
- negate_Xsig(&accumulatore); /* This does 1 - accumulator */
-
- /* Now find the ratio. */
- if (accumulatore.msw == 0) {
- /* accumulatoro must contain 1.0 here, (actually, 0) but it
- really doesn't matter what value we use because it will
- have negligible effect in later calculations
- */
- XSIG_LL(accum) = 0x8000000000000000LL;
- accum.lsw = 0;
- } else {
- div_Xsig(&accumulatoro, &accumulatore, &accum);
- }
-
- /* Multiply by 1/3 * arg^3 */
- mul64_Xsig(&accum, &XSIG_LL(argSignif));
- mul64_Xsig(&accum, &XSIG_LL(argSignif));
- mul64_Xsig(&accum, &XSIG_LL(argSignif));
- mul64_Xsig(&accum, &twothirds);
- shr_Xsig(&accum, -2 * (exponent + 1));
-
- /* tan(arg) = arg + accum */
- add_two_Xsig(&accum, &argSignif, &exponent);
-
- if (invert) {
- /* We now have the value of tan(pi_2 - arg) where pi_2 is an
- approximation for pi/2
- */
- /* The next step is to fix the answer to compensate for the
- error due to the approximation used for pi/2
- */
-
- /* This is (approx) delta, the error in our approx for pi/2
- (see above). It has an exponent of -65
- */
- XSIG_LL(fix_up) = 0x898cc51701b839a2LL;
- fix_up.lsw = 0;
-
- if (exponent == 0)
- adj = 0xffffffff; /* We want approx 1.0 here, but
- this is close enough. */
- else if (exponent > -30) {
- adj = accum.msw >> -(exponent + 1); /* tan */
- adj = mul_32_32(adj, adj); /* tan^2 */
- } else
- adj = 0;
- adj = mul_32_32(0x898cc517, adj); /* delta * tan^2 */
-
- fix_up.msw += adj;
- if (!(fix_up.msw & 0x80000000)) { /* did fix_up overflow ? */
- /* Yes, we need to add an msb */
- shr_Xsig(&fix_up, 1);
- fix_up.msw |= 0x80000000;
- shr_Xsig(&fix_up, 64 + exponent);
- } else
- shr_Xsig(&fix_up, 65 + exponent);
-
- add_two_Xsig(&accum, &fix_up, &exponent);
-
- /* accum now contains tan(pi/2 - arg).
- Use tan(arg) = 1.0 / tan(pi/2 - arg)
- */
- accumulatoro.lsw = accumulatoro.midw = 0;
- accumulatoro.msw = 0x80000000;
- div_Xsig(&accumulatoro, &accum, &accum);
- exponent = -exponent - 1;
- }
-
- /* Transfer the result */
- round_Xsig(&accum);
- FPU_settag0(TAG_Valid);
- significand(st0_ptr) = XSIG_LL(accum);
- setexponent16(st0_ptr, exponent + EXTENDED_Ebias); /* Result is positive. */
-
-}
diff --git a/arch/x86/math-emu/polynom_Xsig.S b/arch/x86/math-emu/polynom_Xsig.S
deleted file mode 100644
index 35fd723..0000000
--- a/arch/x86/math-emu/polynom_Xsig.S
+++ /dev/null
@@ -1,137 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | polynomial_Xsig.S |
- | |
- | Fixed point arithmetic polynomial evaluation. |
- | |
- | Copyright (C) 1992,1993,1994,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Call from C as: |
- | void polynomial_Xsig(Xsig *accum, unsigned long long x, |
- | unsigned long long terms[], int n) |
- | |
- | Computes: |
- | terms[0] + (terms[1] + (terms[2] + ... + (terms[n-1]*x)*x)*x)*x) ... )*x |
- | and adds the result to the 12 byte Xsig. |
- | The terms[] are each 8 bytes, but all computation is performed to 12 byte |
- | precision. |
- | |
- | This function must be used carefully: most overflow of intermediate |
- | results is controlled, but overflow of the result is not. |
- | |
- +---------------------------------------------------------------------------*/
- .file "polynomial_Xsig.S"
-
-#include "fpu_emu.h"
-
-
-#define TERM_SIZE $8
-#define SUM_MS -20(%ebp) /* sum ms long */
-#define SUM_MIDDLE -24(%ebp) /* sum middle long */
-#define SUM_LS -28(%ebp) /* sum ls long */
-#define ACCUM_MS -4(%ebp) /* accum ms long */
-#define ACCUM_MIDDLE -8(%ebp) /* accum middle long */
-#define ACCUM_LS -12(%ebp) /* accum ls long */
-#define OVERFLOWED -16(%ebp) /* addition overflow flag */
-
-.text
-SYM_FUNC_START(polynomial_Xsig)
- pushl %ebp
- movl %esp,%ebp
- subl $32,%esp
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM2,%esi /* x */
- movl PARAM3,%edi /* terms */
-
- movl TERM_SIZE,%eax
- mull PARAM4 /* n */
- addl %eax,%edi
-
- movl 4(%edi),%edx /* terms[n] */
- movl %edx,SUM_MS
- movl (%edi),%edx /* terms[n] */
- movl %edx,SUM_MIDDLE
- xor %eax,%eax
- movl %eax,SUM_LS
- movb %al,OVERFLOWED
-
- subl TERM_SIZE,%edi
- decl PARAM4
- js L_accum_done
-
-L_accum_loop:
- xor %eax,%eax
- movl %eax,ACCUM_MS
- movl %eax,ACCUM_MIDDLE
-
- movl SUM_MIDDLE,%eax
- mull (%esi) /* x ls long */
- movl %edx,ACCUM_LS
-
- movl SUM_MIDDLE,%eax
- mull 4(%esi) /* x ms long */
- addl %eax,ACCUM_LS
- adcl %edx,ACCUM_MIDDLE
- adcl $0,ACCUM_MS
-
- movl SUM_MS,%eax
- mull (%esi) /* x ls long */
- addl %eax,ACCUM_LS
- adcl %edx,ACCUM_MIDDLE
- adcl $0,ACCUM_MS
-
- movl SUM_MS,%eax
- mull 4(%esi) /* x ms long */
- addl %eax,ACCUM_MIDDLE
- adcl %edx,ACCUM_MS
-
- testb $0xff,OVERFLOWED
- jz L_no_overflow
-
- movl (%esi),%eax
- addl %eax,ACCUM_MIDDLE
- movl 4(%esi),%eax
- adcl %eax,ACCUM_MS /* This could overflow too */
-
-L_no_overflow:
-
-/*
- * Now put the sum of next term and the accumulator
- * into the sum register
- */
- movl ACCUM_LS,%eax
- addl (%edi),%eax /* term ls long */
- movl %eax,SUM_LS
- movl ACCUM_MIDDLE,%eax
- adcl (%edi),%eax /* term ls long */
- movl %eax,SUM_MIDDLE
- movl ACCUM_MS,%eax
- adcl 4(%edi),%eax /* term ms long */
- movl %eax,SUM_MS
- sbbb %al,%al
- movb %al,OVERFLOWED /* Used in the next iteration */
-
- subl TERM_SIZE,%edi
- decl PARAM4
- jns L_accum_loop
-
-L_accum_done:
- movl PARAM1,%edi /* accum */
- movl SUM_LS,%eax
- addl %eax,(%edi)
- movl SUM_MIDDLE,%eax
- adcl %eax,4(%edi)
- movl SUM_MS,%eax
- adcl %eax,8(%edi)
-
- popl %ebx
- popl %edi
- popl %esi
- leave
- RET
-SYM_FUNC_END(polynomial_Xsig)
diff --git a/arch/x86/math-emu/reg_add_sub.c b/arch/x86/math-emu/reg_add_sub.c
deleted file mode 100644
index 29451dd..0000000
--- a/arch/x86/math-emu/reg_add_sub.c
+++ /dev/null
@@ -1,334 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_add_sub.c |
- | |
- | Functions to add or subtract two registers and put the result in a third. |
- | |
- | Copyright (C) 1992,1993,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | For each function, the destination may be any FPU_REG, including one of |
- | the source FPU_REGs. |
- | Each function returns 0 if the answer is o.k., otherwise a non-zero |
- | value is returned, indicating either an exception condition or an |
- | internal error. |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "fpu_system.h"
-
-static
-int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa,
- FPU_REG const *b, u_char tagb, u_char signb,
- FPU_REG * dest, int deststnr, int control_w);
-
-/*
- Operates on st(0) and st(n), or on st(0) and temporary data.
- The destination must be one of the source st(x).
- */
-int FPU_add(FPU_REG const *b, u_char tagb, int deststnr, int control_w)
-{
- FPU_REG *a = &st(0);
- FPU_REG *dest = &st(deststnr);
- u_char signb = getsign(b);
- u_char taga = FPU_gettag0();
- u_char signa = getsign(a);
- u_char saved_sign = getsign(dest);
- int diff, tag, expa, expb;
-
- if (!(taga | tagb)) {
- expa = exponent(a);
- expb = exponent(b);
-
- valid_add:
- /* Both registers are valid */
- if (!(signa ^ signb)) {
- /* signs are the same */
- tag =
- FPU_u_add(a, b, dest, control_w, signa, expa, expb);
- } else {
- /* The signs are different, so do a subtraction */
- diff = expa - expb;
- if (!diff) {
- diff = a->sigh - b->sigh; /* This works only if the ms bits
- are identical. */
- if (!diff) {
- diff = a->sigl > b->sigl;
- if (!diff)
- diff = -(a->sigl < b->sigl);
- }
- }
-
- if (diff > 0) {
- tag =
- FPU_u_sub(a, b, dest, control_w, signa,
- expa, expb);
- } else if (diff < 0) {
- tag =
- FPU_u_sub(b, a, dest, control_w, signb,
- expb, expa);
- } else {
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
- /* sign depends upon rounding mode */
- setsign(dest, ((control_w & CW_RC) != RC_DOWN)
- ? SIGN_POS : SIGN_NEG);
- return TAG_Zero;
- }
- }
-
- if (tag < 0) {
- setsign(dest, saved_sign);
- return tag;
- }
- FPU_settagi(deststnr, tag);
- return tag;
- }
-
- if (taga == TAG_Special)
- taga = FPU_Special(a);
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
-
- if (((taga == TAG_Valid) && (tagb == TW_Denormal))
- || ((taga == TW_Denormal) && (tagb == TAG_Valid))
- || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
- FPU_REG x, y;
-
- if (denormal_operand() < 0)
- return FPU_Exception;
-
- FPU_to_exp16(a, &x);
- FPU_to_exp16(b, &y);
- a = &x;
- b = &y;
- expa = exponent16(a);
- expb = exponent16(b);
- goto valid_add;
- }
-
- if ((taga == TW_NaN) || (tagb == TW_NaN)) {
- if (deststnr == 0)
- return real_2op_NaN(b, tagb, deststnr, a);
- else
- return real_2op_NaN(a, taga, deststnr, a);
- }
-
- return add_sub_specials(a, taga, signa, b, tagb, signb,
- dest, deststnr, control_w);
-}
-
-/* Subtract b from a. (a-b) -> dest */
-int FPU_sub(int flags, int rm, int control_w)
-{
- FPU_REG const *a, *b;
- FPU_REG *dest;
- u_char taga, tagb, signa, signb, saved_sign, sign;
- int diff, tag = 0, expa, expb, deststnr;
-
- a = &st(0);
- taga = FPU_gettag0();
-
- deststnr = 0;
- if (flags & LOADED) {
- b = (FPU_REG *) rm;
- tagb = flags & 0x0f;
- } else {
- b = &st(rm);
- tagb = FPU_gettagi(rm);
-
- if (flags & DEST_RM)
- deststnr = rm;
- }
-
- signa = getsign(a);
- signb = getsign(b);
-
- if (flags & REV) {
- signa ^= SIGN_NEG;
- signb ^= SIGN_NEG;
- }
-
- dest = &st(deststnr);
- saved_sign = getsign(dest);
-
- if (!(taga | tagb)) {
- expa = exponent(a);
- expb = exponent(b);
-
- valid_subtract:
- /* Both registers are valid */
-
- diff = expa - expb;
-
- if (!diff) {
- diff = a->sigh - b->sigh; /* Works only if ms bits are identical */
- if (!diff) {
- diff = a->sigl > b->sigl;
- if (!diff)
- diff = -(a->sigl < b->sigl);
- }
- }
-
- switch ((((int)signa) * 2 + signb) / SIGN_NEG) {
- case 0: /* P - P */
- case 3: /* N - N */
- if (diff > 0) {
- /* |a| > |b| */
- tag =
- FPU_u_sub(a, b, dest, control_w, signa,
- expa, expb);
- } else if (diff == 0) {
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-
- /* sign depends upon rounding mode */
- setsign(dest, ((control_w & CW_RC) != RC_DOWN)
- ? SIGN_POS : SIGN_NEG);
- return TAG_Zero;
- } else {
- sign = signa ^ SIGN_NEG;
- tag =
- FPU_u_sub(b, a, dest, control_w, sign, expb,
- expa);
- }
- break;
- case 1: /* P - N */
- tag =
- FPU_u_add(a, b, dest, control_w, SIGN_POS, expa,
- expb);
- break;
- case 2: /* N - P */
- tag =
- FPU_u_add(a, b, dest, control_w, SIGN_NEG, expa,
- expb);
- break;
-#ifdef PARANOID
- default:
- EXCEPTION(EX_INTERNAL | 0x111);
- return -1;
-#endif
- }
- if (tag < 0) {
- setsign(dest, saved_sign);
- return tag;
- }
- FPU_settagi(deststnr, tag);
- return tag;
- }
-
- if (taga == TAG_Special)
- taga = FPU_Special(a);
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
-
- if (((taga == TAG_Valid) && (tagb == TW_Denormal))
- || ((taga == TW_Denormal) && (tagb == TAG_Valid))
- || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
- FPU_REG x, y;
-
- if (denormal_operand() < 0)
- return FPU_Exception;
-
- FPU_to_exp16(a, &x);
- FPU_to_exp16(b, &y);
- a = &x;
- b = &y;
- expa = exponent16(a);
- expb = exponent16(b);
-
- goto valid_subtract;
- }
-
- if ((taga == TW_NaN) || (tagb == TW_NaN)) {
- FPU_REG const *d1, *d2;
- if (flags & REV) {
- d1 = b;
- d2 = a;
- } else {
- d1 = a;
- d2 = b;
- }
- if (flags & LOADED)
- return real_2op_NaN(b, tagb, deststnr, d1);
- if (flags & DEST_RM)
- return real_2op_NaN(a, taga, deststnr, d2);
- else
- return real_2op_NaN(b, tagb, deststnr, d2);
- }
-
- return add_sub_specials(a, taga, signa, b, tagb, signb ^ SIGN_NEG,
- dest, deststnr, control_w);
-}
-
-static
-int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa,
- FPU_REG const *b, u_char tagb, u_char signb,
- FPU_REG * dest, int deststnr, int control_w)
-{
- if (((taga == TW_Denormal) || (tagb == TW_Denormal))
- && (denormal_operand() < 0))
- return FPU_Exception;
-
- if (taga == TAG_Zero) {
- if (tagb == TAG_Zero) {
- /* Both are zero, result will be zero. */
- u_char different_signs = signa ^ signb;
-
- FPU_copy_to_regi(a, TAG_Zero, deststnr);
- if (different_signs) {
- /* Signs are different. */
- /* Sign of answer depends upon rounding mode. */
- setsign(dest, ((control_w & CW_RC) != RC_DOWN)
- ? SIGN_POS : SIGN_NEG);
- } else
- setsign(dest, signa); /* signa may differ from the sign of a. */
- return TAG_Zero;
- } else {
- reg_copy(b, dest);
- if ((tagb == TW_Denormal) && (b->sigh & 0x80000000)) {
- /* A pseudoDenormal, convert it. */
- addexponent(dest, 1);
- tagb = TAG_Valid;
- } else if (tagb > TAG_Empty)
- tagb = TAG_Special;
- setsign(dest, signb); /* signb may differ from the sign of b. */
- FPU_settagi(deststnr, tagb);
- return tagb;
- }
- } else if (tagb == TAG_Zero) {
- reg_copy(a, dest);
- if ((taga == TW_Denormal) && (a->sigh & 0x80000000)) {
- /* A pseudoDenormal */
- addexponent(dest, 1);
- taga = TAG_Valid;
- } else if (taga > TAG_Empty)
- taga = TAG_Special;
- setsign(dest, signa); /* signa may differ from the sign of a. */
- FPU_settagi(deststnr, taga);
- return taga;
- } else if (taga == TW_Infinity) {
- if ((tagb != TW_Infinity) || (signa == signb)) {
- FPU_copy_to_regi(a, TAG_Special, deststnr);
- setsign(dest, signa); /* signa may differ from the sign of a. */
- return taga;
- }
- /* Infinity-Infinity is undefined. */
- return arith_invalid(deststnr);
- } else if (tagb == TW_Infinity) {
- FPU_copy_to_regi(b, TAG_Special, deststnr);
- setsign(dest, signb); /* signb may differ from the sign of b. */
- return tagb;
- }
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x101);
-#endif
-
- return FPU_Exception;
-}
diff --git a/arch/x86/math-emu/reg_compare.c b/arch/x86/math-emu/reg_compare.c
deleted file mode 100644
index eacb512..0000000
--- a/arch/x86/math-emu/reg_compare.c
+++ /dev/null
@@ -1,479 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_compare.c |
- | |
- | Compare two floating point registers |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | compare() is the core FPU_REG comparison function |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "status_w.h"
-
-static int compare(FPU_REG const *b, int tagb)
-{
- int diff, exp0, expb;
- u_char st0_tag;
- FPU_REG *st0_ptr;
- FPU_REG x, y;
- u_char st0_sign, signb = getsign(b);
-
- st0_ptr = &st(0);
- st0_tag = FPU_gettag0();
- st0_sign = getsign(st0_ptr);
-
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
- if (st0_tag == TAG_Special)
- st0_tag = FPU_Special(st0_ptr);
-
- if (((st0_tag != TAG_Valid) && (st0_tag != TW_Denormal))
- || ((tagb != TAG_Valid) && (tagb != TW_Denormal))) {
- if (st0_tag == TAG_Zero) {
- if (tagb == TAG_Zero)
- return COMP_A_eq_B;
- if (tagb == TAG_Valid)
- return ((signb ==
- SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B);
- if (tagb == TW_Denormal)
- return ((signb ==
- SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
- | COMP_Denormal;
- } else if (tagb == TAG_Zero) {
- if (st0_tag == TAG_Valid)
- return ((st0_sign ==
- SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
- if (st0_tag == TW_Denormal)
- return ((st0_sign ==
- SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
- | COMP_Denormal;
- }
-
- if (st0_tag == TW_Infinity) {
- if ((tagb == TAG_Valid) || (tagb == TAG_Zero))
- return ((st0_sign ==
- SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
- else if (tagb == TW_Denormal)
- return ((st0_sign ==
- SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
- | COMP_Denormal;
- else if (tagb == TW_Infinity) {
- /* The 80486 book says that infinities can be equal! */
- return (st0_sign == signb) ? COMP_A_eq_B :
- ((st0_sign ==
- SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
- }
- /* Fall through to the NaN code */
- } else if (tagb == TW_Infinity) {
- if ((st0_tag == TAG_Valid) || (st0_tag == TAG_Zero))
- return ((signb ==
- SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B);
- if (st0_tag == TW_Denormal)
- return ((signb ==
- SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
- | COMP_Denormal;
- /* Fall through to the NaN code */
- }
-
- /* The only possibility now should be that one of the arguments
- is a NaN */
- if ((st0_tag == TW_NaN) || (tagb == TW_NaN)) {
- int signalling = 0, unsupported = 0;
- if (st0_tag == TW_NaN) {
- signalling =
- (st0_ptr->sigh & 0xc0000000) == 0x80000000;
- unsupported = !((exponent(st0_ptr) == EXP_OVER)
- && (st0_ptr->
- sigh & 0x80000000));
- }
- if (tagb == TW_NaN) {
- signalling |=
- (b->sigh & 0xc0000000) == 0x80000000;
- unsupported |= !((exponent(b) == EXP_OVER)
- && (b->sigh & 0x80000000));
- }
- if (signalling || unsupported)
- return COMP_No_Comp | COMP_SNaN | COMP_NaN;
- else
- /* Neither is a signaling NaN */
- return COMP_No_Comp | COMP_NaN;
- }
-
- EXCEPTION(EX_Invalid);
- }
-
- if (st0_sign != signb) {
- return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
- | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
- COMP_Denormal : 0);
- }
-
- if ((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) {
- FPU_to_exp16(st0_ptr, &x);
- FPU_to_exp16(b, &y);
- st0_ptr = &x;
- b = &y;
- exp0 = exponent16(st0_ptr);
- expb = exponent16(b);
- } else {
- exp0 = exponent(st0_ptr);
- expb = exponent(b);
- }
-
-#ifdef PARANOID
- if (!(st0_ptr->sigh & 0x80000000))
- EXCEPTION(EX_Invalid);
- if (!(b->sigh & 0x80000000))
- EXCEPTION(EX_Invalid);
-#endif /* PARANOID */
-
- diff = exp0 - expb;
- if (diff == 0) {
- diff = st0_ptr->sigh - b->sigh; /* Works only if ms bits are
- identical */
- if (diff == 0) {
- diff = st0_ptr->sigl > b->sigl;
- if (diff == 0)
- diff = -(st0_ptr->sigl < b->sigl);
- }
- }
-
- if (diff > 0) {
- return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
- | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
- COMP_Denormal : 0);
- }
- if (diff < 0) {
- return ((st0_sign == SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
- | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
- COMP_Denormal : 0);
- }
-
- return COMP_A_eq_B
- | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
- COMP_Denormal : 0);
-
-}
-
-/* This function requires that st(0) is not empty */
-int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag)
-{
- int f, c;
-
- c = compare(loaded_data, loaded_tag);
-
- if (c & COMP_NaN) {
- EXCEPTION(EX_Invalid);
- f = SW_C3 | SW_C2 | SW_C0;
- } else
- switch (c & 7) {
- case COMP_A_lt_B:
- f = SW_C0;
- break;
- case COMP_A_eq_B:
- f = SW_C3;
- break;
- case COMP_A_gt_B:
- f = 0;
- break;
- case COMP_No_Comp:
- f = SW_C3 | SW_C2 | SW_C0;
- break;
- default:
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x121);
-#endif /* PARANOID */
- f = SW_C3 | SW_C2 | SW_C0;
- break;
- }
- setcc(f);
- if (c & COMP_Denormal) {
- return denormal_operand() < 0;
- }
- return 0;
-}
-
-static int compare_st_st(int nr)
-{
- int f, c;
- FPU_REG *st_ptr;
-
- if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
- setcc(SW_C3 | SW_C2 | SW_C0);
- /* Stack fault */
- EXCEPTION(EX_StackUnder);
- return !(control_word & CW_Invalid);
- }
-
- st_ptr = &st(nr);
- c = compare(st_ptr, FPU_gettagi(nr));
- if (c & COMP_NaN) {
- setcc(SW_C3 | SW_C2 | SW_C0);
- EXCEPTION(EX_Invalid);
- return !(control_word & CW_Invalid);
- } else
- switch (c & 7) {
- case COMP_A_lt_B:
- f = SW_C0;
- break;
- case COMP_A_eq_B:
- f = SW_C3;
- break;
- case COMP_A_gt_B:
- f = 0;
- break;
- case COMP_No_Comp:
- f = SW_C3 | SW_C2 | SW_C0;
- break;
- default:
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x122);
-#endif /* PARANOID */
- f = SW_C3 | SW_C2 | SW_C0;
- break;
- }
- setcc(f);
- if (c & COMP_Denormal) {
- return denormal_operand() < 0;
- }
- return 0;
-}
-
-static int compare_i_st_st(int nr)
-{
- int f, c;
- FPU_REG *st_ptr;
-
- if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
- FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
- /* Stack fault */
- EXCEPTION(EX_StackUnder);
- return !(control_word & CW_Invalid);
- }
-
- partial_status &= ~SW_C0;
- st_ptr = &st(nr);
- c = compare(st_ptr, FPU_gettagi(nr));
- if (c & COMP_NaN) {
- FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
- EXCEPTION(EX_Invalid);
- return !(control_word & CW_Invalid);
- }
-
- switch (c & 7) {
- case COMP_A_lt_B:
- f = X86_EFLAGS_CF;
- break;
- case COMP_A_eq_B:
- f = X86_EFLAGS_ZF;
- break;
- case COMP_A_gt_B:
- f = 0;
- break;
- case COMP_No_Comp:
- f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF;
- break;
- default:
-#ifdef PARANOID
- EXCEPTION(EX_INTERNAL | 0x122);
-#endif /* PARANOID */
- f = 0;
- break;
- }
- FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f;
- if (c & COMP_Denormal) {
- return denormal_operand() < 0;
- }
- return 0;
-}
-
-static int compare_u_st_st(int nr)
-{
- int f = 0, c;
- FPU_REG *st_ptr;
-
- if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
- setcc(SW_C3 | SW_C2 | SW_C0);
- /* Stack fault */
- EXCEPTION(EX_StackUnder);
- return !(control_word & CW_Invalid);
- }
-
- st_ptr = &st(nr);
- c = compare(st_ptr, FPU_gettagi(nr));
- if (c & COMP_NaN) {
- setcc(SW_C3 | SW_C2 | SW_C0);
- if (c & COMP_SNaN) { /* This is the only difference between
- un-ordered and ordinary comparisons */
- EXCEPTION(EX_Invalid);
- return !(control_word & CW_Invalid);
- }
- return 0;
- } else
- switch (c & 7) {
- case COMP_A_lt_B:
- f = SW_C0;
- break;
- case COMP_A_eq_B:
- f = SW_C3;
- break;
- case COMP_A_gt_B:
- f = 0;
- break;
- case COMP_No_Comp:
- f = SW_C3 | SW_C2 | SW_C0;
- break;
-#ifdef PARANOID
- default:
- EXCEPTION(EX_INTERNAL | 0x123);
- f = SW_C3 | SW_C2 | SW_C0;
- break;
-#endif /* PARANOID */
- }
- setcc(f);
- if (c & COMP_Denormal) {
- return denormal_operand() < 0;
- }
- return 0;
-}
-
-static int compare_ui_st_st(int nr)
-{
- int f = 0, c;
- FPU_REG *st_ptr;
-
- if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
- FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
- /* Stack fault */
- EXCEPTION(EX_StackUnder);
- return !(control_word & CW_Invalid);
- }
-
- partial_status &= ~SW_C0;
- st_ptr = &st(nr);
- c = compare(st_ptr, FPU_gettagi(nr));
- if (c & COMP_NaN) {
- FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
- if (c & COMP_SNaN) { /* This is the only difference between
- un-ordered and ordinary comparisons */
- EXCEPTION(EX_Invalid);
- return !(control_word & CW_Invalid);
- }
- return 0;
- }
-
- switch (c & 7) {
- case COMP_A_lt_B:
- f = X86_EFLAGS_CF;
- break;
- case COMP_A_eq_B:
- f = X86_EFLAGS_ZF;
- break;
- case COMP_A_gt_B:
- f = 0;
- break;
- case COMP_No_Comp:
- f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF;
- break;
-#ifdef PARANOID
- default:
- EXCEPTION(EX_INTERNAL | 0x123);
- f = 0;
- break;
-#endif /* PARANOID */
- }
- FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f;
- if (c & COMP_Denormal) {
- return denormal_operand() < 0;
- }
- return 0;
-}
-
-/*---------------------------------------------------------------------------*/
-
-void fcom_st(void)
-{
- /* fcom st(i) */
- compare_st_st(FPU_rm);
-}
-
-void fcompst(void)
-{
- /* fcomp st(i) */
- if (!compare_st_st(FPU_rm))
- FPU_pop();
-}
-
-void fcompp(void)
-{
- /* fcompp */
- if (FPU_rm != 1) {
- FPU_illegal();
- return;
- }
- if (!compare_st_st(1))
- poppop();
-}
-
-void fucom_(void)
-{
- /* fucom st(i) */
- compare_u_st_st(FPU_rm);
-
-}
-
-void fucomp(void)
-{
- /* fucomp st(i) */
- if (!compare_u_st_st(FPU_rm))
- FPU_pop();
-}
-
-void fucompp(void)
-{
- /* fucompp */
- if (FPU_rm == 1) {
- if (!compare_u_st_st(1))
- poppop();
- } else
- FPU_illegal();
-}
-
-/* P6+ compare-to-EFLAGS ops */
-
-void fcomi_(void)
-{
- /* fcomi st(i) */
- compare_i_st_st(FPU_rm);
-}
-
-void fcomip(void)
-{
- /* fcomip st(i) */
- if (!compare_i_st_st(FPU_rm))
- FPU_pop();
-}
-
-void fucomi_(void)
-{
- /* fucomi st(i) */
- compare_ui_st_st(FPU_rm);
-}
-
-void fucomip(void)
-{
- /* fucomip st(i) */
- if (!compare_ui_st_st(FPU_rm))
- FPU_pop();
-}
diff --git a/arch/x86/math-emu/reg_constant.c b/arch/x86/math-emu/reg_constant.c
deleted file mode 100644
index 003a0b2..0000000
--- a/arch/x86/math-emu/reg_constant.c
+++ /dev/null
@@ -1,123 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_constant.c |
- | |
- | All of the constant FPU_REGs |
- | |
- | Copyright (C) 1992,1993,1994,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "reg_constant.h"
-#include "control_w.h"
-
-#define MAKE_REG(s, e, l, h) { l, h, \
- (u16)((EXTENDED_Ebias+(e)) | ((SIGN_##s != 0)*0x8000)) }
-
-FPU_REG const CONST_1 = MAKE_REG(POS, 0, 0x00000000, 0x80000000);
-#if 0
-FPU_REG const CONST_2 = MAKE_REG(POS, 1, 0x00000000, 0x80000000);
-FPU_REG const CONST_HALF = MAKE_REG(POS, -1, 0x00000000, 0x80000000);
-#endif /* 0 */
-static FPU_REG const CONST_L2T = MAKE_REG(POS, 1, 0xcd1b8afe, 0xd49a784b);
-static FPU_REG const CONST_L2E = MAKE_REG(POS, 0, 0x5c17f0bc, 0xb8aa3b29);
-FPU_REG const CONST_PI = MAKE_REG(POS, 1, 0x2168c235, 0xc90fdaa2);
-FPU_REG const CONST_PI2 = MAKE_REG(POS, 0, 0x2168c235, 0xc90fdaa2);
-FPU_REG const CONST_PI4 = MAKE_REG(POS, -1, 0x2168c235, 0xc90fdaa2);
-static FPU_REG const CONST_LG2 = MAKE_REG(POS, -2, 0xfbcff799, 0x9a209a84);
-static FPU_REG const CONST_LN2 = MAKE_REG(POS, -1, 0xd1cf79ac, 0xb17217f7);
-
-/* Extra bits to take pi/2 to more than 128 bits precision. */
-FPU_REG const CONST_PI2extra = MAKE_REG(NEG, -66,
- 0xfc8f8cbb, 0xece675d1);
-
-/* Only the sign (and tag) is used in internal zeroes */
-FPU_REG const CONST_Z = MAKE_REG(POS, EXP_UNDER, 0x0, 0x0);
-
-/* Only the sign and significand (and tag) are used in internal NaNs */
-/* The 80486 never generates one of these
-FPU_REG const CONST_SNAN = MAKE_REG(POS, EXP_OVER, 0x00000001, 0x80000000);
- */
-/* This is the real indefinite QNaN */
-FPU_REG const CONST_QNaN = MAKE_REG(NEG, EXP_OVER, 0x00000000, 0xC0000000);
-
-/* Only the sign (and tag) is used in internal infinities */
-FPU_REG const CONST_INF = MAKE_REG(POS, EXP_OVER, 0x00000000, 0x80000000);
-
-static void fld_const(FPU_REG const * c, int adj, u_char tag)
-{
- FPU_REG *st_new_ptr;
-
- if (STACK_OVERFLOW) {
- FPU_stack_overflow();
- return;
- }
- push();
- reg_copy(c, st_new_ptr);
- st_new_ptr->sigl += adj; /* For all our fldxxx constants, we don't need to
- borrow or carry. */
- FPU_settag0(tag);
- clear_C1();
-}
-
-/* A fast way to find out whether x is one of RC_DOWN or RC_CHOP
- (and not one of RC_RND or RC_UP).
- */
-#define DOWN_OR_CHOP(x) (x & RC_DOWN)
-
-static void fld1(int rc)
-{
- fld_const(&CONST_1, 0, TAG_Valid);
-}
-
-static void fldl2t(int rc)
-{
- fld_const(&CONST_L2T, (rc == RC_UP) ? 1 : 0, TAG_Valid);
-}
-
-static void fldl2e(int rc)
-{
- fld_const(&CONST_L2E, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldpi(int rc)
-{
- fld_const(&CONST_PI, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldlg2(int rc)
-{
- fld_const(&CONST_LG2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldln2(int rc)
-{
- fld_const(&CONST_LN2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldz(int rc)
-{
- fld_const(&CONST_Z, 0, TAG_Zero);
-}
-
-typedef void (*FUNC_RC) (int);
-
-static void FPU_RC_illegal(int unused)
-{
- FPU_illegal();
-}
-
-static FUNC_RC constants_table[] = {
- fld1, fldl2t, fldl2e, fldpi, fldlg2, fldln2, fldz, FPU_RC_illegal
-};
-
-void fconst(void)
-{
- (constants_table[FPU_rm]) (control_word & CW_RC);
-}
diff --git a/arch/x86/math-emu/reg_constant.h b/arch/x86/math-emu/reg_constant.h
deleted file mode 100644
index f2fdd34..0000000
--- a/arch/x86/math-emu/reg_constant.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | reg_constant.h |
- | |
- | Copyright (C) 1992 W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@vaxc.cc.monash.edu.au |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _REG_CONSTANT_H_
-#define _REG_CONSTANT_H_
-
-#include "fpu_emu.h"
-
-extern FPU_REG const CONST_1;
-extern FPU_REG const CONST_PI;
-extern FPU_REG const CONST_PI2;
-extern FPU_REG const CONST_PI2extra;
-extern FPU_REG const CONST_PI4;
-extern FPU_REG const CONST_Z;
-extern FPU_REG const CONST_PINF;
-extern FPU_REG const CONST_INF;
-extern FPU_REG const CONST_MINF;
-extern FPU_REG const CONST_QNaN;
-
-#endif /* _REG_CONSTANT_H_ */
diff --git a/arch/x86/math-emu/reg_convert.c b/arch/x86/math-emu/reg_convert.c
deleted file mode 100644
index 2511806..0000000
--- a/arch/x86/math-emu/reg_convert.c
+++ /dev/null
@@ -1,47 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_convert.c |
- | |
- | Convert register representation. |
- | |
- | Copyright (C) 1992,1993,1994,1996,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-int FPU_to_exp16(FPU_REG const *a, FPU_REG *x)
-{
- int sign = getsign(a);
-
- *(long long *)&(x->sigl) = *(const long long *)&(a->sigl);
-
- /* Set up the exponent as a 16 bit quantity. */
- setexponent16(x, exponent(a));
-
- if (exponent16(x) == EXP_UNDER) {
- /* The number is a de-normal or pseudodenormal. */
- /* We only deal with the significand and exponent. */
-
- if (x->sigh & 0x80000000) {
- /* Is a pseudodenormal. */
- /* This is non-80486 behaviour because the number
- loses its 'denormal' identity. */
- addexponent(x, 1);
- } else {
- /* Is a denormal. */
- addexponent(x, 1);
- FPU_normalize_nuo(x);
- }
- }
-
- if (!(x->sigh & 0x80000000)) {
- EXCEPTION(EX_INTERNAL | 0x180);
- }
-
- return sign;
-}
diff --git a/arch/x86/math-emu/reg_divide.c b/arch/x86/math-emu/reg_divide.c
deleted file mode 100644
index 08c2f6d..0000000
--- a/arch/x86/math-emu/reg_divide.c
+++ /dev/null
@@ -1,183 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_divide.c |
- | |
- | Divide one FPU_REG by another and put the result in a destination FPU_REG.|
- | |
- | Copyright (C) 1996 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | The destination may be any FPU_REG, including one of the source FPU_REGs. |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-
-/*
- Divide one register by another and put the result into a third register.
- */
-int FPU_div(int flags, int rm, int control_w)
-{
- FPU_REG x, y;
- FPU_REG const *a, *b, *st0_ptr, *st_ptr;
- FPU_REG *dest;
- u_char taga, tagb, signa, signb, sign, saved_sign;
- int tag, deststnr;
-
- if (flags & DEST_RM)
- deststnr = rm;
- else
- deststnr = 0;
-
- if (flags & REV) {
- b = &st(0);
- st0_ptr = b;
- tagb = FPU_gettag0();
- if (flags & LOADED) {
- a = (FPU_REG *) rm;
- taga = flags & 0x0f;
- } else {
- a = &st(rm);
- st_ptr = a;
- taga = FPU_gettagi(rm);
- }
- } else {
- a = &st(0);
- st0_ptr = a;
- taga = FPU_gettag0();
- if (flags & LOADED) {
- b = (FPU_REG *) rm;
- tagb = flags & 0x0f;
- } else {
- b = &st(rm);
- st_ptr = b;
- tagb = FPU_gettagi(rm);
- }
- }
-
- signa = getsign(a);
- signb = getsign(b);
-
- sign = signa ^ signb;
-
- dest = &st(deststnr);
- saved_sign = getsign(dest);
-
- if (!(taga | tagb)) {
- /* Both regs Valid, this should be the most common case. */
- reg_copy(a, &x);
- reg_copy(b, &y);
- setpositive(&x);
- setpositive(&y);
- tag = FPU_u_div(&x, &y, dest, control_w, sign);
-
- if (tag < 0)
- return tag;
-
- FPU_settagi(deststnr, tag);
- return tag;
- }
-
- if (taga == TAG_Special)
- taga = FPU_Special(a);
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
-
- if (((taga == TAG_Valid) && (tagb == TW_Denormal))
- || ((taga == TW_Denormal) && (tagb == TAG_Valid))
- || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
- if (denormal_operand() < 0)
- return FPU_Exception;
-
- FPU_to_exp16(a, &x);
- FPU_to_exp16(b, &y);
- tag = FPU_u_div(&x, &y, dest, control_w, sign);
- if (tag < 0)
- return tag;
-
- FPU_settagi(deststnr, tag);
- return tag;
- } else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) {
- if (tagb != TAG_Zero) {
- /* Want to find Zero/Valid */
- if (tagb == TW_Denormal) {
- if (denormal_operand() < 0)
- return FPU_Exception;
- }
-
- /* The result is zero. */
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
- setsign(dest, sign);
- return TAG_Zero;
- }
- /* We have an exception condition, either 0/0 or Valid/Zero. */
- if (taga == TAG_Zero) {
- /* 0/0 */
- return arith_invalid(deststnr);
- }
- /* Valid/Zero */
- return FPU_divide_by_zero(deststnr, sign);
- }
- /* Must have infinities, NaNs, etc */
- else if ((taga == TW_NaN) || (tagb == TW_NaN)) {
- if (flags & LOADED)
- return real_2op_NaN((FPU_REG *) rm, flags & 0x0f, 0,
- st0_ptr);
-
- if (flags & DEST_RM) {
- int tag;
- tag = FPU_gettag0();
- if (tag == TAG_Special)
- tag = FPU_Special(st0_ptr);
- return real_2op_NaN(st0_ptr, tag, rm,
- (flags & REV) ? st0_ptr : &st(rm));
- } else {
- int tag;
- tag = FPU_gettagi(rm);
- if (tag == TAG_Special)
- tag = FPU_Special(&st(rm));
- return real_2op_NaN(&st(rm), tag, 0,
- (flags & REV) ? st0_ptr : &st(rm));
- }
- } else if (taga == TW_Infinity) {
- if (tagb == TW_Infinity) {
- /* infinity/infinity */
- return arith_invalid(deststnr);
- } else {
- /* tagb must be Valid or Zero */
- if ((tagb == TW_Denormal) && (denormal_operand() < 0))
- return FPU_Exception;
-
- /* Infinity divided by Zero or Valid does
- not raise and exception, but returns Infinity */
- FPU_copy_to_regi(a, TAG_Special, deststnr);
- setsign(dest, sign);
- return taga;
- }
- } else if (tagb == TW_Infinity) {
- if ((taga == TW_Denormal) && (denormal_operand() < 0))
- return FPU_Exception;
-
- /* The result is zero. */
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
- setsign(dest, sign);
- return TAG_Zero;
- }
-#ifdef PARANOID
- else {
- EXCEPTION(EX_INTERNAL | 0x102);
- return FPU_Exception;
- }
-#endif /* PARANOID */
-
- return 0;
-}
diff --git a/arch/x86/math-emu/reg_ld_str.c b/arch/x86/math-emu/reg_ld_str.c
deleted file mode 100644
index 7e4521f..0000000
--- a/arch/x86/math-emu/reg_ld_str.c
+++ /dev/null
@@ -1,1220 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_ld_str.c |
- | |
- | All of the functions which transfer data between user memory and FPU_REGs.|
- | |
- | Copyright (C) 1992,1993,1994,1996,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note: |
- | The file contains code which accesses user memory. |
- | Emulator static data may change when user memory is accessed, due to |
- | other processes using the emulator while swapping is in progress. |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-#include <linux/uaccess.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "reg_constant.h"
-#include "control_w.h"
-#include "status_w.h"
-
-#define DOUBLE_Emax 1023 /* largest valid exponent */
-#define DOUBLE_Ebias 1023
-#define DOUBLE_Emin (-1022) /* smallest valid exponent */
-
-#define SINGLE_Emax 127 /* largest valid exponent */
-#define SINGLE_Ebias 127
-#define SINGLE_Emin (-126) /* smallest valid exponent */
-
-static u_char normalize_no_excep(FPU_REG *r, int exp, int sign)
-{
- u_char tag;
-
- setexponent16(r, exp);
-
- tag = FPU_normalize_nuo(r);
- stdexp(r);
- if (sign)
- setnegative(r);
-
- return tag;
-}
-
-int FPU_tagof(FPU_REG *ptr)
-{
- int exp;
-
- exp = exponent16(ptr) & 0x7fff;
- if (exp == 0) {
- if (!(ptr->sigh | ptr->sigl)) {
- return TAG_Zero;
- }
- /* The number is a de-normal or pseudodenormal. */
- return TAG_Special;
- }
-
- if (exp == 0x7fff) {
- /* Is an Infinity, a NaN, or an unsupported data type. */
- return TAG_Special;
- }
-
- if (!(ptr->sigh & 0x80000000)) {
- /* Unsupported data type. */
- /* Valid numbers have the ms bit set to 1. */
- /* Unnormal. */
- return TAG_Special;
- }
-
- return TAG_Valid;
-}
-
-/* Get a long double from user memory */
-int FPU_load_extended(long double __user *s, int stnr)
-{
- FPU_REG *sti_ptr = &st(stnr);
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(s, 10);
- FPU_copy_from_user(sti_ptr, s, 10);
- RE_ENTRANT_CHECK_ON;
-
- return FPU_tagof(sti_ptr);
-}
-
-/* Get a double from user memory */
-int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data)
-{
- int exp, tag, negative;
- unsigned m64, l64;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(dfloat, 8);
- FPU_get_user(m64, 1 + (unsigned long __user *)dfloat);
- FPU_get_user(l64, (unsigned long __user *)dfloat);
- RE_ENTRANT_CHECK_ON;
-
- negative = (m64 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
- exp = ((m64 & 0x7ff00000) >> 20) - DOUBLE_Ebias + EXTENDED_Ebias;
- m64 &= 0xfffff;
- if (exp > DOUBLE_Emax + EXTENDED_Ebias) {
- /* Infinity or NaN */
- if ((m64 == 0) && (l64 == 0)) {
- /* +- infinity */
- loaded_data->sigh = 0x80000000;
- loaded_data->sigl = 0x00000000;
- exp = EXP_Infinity + EXTENDED_Ebias;
- tag = TAG_Special;
- } else {
- /* Must be a signaling or quiet NaN */
- exp = EXP_NaN + EXTENDED_Ebias;
- loaded_data->sigh = (m64 << 11) | 0x80000000;
- loaded_data->sigh |= l64 >> 21;
- loaded_data->sigl = l64 << 11;
- tag = TAG_Special; /* The calling function must look for NaNs */
- }
- } else if (exp < DOUBLE_Emin + EXTENDED_Ebias) {
- /* Zero or de-normal */
- if ((m64 == 0) && (l64 == 0)) {
- /* Zero */
- reg_copy(&CONST_Z, loaded_data);
- exp = 0;
- tag = TAG_Zero;
- } else {
- /* De-normal */
- loaded_data->sigh = m64 << 11;
- loaded_data->sigh |= l64 >> 21;
- loaded_data->sigl = l64 << 11;
-
- return normalize_no_excep(loaded_data, DOUBLE_Emin,
- negative)
- | (denormal_operand() < 0 ? FPU_Exception : 0);
- }
- } else {
- loaded_data->sigh = (m64 << 11) | 0x80000000;
- loaded_data->sigh |= l64 >> 21;
- loaded_data->sigl = l64 << 11;
-
- tag = TAG_Valid;
- }
-
- setexponent16(loaded_data, exp | negative);
-
- return tag;
-}
-
-/* Get a float from user memory */
-int FPU_load_single(float __user *single, FPU_REG *loaded_data)
-{
- unsigned m32;
- int exp, tag, negative;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(single, 4);
- FPU_get_user(m32, (unsigned long __user *)single);
- RE_ENTRANT_CHECK_ON;
-
- negative = (m32 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
-
- if (!(m32 & 0x7fffffff)) {
- /* Zero */
- reg_copy(&CONST_Z, loaded_data);
- addexponent(loaded_data, negative);
- return TAG_Zero;
- }
- exp = ((m32 & 0x7f800000) >> 23) - SINGLE_Ebias + EXTENDED_Ebias;
- m32 = (m32 & 0x7fffff) << 8;
- if (exp < SINGLE_Emin + EXTENDED_Ebias) {
- /* De-normals */
- loaded_data->sigh = m32;
- loaded_data->sigl = 0;
-
- return normalize_no_excep(loaded_data, SINGLE_Emin, negative)
- | (denormal_operand() < 0 ? FPU_Exception : 0);
- } else if (exp > SINGLE_Emax + EXTENDED_Ebias) {
- /* Infinity or NaN */
- if (m32 == 0) {
- /* +- infinity */
- loaded_data->sigh = 0x80000000;
- loaded_data->sigl = 0x00000000;
- exp = EXP_Infinity + EXTENDED_Ebias;
- tag = TAG_Special;
- } else {
- /* Must be a signaling or quiet NaN */
- exp = EXP_NaN + EXTENDED_Ebias;
- loaded_data->sigh = m32 | 0x80000000;
- loaded_data->sigl = 0;
- tag = TAG_Special; /* The calling function must look for NaNs */
- }
- } else {
- loaded_data->sigh = m32 | 0x80000000;
- loaded_data->sigl = 0;
- tag = TAG_Valid;
- }
-
- setexponent16(loaded_data, exp | negative); /* Set the sign. */
-
- return tag;
-}
-
-/* Get a long long from user memory */
-int FPU_load_int64(long long __user *_s)
-{
- long long s;
- int sign;
- FPU_REG *st0_ptr = &st(0);
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(_s, 8);
- if (copy_from_user(&s, _s, 8))
- FPU_abort;
- RE_ENTRANT_CHECK_ON;
-
- if (s == 0) {
- reg_copy(&CONST_Z, st0_ptr);
- return TAG_Zero;
- }
-
- if (s > 0)
- sign = SIGN_Positive;
- else {
- s = -s;
- sign = SIGN_Negative;
- }
-
- significand(st0_ptr) = s;
-
- return normalize_no_excep(st0_ptr, 63, sign);
-}
-
-/* Get a long from user memory */
-int FPU_load_int32(long __user *_s, FPU_REG *loaded_data)
-{
- long s;
- int negative;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(_s, 4);
- FPU_get_user(s, _s);
- RE_ENTRANT_CHECK_ON;
-
- if (s == 0) {
- reg_copy(&CONST_Z, loaded_data);
- return TAG_Zero;
- }
-
- if (s > 0)
- negative = SIGN_Positive;
- else {
- s = -s;
- negative = SIGN_Negative;
- }
-
- loaded_data->sigh = s;
- loaded_data->sigl = 0;
-
- return normalize_no_excep(loaded_data, 31, negative);
-}
-
-/* Get a short from user memory */
-int FPU_load_int16(short __user *_s, FPU_REG *loaded_data)
-{
- int s, negative;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(_s, 2);
- /* Cast as short to get the sign extended. */
- FPU_get_user(s, _s);
- RE_ENTRANT_CHECK_ON;
-
- if (s == 0) {
- reg_copy(&CONST_Z, loaded_data);
- return TAG_Zero;
- }
-
- if (s > 0)
- negative = SIGN_Positive;
- else {
- s = -s;
- negative = SIGN_Negative;
- }
-
- loaded_data->sigh = s << 16;
- loaded_data->sigl = 0;
-
- return normalize_no_excep(loaded_data, 15, negative);
-}
-
-/* Get a packed bcd array from user memory */
-int FPU_load_bcd(u_char __user *s)
-{
- FPU_REG *st0_ptr = &st(0);
- int pos;
- u_char bcd;
- long long l = 0;
- int sign;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(s, 10);
- RE_ENTRANT_CHECK_ON;
- for (pos = 8; pos >= 0; pos--) {
- l *= 10;
- RE_ENTRANT_CHECK_OFF;
- FPU_get_user(bcd, s + pos);
- RE_ENTRANT_CHECK_ON;
- l += bcd >> 4;
- l *= 10;
- l += bcd & 0x0f;
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_get_user(sign, s + 9);
- sign = sign & 0x80 ? SIGN_Negative : SIGN_Positive;
- RE_ENTRANT_CHECK_ON;
-
- if (l == 0) {
- reg_copy(&CONST_Z, st0_ptr);
- addexponent(st0_ptr, sign); /* Set the sign. */
- return TAG_Zero;
- } else {
- significand(st0_ptr) = l;
- return normalize_no_excep(st0_ptr, 63, sign);
- }
-}
-
-/*===========================================================================*/
-
-/* Put a long double into user memory */
-int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag,
- long double __user * d)
-{
- /*
- The only exception raised by an attempt to store to an
- extended format is the Invalid Stack exception, i.e.
- attempting to store from an empty register.
- */
-
- if (st0_tag != TAG_Empty) {
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 10);
-
- FPU_put_user(st0_ptr->sigl, (unsigned long __user *)d);
- FPU_put_user(st0_ptr->sigh,
- (unsigned long __user *)((u_char __user *) d + 4));
- FPU_put_user(exponent16(st0_ptr),
- (unsigned short __user *)((u_char __user *) d +
- 8));
- RE_ENTRANT_CHECK_ON;
-
- return 1;
- }
-
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- if (control_word & CW_Invalid) {
- /* The masked response */
- /* Put out the QNaN indefinite */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 10);
- FPU_put_user(0, (unsigned long __user *)d);
- FPU_put_user(0xc0000000, 1 + (unsigned long __user *)d);
- FPU_put_user(0xffff, 4 + (short __user *)d);
- RE_ENTRANT_CHECK_ON;
- return 1;
- } else
- return 0;
-
-}
-
-/* Put a double into user memory */
-int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag, double __user *dfloat)
-{
- unsigned long l[2];
- unsigned long increment = 0; /* avoid gcc warnings */
- int precision_loss;
- int exp;
- FPU_REG tmp;
-
- l[0] = 0;
- l[1] = 0;
- if (st0_tag == TAG_Valid) {
- reg_copy(st0_ptr, &tmp);
- exp = exponent(&tmp);
-
- if (exp < DOUBLE_Emin) { /* It may be a denormal */
- addexponent(&tmp, -DOUBLE_Emin + 52); /* largest exp to be 51 */
-denormal_arg:
- if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
-#ifdef PECULIAR_486
- /* Did it round to a non-denormal ? */
- /* This behaviour might be regarded as peculiar, it appears
- that the 80486 rounds to the dest precision, then
- converts to decide underflow. */
- if (!
- ((tmp.sigh == 0x00100000) && (tmp.sigl == 0)
- && (st0_ptr->sigl & 0x000007ff)))
-#endif /* PECULIAR_486 */
- {
- EXCEPTION(EX_Underflow);
- /* This is a special case: see sec 16.2.5.1 of
- the 80486 book */
- if (!(control_word & CW_Underflow))
- return 0;
- }
- EXCEPTION(precision_loss);
- if (!(control_word & CW_Precision))
- return 0;
- }
- l[0] = tmp.sigl;
- l[1] = tmp.sigh;
- } else {
- if (tmp.sigl & 0x000007ff) {
- precision_loss = 1;
- switch (control_word & CW_RC) {
- case RC_RND:
- /* Rounding can get a little messy.. */
- increment = ((tmp.sigl & 0x7ff) > 0x400) | /* nearest */
- ((tmp.sigl & 0xc00) == 0xc00); /* odd -> even */
- break;
- case RC_DOWN: /* towards -infinity */
- increment =
- signpositive(&tmp) ? 0 : tmp.
- sigl & 0x7ff;
- break;
- case RC_UP: /* towards +infinity */
- increment =
- signpositive(&tmp) ? tmp.
- sigl & 0x7ff : 0;
- break;
- case RC_CHOP:
- increment = 0;
- break;
- }
-
- /* Truncate the mantissa */
- tmp.sigl &= 0xfffff800;
-
- if (increment) {
- if (tmp.sigl >= 0xfffff800) {
- /* the sigl part overflows */
- if (tmp.sigh == 0xffffffff) {
- /* The sigh part overflows */
- tmp.sigh = 0x80000000;
- exp++;
- if (exp >= EXP_OVER)
- goto overflow;
- } else {
- tmp.sigh++;
- }
- tmp.sigl = 0x00000000;
- } else {
- /* We only need to increment sigl */
- tmp.sigl += 0x00000800;
- }
- }
- } else
- precision_loss = 0;
-
- l[0] = (tmp.sigl >> 11) | (tmp.sigh << 21);
- l[1] = ((tmp.sigh >> 11) & 0xfffff);
-
- if (exp > DOUBLE_Emax) {
- overflow:
- EXCEPTION(EX_Overflow);
- if (!(control_word & CW_Overflow))
- return 0;
- set_precision_flag_up();
- if (!(control_word & CW_Precision))
- return 0;
-
- /* This is a special case: see sec 16.2.5.1 of the 80486 book */
- /* Overflow to infinity */
- l[1] = 0x7ff00000; /* Set to + INF */
- } else {
- if (precision_loss) {
- if (increment)
- set_precision_flag_up();
- else
- set_precision_flag_down();
- }
- /* Add the exponent */
- l[1] |= (((exp + DOUBLE_Ebias) & 0x7ff) << 20);
- }
- }
- } else if (st0_tag == TAG_Zero) {
- /* Number is zero */
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if (st0_tag == TW_Denormal) {
- /* A denormal will always underflow. */
-#ifndef PECULIAR_486
- /* An 80486 is supposed to be able to generate
- a denormal exception here, but... */
- /* Underflow has priority. */
- if (control_word & CW_Underflow)
- denormal_operand();
-#endif /* PECULIAR_486 */
- reg_copy(st0_ptr, &tmp);
- goto denormal_arg;
- } else if (st0_tag == TW_Infinity) {
- l[1] = 0x7ff00000;
- } else if (st0_tag == TW_NaN) {
- /* Is it really a NaN ? */
- if ((exponent(st0_ptr) == EXP_OVER)
- && (st0_ptr->sigh & 0x80000000)) {
- /* See if we can get a valid NaN from the FPU_REG */
- l[0] =
- (st0_ptr->sigl >> 11) | (st0_ptr->
- sigh << 21);
- l[1] = ((st0_ptr->sigh >> 11) & 0xfffff);
- if (!(st0_ptr->sigh & 0x40000000)) {
- /* It is a signalling NaN */
- EXCEPTION(EX_Invalid);
- if (!(control_word & CW_Invalid))
- return 0;
- l[1] |= (0x40000000 >> 11);
- }
- l[1] |= 0x7ff00000;
- } else {
- /* It is an unsupported data type */
- EXCEPTION(EX_Invalid);
- if (!(control_word & CW_Invalid))
- return 0;
- l[1] = 0xfff80000;
- }
- }
- } else if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- if (control_word & CW_Invalid) {
- /* The masked response */
- /* Put out the QNaN indefinite */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(dfloat, 8);
- FPU_put_user(0, (unsigned long __user *)dfloat);
- FPU_put_user(0xfff80000,
- 1 + (unsigned long __user *)dfloat);
- RE_ENTRANT_CHECK_ON;
- return 1;
- } else
- return 0;
- }
- if (getsign(st0_ptr))
- l[1] |= 0x80000000;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(dfloat, 8);
- FPU_put_user(l[0], (unsigned long __user *)dfloat);
- FPU_put_user(l[1], 1 + (unsigned long __user *)dfloat);
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/* Put a float into user memory */
-int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single)
-{
- long templ = 0;
- unsigned long increment = 0; /* avoid gcc warnings */
- int precision_loss;
- int exp;
- FPU_REG tmp;
-
- if (st0_tag == TAG_Valid) {
-
- reg_copy(st0_ptr, &tmp);
- exp = exponent(&tmp);
-
- if (exp < SINGLE_Emin) {
- addexponent(&tmp, -SINGLE_Emin + 23); /* largest exp to be 22 */
-
- denormal_arg:
-
- if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
-#ifdef PECULIAR_486
- /* Did it round to a non-denormal ? */
- /* This behaviour might be regarded as peculiar, it appears
- that the 80486 rounds to the dest precision, then
- converts to decide underflow. */
- if (!((tmp.sigl == 0x00800000) &&
- ((st0_ptr->sigh & 0x000000ff)
- || st0_ptr->sigl)))
-#endif /* PECULIAR_486 */
- {
- EXCEPTION(EX_Underflow);
- /* This is a special case: see sec 16.2.5.1 of
- the 80486 book */
- if (!(control_word & CW_Underflow))
- return 0;
- }
- EXCEPTION(precision_loss);
- if (!(control_word & CW_Precision))
- return 0;
- }
- templ = tmp.sigl;
- } else {
- if (tmp.sigl | (tmp.sigh & 0x000000ff)) {
- unsigned long sigh = tmp.sigh;
- unsigned long sigl = tmp.sigl;
-
- precision_loss = 1;
- switch (control_word & CW_RC) {
- case RC_RND:
- increment = ((sigh & 0xff) > 0x80) /* more than half */
- ||(((sigh & 0xff) == 0x80) && sigl) /* more than half */
- ||((sigh & 0x180) == 0x180); /* round to even */
- break;
- case RC_DOWN: /* towards -infinity */
- increment = signpositive(&tmp)
- ? 0 : (sigl | (sigh & 0xff));
- break;
- case RC_UP: /* towards +infinity */
- increment = signpositive(&tmp)
- ? (sigl | (sigh & 0xff)) : 0;
- break;
- case RC_CHOP:
- increment = 0;
- break;
- }
-
- /* Truncate part of the mantissa */
- tmp.sigl = 0;
-
- if (increment) {
- if (sigh >= 0xffffff00) {
- /* The sigh part overflows */
- tmp.sigh = 0x80000000;
- exp++;
- if (exp >= EXP_OVER)
- goto overflow;
- } else {
- tmp.sigh &= 0xffffff00;
- tmp.sigh += 0x100;
- }
- } else {
- tmp.sigh &= 0xffffff00; /* Finish the truncation */
- }
- } else
- precision_loss = 0;
-
- templ = (tmp.sigh >> 8) & 0x007fffff;
-
- if (exp > SINGLE_Emax) {
- overflow:
- EXCEPTION(EX_Overflow);
- if (!(control_word & CW_Overflow))
- return 0;
- set_precision_flag_up();
- if (!(control_word & CW_Precision))
- return 0;
-
- /* This is a special case: see sec 16.2.5.1 of the 80486 book. */
- /* Masked response is overflow to infinity. */
- templ = 0x7f800000;
- } else {
- if (precision_loss) {
- if (increment)
- set_precision_flag_up();
- else
- set_precision_flag_down();
- }
- /* Add the exponent */
- templ |= ((exp + SINGLE_Ebias) & 0xff) << 23;
- }
- }
- } else if (st0_tag == TAG_Zero) {
- templ = 0;
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if (st0_tag == TW_Denormal) {
- reg_copy(st0_ptr, &tmp);
-
- /* A denormal will always underflow. */
-#ifndef PECULIAR_486
- /* An 80486 is supposed to be able to generate
- a denormal exception here, but... */
- /* Underflow has priority. */
- if (control_word & CW_Underflow)
- denormal_operand();
-#endif /* PECULIAR_486 */
- goto denormal_arg;
- } else if (st0_tag == TW_Infinity) {
- templ = 0x7f800000;
- } else if (st0_tag == TW_NaN) {
- /* Is it really a NaN ? */
- if ((exponent(st0_ptr) == EXP_OVER)
- && (st0_ptr->sigh & 0x80000000)) {
- /* See if we can get a valid NaN from the FPU_REG */
- templ = st0_ptr->sigh >> 8;
- if (!(st0_ptr->sigh & 0x40000000)) {
- /* It is a signalling NaN */
- EXCEPTION(EX_Invalid);
- if (!(control_word & CW_Invalid))
- return 0;
- templ |= (0x40000000 >> 8);
- }
- templ |= 0x7f800000;
- } else {
- /* It is an unsupported data type */
- EXCEPTION(EX_Invalid);
- if (!(control_word & CW_Invalid))
- return 0;
- templ = 0xffc00000;
- }
- }
-#ifdef PARANOID
- else {
- EXCEPTION(EX_INTERNAL | 0x164);
- return 0;
- }
-#endif
- } else if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- if (control_word & EX_Invalid) {
- /* The masked response */
- /* Put out the QNaN indefinite */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(single, 4);
- FPU_put_user(0xffc00000,
- (unsigned long __user *)single);
- RE_ENTRANT_CHECK_ON;
- return 1;
- } else
- return 0;
- }
-#ifdef PARANOID
- else {
- EXCEPTION(EX_INTERNAL | 0x163);
- return 0;
- }
-#endif
- if (getsign(st0_ptr))
- templ |= 0x80000000;
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(single, 4);
- FPU_put_user(templ, (unsigned long __user *)single);
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/* Put a long long into user memory */
-int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, long long __user *d)
-{
- FPU_REG t;
- long long tll;
- int precision_loss;
-
- if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- goto invalid_operand;
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
- EXCEPTION(EX_Invalid);
- goto invalid_operand;
- }
- }
-
- reg_copy(st0_ptr, &t);
- precision_loss = FPU_round_to_int(&t, st0_tag);
- ((long *)&tll)[0] = t.sigl;
- ((long *)&tll)[1] = t.sigh;
- if ((precision_loss == 1) ||
- ((t.sigh & 0x80000000) &&
- !((t.sigh == 0x80000000) && (t.sigl == 0) && signnegative(&t)))) {
- EXCEPTION(EX_Invalid);
- /* This is a special case: see sec 16.2.5.1 of the 80486 book */
- invalid_operand:
- if (control_word & EX_Invalid) {
- /* Produce something like QNaN "indefinite" */
- tll = 0x8000000000000000LL;
- } else
- return 0;
- } else {
- if (precision_loss)
- set_precision_flag(precision_loss);
- if (signnegative(&t))
- tll = -tll;
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 8);
- if (copy_to_user(d, &tll, 8))
- FPU_abort;
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/* Put a long into user memory */
-int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d)
-{
- FPU_REG t;
- int precision_loss;
-
- if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- goto invalid_operand;
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
- EXCEPTION(EX_Invalid);
- goto invalid_operand;
- }
- }
-
- reg_copy(st0_ptr, &t);
- precision_loss = FPU_round_to_int(&t, st0_tag);
- if (t.sigh ||
- ((t.sigl & 0x80000000) &&
- !((t.sigl == 0x80000000) && signnegative(&t)))) {
- EXCEPTION(EX_Invalid);
- /* This is a special case: see sec 16.2.5.1 of the 80486 book */
- invalid_operand:
- if (control_word & EX_Invalid) {
- /* Produce something like QNaN "indefinite" */
- t.sigl = 0x80000000;
- } else
- return 0;
- } else {
- if (precision_loss)
- set_precision_flag(precision_loss);
- if (signnegative(&t))
- t.sigl = -(long)t.sigl;
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 4);
- FPU_put_user(t.sigl, (unsigned long __user *)d);
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/* Put a short into user memory */
-int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d)
-{
- FPU_REG t;
- int precision_loss;
-
- if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- goto invalid_operand;
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
- EXCEPTION(EX_Invalid);
- goto invalid_operand;
- }
- }
-
- reg_copy(st0_ptr, &t);
- precision_loss = FPU_round_to_int(&t, st0_tag);
- if (t.sigh ||
- ((t.sigl & 0xffff8000) &&
- !((t.sigl == 0x8000) && signnegative(&t)))) {
- EXCEPTION(EX_Invalid);
- /* This is a special case: see sec 16.2.5.1 of the 80486 book */
- invalid_operand:
- if (control_word & EX_Invalid) {
- /* Produce something like QNaN "indefinite" */
- t.sigl = 0x8000;
- } else
- return 0;
- } else {
- if (precision_loss)
- set_precision_flag(precision_loss);
- if (signnegative(&t))
- t.sigl = -t.sigl;
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 2);
- FPU_put_user((short)t.sigl, d);
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/* Put a packed bcd array into user memory */
-int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d)
-{
- FPU_REG t;
- unsigned long long ll;
- u_char b;
- int i, precision_loss;
- u_char sign = (getsign(st0_ptr) == SIGN_NEG) ? 0x80 : 0;
-
- if (st0_tag == TAG_Empty) {
- /* Empty register (stack underflow) */
- EXCEPTION(EX_StackUnder);
- goto invalid_operand;
- } else if (st0_tag == TAG_Special) {
- st0_tag = FPU_Special(st0_ptr);
- if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
- EXCEPTION(EX_Invalid);
- goto invalid_operand;
- }
- }
-
- reg_copy(st0_ptr, &t);
- precision_loss = FPU_round_to_int(&t, st0_tag);
- ll = significand(&t);
-
- /* Check for overflow, by comparing with 999999999999999999 decimal. */
- if ((t.sigh > 0x0de0b6b3) ||
- ((t.sigh == 0x0de0b6b3) && (t.sigl > 0xa763ffff))) {
- EXCEPTION(EX_Invalid);
- /* This is a special case: see sec 16.2.5.1 of the 80486 book */
- invalid_operand:
- if (control_word & CW_Invalid) {
- /* Produce the QNaN "indefinite" */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 10);
- for (i = 0; i < 7; i++)
- FPU_put_user(0, d + i); /* These bytes "undefined" */
- FPU_put_user(0xc0, d + 7); /* This byte "undefined" */
- FPU_put_user(0xff, d + 8);
- FPU_put_user(0xff, d + 9);
- RE_ENTRANT_CHECK_ON;
- return 1;
- } else
- return 0;
- } else if (precision_loss) {
- /* Precision loss doesn't stop the data transfer */
- set_precision_flag(precision_loss);
- }
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 10);
- RE_ENTRANT_CHECK_ON;
- for (i = 0; i < 9; i++) {
- b = FPU_div_small(&ll, 10);
- b |= (FPU_div_small(&ll, 10)) << 4;
- RE_ENTRANT_CHECK_OFF;
- FPU_put_user(b, d + i);
- RE_ENTRANT_CHECK_ON;
- }
- RE_ENTRANT_CHECK_OFF;
- FPU_put_user(sign, d + 9);
- RE_ENTRANT_CHECK_ON;
-
- return 1;
-}
-
-/*===========================================================================*/
-
-/* r gets mangled such that sig is int, sign:
- it is NOT normalized */
-/* The return value (in eax) is zero if the result is exact,
- if bits are changed due to rounding, truncation, etc, then
- a non-zero value is returned */
-/* Overflow is signaled by a non-zero return value (in eax).
- In the case of overflow, the returned significand always has the
- largest possible value */
-int FPU_round_to_int(FPU_REG *r, u_char tag)
-{
- u_char very_big;
- unsigned eax;
-
- if (tag == TAG_Zero) {
- /* Make sure that zero is returned */
- significand(r) = 0;
- return 0; /* o.k. */
- }
-
- if (exponent(r) > 63) {
- r->sigl = r->sigh = ~0; /* The largest representable number */
- return 1; /* overflow */
- }
-
- eax = FPU_shrxs(&r->sigl, 63 - exponent(r));
- very_big = !(~(r->sigh) | ~(r->sigl)); /* test for 0xfff...fff */
-#define half_or_more (eax & 0x80000000)
-#define frac_part (eax)
-#define more_than_half ((eax & 0x80000001) == 0x80000001)
- switch (control_word & CW_RC) {
- case RC_RND:
- if (more_than_half /* nearest */
- || (half_or_more && (r->sigl & 1))) { /* odd -> even */
- if (very_big)
- return 1; /* overflow */
- significand(r)++;
- return PRECISION_LOST_UP;
- }
- break;
- case RC_DOWN:
- if (frac_part && getsign(r)) {
- if (very_big)
- return 1; /* overflow */
- significand(r)++;
- return PRECISION_LOST_UP;
- }
- break;
- case RC_UP:
- if (frac_part && !getsign(r)) {
- if (very_big)
- return 1; /* overflow */
- significand(r)++;
- return PRECISION_LOST_UP;
- }
- break;
- case RC_CHOP:
- break;
- }
-
- return eax ? PRECISION_LOST_DOWN : 0;
-
-}
-
-/*===========================================================================*/
-
-u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s)
-{
- unsigned short tag_word = 0;
- u_char tag;
- int i;
-
- if ((addr_modes.default_mode == VM86) ||
- ((addr_modes.default_mode == PM16)
- ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(s, 0x0e);
- FPU_get_user(control_word, (unsigned short __user *)s);
- FPU_get_user(partial_status, (unsigned short __user *)(s + 2));
- FPU_get_user(tag_word, (unsigned short __user *)(s + 4));
- FPU_get_user(instruction_address.offset,
- (unsigned short __user *)(s + 6));
- FPU_get_user(instruction_address.selector,
- (unsigned short __user *)(s + 8));
- FPU_get_user(operand_address.offset,
- (unsigned short __user *)(s + 0x0a));
- FPU_get_user(operand_address.selector,
- (unsigned short __user *)(s + 0x0c));
- RE_ENTRANT_CHECK_ON;
- s += 0x0e;
- if (addr_modes.default_mode == VM86) {
- instruction_address.offset
- += (instruction_address.selector & 0xf000) << 4;
- operand_address.offset +=
- (operand_address.selector & 0xf000) << 4;
- }
- } else {
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(s, 0x1c);
- FPU_get_user(control_word, (unsigned short __user *)s);
- FPU_get_user(partial_status, (unsigned short __user *)(s + 4));
- FPU_get_user(tag_word, (unsigned short __user *)(s + 8));
- FPU_get_user(instruction_address.offset,
- (unsigned long __user *)(s + 0x0c));
- FPU_get_user(instruction_address.selector,
- (unsigned short __user *)(s + 0x10));
- FPU_get_user(instruction_address.opcode,
- (unsigned short __user *)(s + 0x12));
- FPU_get_user(operand_address.offset,
- (unsigned long __user *)(s + 0x14));
- FPU_get_user(operand_address.selector,
- (unsigned long __user *)(s + 0x18));
- RE_ENTRANT_CHECK_ON;
- s += 0x1c;
- }
-
-#ifdef PECULIAR_486
- control_word &= ~0xe080;
-#endif /* PECULIAR_486 */
-
- top = (partial_status >> SW_Top_Shift) & 7;
-
- if (partial_status & ~control_word & CW_Exceptions)
- partial_status |= (SW_Summary | SW_Backward);
- else
- partial_status &= ~(SW_Summary | SW_Backward);
-
- for (i = 0; i < 8; i++) {
- tag = tag_word & 3;
- tag_word >>= 2;
-
- if (tag == TAG_Empty)
- /* New tag is empty. Accept it */
- FPU_settag(i, TAG_Empty);
- else if (FPU_gettag(i) == TAG_Empty) {
- /* Old tag is empty and new tag is not empty. New tag is determined
- by old reg contents */
- if (exponent(&fpu_register(i)) == -EXTENDED_Ebias) {
- if (!
- (fpu_register(i).sigl | fpu_register(i).
- sigh))
- FPU_settag(i, TAG_Zero);
- else
- FPU_settag(i, TAG_Special);
- } else if (exponent(&fpu_register(i)) ==
- 0x7fff - EXTENDED_Ebias) {
- FPU_settag(i, TAG_Special);
- } else if (fpu_register(i).sigh & 0x80000000)
- FPU_settag(i, TAG_Valid);
- else
- FPU_settag(i, TAG_Special); /* An Un-normal */
- }
- /* Else old tag is not empty and new tag is not empty. Old tag
- remains correct */
- }
-
- return s;
-}
-
-void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address)
-{
- int i, regnr;
- u_char __user *s = fldenv(addr_modes, data_address);
- int offset = (top & 7) * 10, other = 80 - offset;
-
- /* Copy all registers in stack order. */
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(s, 80);
- FPU_copy_from_user(register_base + offset, s, other);
- if (offset)
- FPU_copy_from_user(register_base, s + other, offset);
- RE_ENTRANT_CHECK_ON;
-
- for (i = 0; i < 8; i++) {
- regnr = (i + top) & 7;
- if (FPU_gettag(regnr) != TAG_Empty)
- /* The loaded data over-rides all other cases. */
- FPU_settag(regnr, FPU_tagof(&st(i)));
- }
-
-}
-
-u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d)
-{
- if ((addr_modes.default_mode == VM86) ||
- ((addr_modes.default_mode == PM16)
- ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 14);
-#ifdef PECULIAR_486
- FPU_put_user(control_word & ~0xe080, (unsigned long __user *)d);
-#else
- FPU_put_user(control_word, (unsigned short __user *)d);
-#endif /* PECULIAR_486 */
- FPU_put_user(status_word(), (unsigned short __user *)(d + 2));
- FPU_put_user(fpu_tag_word, (unsigned short __user *)(d + 4));
- FPU_put_user(instruction_address.offset,
- (unsigned short __user *)(d + 6));
- FPU_put_user(operand_address.offset,
- (unsigned short __user *)(d + 0x0a));
- if (addr_modes.default_mode == VM86) {
- FPU_put_user((instruction_address.
- offset & 0xf0000) >> 4,
- (unsigned short __user *)(d + 8));
- FPU_put_user((operand_address.offset & 0xf0000) >> 4,
- (unsigned short __user *)(d + 0x0c));
- } else {
- FPU_put_user(instruction_address.selector,
- (unsigned short __user *)(d + 8));
- FPU_put_user(operand_address.selector,
- (unsigned short __user *)(d + 0x0c));
- }
- RE_ENTRANT_CHECK_ON;
- d += 0x0e;
- } else {
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 7 * 4);
-#ifdef PECULIAR_486
- control_word &= ~0xe080;
- /* An 80486 sets nearly all of the reserved bits to 1. */
- control_word |= 0xffff0040;
- partial_status = status_word() | 0xffff0000;
- fpu_tag_word |= 0xffff0000;
- I387->soft.fcs &= ~0xf8000000;
- I387->soft.fos |= 0xffff0000;
-#endif /* PECULIAR_486 */
- if (__copy_to_user(d, &control_word, 7 * 4))
- FPU_abort;
- RE_ENTRANT_CHECK_ON;
- d += 0x1c;
- }
-
- control_word |= CW_Exceptions;
- partial_status &= ~(SW_Summary | SW_Backward);
-
- return d;
-}
-
-void fsave(fpu_addr_modes addr_modes, u_char __user *data_address)
-{
- u_char __user *d;
- int offset = (top & 7) * 10, other = 80 - offset;
-
- d = fstenv(addr_modes, data_address);
-
- RE_ENTRANT_CHECK_OFF;
- FPU_access_ok(d, 80);
-
- /* Copy all registers in stack order. */
- if (__copy_to_user(d, register_base + offset, other))
- FPU_abort;
- if (offset)
- if (__copy_to_user(d + other, register_base, offset))
- FPU_abort;
- RE_ENTRANT_CHECK_ON;
-
- finit();
-}
-
-/*===========================================================================*/
diff --git a/arch/x86/math-emu/reg_mul.c b/arch/x86/math-emu/reg_mul.c
deleted file mode 100644
index d696185..0000000
--- a/arch/x86/math-emu/reg_mul.c
+++ /dev/null
@@ -1,116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- | reg_mul.c |
- | |
- | Multiply one FPU_REG by another, put the result in a destination FPU_REG. |
- | |
- | Copyright (C) 1992,1993,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | Returns the tag of the result if no exceptions or errors occurred. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | The destination may be any FPU_REG, including one of the source FPU_REGs. |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_system.h"
-
-/*
- Multiply two registers to give a register result.
- The sources are st(deststnr) and (b,tagb,signb).
- The destination is st(deststnr).
- */
-/* This routine must be called with non-empty source registers */
-int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w)
-{
- FPU_REG *a = &st(deststnr);
- FPU_REG *dest = a;
- u_char taga = FPU_gettagi(deststnr);
- u_char saved_sign = getsign(dest);
- u_char sign = (getsign(a) ^ getsign(b));
- int tag;
-
- if (!(taga | tagb)) {
- /* Both regs Valid, this should be the most common case. */
-
- tag =
- FPU_u_mul(a, b, dest, control_w, sign,
- exponent(a) + exponent(b));
- if (tag < 0) {
- setsign(dest, saved_sign);
- return tag;
- }
- FPU_settagi(deststnr, tag);
- return tag;
- }
-
- if (taga == TAG_Special)
- taga = FPU_Special(a);
- if (tagb == TAG_Special)
- tagb = FPU_Special(b);
-
- if (((taga == TAG_Valid) && (tagb == TW_Denormal))
- || ((taga == TW_Denormal) && (tagb == TAG_Valid))
- || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
- FPU_REG x, y;
- if (denormal_operand() < 0)
- return FPU_Exception;
-
- FPU_to_exp16(a, &x);
- FPU_to_exp16(b, &y);
- tag = FPU_u_mul(&x, &y, dest, control_w, sign,
- exponent16(&x) + exponent16(&y));
- if (tag < 0) {
- setsign(dest, saved_sign);
- return tag;
- }
- FPU_settagi(deststnr, tag);
- return tag;
- } else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) {
- if (((tagb == TW_Denormal) || (taga == TW_Denormal))
- && (denormal_operand() < 0))
- return FPU_Exception;
-
- /* Must have either both arguments == zero, or
- one valid and the other zero.
- The result is therefore zero. */
- FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
- /* The 80486 book says that the answer is +0, but a real
- 80486 behaves this way.
- IEEE-754 apparently says it should be this way. */
- setsign(dest, sign);
- return TAG_Zero;
- }
- /* Must have infinities, NaNs, etc */
- else if ((taga == TW_NaN) || (tagb == TW_NaN)) {
- return real_2op_NaN(b, tagb, deststnr, &st(0));
- } else if (((taga == TW_Infinity) && (tagb == TAG_Zero))
- || ((tagb == TW_Infinity) && (taga == TAG_Zero))) {
- return arith_invalid(deststnr); /* Zero*Infinity is invalid */
- } else if (((taga == TW_Denormal) || (tagb == TW_Denormal))
- && (denormal_operand() < 0)) {
- return FPU_Exception;
- } else if (taga == TW_Infinity) {
- FPU_copy_to_regi(a, TAG_Special, deststnr);
- setsign(dest, sign);
- return TAG_Special;
- } else if (tagb == TW_Infinity) {
- FPU_copy_to_regi(b, TAG_Special, deststnr);
- setsign(dest, sign);
- return TAG_Special;
- }
-#ifdef PARANOID
- else {
- EXCEPTION(EX_INTERNAL | 0x102);
- return FPU_Exception;
- }
-#endif /* PARANOID */
-
- return 0;
-}
diff --git a/arch/x86/math-emu/reg_norm.S b/arch/x86/math-emu/reg_norm.S
deleted file mode 100644
index 594936e..0000000
--- a/arch/x86/math-emu/reg_norm.S
+++ /dev/null
@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | reg_norm.S |
- | |
- | Copyright (C) 1992,1993,1994,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | Normalize the value in a FPU_REG. |
- | |
- | Call from C as: |
- | int FPU_normalize(FPU_REG *n) |
- | |
- | int FPU_normalize_nuo(FPU_REG *n) |
- | |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-
-.text
-SYM_FUNC_START(FPU_normalize)
- pushl %ebp
- movl %esp,%ebp
- pushl %ebx
-
- movl PARAM1,%ebx
-
- movl SIGH(%ebx),%edx
- movl SIGL(%ebx),%eax
-
- orl %edx,%edx /* ms bits */
- js L_done /* Already normalized */
- jnz L_shift_1 /* Shift left 1 - 31 bits */
-
- orl %eax,%eax
- jz L_zero /* The contents are zero */
-
- movl %eax,%edx
- xorl %eax,%eax
- subw $32,EXP(%ebx) /* This can cause an underflow */
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
- bsrl %edx,%ecx /* get the required shift in %ecx */
- subl $31,%ecx
- negl %ecx
- shld %cl,%eax,%edx
- shl %cl,%eax
- subw %cx,EXP(%ebx) /* This can cause an underflow */
-
- movl %edx,SIGH(%ebx)
- movl %eax,SIGL(%ebx)
-
-L_done:
- cmpw EXP_OVER,EXP(%ebx)
- jge L_overflow
-
- cmpw EXP_UNDER,EXP(%ebx)
- jle L_underflow
-
-L_exit_valid:
- movl TAG_Valid,%eax
-
- /* Convert the exponent to 80x87 form. */
- addw EXTENDED_Ebias,EXP(%ebx)
- andw $0x7fff,EXP(%ebx)
-
-L_exit:
- popl %ebx
- leave
- RET
-
-
-L_zero:
- movw $0,EXP(%ebx)
- movl TAG_Zero,%eax
- jmp L_exit
-
-L_underflow:
- /* Convert the exponent to 80x87 form. */
- addw EXTENDED_Ebias,EXP(%ebx)
- push %ebx
- call arith_underflow
- pop %ebx
- jmp L_exit
-
-L_overflow:
- /* Convert the exponent to 80x87 form. */
- addw EXTENDED_Ebias,EXP(%ebx)
- push %ebx
- call arith_overflow
- pop %ebx
- jmp L_exit
-SYM_FUNC_END(FPU_normalize)
-
-
-
-/* Normalise without reporting underflow or overflow */
-SYM_FUNC_START(FPU_normalize_nuo)
- pushl %ebp
- movl %esp,%ebp
- pushl %ebx
-
- movl PARAM1,%ebx
-
- movl SIGH(%ebx),%edx
- movl SIGL(%ebx),%eax
-
- orl %edx,%edx /* ms bits */
- js L_exit_nuo_valid /* Already normalized */
- jnz L_nuo_shift_1 /* Shift left 1 - 31 bits */
-
- orl %eax,%eax
- jz L_exit_nuo_zero /* The contents are zero */
-
- movl %eax,%edx
- xorl %eax,%eax
- subw $32,EXP(%ebx) /* This can cause an underflow */
-
-/* We need to shift left by 1 - 31 bits */
-L_nuo_shift_1:
- bsrl %edx,%ecx /* get the required shift in %ecx */
- subl $31,%ecx
- negl %ecx
- shld %cl,%eax,%edx
- shl %cl,%eax
- subw %cx,EXP(%ebx) /* This can cause an underflow */
-
- movl %edx,SIGH(%ebx)
- movl %eax,SIGL(%ebx)
-
-L_exit_nuo_valid:
- movl TAG_Valid,%eax
-
- popl %ebx
- leave
- RET
-
-L_exit_nuo_zero:
- movl TAG_Zero,%eax
- movw EXP_UNDER,EXP(%ebx)
-
- popl %ebx
- leave
- RET
-SYM_FUNC_END(FPU_normalize_nuo)
diff --git a/arch/x86/math-emu/reg_round.S b/arch/x86/math-emu/reg_round.S
deleted file mode 100644
index 0bb2a09..0000000
--- a/arch/x86/math-emu/reg_round.S
+++ /dev/null
@@ -1,711 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "reg_round.S"
-/*---------------------------------------------------------------------------+
- | reg_round.S |
- | |
- | Rounding/truncation/etc for FPU basic arithmetic functions. |
- | |
- | Copyright (C) 1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | This code has four possible entry points. |
- | The following must be entered by a jmp instruction: |
- | fpu_reg_round, fpu_reg_round_sqrt, and fpu_Arith_exit. |
- | |
- | The FPU_round entry point is intended to be used by C code. |
- | From C, call as: |
- | int FPU_round(FPU_REG *arg, unsigned int extent, unsigned int control_w) |
- | |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- | |
- | For correct "up" and "down" rounding, the argument must have the correct |
- | sign. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Four entry points. |
- | |
- | Needed by both the fpu_reg_round and fpu_reg_round_sqrt entry points: |
- | %eax:%ebx 64 bit significand |
- | %edx 32 bit extension of the significand |
- | %edi pointer to an FPU_REG for the result to be stored |
- | stack calling function must have set up a C stack frame and |
- | pushed %esi, %edi, and %ebx |
- | |
- | Needed just for the fpu_reg_round_sqrt entry point: |
- | %cx A control word in the same format as the FPU control word. |
- | Otherwise, PARAM4 must give such a value. |
- | |
- | |
- | The significand and its extension are assumed to be exact in the |
- | following sense: |
- | If the significand by itself is the exact result then the significand |
- | extension (%edx) must contain 0, otherwise the significand extension |
- | must be non-zero. |
- | If the significand extension is non-zero then the significand is |
- | smaller than the magnitude of the correct exact result by an amount |
- | greater than zero and less than one ls bit of the significand. |
- | The significand extension is only required to have three possible |
- | non-zero values: |
- | less than 0x80000000 <=> the significand is less than 1/2 an ls |
- | bit smaller than the magnitude of the |
- | true exact result. |
- | exactly 0x80000000 <=> the significand is exactly 1/2 an ls bit |
- | smaller than the magnitude of the true |
- | exact result. |
- | greater than 0x80000000 <=> the significand is more than 1/2 an ls |
- | bit smaller than the magnitude of the |
- | true exact result. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | The code in this module has become quite complex, but it should handle |
- | all of the FPU flags which are set at this stage of the basic arithmetic |
- | computations. |
- | There are a few rare cases where the results are not set identically to |
- | a real FPU. These require a bit more thought because at this stage the |
- | results of the code here appear to be more consistent... |
- | This may be changed in a future version. |
- +---------------------------------------------------------------------------*/
-
-
-#include "fpu_emu.h"
-#include "exception.h"
-#include "control_w.h"
-
-/* Flags for FPU_bits_lost */
-#define LOST_DOWN $1
-#define LOST_UP $2
-
-/* Flags for FPU_denormal */
-#define DENORMAL $1
-#define UNMASKED_UNDERFLOW $2
-
-
-#ifndef NON_REENTRANT_FPU
-/* Make the code re-entrant by putting
- local storage on the stack: */
-#define FPU_bits_lost (%esp)
-#define FPU_denormal 1(%esp)
-
-#else
-/* Not re-entrant, so we can gain speed by putting
- local storage in a static area: */
-.data
- .align 4,0
-FPU_bits_lost:
- .byte 0
-FPU_denormal:
- .byte 0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-.globl fpu_reg_round
-.globl fpu_Arith_exit
-
-/* Entry point when called from C */
-SYM_FUNC_START(FPU_round)
- pushl %ebp
- movl %esp,%ebp
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%edi
- movl SIGH(%edi),%eax
- movl SIGL(%edi),%ebx
- movl PARAM2,%edx
-
-fpu_reg_round: /* Normal entry point */
- movl PARAM4,%ecx
-
-#ifndef NON_REENTRANT_FPU
- pushl %ebx /* adjust the stack pointer */
-#endif /* NON_REENTRANT_FPU */
-
-#ifdef PARANOID
-/* Cannot use this here yet */
-/* orl %eax,%eax */
-/* jns L_entry_bugged */
-#endif /* PARANOID */
-
- cmpw EXP_UNDER,EXP(%edi)
- jle L_Make_denorm /* The number is a de-normal */
-
- movb $0,FPU_denormal /* 0 -> not a de-normal */
-
-Denorm_done:
- movb $0,FPU_bits_lost /* No bits yet lost in rounding */
-
- movl %ecx,%esi
- andl CW_PC,%ecx
- cmpl PR_64_BITS,%ecx
- je LRound_To_64
-
- cmpl PR_53_BITS,%ecx
- je LRound_To_53
-
- cmpl PR_24_BITS,%ecx
- je LRound_To_24
-
-#ifdef PECULIAR_486
-/* With the precision control bits set to 01 "(reserved)", a real 80486
- behaves as if the precision control bits were set to 11 "64 bits" */
- cmpl PR_RESERVED_BITS,%ecx
- je LRound_To_64
-#ifdef PARANOID
- jmp L_bugged_denorm_486
-#endif /* PARANOID */
-#else
-#ifdef PARANOID
- jmp L_bugged_denorm /* There is no bug, just a bad control word */
-#endif /* PARANOID */
-#endif /* PECULIAR_486 */
-
-
-/* Round etc to 24 bit precision */
-LRound_To_24:
- movl %esi,%ecx
- andl CW_RC,%ecx
- cmpl RC_RND,%ecx
- je LRound_nearest_24
-
- cmpl RC_CHOP,%ecx
- je LCheck_truncate_24
-
- cmpl RC_UP,%ecx /* Towards +infinity */
- je LUp_24
-
- cmpl RC_DOWN,%ecx /* Towards -infinity */
- je LDown_24
-
-#ifdef PARANOID
- jmp L_bugged_round24
-#endif /* PARANOID */
-
-LUp_24:
- cmpb SIGN_POS,PARAM5
- jne LCheck_truncate_24 /* If negative then up==truncate */
-
- jmp LCheck_24_round_up
-
-LDown_24:
- cmpb SIGN_POS,PARAM5
- je LCheck_truncate_24 /* If positive then down==truncate */
-
-LCheck_24_round_up:
- movl %eax,%ecx
- andl $0x000000ff,%ecx
- orl %ebx,%ecx
- orl %edx,%ecx
- jnz LDo_24_round_up
- jmp L_Re_normalise
-
-LRound_nearest_24:
- /* Do rounding of the 24th bit if needed (nearest or even) */
- movl %eax,%ecx
- andl $0x000000ff,%ecx
- cmpl $0x00000080,%ecx
- jc LCheck_truncate_24 /* less than half, no increment needed */
-
- jne LGreater_Half_24 /* greater than half, increment needed */
-
- /* Possibly half, we need to check the ls bits */
- orl %ebx,%ebx
- jnz LGreater_Half_24 /* greater than half, increment needed */
-
- orl %edx,%edx
- jnz LGreater_Half_24 /* greater than half, increment needed */
-
- /* Exactly half, increment only if 24th bit is 1 (round to even) */
- testl $0x00000100,%eax
- jz LDo_truncate_24
-
-LGreater_Half_24: /* Rounding: increment at the 24th bit */
-LDo_24_round_up:
- andl $0xffffff00,%eax /* Truncate to 24 bits */
- xorl %ebx,%ebx
- movb LOST_UP,FPU_bits_lost
- addl $0x00000100,%eax
- jmp LCheck_Round_Overflow
-
-LCheck_truncate_24:
- movl %eax,%ecx
- andl $0x000000ff,%ecx
- orl %ebx,%ecx
- orl %edx,%ecx
- jz L_Re_normalise /* No truncation needed */
-
-LDo_truncate_24:
- andl $0xffffff00,%eax /* Truncate to 24 bits */
- xorl %ebx,%ebx
- movb LOST_DOWN,FPU_bits_lost
- jmp L_Re_normalise
-
-
-/* Round etc to 53 bit precision */
-LRound_To_53:
- movl %esi,%ecx
- andl CW_RC,%ecx
- cmpl RC_RND,%ecx
- je LRound_nearest_53
-
- cmpl RC_CHOP,%ecx
- je LCheck_truncate_53
-
- cmpl RC_UP,%ecx /* Towards +infinity */
- je LUp_53
-
- cmpl RC_DOWN,%ecx /* Towards -infinity */
- je LDown_53
-
-#ifdef PARANOID
- jmp L_bugged_round53
-#endif /* PARANOID */
-
-LUp_53:
- cmpb SIGN_POS,PARAM5
- jne LCheck_truncate_53 /* If negative then up==truncate */
-
- jmp LCheck_53_round_up
-
-LDown_53:
- cmpb SIGN_POS,PARAM5
- je LCheck_truncate_53 /* If positive then down==truncate */
-
-LCheck_53_round_up:
- movl %ebx,%ecx
- andl $0x000007ff,%ecx
- orl %edx,%ecx
- jnz LDo_53_round_up
- jmp L_Re_normalise
-
-LRound_nearest_53:
- /* Do rounding of the 53rd bit if needed (nearest or even) */
- movl %ebx,%ecx
- andl $0x000007ff,%ecx
- cmpl $0x00000400,%ecx
- jc LCheck_truncate_53 /* less than half, no increment needed */
-
- jnz LGreater_Half_53 /* greater than half, increment needed */
-
- /* Possibly half, we need to check the ls bits */
- orl %edx,%edx
- jnz LGreater_Half_53 /* greater than half, increment needed */
-
- /* Exactly half, increment only if 53rd bit is 1 (round to even) */
- testl $0x00000800,%ebx
- jz LTruncate_53
-
-LGreater_Half_53: /* Rounding: increment at the 53rd bit */
-LDo_53_round_up:
- movb LOST_UP,FPU_bits_lost
- andl $0xfffff800,%ebx /* Truncate to 53 bits */
- addl $0x00000800,%ebx
- adcl $0,%eax
- jmp LCheck_Round_Overflow
-
-LCheck_truncate_53:
- movl %ebx,%ecx
- andl $0x000007ff,%ecx
- orl %edx,%ecx
- jz L_Re_normalise
-
-LTruncate_53:
- movb LOST_DOWN,FPU_bits_lost
- andl $0xfffff800,%ebx /* Truncate to 53 bits */
- jmp L_Re_normalise
-
-
-/* Round etc to 64 bit precision */
-LRound_To_64:
- movl %esi,%ecx
- andl CW_RC,%ecx
- cmpl RC_RND,%ecx
- je LRound_nearest_64
-
- cmpl RC_CHOP,%ecx
- je LCheck_truncate_64
-
- cmpl RC_UP,%ecx /* Towards +infinity */
- je LUp_64
-
- cmpl RC_DOWN,%ecx /* Towards -infinity */
- je LDown_64
-
-#ifdef PARANOID
- jmp L_bugged_round64
-#endif /* PARANOID */
-
-LUp_64:
- cmpb SIGN_POS,PARAM5
- jne LCheck_truncate_64 /* If negative then up==truncate */
-
- orl %edx,%edx
- jnz LDo_64_round_up
- jmp L_Re_normalise
-
-LDown_64:
- cmpb SIGN_POS,PARAM5
- je LCheck_truncate_64 /* If positive then down==truncate */
-
- orl %edx,%edx
- jnz LDo_64_round_up
- jmp L_Re_normalise
-
-LRound_nearest_64:
- cmpl $0x80000000,%edx
- jc LCheck_truncate_64
-
- jne LDo_64_round_up
-
- /* Now test for round-to-even */
- testb $1,%bl
- jz LCheck_truncate_64
-
-LDo_64_round_up:
- movb LOST_UP,FPU_bits_lost
- addl $1,%ebx
- adcl $0,%eax
-
-LCheck_Round_Overflow:
- jnc L_Re_normalise
-
- /* Overflow, adjust the result (significand to 1.0) */
- rcrl $1,%eax
- rcrl $1,%ebx
- incw EXP(%edi)
- jmp L_Re_normalise
-
-LCheck_truncate_64:
- orl %edx,%edx
- jz L_Re_normalise
-
-LTruncate_64:
- movb LOST_DOWN,FPU_bits_lost
-
-L_Re_normalise:
- testb $0xff,FPU_denormal
- jnz Normalise_result
-
-L_Normalised:
- movl TAG_Valid,%edx
-
-L_deNormalised:
- cmpb LOST_UP,FPU_bits_lost
- je L_precision_lost_up
-
- cmpb LOST_DOWN,FPU_bits_lost
- je L_precision_lost_down
-
-L_no_precision_loss:
- /* store the result */
-
-L_Store_significand:
- movl %eax,SIGH(%edi)
- movl %ebx,SIGL(%edi)
-
- cmpw EXP_OVER,EXP(%edi)
- jge L_overflow
-
- movl %edx,%eax
-
- /* Convert the exponent to 80x87 form. */
- addw EXTENDED_Ebias,EXP(%edi)
- andw $0x7fff,EXP(%edi)
-
-fpu_reg_round_signed_special_exit:
-
- cmpb SIGN_POS,PARAM5
- je fpu_reg_round_special_exit
-
- orw $0x8000,EXP(%edi) /* Negative sign for the result. */
-
-fpu_reg_round_special_exit:
-
-#ifndef NON_REENTRANT_FPU
- popl %ebx /* adjust the stack pointer */
-#endif /* NON_REENTRANT_FPU */
-
-fpu_Arith_exit:
- popl %ebx
- popl %edi
- popl %esi
- leave
- RET
-
-
-/*
- * Set the FPU status flags to represent precision loss due to
- * round-up.
- */
-L_precision_lost_up:
- push %edx
- push %eax
- call set_precision_flag_up
- popl %eax
- popl %edx
- jmp L_no_precision_loss
-
-/*
- * Set the FPU status flags to represent precision loss due to
- * truncation.
- */
-L_precision_lost_down:
- push %edx
- push %eax
- call set_precision_flag_down
- popl %eax
- popl %edx
- jmp L_no_precision_loss
-
-
-/*
- * The number is a denormal (which might get rounded up to a normal)
- * Shift the number right the required number of bits, which will
- * have to be undone later...
- */
-L_Make_denorm:
- /* The action to be taken depends upon whether the underflow
- exception is masked */
- testb CW_Underflow,%cl /* Underflow mask. */
- jz Unmasked_underflow /* Do not make a denormal. */
-
- movb DENORMAL,FPU_denormal
-
- pushl %ecx /* Save */
- movw EXP_UNDER+1,%cx
- subw EXP(%edi),%cx
-
- cmpw $64,%cx /* shrd only works for 0..31 bits */
- jnc Denorm_shift_more_than_63
-
- cmpw $32,%cx /* shrd only works for 0..31 bits */
- jnc Denorm_shift_more_than_32
-
-/*
- * We got here without jumps by assuming that the most common requirement
- * is for a small de-normalising shift.
- * Shift by [1..31] bits
- */
- addw %cx,EXP(%edi)
- orl %edx,%edx /* extension */
- setne %ch /* Save whether %edx is non-zero */
- xorl %edx,%edx
- shrd %cl,%ebx,%edx
- shrd %cl,%eax,%ebx
- shr %cl,%eax
- orb %ch,%dl
- popl %ecx
- jmp Denorm_done
-
-/* Shift by [32..63] bits */
-Denorm_shift_more_than_32:
- addw %cx,EXP(%edi)
- subb $32,%cl
- orl %edx,%edx
- setne %ch
- orb %ch,%bl
- xorl %edx,%edx
- shrd %cl,%ebx,%edx
- shrd %cl,%eax,%ebx
- shr %cl,%eax
- orl %edx,%edx /* test these 32 bits */
- setne %cl
- orb %ch,%bl
- orb %cl,%bl
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- popl %ecx
- jmp Denorm_done
-
-/* Shift by [64..) bits */
-Denorm_shift_more_than_63:
- cmpw $64,%cx
- jne Denorm_shift_more_than_64
-
-/* Exactly 64 bit shift */
- addw %cx,EXP(%edi)
- xorl %ecx,%ecx
- orl %edx,%edx
- setne %cl
- orl %ebx,%ebx
- setne %ch
- orb %ch,%cl
- orb %cl,%al
- movl %eax,%edx
- xorl %eax,%eax
- xorl %ebx,%ebx
- popl %ecx
- jmp Denorm_done
-
-Denorm_shift_more_than_64:
- movw EXP_UNDER+1,EXP(%edi)
-/* This is easy, %eax must be non-zero, so.. */
- movl $1,%edx
- xorl %eax,%eax
- xorl %ebx,%ebx
- popl %ecx
- jmp Denorm_done
-
-
-Unmasked_underflow:
- movb UNMASKED_UNDERFLOW,FPU_denormal
- jmp Denorm_done
-
-
-/* Undo the de-normalisation. */
-Normalise_result:
- cmpb UNMASKED_UNDERFLOW,FPU_denormal
- je Signal_underflow
-
-/* The number must be a denormal if we got here. */
-#ifdef PARANOID
- /* But check it... just in case. */
- cmpw EXP_UNDER+1,EXP(%edi)
- jne L_norm_bugged
-#endif /* PARANOID */
-
-#ifdef PECULIAR_486
- /*
- * This implements a special feature of 80486 behaviour.
- * Underflow will be signaled even if the number is
- * not a denormal after rounding.
- * This difference occurs only for masked underflow, and not
- * in the unmasked case.
- * Actual 80486 behaviour differs from this in some circumstances.
- */
- orl %eax,%eax /* ms bits */
- js LPseudoDenormal /* Will be masked underflow */
-#else
- orl %eax,%eax /* ms bits */
- js L_Normalised /* No longer a denormal */
-#endif /* PECULIAR_486 */
-
- jnz LDenormal_adj_exponent
-
- orl %ebx,%ebx
- jz L_underflow_to_zero /* The contents are zero */
-
-LDenormal_adj_exponent:
- decw EXP(%edi)
-
-LPseudoDenormal:
- testb $0xff,FPU_bits_lost /* bits lost == underflow */
- movl TAG_Special,%edx
- jz L_deNormalised
-
- /* There must be a masked underflow */
- push %eax
- pushl EX_Underflow
- call EXCEPTION
- popl %eax
- popl %eax
- movl TAG_Special,%edx
- jmp L_deNormalised
-
-
-/*
- * The operations resulted in a number too small to represent.
- * Masked response.
- */
-L_underflow_to_zero:
- push %eax
- call set_precision_flag_down
- popl %eax
-
- push %eax
- pushl EX_Underflow
- call EXCEPTION
- popl %eax
- popl %eax
-
-/* Reduce the exponent to EXP_UNDER */
- movw EXP_UNDER,EXP(%edi)
- movl TAG_Zero,%edx
- jmp L_Store_significand
-
-
-/* The operations resulted in a number too large to represent. */
-L_overflow:
- addw EXTENDED_Ebias,EXP(%edi) /* Set for unmasked response. */
- push %edi
- call arith_overflow
- pop %edi
- jmp fpu_reg_round_signed_special_exit
-
-
-Signal_underflow:
- /* The number may have been changed to a non-denormal */
- /* by the rounding operations. */
- cmpw EXP_UNDER,EXP(%edi)
- jle Do_unmasked_underflow
-
- jmp L_Normalised
-
-Do_unmasked_underflow:
- /* Increase the exponent by the magic number */
- addw $(3*(1<<13)),EXP(%edi)
- push %eax
- pushl EX_Underflow
- call EXCEPTION
- popl %eax
- popl %eax
- jmp L_Normalised
-
-
-#ifdef PARANOID
-#ifdef PECULIAR_486
-L_bugged_denorm_486:
- pushl EX_INTERNAL|0x236
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-#else
-L_bugged_denorm:
- pushl EX_INTERNAL|0x230
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-#endif /* PECULIAR_486 */
-
-L_bugged_round24:
- pushl EX_INTERNAL|0x231
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-
-L_bugged_round53:
- pushl EX_INTERNAL|0x232
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-
-L_bugged_round64:
- pushl EX_INTERNAL|0x233
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-
-L_norm_bugged:
- pushl EX_INTERNAL|0x234
- call EXCEPTION
- popl %ebx
- jmp L_exception_exit
-
-L_entry_bugged:
- pushl EX_INTERNAL|0x235
- call EXCEPTION
- popl %ebx
-L_exception_exit:
- mov $-1,%eax
- jmp fpu_reg_round_special_exit
-#endif /* PARANOID */
-
-SYM_FUNC_END(FPU_round)
diff --git a/arch/x86/math-emu/reg_u_add.S b/arch/x86/math-emu/reg_u_add.S
deleted file mode 100644
index 0724728..0000000
--- a/arch/x86/math-emu/reg_u_add.S
+++ /dev/null
@@ -1,169 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "reg_u_add.S"
-/*---------------------------------------------------------------------------+
- | reg_u_add.S |
- | |
- | Add two valid (TAG_Valid) FPU_REG numbers, of the same sign, and put the |
- | result in a destination FPU_REG. |
- | |
- | Copyright (C) 1992,1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | Call from C as: |
- | int FPU_u_add(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ, |
- | int control_w) |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*
- | Kernel addition routine FPU_u_add(reg *arg1, reg *arg2, reg *answ).
- | Takes two valid reg f.p. numbers (TAG_Valid), which are
- | treated as unsigned numbers,
- | and returns their sum as a TAG_Valid or TAG_Special f.p. number.
- | The returned number is normalized.
- | Basic checks are performed if PARANOID is defined.
- */
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-.text
-SYM_FUNC_START(FPU_u_add)
- pushl %ebp
- movl %esp,%ebp
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%esi /* source 1 */
- movl PARAM2,%edi /* source 2 */
-
- movl PARAM6,%ecx
- movl %ecx,%edx
- subl PARAM7,%ecx /* exp1 - exp2 */
- jge L_arg1_larger
-
- /* num1 is smaller */
- movl SIGL(%esi),%ebx
- movl SIGH(%esi),%eax
-
- movl %edi,%esi
- movl PARAM7,%edx
- negw %cx
- jmp L_accum_loaded
-
-L_arg1_larger:
- /* num1 has larger or equal exponent */
- movl SIGL(%edi),%ebx
- movl SIGH(%edi),%eax
-
-L_accum_loaded:
- movl PARAM3,%edi /* destination */
- movw %dx,EXP(%edi) /* Copy exponent to destination */
-
- xorl %edx,%edx /* clear the extension */
-
-#ifdef PARANOID
- testl $0x80000000,%eax
- je L_bugged
-
- testl $0x80000000,SIGH(%esi)
- je L_bugged
-#endif /* PARANOID */
-
-/* The number to be shifted is in %eax:%ebx:%edx */
- cmpw $32,%cx /* shrd only works for 0..31 bits */
- jnc L_more_than_31
-
-/* less than 32 bits */
- shrd %cl,%ebx,%edx
- shrd %cl,%eax,%ebx
- shr %cl,%eax
- jmp L_shift_done
-
-L_more_than_31:
- cmpw $64,%cx
- jnc L_more_than_63
-
- subb $32,%cl
- jz L_exactly_32
-
- shrd %cl,%eax,%edx
- shr %cl,%eax
- orl %ebx,%ebx
- jz L_more_31_no_low /* none of the lowest bits is set */
-
- orl $1,%edx /* record the fact in the extension */
-
-L_more_31_no_low:
- movl %eax,%ebx
- xorl %eax,%eax
- jmp L_shift_done
-
-L_exactly_32:
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- jmp L_shift_done
-
-L_more_than_63:
- cmpw $65,%cx
- jnc L_more_than_64
-
- movl %eax,%edx
- orl %ebx,%ebx
- jz L_more_63_no_low
-
- orl $1,%edx
- jmp L_more_63_no_low
-
-L_more_than_64:
- movl $1,%edx /* The shifted nr always at least one '1' */
-
-L_more_63_no_low:
- xorl %ebx,%ebx
- xorl %eax,%eax
-
-L_shift_done:
- /* Now do the addition */
- addl SIGL(%esi),%ebx
- adcl SIGH(%esi),%eax
- jnc L_round_the_result
-
- /* Overflow, adjust the result */
- rcrl $1,%eax
- rcrl $1,%ebx
- rcrl $1,%edx
- jnc L_no_bit_lost
-
- orl $1,%edx
-
-L_no_bit_lost:
- incw EXP(%edi)
-
-L_round_the_result:
- jmp fpu_reg_round /* Round the result */
-
-
-
-#ifdef PARANOID
-/* If we ever get here then we have problems! */
-L_bugged:
- pushl EX_INTERNAL|0x201
- call EXCEPTION
- pop %ebx
- movl $-1,%eax
- jmp L_exit
-
-L_exit:
- popl %ebx
- popl %edi
- popl %esi
- leave
- RET
-#endif /* PARANOID */
-SYM_FUNC_END(FPU_u_add)
diff --git a/arch/x86/math-emu/reg_u_div.S b/arch/x86/math-emu/reg_u_div.S
deleted file mode 100644
index b5a41e2..0000000
--- a/arch/x86/math-emu/reg_u_div.S
+++ /dev/null
@@ -1,474 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "reg_u_div.S"
-/*---------------------------------------------------------------------------+
- | reg_u_div.S |
- | |
- | Divide one FPU_REG by another and put the result in a destination FPU_REG.|
- | |
- | Copyright (C) 1992,1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Call from C as: |
- | int FPU_u_div(FPU_REG *a, FPU_REG *b, FPU_REG *dest, |
- | unsigned int control_word, char *sign) |
- | |
- | Does not compute the destination exponent, but does adjust it. |
- | |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-
-/* #define dSIGL(x) (x) */
-/* #define dSIGH(x) 4(x) */
-
-
-#ifndef NON_REENTRANT_FPU
-/*
- Local storage on the stack:
- Result: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- Overflow flag: ovfl_flag
- */
-#define FPU_accum_3 -4(%ebp)
-#define FPU_accum_2 -8(%ebp)
-#define FPU_accum_1 -12(%ebp)
-#define FPU_accum_0 -16(%ebp)
-#define FPU_result_1 -20(%ebp)
-#define FPU_result_2 -24(%ebp)
-#define FPU_ovfl_flag -28(%ebp)
-
-#else
-.data
-/*
- Local storage in a static area:
- Result: FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- Overflow flag: ovfl_flag
- */
- .align 4,0
-FPU_accum_3:
- .long 0
-FPU_accum_2:
- .long 0
-FPU_accum_1:
- .long 0
-FPU_accum_0:
- .long 0
-FPU_result_1:
- .long 0
-FPU_result_2:
- .long 0
-FPU_ovfl_flag:
- .byte 0
-#endif /* NON_REENTRANT_FPU */
-
-#define REGA PARAM1
-#define REGB PARAM2
-#define DEST PARAM3
-
-.text
-SYM_FUNC_START(FPU_u_div)
- pushl %ebp
- movl %esp,%ebp
-#ifndef NON_REENTRANT_FPU
- subl $28,%esp
-#endif /* NON_REENTRANT_FPU */
-
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl REGA,%esi
- movl REGB,%ebx
- movl DEST,%edi
-
- movswl EXP(%esi),%edx
- movswl EXP(%ebx),%eax
- subl %eax,%edx
- addl EXP_BIAS,%edx
-
- /* A denormal and a large number can cause an exponent underflow */
- cmpl EXP_WAY_UNDER,%edx
- jg xExp_not_underflow
-
- /* Set to a really low value allow correct handling */
- movl EXP_WAY_UNDER,%edx
-
-xExp_not_underflow:
-
- movw %dx,EXP(%edi)
-
-#ifdef PARANOID
-/* testl $0x80000000, SIGH(%esi) // Dividend */
-/* je L_bugged */
- testl $0x80000000, SIGH(%ebx) /* Divisor */
- je L_bugged
-#endif /* PARANOID */
-
-/* Check if the divisor can be treated as having just 32 bits */
- cmpl $0,SIGL(%ebx)
- jnz L_Full_Division /* Can't do a quick divide */
-
-/* We should be able to zip through the division here */
- movl SIGH(%ebx),%ecx /* The divisor */
- movl SIGH(%esi),%edx /* Dividend */
- movl SIGL(%esi),%eax /* Dividend */
-
- cmpl %ecx,%edx
- setaeb FPU_ovfl_flag /* Keep a record */
- jb L_no_adjust
-
- subl %ecx,%edx /* Prevent the overflow */
-
-L_no_adjust:
- /* Divide the 64 bit number by the 32 bit denominator */
- divl %ecx
- movl %eax,FPU_result_2
-
- /* Work on the remainder of the first division */
- xorl %eax,%eax
- divl %ecx
- movl %eax,FPU_result_1
-
- /* Work on the remainder of the 64 bit division */
- xorl %eax,%eax
- divl %ecx
-
- testb $255,FPU_ovfl_flag /* was the num > denom ? */
- je L_no_overflow
-
- /* Do the shifting here */
- /* increase the exponent */
- incw EXP(%edi)
-
- /* shift the mantissa right one bit */
- stc /* To set the ms bit */
- rcrl FPU_result_2
- rcrl FPU_result_1
- rcrl %eax
-
-L_no_overflow:
- jmp LRound_precision /* Do the rounding as required */
-
-
-/*---------------------------------------------------------------------------+
- | Divide: Return arg1/arg2 to arg3. |
- | |
- | This routine does not use the exponents of arg1 and arg2, but does |
- | adjust the exponent of arg3. |
- | |
- | The maximum returned value is (ignoring exponents) |
- | .ffffffff ffffffff |
- | ------------------ = 1.ffffffff fffffffe |
- | .80000000 00000000 |
- | and the minimum is |
- | .80000000 00000000 |
- | ------------------ = .80000000 00000001 (rounded) |
- | .ffffffff ffffffff |
- | |
- +---------------------------------------------------------------------------*/
-
-
-L_Full_Division:
- /* Save extended dividend in local register */
- movl SIGL(%esi),%eax
- movl %eax,FPU_accum_2
- movl SIGH(%esi),%eax
- movl %eax,FPU_accum_3
- xorl %eax,%eax
- movl %eax,FPU_accum_1 /* zero the extension */
- movl %eax,FPU_accum_0 /* zero the extension */
-
- movl SIGL(%esi),%eax /* Get the current num */
- movl SIGH(%esi),%edx
-
-/*----------------------------------------------------------------------*/
-/* Initialization done.
- Do the first 32 bits. */
-
- movb $0,FPU_ovfl_flag
- cmpl SIGH(%ebx),%edx /* Test for imminent overflow */
- jb LLess_than_1
- ja LGreater_than_1
-
- cmpl SIGL(%ebx),%eax
- jb LLess_than_1
-
-LGreater_than_1:
-/* The dividend is greater or equal, would cause overflow */
- setaeb FPU_ovfl_flag /* Keep a record */
-
- subl SIGL(%ebx),%eax
- sbbl SIGH(%ebx),%edx /* Prevent the overflow */
- movl %eax,FPU_accum_2
- movl %edx,FPU_accum_3
-
-LLess_than_1:
-/* At this point, we have a dividend < divisor, with a record of
- adjustment in FPU_ovfl_flag */
-
- /* We will divide by a number which is too large */
- movl SIGH(%ebx),%ecx
- addl $1,%ecx
- jnc LFirst_div_not_1
-
- /* here we need to divide by 100000000h,
- i.e., no division at all.. */
- mov %edx,%eax
- jmp LFirst_div_done
-
-LFirst_div_not_1:
- divl %ecx /* Divide the numerator by the augmented
- denom ms dw */
-
-LFirst_div_done:
- movl %eax,FPU_result_2 /* Put the result in the answer */
-
- mull SIGH(%ebx) /* mul by the ms dw of the denom */
-
- subl %eax,FPU_accum_2 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_3
-
- movl FPU_result_2,%eax /* Get the result back */
- mull SIGL(%ebx) /* now mul the ls dw of the denom */
-
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
- sbbl $0,FPU_accum_3
- je LDo_2nd_32_bits /* Must check for non-zero result here */
-
-#ifdef PARANOID
- jb L_bugged_1
-#endif /* PARANOID */
-
- /* need to subtract another once of the denom */
- incl FPU_result_2 /* Correct the answer */
-
- movl SIGL(%ebx),%eax
- movl SIGH(%ebx),%edx
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
-
-#ifdef PARANOID
- sbbl $0,FPU_accum_3
- jne L_bugged_1 /* Must check for non-zero result here */
-#endif /* PARANOID */
-
-/*----------------------------------------------------------------------*/
-/* Half of the main problem is done, there is just a reduced numerator
- to handle now.
- Work with the second 32 bits, FPU_accum_0 not used from now on */
-LDo_2nd_32_bits:
- movl FPU_accum_2,%edx /* get the reduced num */
- movl FPU_accum_1,%eax
-
- /* need to check for possible subsequent overflow */
- cmpl SIGH(%ebx),%edx
- jb LDo_2nd_div
- ja LPrevent_2nd_overflow
-
- cmpl SIGL(%ebx),%eax
- jb LDo_2nd_div
-
-LPrevent_2nd_overflow:
-/* The numerator is greater or equal, would cause overflow */
- /* prevent overflow */
- subl SIGL(%ebx),%eax
- sbbl SIGH(%ebx),%edx
- movl %edx,FPU_accum_2
- movl %eax,FPU_accum_1
-
- incl FPU_result_2 /* Reflect the subtraction in the answer */
-
-#ifdef PARANOID
- je L_bugged_2 /* Can't bump the result to 1.0 */
-#endif /* PARANOID */
-
-LDo_2nd_div:
- cmpl $0,%ecx /* augmented denom msw */
- jnz LSecond_div_not_1
-
- /* %ecx == 0, we are dividing by 1.0 */
- mov %edx,%eax
- jmp LSecond_div_done
-
-LSecond_div_not_1:
- divl %ecx /* Divide the numerator by the denom ms dw */
-
-LSecond_div_done:
- movl %eax,FPU_result_1 /* Put the result in the answer */
-
- mull SIGH(%ebx) /* mul by the ms dw of the denom */
-
- subl %eax,FPU_accum_1 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
-#endif /* PARANOID */
-
- movl FPU_result_1,%eax /* Get the result back */
- mull SIGL(%ebx) /* now mul the ls dw of the denom */
-
- subl %eax,FPU_accum_0 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_1 /* Subtract from the num local reg */
- sbbl $0,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
-#endif /* PARANOID */
-
- jz LDo_3rd_32_bits
-
-#ifdef PARANOID
- cmpl $1,FPU_accum_2
- jne L_bugged_2
-#endif /* PARANOID */
-
- /* need to subtract another once of the denom */
- movl SIGL(%ebx),%eax
- movl SIGH(%ebx),%edx
- subl %eax,FPU_accum_0 /* Subtract from the num local reg */
- sbbl %edx,FPU_accum_1
- sbbl $0,FPU_accum_2
-
-#ifdef PARANOID
- jc L_bugged_2
- jne L_bugged_2
-#endif /* PARANOID */
-
- addl $1,FPU_result_1 /* Correct the answer */
- adcl $0,FPU_result_2
-
-#ifdef PARANOID
- jc L_bugged_2 /* Must check for non-zero result here */
-#endif /* PARANOID */
-
-/*----------------------------------------------------------------------*/
-/* The division is essentially finished here, we just need to perform
- tidying operations.
- Deal with the 3rd 32 bits */
-LDo_3rd_32_bits:
- movl FPU_accum_1,%edx /* get the reduced num */
- movl FPU_accum_0,%eax
-
- /* need to check for possible subsequent overflow */
- cmpl SIGH(%ebx),%edx /* denom */
- jb LRound_prep
- ja LPrevent_3rd_overflow
-
- cmpl SIGL(%ebx),%eax /* denom */
- jb LRound_prep
-
-LPrevent_3rd_overflow:
- /* prevent overflow */
- subl SIGL(%ebx),%eax
- sbbl SIGH(%ebx),%edx
- movl %edx,FPU_accum_1
- movl %eax,FPU_accum_0
-
- addl $1,FPU_result_1 /* Reflect the subtraction in the answer */
- adcl $0,FPU_result_2
- jne LRound_prep
- jnc LRound_prep
-
- /* This is a tricky spot, there is an overflow of the answer */
- movb $255,FPU_ovfl_flag /* Overflow -> 1.000 */
-
-LRound_prep:
-/*
- * Prepare for rounding.
- * To test for rounding, we just need to compare 2*accum with the
- * denom.
- */
- movl FPU_accum_0,%ecx
- movl FPU_accum_1,%edx
- movl %ecx,%eax
- orl %edx,%eax
- jz LRound_ovfl /* The accumulator contains zero. */
-
- /* Multiply by 2 */
- clc
- rcll $1,%ecx
- rcll $1,%edx
- jc LRound_large /* No need to compare, denom smaller */
-
- subl SIGL(%ebx),%ecx
- sbbl SIGH(%ebx),%edx
- jnc LRound_not_small
-
- movl $0x70000000,%eax /* Denom was larger */
- jmp LRound_ovfl
-
-LRound_not_small:
- jnz LRound_large
-
- movl $0x80000000,%eax /* Remainder was exactly 1/2 denom */
- jmp LRound_ovfl
-
-LRound_large:
- movl $0xff000000,%eax /* Denom was smaller */
-
-LRound_ovfl:
-/* We are now ready to deal with rounding, but first we must get
- the bits properly aligned */
- testb $255,FPU_ovfl_flag /* was the num > denom ? */
- je LRound_precision
-
- incw EXP(%edi)
-
- /* shift the mantissa right one bit */
- stc /* Will set the ms bit */
- rcrl FPU_result_2
- rcrl FPU_result_1
- rcrl %eax
-
-/* Round the result as required */
-LRound_precision:
- decw EXP(%edi) /* binary point between 1st & 2nd bits */
-
- movl %eax,%edx
- movl FPU_result_1,%ebx
- movl FPU_result_2,%eax
- jmp fpu_reg_round
-
-
-#ifdef PARANOID
-/* The logic is wrong if we got here */
-L_bugged:
- pushl EX_INTERNAL|0x202
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_bugged_1:
- pushl EX_INTERNAL|0x203
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_bugged_2:
- pushl EX_INTERNAL|0x204
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_exit:
- movl $-1,%eax
- popl %ebx
- popl %edi
- popl %esi
-
- leave
- RET
-#endif /* PARANOID */
-
-SYM_FUNC_END(FPU_u_div)
diff --git a/arch/x86/math-emu/reg_u_mul.S b/arch/x86/math-emu/reg_u_mul.S
deleted file mode 100644
index e2588b2..0000000
--- a/arch/x86/math-emu/reg_u_mul.S
+++ /dev/null
@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "reg_u_mul.S"
-/*---------------------------------------------------------------------------+
- | reg_u_mul.S |
- | |
- | Core multiplication routine |
- | |
- | Copyright (C) 1992,1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Basic multiplication routine. |
- | Does not check the resulting exponent for overflow/underflow |
- | |
- | FPU_u_mul(FPU_REG *a, FPU_REG *b, FPU_REG *c, unsigned int cw); |
- | |
- | Internal working is at approx 128 bits. |
- | Result is rounded to nearest 53 or 64 bits, using "nearest or even". |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-
-
-#ifndef NON_REENTRANT_FPU
-/* Local storage on the stack: */
-#define FPU_accum_0 -4(%ebp) /* ms word */
-#define FPU_accum_1 -8(%ebp)
-
-#else
-/* Local storage in a static area: */
-.data
- .align 4,0
-FPU_accum_0:
- .long 0
-FPU_accum_1:
- .long 0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-SYM_FUNC_START(FPU_u_mul)
- pushl %ebp
- movl %esp,%ebp
-#ifndef NON_REENTRANT_FPU
- subl $8,%esp
-#endif /* NON_REENTRANT_FPU */
-
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%esi
- movl PARAM2,%edi
-
-#ifdef PARANOID
- testl $0x80000000,SIGH(%esi)
- jz L_bugged
- testl $0x80000000,SIGH(%edi)
- jz L_bugged
-#endif /* PARANOID */
-
- xorl %ecx,%ecx
- xorl %ebx,%ebx
-
- movl SIGL(%esi),%eax
- mull SIGL(%edi)
- movl %eax,FPU_accum_0
- movl %edx,FPU_accum_1
-
- movl SIGL(%esi),%eax
- mull SIGH(%edi)
- addl %eax,FPU_accum_1
- adcl %edx,%ebx
-/* adcl $0,%ecx // overflow here is not possible */
-
- movl SIGH(%esi),%eax
- mull SIGL(%edi)
- addl %eax,FPU_accum_1
- adcl %edx,%ebx
- adcl $0,%ecx
-
- movl SIGH(%esi),%eax
- mull SIGH(%edi)
- addl %eax,%ebx
- adcl %edx,%ecx
-
- /* Get the sum of the exponents. */
- movl PARAM6,%eax
- subl EXP_BIAS-1,%eax
-
- /* Two denormals can cause an exponent underflow */
- cmpl EXP_WAY_UNDER,%eax
- jg Exp_not_underflow
-
- /* Set to a really low value allow correct handling */
- movl EXP_WAY_UNDER,%eax
-
-Exp_not_underflow:
-
-/* Have now finished with the sources */
- movl PARAM3,%edi /* Point to the destination */
- movw %ax,EXP(%edi)
-
-/* Now make sure that the result is normalized */
- testl $0x80000000,%ecx
- jnz LResult_Normalised
-
- /* Normalize by shifting left one bit */
- shll $1,FPU_accum_0
- rcll $1,FPU_accum_1
- rcll $1,%ebx
- rcll $1,%ecx
- decw EXP(%edi)
-
-LResult_Normalised:
- movl FPU_accum_0,%eax
- movl FPU_accum_1,%edx
- orl %eax,%eax
- jz L_extent_zero
-
- orl $1,%edx
-
-L_extent_zero:
- movl %ecx,%eax
- jmp fpu_reg_round
-
-
-#ifdef PARANOID
-L_bugged:
- pushl EX_INTERNAL|0x205
- call EXCEPTION
- pop %ebx
- jmp L_exit
-
-L_exit:
- popl %ebx
- popl %edi
- popl %esi
- leave
- RET
-#endif /* PARANOID */
-
-SYM_FUNC_END(FPU_u_mul)
diff --git a/arch/x86/math-emu/reg_u_sub.S b/arch/x86/math-emu/reg_u_sub.S
deleted file mode 100644
index 4c900c2..0000000
--- a/arch/x86/math-emu/reg_u_sub.S
+++ /dev/null
@@ -1,274 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "reg_u_sub.S"
-/*---------------------------------------------------------------------------+
- | reg_u_sub.S |
- | |
- | Core floating point subtraction routine. |
- | |
- | Copyright (C) 1992,1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@suburbia.net |
- | |
- | Call from C as: |
- | int FPU_u_sub(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ, |
- | int control_w) |
- | Return value is the tag of the answer, or-ed with FPU_Exception if |
- | one was raised, or -1 on internal error. |
- | |
- +---------------------------------------------------------------------------*/
-
-/*
- | Kernel subtraction routine FPU_u_sub(reg *arg1, reg *arg2, reg *answ).
- | Takes two valid reg f.p. numbers (TAG_Valid), which are
- | treated as unsigned numbers,
- | and returns their difference as a TAG_Valid or TAG_Zero f.p.
- | number.
- | The first number (arg1) must be the larger.
- | The returned number is normalized.
- | Basic checks are performed if PARANOID is defined.
- */
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-.text
-SYM_FUNC_START(FPU_u_sub)
- pushl %ebp
- movl %esp,%ebp
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%esi /* source 1 */
- movl PARAM2,%edi /* source 2 */
-
- movl PARAM6,%ecx
- subl PARAM7,%ecx /* exp1 - exp2 */
-
-#ifdef PARANOID
- /* source 2 is always smaller than source 1 */
- js L_bugged_1
-
- testl $0x80000000,SIGH(%edi) /* The args are assumed to be be normalized */
- je L_bugged_2
-
- testl $0x80000000,SIGH(%esi)
- je L_bugged_2
-#endif /* PARANOID */
-
-/*--------------------------------------+
- | Form a register holding the |
- | smaller number |
- +--------------------------------------*/
- movl SIGH(%edi),%eax /* register ms word */
- movl SIGL(%edi),%ebx /* register ls word */
-
- movl PARAM3,%edi /* destination */
- movl PARAM6,%edx
- movw %dx,EXP(%edi) /* Copy exponent to destination */
-
- xorl %edx,%edx /* register extension */
-
-/*--------------------------------------+
- | Shift the temporary register |
- | right the required number of |
- | places. |
- +--------------------------------------*/
-
- cmpw $32,%cx /* shrd only works for 0..31 bits */
- jnc L_more_than_31
-
-/* less than 32 bits */
- shrd %cl,%ebx,%edx
- shrd %cl,%eax,%ebx
- shr %cl,%eax
- jmp L_shift_done
-
-L_more_than_31:
- cmpw $64,%cx
- jnc L_more_than_63
-
- subb $32,%cl
- jz L_exactly_32
-
- shrd %cl,%eax,%edx
- shr %cl,%eax
- orl %ebx,%ebx
- jz L_more_31_no_low /* none of the lowest bits is set */
-
- orl $1,%edx /* record the fact in the extension */
-
-L_more_31_no_low:
- movl %eax,%ebx
- xorl %eax,%eax
- jmp L_shift_done
-
-L_exactly_32:
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- jmp L_shift_done
-
-L_more_than_63:
- cmpw $65,%cx
- jnc L_more_than_64
-
- /* Shift right by 64 bits */
- movl %eax,%edx
- orl %ebx,%ebx
- jz L_more_63_no_low
-
- orl $1,%edx
- jmp L_more_63_no_low
-
-L_more_than_64:
- jne L_more_than_65
-
- /* Shift right by 65 bits */
- /* Carry is clear if we get here */
- movl %eax,%edx
- rcrl %edx
- jnc L_shift_65_nc
-
- orl $1,%edx
- jmp L_more_63_no_low
-
-L_shift_65_nc:
- orl %ebx,%ebx
- jz L_more_63_no_low
-
- orl $1,%edx
- jmp L_more_63_no_low
-
-L_more_than_65:
- movl $1,%edx /* The shifted nr always at least one '1' */
-
-L_more_63_no_low:
- xorl %ebx,%ebx
- xorl %eax,%eax
-
-L_shift_done:
-L_subtr:
-/*------------------------------+
- | Do the subtraction |
- +------------------------------*/
- xorl %ecx,%ecx
- subl %edx,%ecx
- movl %ecx,%edx
- movl SIGL(%esi),%ecx
- sbbl %ebx,%ecx
- movl %ecx,%ebx
- movl SIGH(%esi),%ecx
- sbbl %eax,%ecx
- movl %ecx,%eax
-
-#ifdef PARANOID
- /* We can never get a borrow */
- jc L_bugged
-#endif /* PARANOID */
-
-/*--------------------------------------+
- | Normalize the result |
- +--------------------------------------*/
- testl $0x80000000,%eax
- jnz L_round /* no shifting needed */
-
- orl %eax,%eax
- jnz L_shift_1 /* shift left 1 - 31 bits */
-
- orl %ebx,%ebx
- jnz L_shift_32 /* shift left 32 - 63 bits */
-
-/*
- * A rare case, the only one which is non-zero if we got here
- * is: 1000000 .... 0000
- * -0111111 .... 1111 1
- * --------------------
- * 0000000 .... 0000 1
- */
-
- cmpl $0x80000000,%edx
- jnz L_must_be_zero
-
- /* Shift left 64 bits */
- subw $64,EXP(%edi)
- xchg %edx,%eax
- jmp fpu_reg_round
-
-L_must_be_zero:
-#ifdef PARANOID
- orl %edx,%edx
- jnz L_bugged_3
-#endif /* PARANOID */
-
- /* The result is zero */
- movw $0,EXP(%edi) /* exponent */
- movl $0,SIGL(%edi)
- movl $0,SIGH(%edi)
- movl TAG_Zero,%eax
- jmp L_exit
-
-L_shift_32:
- movl %ebx,%eax
- movl %edx,%ebx
- movl $0,%edx
- subw $32,EXP(%edi) /* Can get underflow here */
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
- bsrl %eax,%ecx /* get the required shift in %ecx */
- subl $31,%ecx
- negl %ecx
- shld %cl,%ebx,%eax
- shld %cl,%edx,%ebx
- shl %cl,%edx
- subw %cx,EXP(%edi) /* Can get underflow here */
-
-L_round:
- jmp fpu_reg_round /* Round the result */
-
-
-#ifdef PARANOID
-L_bugged_1:
- pushl EX_INTERNAL|0x206
- call EXCEPTION
- pop %ebx
- jmp L_error_exit
-
-L_bugged_2:
- pushl EX_INTERNAL|0x209
- call EXCEPTION
- pop %ebx
- jmp L_error_exit
-
-L_bugged_3:
- pushl EX_INTERNAL|0x210
- call EXCEPTION
- pop %ebx
- jmp L_error_exit
-
-L_bugged_4:
- pushl EX_INTERNAL|0x211
- call EXCEPTION
- pop %ebx
- jmp L_error_exit
-
-L_bugged:
- pushl EX_INTERNAL|0x212
- call EXCEPTION
- pop %ebx
- jmp L_error_exit
-
-L_error_exit:
- movl $-1,%eax
-
-#endif /* PARANOID */
-
-L_exit:
- popl %ebx
- popl %edi
- popl %esi
- leave
- RET
-SYM_FUNC_END(FPU_u_sub)
diff --git a/arch/x86/math-emu/round_Xsig.S b/arch/x86/math-emu/round_Xsig.S
deleted file mode 100644
index 126c404..0000000
--- a/arch/x86/math-emu/round_Xsig.S
+++ /dev/null
@@ -1,142 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | round_Xsig.S |
- | |
- | Copyright (C) 1992,1993,1994,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Normalize and round a 12 byte quantity. |
- | Call from C as: |
- | int round_Xsig(Xsig *n) |
- | |
- | Normalize a 12 byte quantity. |
- | Call from C as: |
- | int norm_Xsig(Xsig *n) |
- | |
- | Each function returns the size of the shift (nr of bits). |
- | |
- +---------------------------------------------------------------------------*/
- .file "round_Xsig.S"
-
-#include "fpu_emu.h"
-
-
-.text
-SYM_FUNC_START(round_Xsig)
- pushl %ebp
- movl %esp,%ebp
- pushl %ebx /* Reserve some space */
- pushl %ebx
- pushl %esi
-
- movl PARAM1,%esi
-
- movl 8(%esi),%edx
- movl 4(%esi),%ebx
- movl (%esi),%eax
-
- movl $0,-4(%ebp)
-
- orl %edx,%edx /* ms bits */
- js L_round /* Already normalized */
- jnz L_shift_1 /* Shift left 1 - 31 bits */
-
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- movl $-32,-4(%ebp)
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
- bsrl %edx,%ecx /* get the required shift in %ecx */
- subl $31,%ecx
- negl %ecx
- subl %ecx,-4(%ebp)
- shld %cl,%ebx,%edx
- shld %cl,%eax,%ebx
- shl %cl,%eax
-
-L_round:
- testl $0x80000000,%eax
- jz L_exit
-
- addl $1,%ebx
- adcl $0,%edx
- jnz L_exit
-
- movl $0x80000000,%edx
- incl -4(%ebp)
-
-L_exit:
- movl %edx,8(%esi)
- movl %ebx,4(%esi)
- movl %eax,(%esi)
-
- movl -4(%ebp),%eax
-
- popl %esi
- popl %ebx
- leave
- RET
-SYM_FUNC_END(round_Xsig)
-
-
-
-SYM_FUNC_START(norm_Xsig)
- pushl %ebp
- movl %esp,%ebp
- pushl %ebx /* Reserve some space */
- pushl %ebx
- pushl %esi
-
- movl PARAM1,%esi
-
- movl 8(%esi),%edx
- movl 4(%esi),%ebx
- movl (%esi),%eax
-
- movl $0,-4(%ebp)
-
- orl %edx,%edx /* ms bits */
- js L_n_exit /* Already normalized */
- jnz L_n_shift_1 /* Shift left 1 - 31 bits */
-
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- movl $-32,-4(%ebp)
-
- orl %edx,%edx /* ms bits */
- js L_n_exit /* Normalized now */
- jnz L_n_shift_1 /* Shift left 1 - 31 bits */
-
- movl %ebx,%edx
- movl %eax,%ebx
- xorl %eax,%eax
- addl $-32,-4(%ebp)
- jmp L_n_exit /* Might not be normalized,
- but shift no more. */
-
-/* We need to shift left by 1 - 31 bits */
-L_n_shift_1:
- bsrl %edx,%ecx /* get the required shift in %ecx */
- subl $31,%ecx
- negl %ecx
- subl %ecx,-4(%ebp)
- shld %cl,%ebx,%edx
- shld %cl,%eax,%ebx
- shl %cl,%eax
-
-L_n_exit:
- movl %edx,8(%esi)
- movl %ebx,4(%esi)
- movl %eax,(%esi)
-
- movl -4(%ebp),%eax
-
- popl %esi
- popl %ebx
- leave
- RET
-SYM_FUNC_END(norm_Xsig)
diff --git a/arch/x86/math-emu/shr_Xsig.S b/arch/x86/math-emu/shr_Xsig.S
deleted file mode 100644
index f726bf6..0000000
--- a/arch/x86/math-emu/shr_Xsig.S
+++ /dev/null
@@ -1,89 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "shr_Xsig.S"
-/*---------------------------------------------------------------------------+
- | shr_Xsig.S |
- | |
- | 12 byte right shift function |
- | |
- | Copyright (C) 1992,1994,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Call from C as: |
- | void shr_Xsig(Xsig *arg, unsigned nr) |
- | |
- | Extended shift right function. |
- | Fastest for small shifts. |
- | Shifts the 12 byte quantity pointed to by the first arg (arg) |
- | right by the number of bits specified by the second arg (nr). |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(shr_Xsig)
- push %ebp
- movl %esp,%ebp
- pushl %esi
- movl PARAM2,%ecx
- movl PARAM1,%esi
- cmpl $32,%ecx /* shrd only works for 0..31 bits */
- jnc L_more_than_31
-
-/* less than 32 bits */
- pushl %ebx
- movl (%esi),%eax /* lsl */
- movl 4(%esi),%ebx /* midl */
- movl 8(%esi),%edx /* msl */
- shrd %cl,%ebx,%eax
- shrd %cl,%edx,%ebx
- shr %cl,%edx
- movl %eax,(%esi)
- movl %ebx,4(%esi)
- movl %edx,8(%esi)
- popl %ebx
- popl %esi
- leave
- RET
-
-L_more_than_31:
- cmpl $64,%ecx
- jnc L_more_than_63
-
- subb $32,%cl
- movl 4(%esi),%eax /* midl */
- movl 8(%esi),%edx /* msl */
- shrd %cl,%edx,%eax
- shr %cl,%edx
- movl %eax,(%esi)
- movl %edx,4(%esi)
- movl $0,8(%esi)
- popl %esi
- leave
- RET
-
-L_more_than_63:
- cmpl $96,%ecx
- jnc L_more_than_95
-
- subb $64,%cl
- movl 8(%esi),%eax /* msl */
- shr %cl,%eax
- xorl %edx,%edx
- movl %eax,(%esi)
- movl %edx,4(%esi)
- movl %edx,8(%esi)
- popl %esi
- leave
- RET
-
-L_more_than_95:
- xorl %eax,%eax
- movl %eax,(%esi)
- movl %eax,4(%esi)
- movl %eax,8(%esi)
- popl %esi
- leave
- RET
-SYM_FUNC_END(shr_Xsig)
diff --git a/arch/x86/math-emu/status_w.h b/arch/x86/math-emu/status_w.h
deleted file mode 100644
index f642957..0000000
--- a/arch/x86/math-emu/status_w.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- | status_w.h |
- | |
- | Copyright (C) 1992,1993 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@vaxc.cc.monash.edu.au |
- | |
- +---------------------------------------------------------------------------*/
-
-#ifndef _STATUS_H_
-#define _STATUS_H_
-
-#include "fpu_emu.h" /* for definition of PECULIAR_486 */
-
-#ifdef __ASSEMBLER__
-#define Const__(x) $##x
-#else
-#define Const__(x) x
-#endif
-
-#define SW_Backward Const__(0x8000) /* backward compatibility */
-#define SW_C3 Const__(0x4000) /* condition bit 3 */
-#define SW_Top Const__(0x3800) /* top of stack */
-#define SW_Top_Shift Const__(11) /* shift for top of stack bits */
-#define SW_C2 Const__(0x0400) /* condition bit 2 */
-#define SW_C1 Const__(0x0200) /* condition bit 1 */
-#define SW_C0 Const__(0x0100) /* condition bit 0 */
-#define SW_Summary Const__(0x0080) /* exception summary */
-#define SW_Stack_Fault Const__(0x0040) /* stack fault */
-#define SW_Precision Const__(0x0020) /* loss of precision */
-#define SW_Underflow Const__(0x0010) /* underflow */
-#define SW_Overflow Const__(0x0008) /* overflow */
-#define SW_Zero_Div Const__(0x0004) /* divide by zero */
-#define SW_Denorm_Op Const__(0x0002) /* denormalized operand */
-#define SW_Invalid Const__(0x0001) /* invalid operation */
-
-#define SW_Exc_Mask Const__(0x27f) /* Status word exception bit mask */
-
-#ifndef __ASSEMBLER__
-
-#define COMP_A_gt_B 1
-#define COMP_A_eq_B 2
-#define COMP_A_lt_B 3
-#define COMP_No_Comp 4
-#define COMP_Denormal 0x20
-#define COMP_NaN 0x40
-#define COMP_SNaN 0x80
-
-#define status_word() \
- ((partial_status & ~SW_Top & 0xffff) | ((top << SW_Top_Shift) & SW_Top))
-static inline void setcc(int cc)
-{
- partial_status &= ~(SW_C0 | SW_C1 | SW_C2 | SW_C3);
- partial_status |= (cc) & (SW_C0 | SW_C1 | SW_C2 | SW_C3);
-}
-
-#ifdef PECULIAR_486
- /* Default, this conveys no information, but an 80486 does it. */
- /* Clear the SW_C1 bit, "other bits undefined". */
-# define clear_C1() { partial_status &= ~SW_C1; }
-# else
-# define clear_C1()
-#endif /* PECULIAR_486 */
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _STATUS_H_ */
diff --git a/arch/x86/math-emu/version.h b/arch/x86/math-emu/version.h
deleted file mode 100644
index a0d73a1..0000000
--- a/arch/x86/math-emu/version.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*---------------------------------------------------------------------------+
- | version.h |
- | |
- | |
- | Copyright (C) 1992,1993,1994,1996,1997,1999 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- | E-mail billm@melbpc.org.au |
- | |
- | |
- +---------------------------------------------------------------------------*/
-
-#define FPU_VERSION "wm-FPU-emu version 2.01"
diff --git a/arch/x86/math-emu/wm_shrx.S b/arch/x86/math-emu/wm_shrx.S
deleted file mode 100644
index f608a28..0000000
--- a/arch/x86/math-emu/wm_shrx.S
+++ /dev/null
@@ -1,207 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "wm_shrx.S"
-/*---------------------------------------------------------------------------+
- | wm_shrx.S |
- | |
- | 64 bit right shift functions |
- | |
- | Copyright (C) 1992,1995 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@jacobi.maths.monash.edu.au |
- | |
- | Call from C as: |
- | unsigned FPU_shrx(void *arg1, unsigned arg2) |
- | and |
- | unsigned FPU_shrxs(void *arg1, unsigned arg2) |
- | |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-/*---------------------------------------------------------------------------+
- | unsigned FPU_shrx(void *arg1, unsigned arg2) |
- | |
- | Extended shift right function. |
- | Fastest for small shifts. |
- | Shifts the 64 bit quantity pointed to by the first arg (arg1) |
- | right by the number of bits specified by the second arg (arg2). |
- | Forms a 96 bit quantity from the 64 bit arg and eax: |
- | [ 64 bit arg ][ eax ] |
- | shift right ---------> |
- | The eax register is initialized to 0 before the shifting. |
- | Results returned in the 64 bit arg and eax. |
- +---------------------------------------------------------------------------*/
-
-SYM_FUNC_START(FPU_shrx)
- push %ebp
- movl %esp,%ebp
- pushl %esi
- movl PARAM2,%ecx
- movl PARAM1,%esi
- cmpl $32,%ecx /* shrd only works for 0..31 bits */
- jnc L_more_than_31
-
-/* less than 32 bits */
- pushl %ebx
- movl (%esi),%ebx /* lsl */
- movl 4(%esi),%edx /* msl */
- xorl %eax,%eax /* extension */
- shrd %cl,%ebx,%eax
- shrd %cl,%edx,%ebx
- shr %cl,%edx
- movl %ebx,(%esi)
- movl %edx,4(%esi)
- popl %ebx
- popl %esi
- leave
- RET
-
-L_more_than_31:
- cmpl $64,%ecx
- jnc L_more_than_63
-
- subb $32,%cl
- movl (%esi),%eax /* lsl */
- movl 4(%esi),%edx /* msl */
- shrd %cl,%edx,%eax
- shr %cl,%edx
- movl %edx,(%esi)
- movl $0,4(%esi)
- popl %esi
- leave
- RET
-
-L_more_than_63:
- cmpl $96,%ecx
- jnc L_more_than_95
-
- subb $64,%cl
- movl 4(%esi),%eax /* msl */
- shr %cl,%eax
- xorl %edx,%edx
- movl %edx,(%esi)
- movl %edx,4(%esi)
- popl %esi
- leave
- RET
-
-L_more_than_95:
- xorl %eax,%eax
- movl %eax,(%esi)
- movl %eax,4(%esi)
- popl %esi
- leave
- RET
-SYM_FUNC_END(FPU_shrx)
-
-
-/*---------------------------------------------------------------------------+
- | unsigned FPU_shrxs(void *arg1, unsigned arg2) |
- | |
- | Extended shift right function (optimized for small floating point |
- | integers). |
- | Shifts the 64 bit quantity pointed to by the first arg (arg1) |
- | right by the number of bits specified by the second arg (arg2). |
- | Forms a 96 bit quantity from the 64 bit arg and eax: |
- | [ 64 bit arg ][ eax ] |
- | shift right ---------> |
- | The eax register is initialized to 0 before the shifting. |
- | The lower 8 bits of eax are lost and replaced by a flag which is |
- | set (to 0x01) if any bit, apart from the first one, is set in the |
- | part which has been shifted out of the arg. |
- | Results returned in the 64 bit arg and eax. |
- +---------------------------------------------------------------------------*/
-SYM_FUNC_START(FPU_shrxs)
- push %ebp
- movl %esp,%ebp
- pushl %esi
- pushl %ebx
- movl PARAM2,%ecx
- movl PARAM1,%esi
- cmpl $64,%ecx /* shrd only works for 0..31 bits */
- jnc Ls_more_than_63
-
- cmpl $32,%ecx /* shrd only works for 0..31 bits */
- jc Ls_less_than_32
-
-/* We got here without jumps by assuming that the most common requirement
- is for small integers */
-/* Shift by [32..63] bits */
- subb $32,%cl
- movl (%esi),%eax /* lsl */
- movl 4(%esi),%edx /* msl */
- xorl %ebx,%ebx
- shrd %cl,%eax,%ebx
- shrd %cl,%edx,%eax
- shr %cl,%edx
- orl %ebx,%ebx /* test these 32 bits */
- setne %bl
- test $0x7fffffff,%eax /* and 31 bits here */
- setne %bh
- orw %bx,%bx /* Any of the 63 bit set ? */
- setne %al
- movl %edx,(%esi)
- movl $0,4(%esi)
- popl %ebx
- popl %esi
- leave
- RET
-
-/* Shift by [0..31] bits */
-Ls_less_than_32:
- movl (%esi),%ebx /* lsl */
- movl 4(%esi),%edx /* msl */
- xorl %eax,%eax /* extension */
- shrd %cl,%ebx,%eax
- shrd %cl,%edx,%ebx
- shr %cl,%edx
- test $0x7fffffff,%eax /* only need to look at eax here */
- setne %al
- movl %ebx,(%esi)
- movl %edx,4(%esi)
- popl %ebx
- popl %esi
- leave
- RET
-
-/* Shift by [64..95] bits */
-Ls_more_than_63:
- cmpl $96,%ecx
- jnc Ls_more_than_95
-
- subb $64,%cl
- movl (%esi),%ebx /* lsl */
- movl 4(%esi),%eax /* msl */
- xorl %edx,%edx /* extension */
- shrd %cl,%ebx,%edx
- shrd %cl,%eax,%ebx
- shr %cl,%eax
- orl %ebx,%edx
- setne %bl
- test $0x7fffffff,%eax /* only need to look at eax here */
- setne %bh
- orw %bx,%bx
- setne %al
- xorl %edx,%edx
- movl %edx,(%esi) /* set to zero */
- movl %edx,4(%esi) /* set to zero */
- popl %ebx
- popl %esi
- leave
- RET
-
-Ls_more_than_95:
-/* Shift by [96..inf) bits */
- xorl %eax,%eax
- movl (%esi),%ebx
- orl 4(%esi),%ebx
- setne %al
- xorl %ebx,%ebx
- movl %ebx,(%esi)
- movl %ebx,4(%esi)
- popl %ebx
- popl %esi
- leave
- RET
-SYM_FUNC_END(FPU_shrxs)
diff --git a/arch/x86/math-emu/wm_sqrt.S b/arch/x86/math-emu/wm_sqrt.S
deleted file mode 100644
index 40526dd..0000000
--- a/arch/x86/math-emu/wm_sqrt.S
+++ /dev/null
@@ -1,472 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
- .file "wm_sqrt.S"
-/*---------------------------------------------------------------------------+
- | wm_sqrt.S |
- | |
- | Fixed point arithmetic square root evaluation. |
- | |
- | Copyright (C) 1992,1993,1995,1997 |
- | W. Metzenthen, 22 Parker St, Ormond, Vic 3163, |
- | Australia. E-mail billm@suburbia.net |
- | |
- | Call from C as: |
- | int wm_sqrt(FPU_REG *n, unsigned int control_word) |
- | |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | wm_sqrt(FPU_REG *n, unsigned int control_word) |
- | returns the square root of n in n. |
- | |
- | Use Newton's method to compute the square root of a number, which must |
- | be in the range [1.0 .. 4.0), to 64 bits accuracy. |
- | Does not check the sign or tag of the argument. |
- | Sets the exponent, but not the sign or tag of the result. |
- | |
- | The guess is kept in %esi:%edi |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-
-#ifndef NON_REENTRANT_FPU
-/* Local storage on the stack: */
-#define FPU_accum_3 -4(%ebp) /* ms word */
-#define FPU_accum_2 -8(%ebp)
-#define FPU_accum_1 -12(%ebp)
-#define FPU_accum_0 -16(%ebp)
-
-/*
- * The de-normalised argument:
- * sq_2 sq_1 sq_0
- * b b b b b b b ... b b b b b b .... b b b b 0 0 0 ... 0
- * ^ binary point here
- */
-#define FPU_fsqrt_arg_2 -20(%ebp) /* ms word */
-#define FPU_fsqrt_arg_1 -24(%ebp)
-#define FPU_fsqrt_arg_0 -28(%ebp) /* ls word, at most the ms bit is set */
-
-#else
-/* Local storage in a static area: */
-.data
- .align 4,0
-FPU_accum_3:
- .long 0 /* ms word */
-FPU_accum_2:
- .long 0
-FPU_accum_1:
- .long 0
-FPU_accum_0:
- .long 0
-
-/* The de-normalised argument:
- sq_2 sq_1 sq_0
- b b b b b b b ... b b b b b b .... b b b b 0 0 0 ... 0
- ^ binary point here
- */
-FPU_fsqrt_arg_2:
- .long 0 /* ms word */
-FPU_fsqrt_arg_1:
- .long 0
-FPU_fsqrt_arg_0:
- .long 0 /* ls word, at most the ms bit is set */
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-SYM_FUNC_START(wm_sqrt)
- pushl %ebp
- movl %esp,%ebp
-#ifndef NON_REENTRANT_FPU
- subl $28,%esp
-#endif /* NON_REENTRANT_FPU */
- pushl %esi
- pushl %edi
- pushl %ebx
-
- movl PARAM1,%esi
-
- movl SIGH(%esi),%eax
- movl SIGL(%esi),%ecx
- xorl %edx,%edx
-
-/* We use a rough linear estimate for the first guess.. */
-
- cmpw EXP_BIAS,EXP(%esi)
- jnz sqrt_arg_ge_2
-
- shrl $1,%eax /* arg is in the range [1.0 .. 2.0) */
- rcrl $1,%ecx
- rcrl $1,%edx
-
-sqrt_arg_ge_2:
-/* From here on, n is never accessed directly again until it is
- replaced by the answer. */
-
- movl %eax,FPU_fsqrt_arg_2 /* ms word of n */
- movl %ecx,FPU_fsqrt_arg_1
- movl %edx,FPU_fsqrt_arg_0
-
-/* Make a linear first estimate */
- shrl $1,%eax
- addl $0x40000000,%eax
- movl $0xaaaaaaaa,%ecx
- mull %ecx
- shll %edx /* max result was 7fff... */
- testl $0x80000000,%edx /* but min was 3fff... */
- jnz sqrt_prelim_no_adjust
-
- movl $0x80000000,%edx /* round up */
-
-sqrt_prelim_no_adjust:
- movl %edx,%esi /* Our first guess */
-
-/* We have now computed (approx) (2 + x) / 3, which forms the basis
- for a few iterations of Newton's method */
-
- movl FPU_fsqrt_arg_2,%ecx /* ms word */
-
-/*
- * From our initial estimate, three iterations are enough to get us
- * to 30 bits or so. This will then allow two iterations at better
- * precision to complete the process.
- */
-
-/* Compute (g + n/g)/2 at each iteration (g is the guess). */
- shrl %ecx /* Doing this first will prevent a divide */
- /* overflow later. */
-
- movl %ecx,%edx /* msw of the arg / 2 */
- divl %esi /* current estimate */
- shrl %esi /* divide by 2 */
- addl %eax,%esi /* the new estimate */
-
- movl %ecx,%edx
- divl %esi
- shrl %esi
- addl %eax,%esi
-
- movl %ecx,%edx
- divl %esi
- shrl %esi
- addl %eax,%esi
-
-/*
- * Now that an estimate accurate to about 30 bits has been obtained (in %esi),
- * we improve it to 60 bits or so.
- *
- * The strategy from now on is to compute new estimates from
- * guess := guess + (n - guess^2) / (2 * guess)
- */
-
-/* First, find the square of the guess */
- movl %esi,%eax
- mull %esi
-/* guess^2 now in %edx:%eax */
-
- movl FPU_fsqrt_arg_1,%ecx
- subl %ecx,%eax
- movl FPU_fsqrt_arg_2,%ecx /* ms word of normalized n */
- sbbl %ecx,%edx
- jnc sqrt_stage_2_positive
-
-/* Subtraction gives a negative result,
- negate the result before division. */
- notl %edx
- notl %eax
- addl $1,%eax
- adcl $0,%edx
-
- divl %esi
- movl %eax,%ecx
-
- movl %edx,%eax
- divl %esi
- jmp sqrt_stage_2_finish
-
-sqrt_stage_2_positive:
- divl %esi
- movl %eax,%ecx
-
- movl %edx,%eax
- divl %esi
-
- notl %ecx
- notl %eax
- addl $1,%eax
- adcl $0,%ecx
-
-sqrt_stage_2_finish:
- sarl $1,%ecx /* divide by 2 */
- rcrl $1,%eax
-
- /* Form the new estimate in %esi:%edi */
- movl %eax,%edi
- addl %ecx,%esi
-
- jnz sqrt_stage_2_done /* result should be [1..2) */
-
-#ifdef PARANOID
-/* It should be possible to get here only if the arg is ffff....ffff */
- cmpl $0xffffffff,FPU_fsqrt_arg_1
- jnz sqrt_stage_2_error
-#endif /* PARANOID */
-
-/* The best rounded result. */
- xorl %eax,%eax
- decl %eax
- movl %eax,%edi
- movl %eax,%esi
- movl $0x7fffffff,%eax
- jmp sqrt_round_result
-
-#ifdef PARANOID
-sqrt_stage_2_error:
- pushl EX_INTERNAL|0x213
- call EXCEPTION
-#endif /* PARANOID */
-
-sqrt_stage_2_done:
-
-/* Now the square root has been computed to better than 60 bits. */
-
-/* Find the square of the guess. */
- movl %edi,%eax /* ls word of guess */
- mull %edi
- movl %edx,FPU_accum_1
-
- movl %esi,%eax
- mull %esi
- movl %edx,FPU_accum_3
- movl %eax,FPU_accum_2
-
- movl %edi,%eax
- mull %esi
- addl %eax,FPU_accum_1
- adcl %edx,FPU_accum_2
- adcl $0,FPU_accum_3
-
-/* movl %esi,%eax */
-/* mull %edi */
- addl %eax,FPU_accum_1
- adcl %edx,FPU_accum_2
- adcl $0,FPU_accum_3
-
-/* guess^2 now in FPU_accum_3:FPU_accum_2:FPU_accum_1 */
-
- movl FPU_fsqrt_arg_0,%eax /* get normalized n */
- subl %eax,FPU_accum_1
- movl FPU_fsqrt_arg_1,%eax
- sbbl %eax,FPU_accum_2
- movl FPU_fsqrt_arg_2,%eax /* ms word of normalized n */
- sbbl %eax,FPU_accum_3
- jnc sqrt_stage_3_positive
-
-/* Subtraction gives a negative result,
- negate the result before division */
- notl FPU_accum_1
- notl FPU_accum_2
- notl FPU_accum_3
- addl $1,FPU_accum_1
- adcl $0,FPU_accum_2
-
-#ifdef PARANOID
- adcl $0,FPU_accum_3 /* This must be zero */
- jz sqrt_stage_3_no_error
-
-sqrt_stage_3_error:
- pushl EX_INTERNAL|0x207
- call EXCEPTION
-
-sqrt_stage_3_no_error:
-#endif /* PARANOID */
-
- movl FPU_accum_2,%edx
- movl FPU_accum_1,%eax
- divl %esi
- movl %eax,%ecx
-
- movl %edx,%eax
- divl %esi
-
- sarl $1,%ecx /* divide by 2 */
- rcrl $1,%eax
-
- /* prepare to round the result */
-
- addl %ecx,%edi
- adcl $0,%esi
-
- jmp sqrt_stage_3_finished
-
-sqrt_stage_3_positive:
- movl FPU_accum_2,%edx
- movl FPU_accum_1,%eax
- divl %esi
- movl %eax,%ecx
-
- movl %edx,%eax
- divl %esi
-
- sarl $1,%ecx /* divide by 2 */
- rcrl $1,%eax
-
- /* prepare to round the result */
-
- notl %eax /* Negate the correction term */
- notl %ecx
- addl $1,%eax
- adcl $0,%ecx /* carry here ==> correction == 0 */
- adcl $0xffffffff,%esi
-
- addl %ecx,%edi
- adcl $0,%esi
-
-sqrt_stage_3_finished:
-
-/*
- * The result in %esi:%edi:%esi should be good to about 90 bits here,
- * and the rounding information here does not have sufficient accuracy
- * in a few rare cases.
- */
- cmpl $0xffffffe0,%eax
- ja sqrt_near_exact_x
-
- cmpl $0x00000020,%eax
- jb sqrt_near_exact
-
- cmpl $0x7fffffe0,%eax
- jb sqrt_round_result
-
- cmpl $0x80000020,%eax
- jb sqrt_get_more_precision
-
-sqrt_round_result:
-/* Set up for rounding operations */
- movl %eax,%edx
- movl %esi,%eax
- movl %edi,%ebx
- movl PARAM1,%edi
- movw EXP_BIAS,EXP(%edi) /* Result is in [1.0 .. 2.0) */
- jmp fpu_reg_round
-
-
-sqrt_near_exact_x:
-/* First, the estimate must be rounded up. */
- addl $1,%edi
- adcl $0,%esi
-
-sqrt_near_exact:
-/*
- * This is an easy case because x^1/2 is monotonic.
- * We need just find the square of our estimate, compare it
- * with the argument, and deduce whether our estimate is
- * above, below, or exact. We use the fact that the estimate
- * is known to be accurate to about 90 bits.
- */
- movl %edi,%eax /* ls word of guess */
- mull %edi
- movl %edx,%ebx /* 2nd ls word of square */
- movl %eax,%ecx /* ls word of square */
-
- movl %edi,%eax
- mull %esi
- addl %eax,%ebx
- addl %eax,%ebx
-
-#ifdef PARANOID
- cmp $0xffffffb0,%ebx
- jb sqrt_near_exact_ok
-
- cmp $0x00000050,%ebx
- ja sqrt_near_exact_ok
-
- pushl EX_INTERNAL|0x214
- call EXCEPTION
-
-sqrt_near_exact_ok:
-#endif /* PARANOID */
-
- or %ebx,%ebx
- js sqrt_near_exact_small
-
- jnz sqrt_near_exact_large
-
- or %ebx,%edx
- jnz sqrt_near_exact_large
-
-/* Our estimate is exactly the right answer */
- xorl %eax,%eax
- jmp sqrt_round_result
-
-sqrt_near_exact_small:
-/* Our estimate is too small */
- movl $0x000000ff,%eax
- jmp sqrt_round_result
-
-sqrt_near_exact_large:
-/* Our estimate is too large, we need to decrement it */
- subl $1,%edi
- sbbl $0,%esi
- movl $0xffffff00,%eax
- jmp sqrt_round_result
-
-
-sqrt_get_more_precision:
-/* This case is almost the same as the above, except we start
- with an extra bit of precision in the estimate. */
- stc /* The extra bit. */
- rcll $1,%edi /* Shift the estimate left one bit */
- rcll $1,%esi
-
- movl %edi,%eax /* ls word of guess */
- mull %edi
- movl %edx,%ebx /* 2nd ls word of square */
- movl %eax,%ecx /* ls word of square */
-
- movl %edi,%eax
- mull %esi
- addl %eax,%ebx
- addl %eax,%ebx
-
-/* Put our estimate back to its original value */
- stc /* The ms bit. */
- rcrl $1,%esi /* Shift the estimate left one bit */
- rcrl $1,%edi
-
-#ifdef PARANOID
- cmp $0xffffff60,%ebx
- jb sqrt_more_prec_ok
-
- cmp $0x000000a0,%ebx
- ja sqrt_more_prec_ok
-
- pushl EX_INTERNAL|0x215
- call EXCEPTION
-
-sqrt_more_prec_ok:
-#endif /* PARANOID */
-
- or %ebx,%ebx
- js sqrt_more_prec_small
-
- jnz sqrt_more_prec_large
-
- or %ebx,%ecx
- jnz sqrt_more_prec_large
-
-/* Our estimate is exactly the right answer */
- movl $0x80000000,%eax
- jmp sqrt_round_result
-
-sqrt_more_prec_small:
-/* Our estimate is too small */
- movl $0x800000ff,%eax
- jmp sqrt_round_result
-
-sqrt_more_prec_large:
-/* Our estimate is too large */
- movl $0x7fffff00,%eax
- jmp sqrt_round_result
-SYM_FUNC_END(wm_sqrt)
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index f0e77e0..250942e 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -169,6 +169,11 @@ is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
}
DEFINE_SPINLOCK(pgd_lock);
+/*
+ * List of all process' PGD pages (excluding init_mm and some other special
+ * system-wide PGDs). Mainly used to keep the kernel portion of the address
+ * space in sync across mms.
+ */
LIST_HEAD(pgd_list);
#ifdef CONFIG_X86_32
@@ -686,7 +691,7 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code,
* avoid hanging the system.
*/
if (IS_ENABLED(CONFIG_EFI))
- efi_crash_gracefully_on_page_fault(address);
+ efi_crash_gracefully_on_page_fault(address, regs);
/* Only not-present faults should be handled by KFENCE. */
if (!(error_code & X86_PF_PROT) &&
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index df2261f..7e20b22 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1014,7 +1014,7 @@ static void __meminit free_pagetable(struct page *page, int order)
#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
enum bootmem_type type = bootmem_type(page);
- if (type == SECTION_INFO || type == MIX_SECTION_INFO) {
+ if (type == MIX_SECTION_INFO) {
while (nr_pages--)
put_page_bootmem(page++);
} else {
@@ -1028,13 +1028,24 @@ static void __meminit free_pagetable(struct page *page, int order)
}
}
-static void __meminit free_hugepage_table(struct page *page,
+static void __meminit free_vmemmap_pages(struct page *page, unsigned int order,
struct vmem_altmap *altmap)
{
- if (altmap)
- vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE);
- else
- free_pagetable(page, get_order(PMD_SIZE));
+ unsigned long nr_pages = 1u << order;
+
+ if (altmap) {
+ vmem_altmap_free(altmap, nr_pages);
+ } else if (PageReserved(page)) {
+ if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) &&
+ bootmem_type(page) == SECTION_INFO) {
+ while (nr_pages--)
+ put_page_bootmem(page++);
+ } else {
+ free_reserved_pages(page, nr_pages);
+ }
+ } else {
+ __free_pages(page, order);
+ }
}
static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
@@ -1118,7 +1129,8 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
return;
if (!direct)
- free_pagetable(pte_page(*pte), 0);
+ /* We never populate base pages from the altmap. */
+ free_vmemmap_pages(pte_page(*pte), 0, NULL);
spin_lock(&init_mm.page_table_lock);
pte_clear(&init_mm, addr, pte);
@@ -1153,19 +1165,19 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
if (IS_ALIGNED(addr, PMD_SIZE) &&
IS_ALIGNED(next, PMD_SIZE)) {
if (!direct)
- free_hugepage_table(pmd_page(*pmd),
- altmap);
+ free_vmemmap_pages(pmd_page(*pmd),
+ PMD_ORDER, altmap);
spin_lock(&init_mm.page_table_lock);
pmd_clear(pmd);
spin_unlock(&init_mm.page_table_lock);
pages++;
} else if (vmemmap_pmd_is_unused(addr, next)) {
- free_hugepage_table(pmd_page(*pmd),
- altmap);
- spin_lock(&init_mm.page_table_lock);
- pmd_clear(pmd);
- spin_unlock(&init_mm.page_table_lock);
+ free_vmemmap_pages(pmd_page(*pmd), PMD_ORDER,
+ altmap);
+ spin_lock(&init_mm.page_table_lock);
+ pmd_clear(pmd);
+ spin_unlock(&init_mm.page_table_lock);
}
continue;
}
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index da7f0a0..f32facd 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -99,17 +99,6 @@ static void pgd_dtor(pgd_t *pgd)
spin_unlock(&pgd_lock);
}
-/*
- * List of all pgd's needed for non-PAE so it can invalidate entries
- * in both cached and uncached pgd's; not needed for PAE since the
- * kernel pmd is shared. If PAE were not to share the pmd a similar
- * tactic would be needed. This is essentially codepath-based locking
- * against pageattr.c; it is the unique case in which a valid change
- * of kernel pagetables can't be lazily synchronized by vmalloc faults.
- * vmalloc faults work because attached pagetables are never freed.
- * -- nyc
- */
-
#ifdef CONFIG_X86_PAE
/*
* In PAE mode, we need to do a cr3 reload (=tlb flush) when
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631f037..598f553 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -31,6 +31,7 @@
#include <asm/cpufeature.h>
#include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
#include <asm/vsyscall.h>
#include <asm/cmdline.h>
#include <asm/pti.h>
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index af43d17..1023aca 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -1123,7 +1123,7 @@ static void flush_tlb_func(void *info)
VM_WARN_ON(!irqs_disabled());
if (!local) {
- inc_irq_stat(irq_tlb_count);
+ inc_irq_stat(TLB);
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
}
@@ -1769,7 +1769,7 @@ bool nmi_uaccess_okay(void)
}
static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
- size_t count, loff_t *ppos)
+ size_t count, loff_t *ppos)
{
char buf[32];
unsigned int len;
@@ -1778,20 +1778,15 @@ static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
return simple_read_from_buffer(user_buf, count, ppos, buf, len);
}
-static ssize_t tlbflush_write_file(struct file *file,
- const char __user *user_buf, size_t count, loff_t *ppos)
+static ssize_t tlbflush_write_file(struct file *file, const char __user *user_buf,
+ size_t count, loff_t *ppos)
{
- char buf[32];
- ssize_t len;
int ceiling;
+ int err;
- len = min(count, sizeof(buf) - 1);
- if (copy_from_user(buf, user_buf, len))
- return -EFAULT;
-
- buf[len] = '\0';
- if (kstrtoint(buf, 0, &ceiling))
- return -EINVAL;
+ err = kstrtoint_from_user(user_buf, count, 0, &ceiling);
+ if (err)
+ return err;
if (ceiling < 0)
return -EINVAL;
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index 6818515..550c631 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -18,6 +18,7 @@
#include <linux/io.h>
#include <asm/io_apic.h>
#include <asm/pci_x86.h>
+#include <asm/cpuid/api.h>
#include <asm/xen/hypervisor.h>
diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c
index df24ffc..90a065f 100644
--- a/arch/x86/platform/efi/quirks.c
+++ b/arch/x86/platform/efi/quirks.c
@@ -761,7 +761,8 @@ int efi_capsule_setup_info(struct capsule_info *cap_info, void *kbuff,
* @return: Returns, if the page fault is not handled. This function
* will never return if the page fault is handled successfully.
*/
-void efi_crash_gracefully_on_page_fault(unsigned long phys_addr)
+void efi_crash_gracefully_on_page_fault(unsigned long phys_addr,
+ const struct pt_regs *regs)
{
if (!IS_ENABLED(CONFIG_X86_64))
return;
@@ -770,7 +771,7 @@ void efi_crash_gracefully_on_page_fault(unsigned long phys_addr)
* If we get an interrupt/NMI while processing an EFI runtime service
* then this is a regular OOPS, not an EFI failure.
*/
- if (in_interrupt())
+ if (!in_task())
return;
/*
@@ -811,6 +812,14 @@ void efi_crash_gracefully_on_page_fault(unsigned long phys_addr)
}
/*
+ * The API does not permit entering a kernel mode FPU section with
+ * interrupts enabled and leaving it with interrupts disabled. So
+ * re-enable interrupts now if they were enabled when the page fault
+ * occurred.
+ */
+ local_irq_restore(regs->flags);
+
+ /*
* Before calling EFI Runtime Service, the kernel has switched the
* calling process to efi_mm. Hence, switch back to task_mm.
*/
diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c
index 5c50e55..565ab43 100644
--- a/arch/x86/platform/uv/uv_nmi.c
+++ b/arch/x86/platform/uv/uv_nmi.c
@@ -18,6 +18,7 @@
#include <linux/sched/debug.h>
#include <linux/slab.h>
#include <linux/string.h>
+#include <linux/string_choices.h>
#include <linux/clocksource.h>
#include <asm/apic.h>
@@ -340,7 +341,7 @@ static void uv_nmi_setup_hubless_intr(void)
uv_pch_intr_now_enabled ? GPIROUTNMI : 0);
nmi_debug("UV:NMI: GPP_D_0 interrupt %s\n",
- uv_pch_intr_now_enabled ? "enabled" : "disabled");
+ str_enabled_disabled(uv_pch_intr_now_enabled));
}
static struct init_nmi {
diff --git a/arch/x86/tools/vdso2c.c b/arch/x86/tools/vdso2c.c
index b8a5557..64a636b 100644
--- a/arch/x86/tools/vdso2c.c
+++ b/arch/x86/tools/vdso2c.c
@@ -75,12 +75,16 @@ struct vdso_sym {
};
struct vdso_sym required_syms[] = {
- {"__kernel_vsyscall", true},
- {"__kernel_sigreturn", true},
- {"__kernel_rt_sigreturn", true},
- {"int80_landing_pad", true},
- {"vdso32_rt_sigreturn_landing_pad", true},
- {"vdso32_sigreturn_landing_pad", true},
+ {"__kernel_vsyscall", true},
+ {"__kernel_sigreturn", true},
+ {"__kernel_rt_sigreturn", true},
+ {"int80_landing_pad", true},
+ {"vdso32_rt_sigreturn_landing_pad", true},
+ {"vdso32_sigreturn_landing_pad", true},
+ {"__futex_list64_try_unlock_cs_start", true},
+ {"__futex_list64_try_unlock_cs_end", true},
+ {"__futex_list32_try_unlock_cs_start", true},
+ {"__futex_list32_try_unlock_cs_end", true},
};
__attribute__((format(printf, 1, 2))) __attribute__((noreturn))
diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
index f647557..7e9091c 100644
--- a/arch/x86/virt/hw.c
+++ b/arch/x86/virt/hw.c
@@ -49,7 +49,20 @@ static void x86_virt_invoke_kvm_emergency_callback(void)
{
cpu_emergency_virt_cb *kvm_callback;
- kvm_callback = rcu_dereference(kvm_emergency_callback);
+ /*
+ * RCU may not be watching the crashing CPU here, so rcu_dereference()
+ * triggers a suspicious-RCU-usage splat. In principle, a concurrent
+ * KVM module unload could race with this read; see commit 2baa33a8ddd6
+ * ("KVM: x86: Leave user-return notifier registered on reboot/shutdown")
+ * which notes that nothing prevents module unload during panic/reboot.
+ *
+ * However, taking a lock here would be riskier than the current race:
+ * the system is going down via NMI shootdown, and any lock could be
+ * held by an already-stopped CPU. Use rcu_dereference_raw() to silence
+ * the lockdep splat and accept the comically small remaining race;
+ * panic context inherently cannot guarantee complete correctness.
+ */
+ kvm_callback = rcu_dereference_raw(kvm_emergency_callback);
if (kvm_callback)
kvm_callback();
}
diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index 41f76f1..8bcdce9 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -511,8 +511,9 @@ static void clear_hsave_pa(void *arg)
wrmsrq(MSR_VM_HSAVE_PA, 0);
}
-void snp_prepare(void)
+int snp_prepare(void)
{
+ int ret;
u64 val;
/*
@@ -521,12 +522,20 @@ void snp_prepare(void)
*/
rdmsrq(MSR_AMD64_SYSCFG, val);
if (val & MSR_AMD64_SYSCFG_SNP_EN)
- return;
+ return 0;
clear_rmp();
cpus_read_lock();
+ if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) {
+ ret = -EOPNOTSUPP;
+ pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n",
+ cpumask_pr_args(cpu_online_mask),
+ cpumask_pr_args(cpu_present_mask));
+ goto unlock;
+ }
+
/*
* MtrrFixDramModEn is not shared between threads on a core,
* therefore it must be set on all CPUs prior to enabling SNP.
@@ -537,7 +546,12 @@ void snp_prepare(void)
/* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
on_each_cpu(clear_hsave_pa, NULL, 1);
+ ret = 0;
+
+unlock:
cpus_read_unlock();
+
+ return ret;
}
EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp");
diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
index aa4040f..dc38f95 100644
--- a/arch/x86/xen/Kconfig
+++ b/arch/x86/xen/Kconfig
@@ -11,7 +11,7 @@
select HIBERNATE_CALLBACKS
depends on X86_64 || (X86_32 && X86_PAE)
depends on X86_64 || (X86_GENERIC || MPENTIUM4 || MATOM)
- depends on X86_LOCAL_APIC && X86_TSC
+ depends on X86_LOCAL_APIC
help
This is the Linux Xen port. Enabling this will allow the
kernel to boot in a paravirtualized environment under the
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 23b91bf..cf061ed 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -17,7 +17,7 @@
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <asm/cpu.h>
-#include <asm/e820/api.h>
+#include <asm/e820/api.h>
#include <asm/setup.h>
#include "xen-ops.h"
@@ -76,6 +76,7 @@ unsigned long xen_released_pages;
static __ref void xen_get_vendor(void)
{
init_cpu_devs();
+ cpuid_scan_cpu(&boot_cpu_data);
cpu_detect(&boot_cpu_data);
get_cpu_vendor(&boot_cpu_data);
}
diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c
index 2f9fa27..89ae284 100644
--- a/arch/x86/xen/enlighten_hvm.c
+++ b/arch/x86/xen/enlighten_hvm.c
@@ -20,6 +20,7 @@
#include <asm/setup.h>
#include <asm/idtentry.h>
#include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
#include <asm/e820/api.h>
#include <asm/early_ioremap.h>
@@ -125,7 +126,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback)
if (xen_percpu_upcall)
apic_eoi();
- inc_irq_stat(irq_hv_callback_count);
+ inc_irq_stat(HYPERVISOR_CALLBACK);
xen_evtchn_do_upcall();
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index ed2d7a3..b8a7c6b 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -728,7 +728,7 @@ static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
- inc_irq_stat(irq_hv_callback_count);
+ inc_irq_stat(HYPERVISOR_CALLBACK);
xen_evtchn_do_upcall();
@@ -1429,6 +1429,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
xen_build_dynamic_phys_to_machine();
/* Work out if we support NX */
+ cpuid_scan_cpu(&boot_cpu_data);
get_cpu_cap(&boot_cpu_data);
x86_configure_nx();
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index c80d005..3eee5f8 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -2145,7 +2145,10 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
static void xen_enter_lazy_mmu(void)
{
- enter_lazy(XEN_LAZY_MMU);
+ preempt_disable();
+ if (xen_get_lazy_mode() != XEN_LAZY_MMU)
+ enter_lazy(XEN_LAZY_MMU);
+ preempt_enable();
}
static void xen_flush_lazy_mmu(void)
@@ -2182,7 +2185,8 @@ static void xen_leave_lazy_mmu(void)
{
preempt_disable();
xen_mc_flush();
- leave_lazy(XEN_LAZY_MMU);
+ if (xen_get_lazy_mode() != XEN_LAZY_NONE)
+ leave_lazy(XEN_LAZY_MMU);
preempt_enable();
}
diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index 8f89ce0..5f50a3e 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -2,6 +2,7 @@
#include <linux/types.h>
#include <linux/interrupt.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/xen/hypercall.h>
#include <xen/xen.h>
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index ac8021c..41251d4 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -655,7 +655,7 @@ static void __init xen_e820_swap_entry_with_ram(struct e820_entry *swap_entry)
/* Fill new entry (keep size and page offset). */
entry->type = swap_entry->type;
entry->addr = entry_end - swap_size +
- swap_addr - swap_entry->addr;
+ swap_entry->addr - swap_addr;
entry->size = swap_entry->size;
/* Convert old entry to RAM, align to pages. */
@@ -695,17 +695,22 @@ static void __init xen_e820_resolve_conflicts(phys_addr_t start,
return;
end = start + size;
- entry = xen_e820_table.entries;
+ mapcnt = 0;
- for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++) {
+ while (mapcnt < xen_e820_table.nr_entries) {
+ entry = xen_e820_table.entries + mapcnt;
if (entry->addr >= end)
return;
if (entry->addr + entry->size > start &&
- entry->type == E820_TYPE_NVS)
+ entry->type == E820_TYPE_NVS) {
xen_e820_swap_entry_with_ram(entry);
+ /* E820 map has been changed, restart loop! */
+ mapcnt = 0;
+ continue;
+ }
- entry++;
+ mapcnt++;
}
}
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 05f92c8..05ee0d3 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -23,7 +23,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
*/
static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
{
- inc_irq_stat(irq_resched_count);
+ inc_irq_stat(RESCHEDULE);
scheduler_ipi();
return IRQ_HANDLED;
@@ -254,7 +254,7 @@ void xen_send_IPI_allbutself(int vector)
static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
{
generic_smp_call_function_interrupt();
- inc_irq_stat(irq_call_count);
+ inc_irq_stat(CALL_FUNCTION);
return IRQ_HANDLED;
}
@@ -262,7 +262,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
{
generic_smp_call_function_single_interrupt();
- inc_irq_stat(irq_call_count);
+ inc_irq_stat(CALL_FUNCTION);
return IRQ_HANDLED;
}
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index db9b8e2..c2812f8 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -400,7 +400,7 @@ static void xen_pv_stop_other_cpus(int wait)
static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
{
irq_work_run();
- inc_irq_stat(apic_irq_work_irqs);
+ inc_irq_stat(IRQ_WORK);
return IRQ_HANDLED;
}
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index 6f9f665..d62c143 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -18,6 +18,7 @@
#include <linux/timekeeper_internal.h>
#include <linux/sched/cputime.h>
+#include <asm/cpuid/api.h>
#include <asm/pvclock.h>
#include <asm/timer.h>
#include <asm/xen/hypervisor.h>
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index 13fe45d..e57af61 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -6,5 +6,6 @@
generic-y += parport.h
generic-y += qrwlock.h
generic-y += qspinlock.h
+generic-y += ring_buffer.h
generic-y += user.h
generic-y += text-patching.h
diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c
index b1e410f..6f01f53 100644
--- a/arch/xtensa/kernel/irq.c
+++ b/arch/xtensa/kernel/irq.c
@@ -59,7 +59,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%*s:", prec, "NMI");
for_each_online_cpu(cpu)
seq_printf(p, " %10lu", per_cpu(nmi_count, cpu));
- seq_puts(p, " Non-maskable interrupts\n");
+ seq_puts(p, " Non-maskable interrupts\n");
#endif
return 0;
}
diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c
index acb1e5f..0daa42d 100644
--- a/block/bio-integrity-fs.c
+++ b/block/bio-integrity-fs.c
@@ -55,6 +55,10 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
{
struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
struct bio_integrity_payload *bip = bio_integrity(bio);
+ struct bvec_iter data_iter = {
+ .bi_sector = sector,
+ .bi_size = size,
+ };
/*
* Reinitialize bip->bip_iter.
@@ -65,7 +69,7 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
memset(&bip->bip_iter, 0, sizeof(bip->bip_iter));
bip->bip_iter.bi_sector = sector;
bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT);
- return blk_status_to_errno(bio_integrity_verify(bio, &bip->bip_iter));
+ return blk_status_to_errno(bio_integrity_verify(bio, &data_iter));
}
static int __init fs_bio_integrity_init(void)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index e54c6e0..e796de1 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -308,7 +308,6 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
}
bip->bip_flags |= BIP_COPY_USER;
- bip->bip_vcnt = nr_vecs;
return 0;
free_bip:
bio_integrity_free(bio);
@@ -403,6 +402,24 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
if (unlikely(ret < 0))
goto free_bvec;
+ /*
+ * Handle partial pinning. This can happen when pin_user_pages_fast()
+ * returns fewer pages than requested.
+ */
+ if (user_backed_iter(iter) && unlikely(ret != bytes)) {
+ if (ret > 0) {
+ int npinned = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
+ int i;
+
+ for (i = 0; i < npinned; i++)
+ unpin_user_page(pages[i]);
+ }
+ if (pages != stack_pages)
+ kvfree(pages);
+ ret = -EFAULT;
+ goto free_bvec;
+ }
+
nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset,
&is_p2p);
if (pages != stack_pages)
diff --git a/block/bio.c b/block/bio.c
index b8972db..5f10900 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1279,11 +1279,12 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
return bio_iov_iter_align_down(bio, iter, len_align_mask);
}
-static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size)
+static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
+ size_t minsize)
{
struct folio *folio;
- while (*size > PAGE_SIZE) {
+ while (*size > minsize) {
folio = folio_alloc(gfp | __GFP_NORETRY, get_order(*size));
if (folio)
return folio;
@@ -1307,7 +1308,7 @@ static void bio_free_folios(struct bio *bio)
}
static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
- size_t maxlen)
+ size_t maxlen, size_t minsize)
{
size_t total_len = min(maxlen, iov_iter_count(iter));
@@ -1322,13 +1323,13 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
size_t this_len = min(total_len, SZ_1M);
struct folio *folio;
- if (this_len > PAGE_SIZE * 2)
+ if (this_len > minsize * 2)
this_len = rounddown_pow_of_two(this_len);
if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len)
break;
- folio = folio_alloc_greedy(GFP_KERNEL, &this_len);
+ folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize);
if (!folio)
break;
bio_add_folio_nofail(bio, folio, this_len, 0);
@@ -1344,16 +1345,16 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
if (!bio->bi_iter.bi_size)
return -ENOMEM;
- return 0;
+ return bio_iov_iter_align_down(bio, iter, minsize - 1);
}
static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
- size_t maxlen)
+ size_t maxlen, size_t minsize)
{
size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
struct folio *folio;
- folio = folio_alloc_greedy(GFP_KERNEL, &len);
+ folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize);
if (!folio)
return -ENOMEM;
@@ -1382,7 +1383,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0);
if (iov_iter_extract_will_pin(iter))
bio_set_flag(bio, BIO_PAGE_PINNED);
- return 0;
+ return bio_iov_iter_align_down(bio, iter, minsize - 1);
}
/**
@@ -1390,6 +1391,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
* @bio: bio to send
* @iter: iter to read from / write into
* @maxlen: maximum size to bounce
+ * @minsize: minimum folio allocation size
*
* Helper for direct I/O implementations that need to bounce buffer because
* we need to checksum the data or perform other operations that require
@@ -1397,11 +1399,12 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
* copies the data into it. Needs to be paired with bio_iov_iter_unbounce()
* called on completion.
*/
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen)
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen,
+ size_t minsize)
{
if (op_is_write(bio_op(bio)))
- return bio_iov_iter_bounce_write(bio, iter, maxlen);
- return bio_iov_iter_bounce_read(bio, iter, maxlen);
+ return bio_iov_iter_bounce_write(bio, iter, maxlen, minsize);
+ return bio_iov_iter_bounce_read(bio, iter, maxlen, minsize);
}
static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 554c87b..bc63bd2 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -2241,7 +2241,7 @@ void blk_cgroup_bio_start(struct bio *bio)
}
u64_stats_update_end_irqrestore(&bis->sync, flags);
- css_rstat_updated(&blkcg->css, cpu);
+ __css_rstat_updated(&blkcg->css, cpu);
put_cpu();
}
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 4c5c16c..a241754 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -3077,7 +3077,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q,
/*
* Check if there is a suitable cached request and return it.
*/
-static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,
+static struct request *blk_mq_get_cached_request(struct blk_plug *plug,
struct request_queue *q, blk_opf_t opf)
{
enum hctx_type type = blk_mq_get_hctx_type(opf);
@@ -3093,27 +3093,10 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,
return NULL;
if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
return NULL;
+ rq_list_pop(&plug->cached_rqs);
return rq;
}
-static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
- struct bio *bio)
-{
- if (rq_list_pop(&plug->cached_rqs) != rq)
- WARN_ON_ONCE(1);
-
- /*
- * If any qos ->throttle() end up blocking, we will have flushed the
- * plug and hence killed the cached_rq list as well. Pop this entry
- * before we throttle.
- */
- rq_qos_throttle(rq->q, bio);
-
- blk_mq_rq_time_init(rq, blk_time_get_ns());
- rq->cmd_flags = bio->bi_opf;
- INIT_LIST_HEAD(&rq->queuelist);
-}
-
static bool bio_unaligned(const struct bio *bio, struct request_queue *q)
{
unsigned int bs_mask = queue_logical_block_size(q) - 1;
@@ -3152,7 +3135,7 @@ void blk_mq_submit_bio(struct bio *bio)
/*
* If the plug has a cached request for this queue, try to use it.
*/
- rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
+ rq = blk_mq_get_cached_request(plug, q, bio->bi_opf);
/*
* A BIO that was released from a zone write plug has already been
@@ -3211,7 +3194,10 @@ void blk_mq_submit_bio(struct bio *bio)
new_request:
if (rq) {
- blk_mq_use_cached_rq(rq, plug, bio);
+ rq_qos_throttle(rq->q, bio);
+ blk_mq_rq_time_init(rq, blk_time_get_ns());
+ rq->cmd_flags = bio->bi_opf;
+ INIT_LIST_HEAD(&rq->queuelist);
} else {
rq = blk_mq_get_new_requests(q, plug, bio);
if (unlikely(!rq)) {
@@ -3257,12 +3243,10 @@ void blk_mq_submit_bio(struct bio *bio)
return;
queue_exit:
- /*
- * Don't drop the queue reference if we were trying to use a cached
- * request and thus didn't acquire one.
- */
if (!rq)
blk_queue_exit(q);
+ else
+ rq_list_add_head(&plug->cached_rqs, rq);
}
#ifdef CONFIG_BLK_MQ_STACKING
@@ -3307,6 +3291,25 @@ blk_status_t blk_insert_cloned_request(struct request *rq)
return BLK_STS_IOERR;
}
+ /*
+ * Integrity segment counting depends on the same queue limits
+ * (virt_boundary_mask, seg_boundary_mask, max_segment_size) that
+ * vary across stacked queues, so recompute against the bottom
+ * queue just like nr_phys_segments above.
+ */
+ if (blk_integrity_rq(rq) && rq->bio) {
+ unsigned short max_int_segs = queue_max_integrity_segments(q);
+
+ rq->nr_integrity_segments =
+ blk_rq_count_integrity_sg(rq->q, rq->bio);
+ if (rq->nr_integrity_segments > max_int_segs) {
+ printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n",
+ __func__, rq->nr_integrity_segments,
+ max_int_segs);
+ return BLK_STS_IOERR;
+ }
+ }
+
if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
return BLK_STS_IOERR;
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index 30cad2b..6a221c18 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -623,6 +623,28 @@ static void disk_mark_zone_wplug_dead(struct blk_zone_wplug *zwplug)
}
}
+static inline bool disk_check_zone_wplug_dead(struct blk_zone_wplug *zwplug)
+{
+ if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD))
+ return false;
+
+ /*
+ * If a new write is received right after a zone reset completes and
+ * while the disk_zone_wplugs_worker() thread has not yet released the
+ * reference on the zone write plug after processing the last write to
+ * the zone, then the new write BIO will see the zone write plug marked
+ * as dead. This case is however a false positive and a perfectly valid
+ * pattern. In such case, restore the zone write plug to a live one.
+ */
+ if (!zwplug->wp_offset && bio_list_empty(&zwplug->bio_list)) {
+ zwplug->flags &= ~BLK_ZONE_WPLUG_DEAD;
+ refcount_inc(&zwplug->ref);
+ return false;
+ }
+
+ return true;
+}
+
static bool disk_zone_wplug_submit_bio(struct gendisk *disk,
struct blk_zone_wplug *zwplug);
@@ -1444,12 +1466,12 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
spin_lock_irqsave(&zwplug->lock, flags);
/*
- * If we got a zone write plug marked as dead, then the user is issuing
- * writes to a full zone, or without synchronizing with zone reset or
- * zone finish operations. In such case, fail the BIO to signal this
- * invalid usage.
+ * Check if we got a zone write plug marked as dead. If yes, then the
+ * user is likely issuing writes to a full zone, or without
+ * synchronizing with zone reset or zone finish operations. In such
+ * case, fail the BIO to signal this invalid usage.
*/
- if (zwplug->flags & BLK_ZONE_WPLUG_DEAD) {
+ if (disk_check_zone_wplug_dead(zwplug)) {
spin_unlock_irqrestore(&zwplug->lock, flags);
disk_put_zone_wplug(zwplug);
bio_io_error(bio);
@@ -1979,8 +2001,10 @@ static void disk_set_zones_cond_array(struct gendisk *disk, u8 *zones_cond)
void disk_free_zone_resources(struct gendisk *disk)
{
- if (disk->zone_wplugs_worker)
+ if (disk->zone_wplugs_worker) {
kthread_stop(disk->zone_wplugs_worker);
+ disk->zone_wplugs_worker = NULL;
+ }
WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list));
if (disk->zone_wplugs_wq) {
@@ -2113,9 +2137,6 @@ static int disk_update_zone_resources(struct gendisk *disk,
ret = queue_limits_commit_update(q, &lim);
unfreeze:
- if (ret)
- disk_free_zone_resources(disk);
-
blk_mq_unfreeze_queue(q, memflags);
return ret;
diff --git a/block/ioctl.c b/block/ioctl.c
index fc3be05..ab2c9ed 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -857,6 +857,8 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg)
#endif
struct blk_iou_cmd {
+ u64 start;
+ u64 len;
int res;
bool nowait;
};
@@ -946,23 +948,27 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
{
struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host);
struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd);
- const struct io_uring_sqe *sqe = cmd->sqe;
u32 cmd_op = cmd->cmd_op;
- uint64_t start, len;
- if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len ||
- sqe->rw_flags || sqe->file_index))
- return -EINVAL;
+ /* Read what we need from the SQE on the first issue */
+ if (!(issue_flags & IORING_URING_CMD_REISSUE)) {
+ const struct io_uring_sqe *sqe = cmd->sqe;
+
+ if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len ||
+ sqe->rw_flags || sqe->file_index))
+ return -EINVAL;
+
+ bic->start = READ_ONCE(sqe->addr);
+ bic->len = READ_ONCE(sqe->addr3);
+ }
bic->res = 0;
bic->nowait = issue_flags & IO_URING_F_NONBLOCK;
- start = READ_ONCE(sqe->addr);
- len = READ_ONCE(sqe->addr3);
-
switch (cmd_op) {
case BLOCK_URING_CMD_DISCARD:
- return blkdev_cmd_discard(cmd, bdev, start, len, bic->nowait);
+ return blkdev_cmd_discard(cmd, bdev, bic->start, bic->len,
+ bic->nowait);
}
return -EINVAL;
}
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index af3d584..522df41 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -390,6 +390,11 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl,
auth = crypto_spawn_ahash_alg(&ctx->auth);
auth_base = &auth->base;
+ if (auth->digestsize > 0 && auth->digestsize < 4) {
+ err = -EINVAL;
+ goto err_free_inst;
+ }
+
err = crypto_grab_skcipher(&ctx->enc, aead_crypto_instance(inst),
crypto_attr_alg_name(tb[2]), 0, mask);
if (err)
diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c
index 23026d4..c7ea40f9 100644
--- a/crypto/krb5/krb5_api.c
+++ b/crypto/krb5/krb5_api.c
@@ -134,27 +134,69 @@ EXPORT_SYMBOL(crypto_krb5_how_much_data);
* Find the offset and size of the data in a secure message so that this
* information can be used in the metadata buffer which will get added to the
* digest by crypto_krb5_verify_mic().
+ *
+ * Return: 0 if successful, -EBADMSG if the message is too short or -EINVAL if
+ * the mode is unsupported.
*/
-void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
- enum krb5_crypto_mode mode,
- size_t *_offset, size_t *_len)
+int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t *_offset, size_t *_len)
{
switch (mode) {
case KRB5_CHECKSUM_MODE:
+ if (*_len < krb5->cksum_len)
+ return -EBADMSG;
*_offset += krb5->cksum_len;
*_len -= krb5->cksum_len;
- return;
+ return 0;
case KRB5_ENCRYPT_MODE:
+ if (*_len < krb5->conf_len + krb5->cksum_len)
+ return -EBADMSG;
*_offset += krb5->conf_len;
*_len -= krb5->conf_len + krb5->cksum_len;
- return;
+ return 0;
default:
WARN_ON_ONCE(1);
- return;
+ return -EINVAL;
}
}
EXPORT_SYMBOL(crypto_krb5_where_is_the_data);
+/**
+ * crypto_krb5_check_data_len - Check a message is big enough
+ * @krb5: The encoding to use.
+ * @mode: Mode of operation.
+ * @len: The length of the secure blob.
+ * @min_content: Minimum length of the content inside the blob.
+ *
+ * Check that a message is large enough to hold whatever bits the encryption
+ * type wants to glue on (nonce, checksum) plus a minimum amount of content.
+ *
+ * Return: 0 if successful, -EBADMSG if the message is too short or -EINVAL if
+ * the mode is unsupported.
+ */
+int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t len, size_t min_content)
+{
+ switch (mode) {
+ case KRB5_CHECKSUM_MODE:
+ if (len < krb5->cksum_len ||
+ len - krb5->cksum_len < min_content)
+ return -EBADMSG;
+ return 0;
+ case KRB5_ENCRYPT_MODE:
+ if (len < krb5->conf_len + krb5->cksum_len ||
+ len - (krb5->conf_len + krb5->cksum_len) < min_content)
+ return -EBADMSG;
+ return 0;
+ default:
+ WARN_ON_ONCE(1);
+ return -EINVAL;
+ }
+}
+EXPORT_SYMBOL(crypto_krb5_check_data_len);
+
/*
* Prepare the encryption with derived key data.
*/
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index f1ac4e0..4500b9c 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -511,6 +511,11 @@ static int aie2_init(struct amdxdna_dev *xdna)
return -EINVAL;
}
+ if (!xdna->group) {
+ XDNA_ERR(xdna, "Running without IOMMU not supported");
+ return -EINVAL;
+ }
+
ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
if (!ndev)
return -ENOMEM;
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 238ee24..6e367dd 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -490,6 +490,9 @@ static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int
struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+ if (abo->private_buffer)
+ return ERR_PTR(-EOPNOTSUPP);
+
if (abo->dma_buf) {
get_dma_buf(abo->dma_buf);
return abo->dma_buf;
@@ -685,6 +688,7 @@ amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create
{
struct amdxdna_dev *xdna = to_xdna_dev(dev);
struct amdxdna_drm_va_tbl va_tbl;
+ struct amdxdna_gem_obj *abo;
struct drm_gem_object *gobj;
struct dma_buf *dma_buf;
@@ -711,7 +715,10 @@ amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create
dma_buf_put(dma_buf);
- return to_xdna_obj(gobj);
+ abo = to_xdna_obj(gobj);
+ abo->private_buffer = true;
+
+ return abo;
}
struct drm_gem_object *
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index 4fc48a1..957305c 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -54,6 +54,8 @@ struct amdxdna_gem_obj {
/* True, if BO is managed by XRT, not application */
bool internal;
+ /* True, if BO is not exportable */
+ bool private_buffer;
};
#define to_gobj(obj) (&(obj)->base.base)
diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c
index fb999aa..85390e3 100644
--- a/drivers/accel/amdxdna/amdxdna_ubuf.c
+++ b/drivers/accel/amdxdna/amdxdna_ubuf.c
@@ -69,60 +69,10 @@ static void amdxdna_ubuf_release(struct dma_buf *dbuf)
kfree(ubuf);
}
-static vm_fault_t amdxdna_ubuf_vm_fault(struct vm_fault *vmf)
-{
- struct vm_area_struct *vma = vmf->vma;
- struct amdxdna_ubuf_priv *ubuf;
- unsigned long pfn;
- pgoff_t pgoff;
-
- ubuf = vma->vm_private_data;
- pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-
- pfn = page_to_pfn(ubuf->pages[pgoff]);
- return vmf_insert_pfn(vma, vmf->address, pfn);
-}
-
-static const struct vm_operations_struct amdxdna_ubuf_vm_ops = {
- .fault = amdxdna_ubuf_vm_fault,
-};
-
-static int amdxdna_ubuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma)
-{
- struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
-
- vma->vm_ops = &amdxdna_ubuf_vm_ops;
- vma->vm_private_data = ubuf;
- vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-
- return 0;
-}
-
-static int amdxdna_ubuf_vmap(struct dma_buf *dbuf, struct iosys_map *map)
-{
- struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
- void *kva;
-
- kva = vmap(ubuf->pages, ubuf->nr_pages, VM_MAP, PAGE_KERNEL);
- if (!kva)
- return -EINVAL;
-
- iosys_map_set_vaddr(map, kva);
- return 0;
-}
-
-static void amdxdna_ubuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map)
-{
- vunmap(map->vaddr);
-}
-
static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = {
.map_dma_buf = amdxdna_ubuf_map,
.unmap_dma_buf = amdxdna_ubuf_unmap,
.release = amdxdna_ubuf_release,
- .mmap = amdxdna_ubuf_mmap,
- .vmap = amdxdna_ubuf_vmap,
- .vunmap = amdxdna_ubuf_vunmap,
};
struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c
index 189dbe9..dc20bc7 100644
--- a/drivers/accel/ivpu/ivpu_debugfs.c
+++ b/drivers/accel/ivpu/ivpu_debugfs.c
@@ -450,7 +450,7 @@ priority_bands_fops_write(struct file *file, const char __user *user_buf, size_t
u32 band;
int ret;
- if (size >= sizeof(buf))
+ if (*pos != 0 || size >= sizeof(buf))
return -EINVAL;
ret = simple_write_to_buffer(buf, sizeof(buf) - 1, pos, user_buf, size);
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 2801378..3b7b008 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -537,6 +537,26 @@ static const struct file_operations ivpu_fops = {
#endif
};
+static int ivpu_gem_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file_priv,
+ u32 handle, u32 flags, int *prime_fd)
+{
+ struct drm_gem_object *obj;
+
+ obj = drm_gem_object_lookup(file_priv, handle);
+ if (!obj)
+ return -ENOENT;
+
+ if (drm_gem_is_imported(obj)) {
+ /* Do not allow re-exporting */
+ drm_gem_object_put(obj);
+ return -EOPNOTSUPP;
+ }
+
+ drm_gem_object_put(obj);
+
+ return drm_gem_prime_handle_to_fd(dev, file_priv, handle, flags, prime_fd);
+}
+
static const struct drm_driver driver = {
.driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
@@ -545,6 +565,7 @@ static const struct drm_driver driver = {
.gem_create_object = ivpu_gem_create_object,
.gem_prime_import = ivpu_gem_prime_import,
+ .prime_handle_to_fd = ivpu_gem_prime_handle_to_fd,
.ioctls = ivpu_drm_ioctls,
.num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c
index 95300c2..1e4c579 100644
--- a/drivers/accel/qaic/qaic_data.c
+++ b/drivers/accel/qaic/qaic_data.c
@@ -606,8 +606,11 @@ static const struct vm_operations_struct drm_vm_ops = {
static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
{
struct qaic_bo *bo = to_qaic_bo(obj);
+ unsigned long remap_start;
unsigned long offset = 0;
+ unsigned long remap_end;
struct scatterlist *sg;
+ unsigned long length;
int ret = 0;
if (drm_gem_is_imported(obj))
@@ -615,11 +618,27 @@ static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc
for (sg = bo->sgt->sgl; sg; sg = sg_next(sg)) {
if (sg_page(sg)) {
+ /* if sg is too large for the VMA, so truncate it to fit */
+ if (check_add_overflow(vma->vm_start, offset, &remap_start))
+ return -EINVAL;
+ if (check_add_overflow(remap_start, sg->length, &remap_end))
+ return -EINVAL;
+
+ if (remap_end > vma->vm_end) {
+ if (check_sub_overflow(vma->vm_end, remap_start, &length))
+ return -EINVAL;
+ } else {
+ length = sg->length;
+ }
+
+ if (length == 0)
+ goto out;
+
ret = remap_pfn_range(vma, vma->vm_start + offset, page_to_pfn(sg_page(sg)),
- sg->length, vma->vm_page_prot);
+ length, vma->vm_page_prot);
if (ret)
goto out;
- offset += sg->length;
+ offset += length;
}
}
diff --git a/drivers/accel/qaic/qaic_ras.c b/drivers/accel/qaic/qaic_ras.c
index cc0b754..6791af3 100644
--- a/drivers/accel/qaic/qaic_ras.c
+++ b/drivers/accel/qaic/qaic_ras.c
@@ -497,11 +497,11 @@ static void decode_ras_msg(struct qaic_device *qdev, struct ras_data *msg)
qdev->ce_count++;
break;
case UE:
- if (qdev->ce_count != UINT_MAX)
+ if (qdev->ue_count != UINT_MAX)
qdev->ue_count++;
break;
case UE_NF:
- if (qdev->ce_count != UINT_MAX)
+ if (qdev->ue_nf_count != UINT_MAX)
qdev->ue_nf_count++;
break;
default:
diff --git a/drivers/accel/rocket/rocket_gem.c b/drivers/accel/rocket/rocket_gem.c
index b6a385d..a5fffa5 100644
--- a/drivers/accel/rocket/rocket_gem.c
+++ b/drivers/accel/rocket/rocket_gem.c
@@ -79,11 +79,6 @@ int rocket_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_file *
rkt_obj->size = args->size;
rkt_obj->offset = 0;
- ret = drm_gem_handle_create(file, gem_obj, &args->handle);
- drm_gem_object_put(gem_obj);
- if (ret)
- goto err;
-
sgt = drm_gem_shmem_get_pages_sgt(shmem_obj);
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
@@ -95,6 +90,8 @@ int rocket_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_file *
rkt_obj->size, PAGE_SIZE,
0, 0);
mutex_unlock(&rocket_priv->mm_lock);
+ if (ret)
+ goto err;
ret = iommu_map_sgtable(rocket_priv->domain->domain,
rkt_obj->mm.start,
@@ -112,8 +109,18 @@ int rocket_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_file *
args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
args->dma_address = rkt_obj->mm.start;
+ ret = drm_gem_handle_create(file, gem_obj, &args->handle);
+ if (ret)
+ goto err_unmap;
+
+ drm_gem_object_put(gem_obj);
+
return 0;
+err_unmap:
+ iommu_unmap(rocket_priv->domain->domain,
+ rkt_obj->mm.start, rkt_obj->size);
+
err_remove_node:
mutex_lock(&rocket_priv->mm_lock);
drm_mm_remove_node(&rkt_obj->mm);
@@ -145,6 +152,8 @@ int rocket_ioctl_prep_bo(struct drm_device *dev, void *data, struct drm_file *fi
ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_WRITE, true, timeout);
if (!ret)
ret = timeout ? -ETIMEDOUT : -EBUSY;
+ else if (ret > 0)
+ ret = 0;
shmem_obj = &to_rocket_bo(gem_obj)->base;
diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c
index e9e970f..27f3174 100644
--- a/drivers/acpi/ac.c
+++ b/drivers/acpi/ac.c
@@ -192,11 +192,15 @@ static const struct dmi_system_id ac_dmi_table[] __initconst = {
static int acpi_ac_probe(struct platform_device *pdev)
{
- struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
struct power_supply_config psy_cfg = {};
+ struct acpi_device *adev;
struct acpi_ac *ac;
int result;
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
+
ac = kzalloc_obj(struct acpi_ac);
if (!ac)
return -ENOMEM;
diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c
index 0a8e02b..ec94b09 100644
--- a/drivers/acpi/acpi_pad.c
+++ b/drivers/acpi/acpi_pad.c
@@ -423,7 +423,11 @@ static void acpi_pad_notify(acpi_handle handle, u32 event, void *data)
static int acpi_pad_probe(struct platform_device *pdev)
{
- struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *adev;
+
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
return acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
acpi_pad_notify, adev);
diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c
index b406d7a..386fc1a 100644
--- a/drivers/acpi/acpi_tad.c
+++ b/drivers/acpi/acpi_tad.c
@@ -605,15 +605,12 @@ static umode_t acpi_tad_attr_is_visible(struct kobject *kobj,
return 0;
}
-static const struct attribute_group acpi_tad_attr_group = {
+static const struct attribute_group acpi_tad_group = {
.attrs = acpi_tad_attrs,
.is_visible = acpi_tad_attr_is_visible,
};
-static const struct attribute_group *acpi_tad_attr_groups[] = {
- &acpi_tad_attr_group,
- NULL,
-};
+__ATTRIBUTE_GROUPS(acpi_tad);
#ifdef CONFIG_RTC_CLASS
/* RTC class device interface */
@@ -683,9 +680,8 @@ static int acpi_tad_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *t)
acpi_tad_rt_to_tm(&rt, &tm_now);
- value = ktime_divns(ktime_sub(rtc_tm_to_ktime(t->time),
- rtc_tm_to_ktime(tm_now)), NSEC_PER_SEC);
- if (value <= 0 || value > U32_MAX)
+ value = rtc_tm_to_time64(&t->time) - rtc_tm_to_time64(&tm_now);
+ if (value <= 0 || value >= U32_MAX)
return -EINVAL;
}
@@ -748,8 +744,7 @@ static int acpi_tad_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *t)
if (retval != ACPI_TAD_WAKE_DISABLED) {
t->enabled = 1;
- t->time = rtc_ktime_to_tm(ktime_add_ns(rtc_tm_to_ktime(tm_now),
- (u64)retval * NSEC_PER_SEC));
+ rtc_time64_to_tm(rtc_tm_to_time64(&tm_now) + retval, &t->time);
} else {
t->enabled = 0;
t->time = tm_now;
@@ -795,9 +790,9 @@ static int acpi_tad_disable_timer(struct device *dev, u32 timer_id)
return acpi_tad_wake_set(dev, "_STV", timer_id, ACPI_TAD_WAKE_DISABLED);
}
-static void acpi_tad_remove(struct platform_device *pdev)
+static void acpi_tad_remove(void *data)
{
- struct device *dev = &pdev->dev;
+ struct device *dev = data;
struct acpi_tad_driver_data *dd = dev_get_drvdata(dev);
device_init_wakeup(dev, false);
@@ -820,10 +815,15 @@ static void acpi_tad_remove(struct platform_device *pdev)
static int acpi_tad_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
- acpi_handle handle = ACPI_HANDLE(dev);
struct acpi_tad_driver_data *dd;
+ acpi_handle handle;
acpi_status status;
unsigned long long caps;
+ int ret;
+
+ handle = ACPI_HANDLE(dev);
+ if (!handle)
+ return -ENODEV;
/*
* Initialization failure messages are mostly about firmware issues, so
@@ -863,13 +863,21 @@ static int acpi_tad_probe(struct platform_device *pdev)
}
/*
- * The platform bus type layer tells the ACPI PM domain powers up the
- * device, so set the runtime PM status of it to "active".
+ * The platform bus type probe callback tells the ACPI PM domain to
+ * power up the device, so set the runtime PM status of it to "active".
*/
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
pm_runtime_suspend(dev);
+ /*
+ * acpi_tad_remove() needs to run after unregistering the RTC class
+ * device to avoid racing with the latter's callbacks.
+ */
+ ret = devm_add_action_or_reset(&pdev->dev, acpi_tad_remove, &pdev->dev);
+ if (ret)
+ return ret;
+
if (caps & ACPI_TAD_RT)
acpi_tad_register_rtc(dev, caps);
@@ -885,10 +893,9 @@ static struct platform_driver acpi_tad_driver = {
.driver = {
.name = "acpi-tad",
.acpi_match_table = acpi_tad_ids,
- .dev_groups = acpi_tad_attr_groups,
+ .dev_groups = acpi_tad_groups,
},
.probe = acpi_tad_probe,
- .remove = acpi_tad_remove,
};
MODULE_DEVICE_TABLE(acpi, acpi_tad_ids);
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 60dacec..4074b59 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -78,18 +78,22 @@ ACPI_EXPORT_SYMBOL(acpi_update_all_gpes)
/*******************************************************************************
*
- * FUNCTION: acpi_enable_gpe
+ * FUNCTION: acpi_enable_gpe_cond
*
* PARAMETERS: gpe_device - Parent GPE Device. NULL for GPE0/GPE1
* gpe_number - GPE level within the GPE block
+ * dispatch_type - GPE dispatch type to match
*
* RETURN: Status
*
- * DESCRIPTION: Add a reference to a GPE. On the first reference, the GPE is
- * hardware-enabled.
+ * DESCRIPTION: Add a reference to a GPE so long as its dispatch type matches
+ * the supplied one, or it is different from ACPI_GPE_DISPATCH_NONE
+ * if the supplied one is ACPI_GPE_DISPATCH_MASK. On the first
+ * reference, the GPE is hardware-enabled.
*
******************************************************************************/
-acpi_status acpi_enable_gpe(acpi_handle gpe_device, u32 gpe_number)
+acpi_status acpi_enable_gpe_cond(acpi_handle gpe_device, u32 gpe_number,
+ u8 dispatch_type)
{
acpi_status status = AE_BAD_PARAMETER;
struct acpi_gpe_event_info *gpe_event_info;
@@ -100,14 +104,18 @@ acpi_status acpi_enable_gpe(acpi_handle gpe_device, u32 gpe_number)
flags = acpi_os_acquire_lock(acpi_gbl_gpe_lock);
/*
- * Ensure that we have a valid GPE number and that there is some way
- * of handling the GPE (handler or a GPE method). In other words, we
- * won't allow a valid GPE to be enabled if there is no way to handle it.
+ * Ensure that we have a valid GPE number and that the dispatch type of
+ * the GPE matches the supplied one (or it is not ACPI_GPE_DISPATCH_NONE
+ * if the supplied one is ACPI_GPE_DISPATCH_MASK).
*/
gpe_event_info = acpi_ev_get_gpe_event_info(gpe_device, gpe_number);
if (gpe_event_info) {
- if (ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) !=
- ACPI_GPE_DISPATCH_NONE) {
+ if (dispatch_type == ACPI_GPE_DISPATCH_MASK)
+ dispatch_type = ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags);
+ else if (dispatch_type != ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags))
+ dispatch_type = ACPI_GPE_DISPATCH_NONE;
+
+ if (dispatch_type != ACPI_GPE_DISPATCH_NONE) {
status = acpi_ev_add_gpe_reference(gpe_event_info, TRUE);
if (ACPI_SUCCESS(status) &&
ACPI_GPE_IS_POLLING_NEEDED(gpe_event_info)) {
@@ -128,6 +136,30 @@ acpi_status acpi_enable_gpe(acpi_handle gpe_device, u32 gpe_number)
acpi_os_release_lock(acpi_gbl_gpe_lock, flags);
return_ACPI_STATUS(status);
}
+ACPI_EXPORT_SYMBOL(acpi_enable_gpe_cond)
+
+/*******************************************************************************
+ *
+ * FUNCTION: acpi_enable_gpe
+ *
+ * PARAMETERS: gpe_device - Parent GPE Device. NULL for GPE0/GPE1
+ * gpe_number - GPE level within the GPE block
+ *
+ * RETURN: Status
+ *
+ * DESCRIPTION: Add a reference to a GPE. On the first reference, the GPE is
+ * hardware-enabled.
+ *
+ ******************************************************************************/
+acpi_status acpi_enable_gpe(acpi_handle gpe_device, u32 gpe_number)
+{
+ /*
+ * Ensure that there is some way of handling the GPE (handler or a GPE
+ * method). In other words, we won't allow a valid GPE to be enabled if
+ * there is no way to handle it.
+ */
+ return acpi_enable_gpe_cond(gpe_device, gpe_number, ACPI_GPE_DISPATCH_MASK);
+}
ACPI_EXPORT_SYMBOL(acpi_enable_gpe)
/*******************************************************************************
diff --git a/drivers/acpi/apei/einj-core.c b/drivers/acpi/apei/einj-core.c
index a9248af0..1f3fa22 100644
--- a/drivers/acpi/apei/einj-core.c
+++ b/drivers/acpi/apei/einj-core.c
@@ -401,8 +401,18 @@ static struct acpi_generic_address *einj_get_trigger_parameter_region(
return NULL;
}
+
+static bool is_memory_injection(u32 type, u32 flags)
+{
+ if (flags & SETWA_FLAGS_EINJV2)
+ return !!(type & ACPI_EINJV2_MEMORY);
+ if (type & ACPI5_VENDOR_BIT)
+ return !!(vendor_flags & SETWA_FLAGS_MEM);
+ return !!(type & MEM_ERROR_MASK) || !!(flags & SETWA_FLAGS_MEM);
+}
+
/* Execute instructions in trigger error action table */
-static int __einj_error_trigger(u64 trigger_paddr, u32 type,
+static int __einj_error_trigger(u64 trigger_paddr, u32 type, u32 flags,
u64 param1, u64 param2)
{
struct acpi_einj_trigger trigger_tab;
@@ -480,7 +490,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type,
* This will cause resource conflict with regular memory. So
* remove it from trigger table resources.
*/
- if ((param_extension || acpi5) && (type & MEM_ERROR_MASK) && param2) {
+ if ((param_extension || acpi5) && is_memory_injection(type, flags)) {
struct apei_resources addr_resources;
apei_resources_init(&addr_resources);
@@ -660,7 +670,7 @@ static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2,
return rc;
trigger_paddr = apei_exec_ctx_get_output(&ctx);
if (notrigger == 0) {
- rc = __einj_error_trigger(trigger_paddr, type, param1, param2);
+ rc = __einj_error_trigger(trigger_paddr, type, flags, param1, param2);
if (rc)
return rc;
}
@@ -718,28 +728,6 @@ int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3,
SETWA_FLAGS_PCIE_SBDF | SETWA_FLAGS_EINJV2)))
return -EINVAL;
- /* check if type is a valid EINJv2 error type */
- if (is_v2) {
- if (!(type & available_error_type_v2))
- return -EINVAL;
- }
- /*
- * We need extra sanity checks for memory errors.
- * Other types leap directly to injection.
- */
-
- /* ensure param1/param2 existed */
- if (!(param_extension || acpi5))
- goto inject;
-
- /* ensure injection is memory related */
- if (type & ACPI5_VENDOR_BIT) {
- if (vendor_flags != SETWA_FLAGS_MEM)
- goto inject;
- } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) {
- goto inject;
- }
-
/*
* Injections targeting a CXL 1.0/1.1 port have to be injected
* via the einj_cxl_rch_error_inject() path as that does the proper
@@ -748,6 +736,23 @@ int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3,
if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM))
return -EINVAL;
+ /* check if type is a valid EINJv2 error type */
+ if (is_v2) {
+ if (!(type & available_error_type_v2))
+ return -EINVAL;
+ }
+
+ /* ensure param1/param2 existed */
+ if (!(param_extension || acpi5))
+ goto inject;
+
+ /*
+ * We need extra sanity checks for memory errors.
+ * Other types leap directly to injection.
+ */
+ if (!is_memory_injection(type, flags))
+ goto inject;
+
/*
* Disallow crazy address masks that give BIOS leeway to pick
* injection address almost anywhere. Insist on page or
diff --git a/drivers/acpi/arm64/cpuidle.c b/drivers/acpi/arm64/cpuidle.c
index 801f9c4..c68a5db 100644
--- a/drivers/acpi/arm64/cpuidle.c
+++ b/drivers/acpi/arm64/cpuidle.c
@@ -16,7 +16,7 @@
static int psci_acpi_cpu_init_idle(unsigned int cpu)
{
- int i, count;
+ int i;
struct acpi_lpi_state *lpi;
struct acpi_processor *pr = per_cpu(processors, cpu);
@@ -30,14 +30,10 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu)
if (!psci_ops.cpu_suspend)
return -EOPNOTSUPP;
- count = pr->power.count - 1;
- if (count <= 0)
- return -ENODEV;
-
- for (i = 0; i < count; i++) {
+ for (i = 1; i < pr->power.count; i++) {
u32 state;
- lpi = &pr->power.lpi_states[i + 1];
+ lpi = &pr->power.lpi_states[i];
/*
* Only bits[31:0] represent a PSCI power_state while
* bits[63:32] must be 0x0 as per ARM ACPI FFH Specification
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index b4c2547..b82dd67 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -94,6 +94,7 @@ struct acpi_battery {
struct power_supply *bat;
struct power_supply_desc bat_desc;
struct acpi_device *device;
+ struct device *phys_dev;
struct notifier_block pm_nb;
struct list_head list;
unsigned long update_time;
@@ -1033,7 +1034,7 @@ static int acpi_battery_update(struct acpi_battery *battery, bool resume)
if ((battery->state & ACPI_BATTERY_STATE_CRITICAL) ||
(test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags) &&
(battery->capacity_now <= battery->alarm)))
- acpi_pm_wakeup_event(&battery->device->dev);
+ acpi_pm_wakeup_event(battery->phys_dev);
return result;
}
@@ -1214,10 +1215,14 @@ static void sysfs_battery_cleanup(struct acpi_battery *battery)
static int acpi_battery_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct acpi_battery *battery;
+ struct acpi_device *device;
int result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
if (device->dep_unmet)
return -EPROBE_DEFER;
@@ -1227,6 +1232,7 @@ static int acpi_battery_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, battery);
+ battery->phys_dev = &pdev->dev;
battery->device = device;
result = devm_mutex_init(&pdev->dev, &battery->update_lock);
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index dc064a3..d802763 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -179,6 +179,7 @@ struct acpi_button {
ktime_t last_time;
bool suspended;
bool lid_state_initialized;
+ bool gpe_enabled;
};
static struct acpi_device *lid_device;
@@ -531,15 +532,20 @@ static int acpi_lid_input_open(struct input_dev *input)
static int acpi_button_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
acpi_notify_handler handler;
+ struct acpi_device *device;
struct acpi_button *button;
struct input_dev *input;
- const char *hid = acpi_device_hid(device);
acpi_status status;
char *name, *class;
+ const char *hid;
int error = 0;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
+ hid = acpi_device_hid(device);
if (!strcmp(hid, ACPI_BUTTON_HID_LID) &&
lid_init_state == ACPI_BUTTON_LID_INIT_DISABLED)
return -ENODEV;
@@ -641,6 +647,21 @@ static int acpi_button_probe(struct platform_device *pdev)
status = acpi_install_notify_handler(device->handle,
ACPI_ALL_NOTIFY, handler,
button);
+ if (ACPI_SUCCESS(status) && device->wakeup.flags.valid) {
+ acpi_status st;
+
+ /*
+ * If the wakeup GPE has a handler method, enable it in
+ * case it is also used for signaling runtime events.
+ */
+ st = acpi_enable_gpe_cond(device->wakeup.gpe_device,
+ device->wakeup.gpe_number,
+ ACPI_GPE_DISPATCH_METHOD);
+ button->gpe_enabled = ACPI_SUCCESS(st);
+ if (button->gpe_enabled)
+ dev_dbg(button->dev, "Enabled ACPI GPE%02llx\n",
+ device->wakeup.gpe_number);
+ }
break;
}
if (ACPI_FAILURE(status)) {
@@ -666,6 +687,7 @@ static int acpi_button_probe(struct platform_device *pdev)
acpi_button_remove_fs(button);
err_free_button:
kfree(button);
+ memset(acpi_device_class(device), 0, sizeof(acpi_device_class));
return error;
}
@@ -684,7 +706,13 @@ static void acpi_button_remove(struct platform_device *pdev)
acpi_button_event);
break;
default:
- acpi_remove_notify_handler(adev->handle, ACPI_DEVICE_NOTIFY,
+ if (button->gpe_enabled) {
+ dev_dbg(button->dev, "Disabling ACPI GPE%02llx\n",
+ adev->wakeup.gpe_number);
+ acpi_disable_gpe(adev->wakeup.gpe_device,
+ adev->wakeup.gpe_number);
+ }
+ acpi_remove_notify_handler(adev->handle, ACPI_ALL_NOTIFY,
button->type == ACPI_BUTTON_TYPE_LID ?
acpi_lid_notify :
acpi_button_notify);
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 2e91c5a..f370be8 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd)
end:
if (cmd == CMD_WRITE) {
if (unlikely(ret)) {
- for_each_online_cpu(i) {
+ for_each_possible_cpu(i) {
struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i);
if (!desc)
@@ -524,13 +524,13 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data)
else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY)
cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY;
- for_each_online_cpu(i) {
+ for_each_possible_cpu(i) {
if (i == cpu)
continue;
match_cpc_ptr = per_cpu(cpc_desc_ptr, i);
if (!match_cpc_ptr)
- goto err_fault;
+ continue;
match_pdomain = &(match_cpc_ptr->domain_info);
if (match_pdomain->domain != pdomain->domain)
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 4520453..64ad4cf 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -1676,10 +1676,14 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca
static int acpi_ec_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
struct acpi_ec *ec;
int ret;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
if (boot_ec && (boot_ec->handle == device->handle ||
!strcmp(acpi_device_hid(device), ACPI_ECDT_HID))) {
/* Fast path: this device corresponds to the boot EC. */
diff --git a/drivers/acpi/hed.c b/drivers/acpi/hed.c
index 4d5e12e..060e8d6 100644
--- a/drivers/acpi/hed.c
+++ b/drivers/acpi/hed.c
@@ -50,9 +50,13 @@ static void acpi_hed_notify(acpi_handle handle, u32 event, void *data)
static int acpi_hed_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
int err;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
/* Only one hardware error device */
if (hed_handle)
return -EINVAL;
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index d13264f..9304ac9 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -3341,12 +3341,16 @@ static int acpi_nfit_probe(struct platform_device *pdev)
struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
struct acpi_nfit_desc *acpi_desc;
struct device *dev = &pdev->dev;
- struct acpi_device *adev = ACPI_COMPANION(dev);
struct acpi_table_header *tbl;
+ struct acpi_device *adev;
acpi_status status = AE_OK;
acpi_size sz;
int rc = 0;
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
+
rc = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
acpi_nfit_notify, dev);
if (rc)
diff --git a/drivers/acpi/pfr_telemetry.c b/drivers/acpi/pfr_telemetry.c
index 32bdf8c..2387376 100644
--- a/drivers/acpi/pfr_telemetry.c
+++ b/drivers/acpi/pfr_telemetry.c
@@ -360,10 +360,14 @@ static void pfrt_log_put_idx(void *data)
static int acpi_pfrt_log_probe(struct platform_device *pdev)
{
- acpi_handle handle = ACPI_HANDLE(&pdev->dev);
struct pfrt_log_device *pfrt_log_dev;
+ acpi_handle handle;
int ret;
+ handle = ACPI_HANDLE(&pdev->dev);
+ if (!handle)
+ return -ENODEV;
+
if (!acpi_has_method(handle, "_DSM")) {
dev_dbg(&pdev->dev, "Missing _DSM\n");
return -ENODEV;
diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c
index 11b1c28..6283105 100644
--- a/drivers/acpi/pfr_update.c
+++ b/drivers/acpi/pfr_update.c
@@ -538,10 +538,14 @@ static void pfru_put_idx(void *data)
static int acpi_pfru_probe(struct platform_device *pdev)
{
- acpi_handle handle = ACPI_HANDLE(&pdev->dev);
struct pfru_device *pfru_dev;
+ acpi_handle handle;
int ret;
+ handle = ACPI_HANDLE(&pdev->dev);
+ if (!handle)
+ return -ENODEV;
+
if (!acpi_has_method(handle, "_DSM")) {
dev_dbg(&pdev->dev, "Missing _DSM\n");
return -ENODEV;
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index 440f1d6..86b7c79 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -629,11 +629,15 @@ static void acpi_sbs_callback(void *context)
static int acpi_sbs_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
struct acpi_sbs *sbs;
int result = 0;
int id;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
sbs = kzalloc_obj(struct acpi_sbs);
if (!sbs) {
result = -ENOMEM;
diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c
index f413270..c0ffa26 100644
--- a/drivers/acpi/sbshc.c
+++ b/drivers/acpi/sbshc.c
@@ -237,11 +237,15 @@ static int smbus_alarm(void *context)
static int acpi_smbus_hc_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
int status;
unsigned long long val;
struct acpi_smb_hc *hc;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
status = acpi_evaluate_integer(device->handle, "_EC", NULL, &val);
if (ACPI_FAILURE(status)) {
pr_err("error obtaining _EC.\n");
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index b8b487d..dfc7daa 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -789,7 +789,7 @@ static int acpi_thermal_probe(struct platform_device *pdev)
int i;
if (!device)
- return -EINVAL;
+ return -ENODEV;
tz = kzalloc_obj(struct acpi_thermal);
if (!tz)
diff --git a/drivers/acpi/tiny-power-button.c b/drivers/acpi/tiny-power-button.c
index 531e65b..92516ef 100644
--- a/drivers/acpi/tiny-power-button.c
+++ b/drivers/acpi/tiny-power-button.c
@@ -38,9 +38,13 @@ static u32 acpi_tiny_power_button_event(void *not_used)
static int acpi_tiny_power_button_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
acpi_status status;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
if (device->device_type == ACPI_BUS_TYPE_POWER_BUTTON) {
status = acpi_install_fixed_event_handler(ACPI_EVENT_POWER_BUTTON,
acpi_tiny_power_button_event,
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index 0a3c823..458efa4 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -916,6 +916,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
DMI_MATCH(DMI_PRODUCT_NAME, "82K8"),
},
},
+ {
+ .callback = video_detect_force_native,
+ /* HP OMEN Gaming Laptop 16-n0xxx */
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "HP"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "OMEN by HP Gaming Laptop 16-n0xxx"),
+ },
+ },
/*
* x86 android tablets which directly control the backlight through
diff --git a/drivers/android/binder/allocation.rs b/drivers/android/binder/allocation.rs
index 0cab959..b7b05e7 100644
--- a/drivers/android/binder/allocation.rs
+++ b/drivers/android/binder/allocation.rs
@@ -157,6 +157,14 @@ pub(crate) fn set_info_target_node(&mut self, target_node: NodeRef) {
self.get_or_init_info().target_node = Some(target_node);
}
+ pub(crate) fn take_oneway_node(&mut self) -> Option<DArc<Node>> {
+ if let Some(info) = self.allocation_info.as_mut() {
+ info.oneway_node.take()
+ } else {
+ None
+ }
+ }
+
/// Reserve enough space to push at least `num_fds` fds.
pub(crate) fn info_add_fd_reserve(&mut self, num_fds: usize) -> Result {
self.get_or_init_info()
diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs
index 820cbd5..96b8440 100644
--- a/drivers/android/binder/process.rs
+++ b/drivers/android/binder/process.rs
@@ -1402,7 +1402,12 @@ fn deferred_release(self: Arc<Self>) {
// Clear delivered_deaths list.
//
// Scope ensures that MutexGuard is dropped while executing the body.
- while let Some(delivered_death) = { self.inner.lock().delivered_deaths.pop_front() } {
+ while let Some(delivered_death) = {
+ // Explicitly bind to avoid tail expression lifetime extension of the lockguard
+ // Can be removed when the kernel moves to edition 2024
+ let maybe_death = self.inner.lock().delivered_deaths.pop_front();
+ maybe_death
+ } {
drop(delivered_death);
}
diff --git a/drivers/android/binder/range_alloc/array.rs b/drivers/android/binder/range_alloc/array.rs
index ada1d1b..081d19b 100644
--- a/drivers/android/binder/range_alloc/array.rs
+++ b/drivers/android/binder/range_alloc/array.rs
@@ -204,7 +204,6 @@ pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result<FreedRange>
// caller will mark them as unused, which means that they can be freed if the system comes
// under memory pressure.
let mut freed_range = FreedRange::interior_pages(offset, size);
- #[expect(clippy::collapsible_if)] // reads better like this
if offset % PAGE_SIZE != 0 {
if i == 0 || self.ranges[i - 1].endpoint() <= (offset & PAGE_MASK) {
freed_range.start_page_idx -= 1;
diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs
index 47d5e4d..1d9b669 100644
--- a/drivers/android/binder/transaction.rs
+++ b/drivers/android/binder/transaction.rs
@@ -270,7 +270,8 @@ fn drop_outstanding_txn(&self) {
/// Not used for replies.
pub(crate) fn submit(self: DLArc<Self>, info: &mut TransactionInfo) -> BinderResult {
// Defined before `process_inner` so that the destructor runs after releasing the lock.
- let mut _t_outdated;
+ let _t_outdated;
+ let _oneway_node;
let oneway = self.flags & TF_ONE_WAY != 0;
let process = self.to.clone();
@@ -287,6 +288,14 @@ pub(crate) fn submit(self: DLArc<Self>, info: &mut TransactionInfo) -> BinderRes
if let Some(t_outdated) =
target_node.take_outdated_transaction(&self, &mut process_inner)
{
+ let mut alloc_guard = t_outdated.allocation.lock();
+ if let Some(alloc) = (*alloc_guard).as_mut() {
+ // Take the oneway node to prevent `Allocation::drop` from calling
+ // `pending_oneway_finished()`, which would be incorrect as this
+ // transaction is not being submitted.
+ _oneway_node = alloc.take_oneway_node();
+ }
+ drop(alloc_guard);
// Save the transaction to be dropped after locks are released.
_t_outdated = t_outdated;
}
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index e76d154..3d0027e 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -5584,6 +5584,7 @@ void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp)
link->pmp = pmp;
link->active_tag = ATA_TAG_POISON;
link->hw_sata_spd_limit = UINT_MAX;
+ INIT_WORK(&link->deferred_qc_work, ata_scsi_deferred_qc_work);
/* can't use iterator, ap isn't initialized yet */
for (i = 0; i < ATA_MAX_DEVICES; i++) {
@@ -5666,7 +5667,6 @@ struct ata_port *ata_port_alloc(struct ata_host *host)
mutex_init(&ap->scsi_scan_mutex);
INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug);
INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan);
- INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work);
INIT_LIST_HEAD(&ap->eh_done_q);
init_waitqueue_head(&ap->eh_wait_q);
init_completion(&ap->park_req_pending);
@@ -6291,12 +6291,15 @@ static void ata_port_detach(struct ata_port *ap)
/* It better be dead now and not have any remaining deferred qc. */
WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED));
- WARN_ON(ap->deferred_qc);
- cancel_work_sync(&ap->deferred_qc_work);
cancel_delayed_work_sync(&ap->hotplug_task);
cancel_delayed_work_sync(&ap->scsi_rescan_task);
+ ata_for_each_link(link, ap, PMP_FIRST) {
+ WARN_ON(link->deferred_qc);
+ cancel_work_sync(&link->deferred_qc_work);
+ }
+
/* Delete port multiplier link transport devices */
if (ap->pmp_link) {
int i;
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 9a4b67b9..d623eb3 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -651,11 +651,11 @@ int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
if (qc->scsicmd != scmd)
continue;
if ((qc->flags & ATA_QCFLAG_ACTIVE) ||
- qc == ap->deferred_qc)
+ qc == qc->dev->link->deferred_qc)
break;
}
- if (i < ATA_MAX_QUEUE && qc == ap->deferred_qc) {
+ if (i < ATA_MAX_QUEUE && qc == qc->dev->link->deferred_qc) {
/*
* This is a deferred command that timed out while
* waiting for the command queue to drain. Since the qc
@@ -666,8 +666,8 @@ int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
* deferred qc work from issuing this qc.
*/
WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE);
- ap->deferred_qc = NULL;
- cancel_work(&ap->deferred_qc_work);
+ qc->dev->link->deferred_qc = NULL;
+ cancel_work(&qc->dev->link->deferred_qc_work);
set_host_byte(scmd, DID_TIME_OUT);
scsi_eh_finish_cmd(scmd, &ap->eh_done_q);
} else if (i < ATA_MAX_QUEUE) {
diff --git a/drivers/ata/libata-pmp.c b/drivers/ata/libata-pmp.c
index e3adc00..e854093 100644
--- a/drivers/ata/libata-pmp.c
+++ b/drivers/ata/libata-pmp.c
@@ -110,13 +110,24 @@ int sata_pmp_qc_defer_cmd_switch(struct ata_queued_cmd *qc)
{
struct ata_link *link = qc->dev->link;
struct ata_port *ap = link->ap;
+ int ret;
if (ap->excl_link == NULL || ap->excl_link == link) {
if (ap->nr_active_links == 0 || ata_link_active(link)) {
qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
- return ata_std_qc_defer(qc);
+ ret = ata_std_qc_defer(qc);
+ if (ret == ATA_DEFER_LINK)
+ return ATA_DEFER_LINK_EXCL;
+ return ret;
}
+ /*
+ * Note: ap->excl_link contains the link that is next in line,
+ * i.e. implicit round robin. If there is only one link
+ * dispatching, ap->excl_link will be left unclaimed, allowing
+ * other links to set ap->excl_link, ensuring that the currently
+ * active link cannot queue any more.
+ */
ap->excl_link = link;
}
@@ -571,8 +582,11 @@ static void sata_pmp_detach(struct ata_device *dev)
if (ap->ops->pmp_detach)
ap->ops->pmp_detach(ap);
- ata_for_each_link(tlink, ap, EDGE)
+ ata_for_each_link(tlink, ap, EDGE) {
+ WARN_ON(tlink->deferred_qc);
+ cancel_work_sync(&tlink->deferred_qc_work);
ata_eh_detach_dev(tlink->device);
+ }
spin_lock_irqsave(ap->lock, flags);
ap->nr_pmp_links = 0;
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index f44612e..d43207c 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1664,8 +1664,9 @@ static void ata_scsi_qc_done(struct ata_queued_cmd *qc, bool set_result,
void ata_scsi_deferred_qc_work(struct work_struct *work)
{
- struct ata_port *ap =
- container_of(work, struct ata_port, deferred_qc_work);
+ struct ata_link *link =
+ container_of(work, struct ata_link, deferred_qc_work);
+ struct ata_port *ap = link->ap;
struct ata_queued_cmd *qc;
unsigned long flags;
@@ -1676,10 +1677,10 @@ void ata_scsi_deferred_qc_work(struct work_struct *work)
* such case, we should not need any more deferring the qc, so warn if
* qc_defer() says otherwise.
*/
- qc = ap->deferred_qc;
+ qc = link->deferred_qc;
if (qc && !ata_port_eh_scheduled(ap)) {
WARN_ON_ONCE(ap->ops->qc_defer(qc));
- ap->deferred_qc = NULL;
+ link->deferred_qc = NULL;
ata_qc_issue(qc);
}
@@ -1688,7 +1689,7 @@ void ata_scsi_deferred_qc_work(struct work_struct *work)
void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
{
- struct ata_queued_cmd *qc = ap->deferred_qc;
+ struct ata_link *link;
lockdep_assert_held(ap->lock);
@@ -1697,16 +1698,21 @@ void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
* do not try to be smart about what to do with this deferred command
* and simply requeue it by completing it with DID_REQUEUE.
*/
- if (qc) {
- ap->deferred_qc = NULL;
- cancel_work(&ap->deferred_qc_work);
- ata_scsi_qc_done(qc, true, DID_REQUEUE << 16);
+ ata_for_each_link(link, ap, PMP_FIRST) {
+ struct ata_queued_cmd *qc = link->deferred_qc;
+
+ if (qc) {
+ link->deferred_qc = NULL;
+ cancel_work(&link->deferred_qc_work);
+ ata_scsi_qc_done(qc, true, DID_REQUEUE << 16);
+ }
}
}
-static void ata_scsi_schedule_deferred_qc(struct ata_port *ap)
+static void ata_scsi_schedule_deferred_qc(struct ata_link *link)
{
- struct ata_queued_cmd *qc = ap->deferred_qc;
+ struct ata_queued_cmd *qc = link->deferred_qc;
+ struct ata_port *ap = link->ap;
lockdep_assert_held(ap->lock);
@@ -1723,12 +1729,12 @@ static void ata_scsi_schedule_deferred_qc(struct ata_port *ap)
return;
}
if (!ap->ops->qc_defer(qc))
- queue_work(system_highpri_wq, &ap->deferred_qc_work);
+ queue_work(system_highpri_wq, &link->deferred_qc_work);
}
static void ata_scsi_qc_complete(struct ata_queued_cmd *qc)
{
- struct ata_port *ap = qc->ap;
+ struct ata_link *link = qc->dev->link;
struct scsi_cmnd *cmd = qc->scsicmd;
u8 *cdb = cmd->cmnd;
bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID;
@@ -1759,22 +1765,23 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc)
ata_scsi_qc_done(qc, false, 0);
- ata_scsi_schedule_deferred_qc(ap);
+ ata_scsi_schedule_deferred_qc(link);
}
static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc)
{
+ struct ata_link *link = qc->dev->link;
int ret;
if (!ap->ops->qc_defer)
- goto issue;
+ goto issue_qc;
/*
* If we already have a deferred qc, then rely on the SCSI layer to
* requeue and defer all incoming commands until the deferred qc is
* processed, once all on-going commands complete.
*/
- if (ap->deferred_qc) {
+ if (link->deferred_qc) {
ata_qc_free(qc);
return SCSI_MLQUEUE_DEVICE_BUSY;
}
@@ -1786,38 +1793,46 @@ static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc)
break;
case ATA_DEFER_LINK:
ret = SCSI_MLQUEUE_DEVICE_BUSY;
- break;
+ goto defer_qc;
+ case ATA_DEFER_LINK_EXCL:
+ /*
+ * Drivers making use of ap->excl_link cannot store the QC in
+ * link->deferred_qc, because the ap->excl_link handling is
+ * incompatible with the link->deferred_qc workqueue handling.
+ */
+ ret = SCSI_MLQUEUE_DEVICE_BUSY;
+ goto free_qc;
case ATA_DEFER_PORT:
ret = SCSI_MLQUEUE_HOST_BUSY;
- break;
+ goto free_qc;
default:
WARN_ON_ONCE(1);
ret = SCSI_MLQUEUE_HOST_BUSY;
- break;
+ goto free_qc;
}
- if (ret) {
- /*
- * We must defer this qc: if this is not an NCQ command, keep
- * this qc as a deferred one and report to the SCSI layer that
- * we issued it so that it is not requeued. The deferred qc will
- * be issued with the port deferred_qc_work once all on-going
- * commands complete.
- */
- if (!ata_is_ncq(qc->tf.protocol)) {
- ap->deferred_qc = qc;
- return 0;
- }
-
- /* Force a requeue of the command to defer its execution. */
- ata_qc_free(qc);
- return ret;
- }
-
-issue:
+issue_qc:
ata_qc_issue(qc);
-
return 0;
+
+defer_qc:
+ /*
+ * We must defer this qc: if this is not an NCQ command, keep
+ * this qc as a deferred one and report to the SCSI layer that
+ * we issued it so that it is not requeued. The deferred qc will
+ * be issued with the port deferred_qc_work once all on-going
+ * commands complete.
+ */
+ if (!ata_is_ncq(qc->tf.protocol)) {
+ link->deferred_qc = qc;
+ return 0;
+ }
+
+free_qc:
+ /* Force a requeue of the command to defer its execution. */
+ ata_qc_free(qc);
+
+ return ret;
}
/**
diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c
index a5b9598..40baeac 100644
--- a/drivers/ata/pata_parport/pata_parport.c
+++ b/drivers/ata/pata_parport/pata_parport.c
@@ -459,19 +459,11 @@ static void pata_parport_dev_release(struct device *dev)
kfree(pi);
}
-static void pata_parport_bus_release(struct device *dev)
-{
- /* nothing to do here but required to avoid warning on device removal */
-}
-
static const struct bus_type pata_parport_bus_type = {
.name = DRV_NAME,
};
-static struct device pata_parport_bus = {
- .init_name = DRV_NAME,
- .release = pata_parport_bus_release,
-};
+static struct device *pata_parport_bus;
static const struct scsi_host_template pata_parport_sht = {
PATA_PARPORT_SHT("pata_parport")
@@ -518,7 +510,7 @@ static struct pi_adapter *pi_init_one(struct parport *parport,
}
/* set up pi->dev before pi_probe_unit() so it can use dev_printk() */
- pi->dev.parent = &pata_parport_bus;
+ pi->dev.parent = pata_parport_bus;
pi->dev.bus = &pata_parport_bus_type;
pi->dev.driver = &pr->driver;
pi->dev.release = pata_parport_dev_release;
@@ -780,8 +772,9 @@ static __init int pata_parport_init(void)
return error;
}
- error = device_register(&pata_parport_bus);
- if (error) {
+ pata_parport_bus = root_device_register(DRV_NAME);
+ if (IS_ERR(pata_parport_bus)) {
+ error = PTR_ERR(pata_parport_bus);
pr_err("failed to register pata_parport bus, error: %d\n", error);
goto out_unregister_bus;
}
@@ -811,7 +804,7 @@ static __init int pata_parport_init(void)
out_remove_new:
bus_remove_file(&pata_parport_bus_type, &bus_attr_new_device);
out_unregister_dev:
- device_unregister(&pata_parport_bus);
+ root_device_unregister(pata_parport_bus);
out_unregister_bus:
bus_unregister(&pata_parport_bus_type);
return error;
@@ -822,7 +815,7 @@ static __exit void pata_parport_exit(void)
parport_unregister_driver(&pata_parport_driver);
bus_remove_file(&pata_parport_bus_type, &bus_attr_new_device);
bus_remove_file(&pata_parport_bus_type, &bus_attr_delete_device);
- device_unregister(&pata_parport_bus);
+ root_device_unregister(pata_parport_bus);
bus_unregister(&pata_parport_bus_type);
}
diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c
index d642ece..57f1081 100644
--- a/drivers/ata/sata_sil24.c
+++ b/drivers/ata/sata_sil24.c
@@ -789,6 +789,7 @@ static int sil24_qc_defer(struct ata_queued_cmd *qc)
struct ata_link *link = qc->dev->link;
struct ata_port *ap = link->ap;
u8 prot = qc->tf.protocol;
+ int ret;
/*
* There is a bug in the chip:
@@ -826,7 +827,10 @@ static int sil24_qc_defer(struct ata_queued_cmd *qc)
qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
}
- return ata_std_qc_defer(qc);
+ ret = ata_std_qc_defer(qc);
+ if (ret == ATA_DEFER_LINK)
+ return ATA_DEFER_LINK_EXCL;
+ return ret;
}
static enum ata_completion_errors sil24_qc_prep(struct ata_queued_cmd *qc)
diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig
index bedc613..1ea7c03 100644
--- a/drivers/auxdisplay/Kconfig
+++ b/drivers/auxdisplay/Kconfig
@@ -327,7 +327,7 @@
say 'N' and keep the default message with the version.
config PANEL_BOOT_MESSAGE
- depends on PANEL_CHANGE_MESSAGE="y"
+ depends on PANEL_CHANGE_MESSAGE
string "New initialization message"
default ""
help
diff --git a/drivers/auxdisplay/line-display.c b/drivers/auxdisplay/line-display.c
index fb6d9294..915eb5c 100644
--- a/drivers/auxdisplay/line-display.c
+++ b/drivers/auxdisplay/line-display.c
@@ -173,7 +173,7 @@ static int linedisp_display(struct linedisp *linedisp, const char *msg,
count = strlen(msg);
/* if the string ends with a newline, trim it */
- if (msg[count - 1] == '\n')
+ if (count && msg[count - 1] == '\n')
count--;
if (!count) {
diff --git a/drivers/auxdisplay/max6959.c b/drivers/auxdisplay/max6959.c
index 6bbc8d48..3bdef09 100644
--- a/drivers/auxdisplay/max6959.c
+++ b/drivers/auxdisplay/max6959.c
@@ -86,10 +86,7 @@ static const struct linedisp_ops max6959_linedisp_ops = {
static int max6959_enable(struct max6959_priv *priv, bool enable)
{
- u8 mask = REG_CONFIGURATION_S_BIT;
- u8 value = enable ? mask : 0;
-
- return regmap_update_bits(priv->regmap, REG_CONFIGURATION, mask, value);
+ return regmap_assign_bits(priv->regmap, REG_CONFIGURATION, REG_CONFIGURATION_S_BIT, enable);
}
static void max6959_power_off(void *priv)
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 391ac5e..70701d3 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -17,6 +17,7 @@
#include <linux/init.h>
#include <linux/of.h>
#include <linux/sched.h>
+#include <linux/sched/topology.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/sysfs.h>
@@ -68,6 +69,24 @@ bool last_level_cache_is_valid(unsigned int cpu)
}
+/*
+ * Get the cacheinfo of the LLC associated with @cpu.
+ * Derived from update_per_cpu_data_slice_size_cpu().
+ */
+struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
+{
+ struct cacheinfo *llc;
+
+ if (!last_level_cache_is_valid(cpu))
+ return NULL;
+
+ llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
+ if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
+ return NULL;
+
+ return llc;
+}
+
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
{
struct cacheinfo *llc_x, *llc_y;
@@ -1018,6 +1037,7 @@ static int cacheinfo_cpu_online(unsigned int cpu)
goto err;
if (cpu_map_shared_cache(true, cpu, &cpu_map))
update_per_cpu_data_slice_size(true, cpu, cpu_map);
+ sched_update_llc_bytes(cpu);
return 0;
err:
free_cache_attributes(cpu);
@@ -1036,6 +1056,9 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
free_cache_attributes(cpu);
if (nr_shared > 1)
update_per_cpu_data_slice_size(false, cpu, cpu_map);
+
+ sched_update_llc_bytes(cpu);
+
return 0;
}
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index f806a68..6981b55 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -1230,8 +1230,10 @@ void memblk_nr_poison_inc(unsigned long pfn)
const unsigned long block_id = pfn_to_block_id(pfn);
struct memory_block *mem = find_memory_block_by_id(block_id);
- if (mem)
+ if (mem) {
atomic_long_inc(&mem->nr_hwpoison);
+ put_device(&mem->dev);
+ }
}
void memblk_nr_poison_sub(unsigned long pfn, long i)
@@ -1239,8 +1241,10 @@ void memblk_nr_poison_sub(unsigned long pfn, long i)
const unsigned long block_id = pfn_to_block_id(pfn);
struct memory_block *mem = find_memory_block_by_id(block_id);
- if (mem)
+ if (mem) {
atomic_long_sub(i, &mem->nr_hwpoison);
+ put_device(&mem->dev);
+ }
}
static unsigned long memblk_nr_poison(struct memory_block *mem)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 75b4698..a19dd22 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -606,6 +606,12 @@ static void platform_device_release(struct device *dev)
kfree(pa);
}
+static void platform_device_release_full(struct device *dev)
+{
+ device_remove_software_node(dev);
+ platform_device_release(dev);
+}
+
/**
* platform_device_alloc - create a platform device
* @name: base name of the device we're adding
@@ -848,7 +854,13 @@ struct platform_device *platform_device_register_full(const struct platform_devi
int ret;
struct platform_device *pdev;
- if (pdevinfo->swnode && pdevinfo->properties)
+ /*
+ * Only one software node per device is allowed. Make sure we don't
+ * accept or create two.
+ */
+ if ((pdevinfo->swnode && pdevinfo->properties) ||
+ (pdevinfo->swnode && is_software_node(pdevinfo->fwnode)) ||
+ (pdevinfo->properties && is_software_node(pdevinfo->fwnode)))
return ERR_PTR(-EINVAL);
pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id);
@@ -878,6 +890,8 @@ struct platform_device *platform_device_register_full(const struct platform_devi
ret = device_add_software_node(&pdev->dev, pdevinfo->swnode);
if (ret)
goto err;
+
+ pdev->dev.release = platform_device_release_full;
} else if (pdevinfo->properties) {
ret = device_create_managed_software_node(&pdev->dev,
pdevinfo->properties, NULL);
diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c
index 4533fe7..2585933 100644
--- a/drivers/base/regmap/regmap-sdw-mbq.c
+++ b/drivers/base/regmap/regmap-sdw-mbq.c
@@ -172,7 +172,7 @@ static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *va
ret = regmap_sdw_mbq_read_impl(slave, reg, val, mbq_size);
if (ret == -ENODATA) {
if (!deferrable)
- dev_warn(dev, "Defer on undeferable control: %x\n", reg);
+ dev_warn(dev, "Defer on undeferrable control: %x\n", reg);
ret = regmap_sdw_mbq_poll_busy(slave, reg, ctx);
if (ret)
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index b2b26f0..e6e022b 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -3257,6 +3257,9 @@ static int _regmap_update_bits(struct regmap *map, unsigned int reg,
*change = false;
if (regmap_volatile(map, reg) && map->reg_update_bits) {
+ if (map->cache_only)
+ return -EBUSY;
+
reg = regmap_reg_addr(map, reg);
ret = map->reg_update_bits(map->bus_context, reg, mask, val);
if (ret == 0 && change)
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 4065336..6c1e734 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -4565,24 +4565,12 @@ static int rbd_register_watch(struct rbd_device *rbd_dev)
return ret;
}
-static void cancel_tasks_sync(struct rbd_device *rbd_dev)
-{
- dout("%s rbd_dev %p\n", __func__, rbd_dev);
-
- cancel_work_sync(&rbd_dev->acquired_lock_work);
- cancel_work_sync(&rbd_dev->released_lock_work);
- cancel_delayed_work_sync(&rbd_dev->lock_dwork);
- cancel_work_sync(&rbd_dev->unlock_work);
-}
-
/*
* header_rwsem must not be held to avoid a deadlock with
* rbd_dev_refresh() when flushing notifies.
*/
static void rbd_unregister_watch(struct rbd_device *rbd_dev)
{
- cancel_tasks_sync(rbd_dev);
-
mutex_lock(&rbd_dev->watch_mutex);
if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED)
__rbd_unregister_watch(rbd_dev);
@@ -6548,10 +6536,18 @@ static int rbd_add_parse_args(const char *buf,
static void rbd_dev_image_unlock(struct rbd_device *rbd_dev)
{
+ dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+ disable_delayed_work_sync(&rbd_dev->lock_dwork);
+ disable_work_sync(&rbd_dev->unlock_work);
+
down_write(&rbd_dev->lock_rwsem);
if (__rbd_is_lock_owner(rbd_dev))
__rbd_release_lock(rbd_dev);
up_write(&rbd_dev->lock_rwsem);
+
+ flush_work(&rbd_dev->acquired_lock_work);
+ flush_work(&rbd_dev->released_lock_work);
}
/*
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 8e5f373..6c041ea 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -900,12 +900,29 @@ static int ublk_validate_params(const struct ublk_device *ub)
if (p->logical_bs_shift > PAGE_SHIFT || p->logical_bs_shift < 9)
return -EINVAL;
+ /*
+ * 256M is a reasonable upper bound for physical block size,
+ * io_min and io_opt; it aligns with the maximum physical
+ * block size possible in NVMe.
+ */
+ if (p->physical_bs_shift > ilog2(SZ_256M))
+ return -EINVAL;
+
+ if (p->io_min_shift > ilog2(SZ_256M))
+ return -EINVAL;
+
+ if (p->io_opt_shift > ilog2(SZ_256M))
+ return -EINVAL;
+
if (p->logical_bs_shift > p->physical_bs_shift)
return -EINVAL;
if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9))
return -EINVAL;
+ if (p->max_sectors < PAGE_SECTORS)
+ return -EINVAL;
+
if (ublk_dev_is_zoned(ub) && !p->chunk_sectors)
return -EINVAL;
} else
@@ -2397,8 +2414,14 @@ static void ublk_reset_ch_dev(struct ublk_device *ub)
{
int i;
- for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
- ublk_queue_reinit(ub, ublk_get_queue(ub, i));
+ for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
+ struct ublk_queue *ubq = ublk_get_queue(ub, i);
+
+ /* Sync with ublk_cancel_cmd() */
+ spin_lock(&ubq->cancel_lock);
+ ublk_queue_reinit(ub, ubq);
+ spin_unlock(&ubq->cancel_lock);
+ }
/* set to NULL, otherwise new tasks cannot mmap io_cmd_buf */
ub->mm = NULL;
@@ -2739,6 +2762,7 @@ static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag,
{
struct ublk_io *io = &ubq->ios[tag];
struct ublk_device *ub = ubq->dev;
+ struct io_uring_cmd *cmd = NULL;
struct request *req;
bool done;
@@ -2761,12 +2785,15 @@ static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag,
spin_lock(&ubq->cancel_lock);
done = !!(io->flags & UBLK_IO_FLAG_CANCELED);
- if (!done)
+ if (!done) {
io->flags |= UBLK_IO_FLAG_CANCELED;
+ cmd = io->cmd;
+ io->cmd = NULL;
+ }
spin_unlock(&ubq->cancel_lock);
- if (!done)
- io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, issue_flags);
+ if (!done && cmd)
+ io_uring_cmd_done(cmd, UBLK_IO_RES_ABORT, issue_flags);
}
/*
@@ -3496,8 +3523,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
- int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
+ int ret = -ECANCELED;
+ if (!tw.cancel)
+ ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
if (ret != -EIOCBQUEUED)
io_uring_cmd_done(cmd, ret, issue_flags);
}
@@ -4990,13 +5019,15 @@ static int ublk_ctrl_set_params(struct ublk_device *ub,
*/
ret = -EACCES;
} else if (copy_from_user(&ub->params, argp, ph.len)) {
+ /* zero out partial copy so no stale params survive */
+ memset(&ub->params, 0, sizeof(ub->params));
ret = -EFAULT;
} else {
/* clear all we don't support yet */
ub->params.types &= UBLK_PARAM_TYPE_ALL;
ret = ublk_validate_params(ub);
if (ret)
- ub->params.types = 0;
+ memset(&ub->params, 0, sizeof(ub->params));
}
mutex_unlock(&ub->mutex);
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index aebc710..0711145 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -33,6 +33,7 @@
#include <linux/cpuhotplug.h>
#include <linux/part_stat.h>
#include <linux/kernel_read_file.h>
+#include <linux/rcupdate.h>
#include "zram_drv.h"
@@ -504,6 +505,7 @@ struct zram_wb_ctl {
wait_queue_head_t done_wait;
spinlock_t done_lock;
atomic_t num_inflight;
+ struct rcu_head rcu;
};
struct zram_wb_req {
@@ -847,7 +849,7 @@ static void release_wb_ctl(struct zram_wb_ctl *wb_ctl)
release_wb_req(req);
}
- kfree(wb_ctl);
+ kfree_rcu(wb_ctl, rcu);
}
static struct zram_wb_ctl *init_wb_ctl(struct zram *zram)
@@ -964,11 +966,13 @@ static void zram_writeback_endio(struct bio *bio)
struct zram_wb_ctl *wb_ctl = bio->bi_private;
unsigned long flags;
+ rcu_read_lock();
spin_lock_irqsave(&wb_ctl->done_lock, flags);
list_add(&req->entry, &wb_ctl->done_reqs);
spin_unlock_irqrestore(&wb_ctl->done_lock, flags);
wake_up(&wb_ctl->done_wait);
+ rcu_read_unlock();
}
static void zram_submit_wb_request(struct zram *zram,
diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c
index 2f59c0d..37e0507 100644
--- a/drivers/bluetooth/btintel_pcie.c
+++ b/drivers/bluetooth/btintel_pcie.c
@@ -289,6 +289,9 @@ static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev)
skb_put_data(skb, buf, strlen(buf));
data->boot_stage_cache = reg;
+ if (reg & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING)
+ bt_dev_warn(hdev, "Controller device warning (boot_stage: 0x%8.8x)", reg);
+
reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_STATUS_REG);
snprintf(buf, sizeof(buf), "ipc status: 0x%8.8x", reg);
skb_put_data(skb, buf, strlen(buf));
@@ -579,12 +582,10 @@ static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data)
reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
- reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
- reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
- if ((reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS) == 0)
+ if (!(reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)) {
reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
-
- btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+ btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+ }
do {
reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
@@ -604,16 +605,10 @@ static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data)
reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
- if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)
+ if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ) {
reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
-
- if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS)
- reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
-
- if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ)
- reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
-
- btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+ btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+ }
}
static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type,
@@ -880,8 +875,11 @@ static inline bool btintel_pcie_in_lockdown(struct btintel_pcie_data *data)
static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data)
{
- return (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_ERR) ||
- (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER);
+ if (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING)
+ bt_dev_warn(data->hdev, "Controller device warning (boot_stage: 0x%8.8x)",
+ data->boot_stage_cache);
+
+ return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER;
}
static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data)
@@ -914,7 +912,8 @@ static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)
data->img_resp_cache = reg;
if (btintel_pcie_in_error(data)) {
- bt_dev_err(data->hdev, "Controller in error state");
+ bt_dev_err(data->hdev, "Controller in error state (boot_stage: 0x%8.8x)",
+ data->boot_stage_cache);
btintel_pcie_dump_debug_registers(data->hdev);
return;
}
diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h
index 3c7bb70..13efef4 100644
--- a/drivers/bluetooth/btintel_pcie.h
+++ b/drivers/bluetooth/btintel_pcie.h
@@ -34,9 +34,6 @@
#define BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS (BIT(20))
#define BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ (BIT(21))
-/* Stop MAC Access disconnection request */
-#define BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS (BIT(22))
-#define BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ (BIT(23))
#define BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS (BIT(28))
#define BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON (BIT(29))
@@ -48,7 +45,7 @@
#define BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW (BIT(2))
#define BTINTEL_PCIE_CSR_BOOT_STAGE_ROM_LOCKDOWN (BIT(10))
#define BTINTEL_PCIE_CSR_BOOT_STAGE_IML_LOCKDOWN (BIT(11))
-#define BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_ERR (BIT(12))
+#define BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING (BIT(12))
#define BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER (BIT(13))
#define BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED (BIT(14))
#define BTINTEL_PCIE_CSR_BOOT_STAGE_MAC_ACCESS_ON (BIT(16))
diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c
index 6fb6ca2..8ff66b2 100644
--- a/drivers/bluetooth/btmtk.c
+++ b/drivers/bluetooth/btmtk.c
@@ -537,6 +537,7 @@ static void btmtk_usb_wmt_recv(struct urb *urb)
return;
} else if (urb->status == -ENOENT) {
/* Avoid suspend failed when usb_kill_urb */
+ kfree(urb->setup_packet);
return;
}
@@ -610,6 +611,7 @@ static int btmtk_usb_submit_wmt_recv_urb(struct hci_dev *hdev)
if (err != -EPERM && err != -ENODEV)
bt_dev_err(hdev, "urb %p submission failed (%d)",
urb, -err);
+ kfree(dr);
usb_unanchor_urb(urb);
}
@@ -695,8 +697,13 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev,
if (data->evt_skb == NULL)
goto err_free_wc;
- /* Parse and handle the return WMT event */
- wmt_evt = (struct btmtk_hci_wmt_evt *)data->evt_skb->data;
+ wmt_evt = skb_pull_data(data->evt_skb, sizeof(*wmt_evt));
+ if (!wmt_evt) {
+ bt_dev_err(hdev, "WMT event too short (%u bytes)",
+ data->evt_skb->len);
+ err = -EINVAL;
+ goto err_free_skb;
+ }
if (wmt_evt->whdr.op != hdr->op) {
bt_dev_err(hdev, "Wrong op received %d expected %d",
wmt_evt->whdr.op, hdr->op);
@@ -712,6 +719,12 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev,
status = BTMTK_WMT_PATCH_DONE;
break;
case BTMTK_WMT_FUNC_CTRL:
+ if (!skb_pull_data(data->evt_skb,
+ sizeof(wmt_evt_funcc->status))) {
+ status = BTMTK_WMT_ON_UNDONE;
+ break;
+ }
+
wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt;
if (be16_to_cpu(wmt_evt_funcc->status) == 0x404)
status = BTMTK_WMT_ON_DONE;
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 7f5fce9..830fefb 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -3540,7 +3540,13 @@ static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev,
"firmware rome 0x%x build 0x%x",
rver_rom, rver_patch, ver_rom, ver_patch);
- if (rver_rom != ver_rom || rver_patch <= ver_patch) {
+ /* Allow rampatch when the patch version equals the firmware version.
+ * A firmware download may be aborted by a transient USB error (e.g.
+ * disconnect) after the controller updates version info but before
+ * completion.
+ * Allowing equal versions enables re-flashing during recovery.
+ */
+ if (rver_rom != ver_rom || rver_patch < ver_patch) {
bt_dev_err(hdev, "rampatch file version did not match with firmware");
err = -EINVAL;
goto done;
diff --git a/drivers/bluetooth/hci_ath.c b/drivers/bluetooth/hci_ath.c
index fa679ad..8201fa7 100644
--- a/drivers/bluetooth/hci_ath.c
+++ b/drivers/bluetooth/hci_ath.c
@@ -191,6 +191,9 @@ static int ath_recv(struct hci_uart *hu, const void *data, int count)
{
struct ath_struct *ath = hu->priv;
+ if (!ath)
+ return -ENODEV;
+
ath->rx_skb = h4_recv_buf(hu, ath->rx_skb, data, count,
ath_recv_pkts, ARRAY_SIZE(ath_recv_pkts));
if (IS_ERR(ath->rx_skb)) {
diff --git a/drivers/bluetooth/hci_bcsp.c b/drivers/bluetooth/hci_bcsp.c
index b386f91..db56eea 100644
--- a/drivers/bluetooth/hci_bcsp.c
+++ b/drivers/bluetooth/hci_bcsp.c
@@ -585,6 +585,9 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count)
if (!test_bit(HCI_UART_REGISTERED, &hu->flags))
return -EUNATCH;
+ if (!bcsp)
+ return -ENODEV;
+
BT_DBG("hu %p count %d rx_state %d rx_count %ld",
hu, count, bcsp->rx_state, bcsp->rx_count);
diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c
index a889a66..7673727 100644
--- a/drivers/bluetooth/hci_h4.c
+++ b/drivers/bluetooth/hci_h4.c
@@ -109,6 +109,9 @@ static int h4_recv(struct hci_uart *hu, const void *data, int count)
{
struct h4_struct *h4 = hu->priv;
+ if (!h4)
+ return -ENODEV;
+
h4->rx_skb = h4_recv_buf(hu, h4->rx_skb, data, count,
h4_recv_pkts, ARRAY_SIZE(h4_recv_pkts));
if (IS_ERR(h4->rx_skb)) {
diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c
index cfdf75d..d353837 100644
--- a/drivers/bluetooth/hci_h5.c
+++ b/drivers/bluetooth/hci_h5.c
@@ -587,6 +587,9 @@ static int h5_recv(struct hci_uart *hu, const void *data, int count)
struct h5 *h5 = hu->priv;
const unsigned char *ptr = data;
+ if (!h5)
+ return -ENODEV;
+
BT_DBG("%s pending %zu count %d", hu->hdev->name, h5->rx_pending,
count);
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index 275ea86..47f4902 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -194,7 +194,15 @@ void hci_uart_init_work(struct work_struct *work)
err = hci_register_dev(hu->hdev);
if (err < 0) {
BT_ERR("Can't register HCI device");
+
+ percpu_down_write(&hu->proto_lock);
clear_bit(HCI_UART_PROTO_READY, &hu->flags);
+ percpu_up_write(&hu->proto_lock);
+
+ /* Safely cancel work after clearing flags */
+ cancel_work_sync(&hu->write_work);
+
+ /* Close protocol before freeing hdev */
hu->proto->close(hu);
hdev = hu->hdev;
hu->hdev = NULL;
@@ -263,8 +271,12 @@ static int hci_uart_open(struct hci_dev *hdev)
/* Close device */
static int hci_uart_close(struct hci_dev *hdev)
{
+ struct hci_uart *hu = hci_get_drvdata(hdev);
+
BT_DBG("hdev %p", hdev);
+ cancel_work_sync(&hu->write_work);
+
hci_uart_flush(hdev);
hdev->flush = NULL;
return 0;
@@ -531,6 +543,7 @@ static void hci_uart_tty_close(struct tty_struct *tty)
{
struct hci_uart *hu = tty->disc_data;
struct hci_dev *hdev;
+ bool proto_ready;
BT_DBG("tty %p", tty);
@@ -540,24 +553,38 @@ static void hci_uart_tty_close(struct tty_struct *tty)
if (!hu)
return;
- hdev = hu->hdev;
- if (hdev)
- hci_uart_close(hdev);
+ /* Wait for init_ready to finish to prevent registration races */
+ cancel_work_sync(&hu->init_ready);
- if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) {
+ proto_ready = test_bit(HCI_UART_PROTO_READY, &hu->flags);
+ if (proto_ready) {
percpu_down_write(&hu->proto_lock);
clear_bit(HCI_UART_PROTO_READY, &hu->flags);
percpu_up_write(&hu->proto_lock);
+ }
- cancel_work_sync(&hu->init_ready);
- cancel_work_sync(&hu->write_work);
+ /*
+ * Unconditionally cancel write_work AFTER clearing PROTO_READY.
+ * This ensures that concurrent protocol timers cannot requeue
+ * write_work via hci_uart_tx_wakeup(), permanently preventing
+ * double-free races and UAFs.
+ */
+ cancel_work_sync(&hu->write_work);
+ hdev = hu->hdev;
+ if (hdev)
+ hci_uart_close(hdev); /* proto->flush is safely skipped */
+
+ if (proto_ready) {
if (hdev) {
if (test_bit(HCI_UART_REGISTERED, &hu->flags))
hci_unregister_dev(hdev);
- hci_free_dev(hdev);
}
+ /* Close protocol before freeing hdev (intrinsically purges queues) */
hu->proto->close(hu);
+
+ if (hdev)
+ hci_free_dev(hdev);
}
clear_bit(HCI_UART_PROTO_SET, &hu->flags);
@@ -625,11 +652,12 @@ static void hci_uart_tty_receive(struct tty_struct *tty, const u8 *data,
* tty caller
*/
hu->proto->recv(hu, data, count);
- percpu_up_read(&hu->proto_lock);
if (hu->hdev)
hu->hdev->stat.byte_rx += count;
+ percpu_up_read(&hu->proto_lock);
+
tty_unthrottle(tty);
}
@@ -695,6 +723,10 @@ static int hci_uart_register_dev(struct hci_uart *hu)
percpu_down_write(&hu->proto_lock);
clear_bit(HCI_UART_PROTO_INIT, &hu->flags);
percpu_up_write(&hu->proto_lock);
+ /* Cancel work after clearing flags */
+ cancel_work_sync(&hu->write_work);
+
+ /* Close protocol before freeing hdev */
hu->proto->close(hu);
hu->hdev = NULL;
hci_free_dev(hdev);
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index cd18342..3450013 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -48,13 +48,12 @@
#define HCI_MAX_IBS_SIZE 10
#define IBS_WAKE_RETRANS_TIMEOUT_MS 100
-#define IBS_BTSOC_TX_IDLE_TIMEOUT_MS 200
+#define IBS_BTSOC_TX_IDLE_TIMEOUT msecs_to_jiffies(200)
#define IBS_HOST_TX_IDLE_TIMEOUT_MS 2000
-#define CMD_TRANS_TIMEOUT_MS 100
-#define MEMDUMP_TIMEOUT_MS 8000
-#define IBS_DISABLE_SSR_TIMEOUT_MS \
- (MEMDUMP_TIMEOUT_MS + FW_DOWNLOAD_TIMEOUT_MS)
-#define FW_DOWNLOAD_TIMEOUT_MS 3000
+#define CMD_TRANS_TIMEOUT msecs_to_jiffies(100)
+#define MEMDUMP_TIMEOUT msecs_to_jiffies(8000)
+#define FW_DOWNLOAD_TIMEOUT msecs_to_jiffies(3000)
+#define IBS_DISABLE_SSR_TIMEOUT (MEMDUMP_TIMEOUT + FW_DOWNLOAD_TIMEOUT)
/* susclk rate */
#define SUSCLK_RATE_32KHZ 32768
@@ -1096,7 +1095,7 @@ static void qca_controller_memdump(struct work_struct *work)
queue_delayed_work(qca->workqueue,
&qca->ctrl_memdump_timeout,
- msecs_to_jiffies(MEMDUMP_TIMEOUT_MS));
+ MEMDUMP_TIMEOUT);
skb_pull(skb, sizeof(qca_memdump->ram_dump_size));
qca_memdump->current_seq_no = 0;
qca_memdump->received_dump = 0;
@@ -1369,7 +1368,7 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate)
if (hu->serdev)
serdev_device_wait_until_sent(hu->serdev,
- msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS));
+ CMD_TRANS_TIMEOUT);
/* Give the controller time to process the request */
switch (qca_soc_type(hu)) {
@@ -1401,8 +1400,8 @@ static inline void host_set_baudrate(struct hci_uart *hu, unsigned int speed)
static int qca_send_power_pulse(struct hci_uart *hu, bool on)
{
+ int timeout = CMD_TRANS_TIMEOUT;
int ret;
- int timeout = msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS);
u8 cmd = on ? QCA_WCN3990_POWERON_PULSE : QCA_WCN3990_POWEROFF_PULSE;
/* These power pulses are single byte command which are sent
@@ -1607,7 +1606,7 @@ static void qca_wait_for_dump_collection(struct hci_dev *hdev)
struct qca_data *qca = hu->priv;
wait_on_bit_timeout(&qca->flags, QCA_MEMDUMP_COLLECTION,
- TASK_UNINTERRUPTIBLE, MEMDUMP_TIMEOUT_MS);
+ TASK_UNINTERRUPTIBLE, MEMDUMP_TIMEOUT);
clear_bit(QCA_MEMDUMP_COLLECTION, &qca->flags);
}
@@ -1681,8 +1680,8 @@ static void qca_hw_error(struct hci_dev *hdev, u8 code)
mod_timer(&qca->tx_idle_timer, jiffies +
msecs_to_jiffies(qca->tx_idle_delay));
- /* Controller reset completion time is 50ms */
- msleep(50);
+ /* Wait for the controller to load the rampatch and NVM. */
+ msleep(100);
clear_bit(QCA_SSR_TRIGGERED, &qca->flags);
clear_bit(QCA_IBS_DISABLED, &qca->flags);
@@ -2591,7 +2590,7 @@ static void qca_serdev_remove(struct serdev_device *serdev)
static void qca_serdev_shutdown(struct serdev_device *serdev)
{
int ret;
- int timeout = msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS);
+ int timeout = CMD_TRANS_TIMEOUT;
struct qca_serdev *qcadev = serdev_device_get_drvdata(serdev);
struct hci_uart *hu = &qcadev->serdev_hu;
struct hci_dev *hdev = hu->hdev;
@@ -2648,7 +2647,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
bool tx_pending = false;
int ret = 0;
u8 cmd;
- u32 wait_timeout = 0;
+ unsigned long wait_timeout = 0;
set_bit(QCA_SUSPENDING, &qca->flags);
@@ -2669,15 +2668,15 @@ static int __maybe_unused qca_suspend(struct device *dev)
if (test_bit(QCA_IBS_DISABLED, &qca->flags) ||
test_bit(QCA_SSR_TRIGGERED, &qca->flags)) {
wait_timeout = test_bit(QCA_SSR_TRIGGERED, &qca->flags) ?
- IBS_DISABLE_SSR_TIMEOUT_MS :
- FW_DOWNLOAD_TIMEOUT_MS;
+ IBS_DISABLE_SSR_TIMEOUT :
+ FW_DOWNLOAD_TIMEOUT;
/* QCA_IBS_DISABLED flag is set to true, During FW download
* and during memory dump collection. It is reset to false,
* After FW download complete.
*/
wait_on_bit_timeout(&qca->flags, QCA_IBS_DISABLED,
- TASK_UNINTERRUPTIBLE, msecs_to_jiffies(wait_timeout));
+ TASK_UNINTERRUPTIBLE, wait_timeout);
if (test_bit(QCA_IBS_DISABLED, &qca->flags)) {
bt_dev_err(hu->hdev, "SSR or FW download time out");
@@ -2729,7 +2728,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
if (tx_pending) {
serdev_device_wait_until_sent(hu->serdev,
- msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS));
+ CMD_TRANS_TIMEOUT);
serial_clock_vote(HCI_IBS_TX_VOTE_CLOCK_OFF, hu);
}
@@ -2738,7 +2737,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
*/
ret = wait_event_interruptible_timeout(qca->suspend_wait_q,
qca->rx_ibs_state == HCI_IBS_RX_ASLEEP,
- msecs_to_jiffies(IBS_BTSOC_TX_IDLE_TIMEOUT_MS));
+ IBS_BTSOC_TX_IDLE_TIMEOUT);
if (ret == 0) {
ret = -ETIMEDOUT;
goto error;
diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c
index 76d61af..140ab55 100644
--- a/drivers/bluetooth/virtio_bt.c
+++ b/drivers/bluetooth/virtio_bt.c
@@ -12,6 +12,7 @@
#include <net/bluetooth/hci_core.h>
#define VERSION "0.1"
+#define VIRTBT_RX_BUF_SIZE 1000
enum {
VIRTBT_VQ_TX,
@@ -33,11 +34,11 @@ static int virtbt_add_inbuf(struct virtio_bluetooth *vbt)
struct sk_buff *skb;
int err;
- skb = alloc_skb(1000, GFP_KERNEL);
+ skb = alloc_skb(VIRTBT_RX_BUF_SIZE, GFP_KERNEL);
if (!skb)
return -ENOMEM;
- sg_init_one(sg, skb->data, 1000);
+ sg_init_one(sg, skb->data, VIRTBT_RX_BUF_SIZE);
err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_KERNEL);
if (err < 0) {
@@ -197,6 +198,7 @@ static int virtbt_shutdown_generic(struct hci_dev *hdev)
static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb)
{
+ size_t min_hdr;
__u8 pkt_type;
pkt_type = *((__u8 *) skb->data);
@@ -204,16 +206,32 @@ static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb)
switch (pkt_type) {
case HCI_EVENT_PKT:
+ min_hdr = sizeof(struct hci_event_hdr);
+ break;
case HCI_ACLDATA_PKT:
+ min_hdr = sizeof(struct hci_acl_hdr);
+ break;
case HCI_SCODATA_PKT:
+ min_hdr = sizeof(struct hci_sco_hdr);
+ break;
case HCI_ISODATA_PKT:
- hci_skb_pkt_type(skb) = pkt_type;
- hci_recv_frame(vbt->hdev, skb);
+ min_hdr = sizeof(struct hci_iso_hdr);
break;
default:
kfree_skb(skb);
- break;
+ return;
}
+
+ if (skb->len < min_hdr) {
+ bt_dev_err_ratelimited(vbt->hdev,
+ "rx pkt_type 0x%02x payload %u < hdr %zu\n",
+ pkt_type, skb->len, min_hdr);
+ kfree_skb(skb);
+ return;
+ }
+
+ hci_skb_pkt_type(skb) = pkt_type;
+ hci_recv_frame(vbt->hdev, skb);
}
static void virtbt_rx_work(struct work_struct *work)
@@ -227,8 +245,15 @@ static void virtbt_rx_work(struct work_struct *work)
if (!skb)
return;
- skb_put(skb, len);
- virtbt_rx_handle(vbt, skb);
+ if (!len || len > VIRTBT_RX_BUF_SIZE) {
+ bt_dev_err_ratelimited(vbt->hdev,
+ "rx reply len %u outside [1, %u]\n",
+ len, VIRTBT_RX_BUF_SIZE);
+ kfree_skb(skb);
+ } else {
+ skb_put(skb, len);
+ virtbt_rx_handle(vbt, skb);
+ }
if (virtbt_add_inbuf(vbt) < 0)
return;
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index fc049612d..62934cf 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -631,6 +631,16 @@ int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi)
WARN_ON(!cdo->generic_packet);
+ /*
+ * Propagate the drive's write support to the block layer so BLKROGET
+ * reflects actual write capability. Drivers that use GET CONFIGURATION
+ * features (CDC_MRW_W, CDC_RAM) must have called
+ * cdrom_probe_write_features() before register_cdrom() so the mask is
+ * complete here.
+ */
+ set_disk_ro(disk, !CDROM_CAN(CDC_DVD_RAM | CDC_MRW_W | CDC_RAM |
+ CDC_CD_RW));
+
cd_dbg(CD_REG_UNREG, "drive \"/dev/%s\" registered\n", cdi->name);
mutex_lock(&cdrom_mutex);
list_add(&cdi->list, &cdrom_list);
@@ -742,6 +752,44 @@ static int cdrom_is_random_writable(struct cdrom_device_info *cdi, int *write)
return 0;
}
+/*
+ * Probe write-related MMC features via GET CONFIGURATION and update
+ * cdi->mask accordingly. Drivers that populate cdi->mask from the MODE SENSE
+ * capabilities page (e.g. sr) should call this after those MODE SENSE bits
+ * have been set but before register_cdrom(), so that the full set of
+ * write-capability bits is known by the time register_cdrom() decides on the
+ * initial read-only state of the disk.
+ */
+void cdrom_probe_write_features(struct cdrom_device_info *cdi)
+{
+ int mrw, mrw_write, ram_write;
+
+ mrw = 0;
+ if (!cdrom_is_mrw(cdi, &mrw_write))
+ mrw = 1;
+
+ if (CDROM_CAN(CDC_MO_DRIVE))
+ ram_write = 1;
+ else
+ (void) cdrom_is_random_writable(cdi, &ram_write);
+
+ if (mrw)
+ cdi->mask &= ~CDC_MRW;
+ else
+ cdi->mask |= CDC_MRW;
+
+ if (mrw_write)
+ cdi->mask &= ~CDC_MRW_W;
+ else
+ cdi->mask |= CDC_MRW_W;
+
+ if (ram_write)
+ cdi->mask &= ~CDC_RAM;
+ else
+ cdi->mask |= CDC_RAM;
+}
+EXPORT_SYMBOL(cdrom_probe_write_features);
+
static int cdrom_media_erasable(struct cdrom_device_info *cdi)
{
disc_information di;
@@ -894,33 +942,8 @@ static int cdrom_is_dvd_rw(struct cdrom_device_info *cdi)
*/
static int cdrom_open_write(struct cdrom_device_info *cdi)
{
- int mrw, mrw_write, ram_write;
int ret = 1;
- mrw = 0;
- if (!cdrom_is_mrw(cdi, &mrw_write))
- mrw = 1;
-
- if (CDROM_CAN(CDC_MO_DRIVE))
- ram_write = 1;
- else
- (void) cdrom_is_random_writable(cdi, &ram_write);
-
- if (mrw)
- cdi->mask &= ~CDC_MRW;
- else
- cdi->mask |= CDC_MRW;
-
- if (mrw_write)
- cdi->mask &= ~CDC_MRW_W;
- else
- cdi->mask |= CDC_MRW_W;
-
- if (ram_write)
- cdi->mask &= ~CDC_RAM;
- else
- cdi->mask |= CDC_RAM;
-
if (CDROM_CAN(CDC_MRW_W))
ret = cdrom_mrw_open_write(cdi);
else if (CDROM_CAN(CDC_DVD_RAM))
diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c
index 0d25bbd..4d0b7d7 100644
--- a/drivers/char/agp/efficeon-agp.c
+++ b/drivers/char/agp/efficeon-agp.c
@@ -27,6 +27,7 @@
#include <linux/gfp.h>
#include <linux/page-flags.h>
#include <linux/mm.h>
+#include <asm/cpuid/api.h>
#include "agp.h"
#include "intel-agp.h"
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index 4a9e9de..9a9d12b 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -168,6 +168,10 @@ struct smi_info {
OEM2_DATA_AVAIL)
unsigned char msg_flags;
+ /* When requesting events and messages, don't do it forever. */
+ unsigned int num_requests_in_a_row;
+ bool last_was_flag_fetch;
+
/* Does the BMC have an event buffer? */
bool has_event_buffer;
@@ -410,7 +414,10 @@ static void start_getting_msg_queue(struct smi_info *smi_info)
start_new_msg(smi_info, smi_info->curr_msg->data,
smi_info->curr_msg->data_size);
- smi_info->si_state = SI_GETTING_MESSAGES;
+ if (smi_info->si_state != SI_GETTING_MESSAGES) {
+ smi_info->num_requests_in_a_row = 0;
+ smi_info->si_state = SI_GETTING_MESSAGES;
+ }
}
static void start_getting_events(struct smi_info *smi_info)
@@ -421,7 +428,10 @@ static void start_getting_events(struct smi_info *smi_info)
start_new_msg(smi_info, smi_info->curr_msg->data,
smi_info->curr_msg->data_size);
- smi_info->si_state = SI_GETTING_EVENTS;
+ if (smi_info->si_state != SI_GETTING_EVENTS) {
+ smi_info->num_requests_in_a_row = 0;
+ smi_info->si_state = SI_GETTING_EVENTS;
+ }
}
/*
@@ -487,15 +497,19 @@ static void handle_flags(struct smi_info *smi_info)
} else if (smi_info->msg_flags & RECEIVE_MSG_AVAIL) {
/* Messages available. */
smi_info->curr_msg = alloc_msg_handle_irq(smi_info);
- if (!smi_info->curr_msg)
+ if (!smi_info->curr_msg) {
+ smi_info->si_state = SI_NORMAL;
return;
+ }
start_getting_msg_queue(smi_info);
} else if (smi_info->msg_flags & EVENT_MSG_BUFFER_FULL) {
/* Events available. */
smi_info->curr_msg = alloc_msg_handle_irq(smi_info);
- if (!smi_info->curr_msg)
+ if (!smi_info->curr_msg) {
+ smi_info->si_state = SI_NORMAL;
return;
+ }
start_getting_events(smi_info);
} else if (smi_info->msg_flags & OEM_DATA_AVAIL &&
@@ -595,6 +609,7 @@ static void handle_transaction_done(struct smi_info *smi_info)
smi_info->si_state = SI_NORMAL;
} else {
smi_info->msg_flags = msg[3];
+ smi_info->last_was_flag_fetch = true;
handle_flags(smi_info);
}
break;
@@ -630,7 +645,13 @@ static void handle_transaction_done(struct smi_info *smi_info)
*/
msg = smi_info->curr_msg;
smi_info->curr_msg = NULL;
- if (msg->rsp[2] != 0) {
+ /*
+ * It appears some BMCs, with no event data, return no
+ * data in the message and not a 0x80 error as the
+ * spec says they should. Shut down processing if
+ * the data is not the right length.
+ */
+ if (msg->rsp[2] != 0 || msg->rsp_size != 19) {
/* Error getting event, probably done. */
msg->done(msg);
@@ -640,6 +661,11 @@ static void handle_transaction_done(struct smi_info *smi_info)
} else {
smi_inc_stat(smi_info, events);
+ smi_info->num_requests_in_a_row++;
+ if (smi_info->num_requests_in_a_row > 10)
+ /* Stop if we do this too many times. */
+ smi_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL;
+
/*
* Do this before we deliver the message
* because delivering the message releases the
@@ -678,6 +704,11 @@ static void handle_transaction_done(struct smi_info *smi_info)
} else {
smi_inc_stat(smi_info, incoming_messages);
+ smi_info->num_requests_in_a_row++;
+ if (smi_info->num_requests_in_a_row > 10)
+ /* Stop if we do this too many times. */
+ smi_info->msg_flags &= ~RECEIVE_MSG_AVAIL;
+
/*
* Do this before we deliver the message
* because delivering the message releases the
@@ -820,6 +851,26 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info,
}
/*
+ * If we are currently idle, or if the last thing that was
+ * done was a flag fetch and there is a message pending, try
+ * to start the next message.
+ *
+ * We do the waiting message check to avoid a stuck flag
+ * completely wedging the driver. Let a message through
+ * in between flag operations if that happens.
+ */
+ if (si_sm_result == SI_SM_IDLE ||
+ (si_sm_result == SI_SM_ATTN && smi_info->waiting_msg &&
+ smi_info->last_was_flag_fetch)) {
+ smi_info->last_was_flag_fetch = false;
+ smi_inc_stat(smi_info, idles);
+
+ si_sm_result = start_next_msg(smi_info);
+ if (si_sm_result != SI_SM_IDLE)
+ goto restart;
+ }
+
+ /*
* We prefer handling attn over new messages. But don't do
* this if there is not yet an upper layer to handle anything.
*/
@@ -846,15 +897,6 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info,
}
}
- /* If we are currently idle, try to start the next message. */
- if (si_sm_result == SI_SM_IDLE) {
- smi_inc_stat(smi_info, idles);
-
- si_sm_result = start_next_msg(smi_info);
- if (si_sm_result != SI_SM_IDLE)
- goto restart;
- }
-
if ((si_sm_result == SI_SM_IDLE)
&& (atomic_read(&smi_info->req_events))) {
/*
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index b49500a..f419b46 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -225,6 +225,9 @@ struct ssif_info {
bool has_event_buffer;
bool supports_alert;
+ /* When requesting events and messages, don't do it forever. */
+ unsigned int num_requests_in_a_row;
+
/*
* Used to tell what we should do with alerts. If we are
* waiting on a response, read the data immediately.
@@ -413,7 +416,10 @@ static void start_event_fetch(struct ssif_info *ssif_info, unsigned long *flags)
}
ssif_info->curr_msg = msg;
- ssif_info->ssif_state = SSIF_GETTING_EVENTS;
+ if (ssif_info->ssif_state != SSIF_GETTING_EVENTS) {
+ ssif_info->num_requests_in_a_row = 0;
+ ssif_info->ssif_state = SSIF_GETTING_EVENTS;
+ }
ipmi_ssif_unlock_cond(ssif_info, flags);
msg->data[0] = (IPMI_NETFN_APP_REQUEST << 2);
@@ -436,7 +442,10 @@ static void start_recv_msg_fetch(struct ssif_info *ssif_info,
}
ssif_info->curr_msg = msg;
- ssif_info->ssif_state = SSIF_GETTING_MESSAGES;
+ if (ssif_info->ssif_state != SSIF_GETTING_MESSAGES) {
+ ssif_info->num_requests_in_a_row = 0;
+ ssif_info->ssif_state = SSIF_GETTING_MESSAGES;
+ }
ipmi_ssif_unlock_cond(ssif_info, flags);
msg->data[0] = (IPMI_NETFN_APP_REQUEST << 2);
@@ -843,6 +852,11 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result,
ssif_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL;
handle_flags(ssif_info, flags);
} else {
+ ssif_info->num_requests_in_a_row++;
+ if (ssif_info->num_requests_in_a_row > 10)
+ /* Stop if we do this too many times. */
+ ssif_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL;
+
handle_flags(ssif_info, flags);
ssif_inc_stat(ssif_info, events);
deliver_recv_msg(ssif_info, msg);
@@ -876,6 +890,11 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result,
ssif_info->msg_flags &= ~RECEIVE_MSG_AVAIL;
handle_flags(ssif_info, flags);
} else {
+ ssif_info->num_requests_in_a_row++;
+ if (ssif_info->num_requests_in_a_row > 10)
+ /* Stop if we do this too many times. */
+ ssif_info->msg_flags &= ~RECEIVE_MSG_AVAIL;
+
ssif_inc_stat(ssif_info, incoming_messages);
handle_flags(ssif_info, flags);
deliver_recv_msg(ssif_info, msg);
@@ -1886,6 +1905,7 @@ static int ssif_probe(struct i2c_client *client)
"kssif%4.4x", thread_num);
if (IS_ERR(ssif_info->thread)) {
rv = PTR_ERR(ssif_info->thread);
+ ssif_info->thread = NULL;
dev_notice(&ssif_info->client->dev,
"Could not start kernel thread: error %d\n",
rv);
diff --git a/drivers/clk/clk-eyeq.c b/drivers/clk/clk-eyeq.c
index c1dcced..d9303c2 100644
--- a/drivers/clk/clk-eyeq.c
+++ b/drivers/clk/clk-eyeq.c
@@ -110,6 +110,7 @@ struct eqc_match_data {
const char *reset_auxdev_name;
const char *pinctrl_auxdev_name;
+ const char *eth_phy_auxdev_name;
unsigned int early_clk_count;
};
@@ -321,38 +322,18 @@ static void eqc_probe_init_fixed_factors(struct device *dev,
}
}
-static void eqc_auxdev_release(struct device *dev)
-{
- struct auxiliary_device *adev = to_auxiliary_dev(dev);
-
- kfree(adev);
-}
-
-static int eqc_auxdev_create(struct device *dev, void __iomem *base,
- const char *name, u32 id)
+static void eqc_auxdev_create_optional(struct device *dev, void __iomem *base,
+ const char *name)
{
struct auxiliary_device *adev;
- int ret;
- adev = kzalloc_obj(*adev);
- if (!adev)
- return -ENOMEM;
-
- adev->name = name;
- adev->dev.parent = dev;
- adev->dev.platform_data = (void __force *)base;
- adev->dev.release = eqc_auxdev_release;
- adev->id = id;
-
- ret = auxiliary_device_init(adev);
- if (ret)
- return ret;
-
- ret = auxiliary_device_add(adev);
- if (ret)
- auxiliary_device_uninit(adev);
-
- return ret;
+ if (name) {
+ adev = devm_auxiliary_device_create(dev, name,
+ (void __force *)base);
+ if (!adev)
+ dev_warn(dev, "failed creating auxiliary device %s.%s\n",
+ KBUILD_MODNAME, name);
+ }
}
static int eqc_probe(struct platform_device *pdev)
@@ -364,7 +345,6 @@ static int eqc_probe(struct platform_device *pdev)
unsigned int i, clk_count;
struct resource *res;
void __iomem *base;
- int ret;
data = device_get_match_data(dev);
if (!data)
@@ -378,21 +358,10 @@ static int eqc_probe(struct platform_device *pdev)
if (!base)
return -ENOMEM;
- /* Init optional reset auxiliary device. */
- if (data->reset_auxdev_name) {
- ret = eqc_auxdev_create(dev, base, data->reset_auxdev_name, 0);
- if (ret)
- dev_warn(dev, "failed creating auxiliary device %s.%s: %d\n",
- KBUILD_MODNAME, data->reset_auxdev_name, ret);
- }
-
- /* Init optional pinctrl auxiliary device. */
- if (data->pinctrl_auxdev_name) {
- ret = eqc_auxdev_create(dev, base, data->pinctrl_auxdev_name, 0);
- if (ret)
- dev_warn(dev, "failed creating auxiliary device %s.%s: %d\n",
- KBUILD_MODNAME, data->pinctrl_auxdev_name, ret);
- }
+ /* Init optional auxiliary devices. */
+ eqc_auxdev_create_optional(dev, base, data->reset_auxdev_name);
+ eqc_auxdev_create_optional(dev, base, data->pinctrl_auxdev_name);
+ eqc_auxdev_create_optional(dev, base, data->eth_phy_auxdev_name);
if (data->pll_count + data->div_count + data->fixed_factor_count == 0)
return 0; /* Zero clocks, we are done. */
@@ -553,6 +522,7 @@ static const struct eqc_match_data eqc_eyeq5_match_data = {
.reset_auxdev_name = "reset",
.pinctrl_auxdev_name = "pinctrl",
+ .eth_phy_auxdev_name = "phy",
.early_clk_count = ARRAY_SIZE(eqc_eyeq5_early_plls) +
ARRAY_SIZE(eqc_eyeq5_early_fixed_factors),
diff --git a/drivers/clk/clk-rk808.c b/drivers/clk/clk-rk808.c
index f7412b1..5a75b5c 100644
--- a/drivers/clk/clk-rk808.c
+++ b/drivers/clk/clk-rk808.c
@@ -153,7 +153,7 @@ static int rk808_clkout_probe(struct platform_device *pdev)
struct rk808_clkout *rk808_clkout;
int ret;
- dev->of_node = pdev->dev.parent->of_node;
+ device_set_of_node_from_dev(dev, dev->parent);
rk808_clkout = devm_kzalloc(dev,
sizeof(*rk808_clkout), GFP_KERNEL);
diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c
index e98afd5..bb8b75b 100644
--- a/drivers/clk/spacemit/ccu-k3.c
+++ b/drivers/clk/spacemit/ccu-k3.c
@@ -846,7 +846,7 @@ static const struct clk_parent_data top_parents[] = {
CCU_PARENT_HW(pll6_d3),
};
CCU_MUX_DIV_GATE_FC_DEFINE(top_dclk, top_parents, APMU_TOP_DCLK_CTRL, 5, 3,
- BIT(8), 2, 3, BIT(1), 0);
+ BIT(8), 2, 3, BIT(1), CLK_IS_CRITICAL);
static const struct clk_parent_data ucie_parents[] = {
CCU_PARENT_HW(pll1_d8_307p2),
diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c
index e9f5034..df56779 100644
--- a/drivers/clocksource/hyperv_timer.c
+++ b/drivers/clocksource/hyperv_timer.c
@@ -444,6 +444,22 @@ static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg)
return read_hv_clock_tsc();
}
+static u64 notrace read_hv_clock_tsc_cs_snapshot(struct clocksource *arg,
+ struct clocksource_hw_snapshot *chs)
+{
+ u64 time;
+
+ if (hv_read_tsc_page_tsc(tsc_page, &chs->hw_cycles, &time)) {
+ chs->hw_csid = CSID_X86_TSC;
+ } else {
+ chs->hw_cycles = 0;
+ chs->hw_csid = CSID_GENERIC;
+ time = read_hv_clock_msr();
+ }
+
+ return time;
+}
+
static u64 noinstr read_hv_sched_clock_tsc(void)
{
return (read_hv_clock_tsc() - hv_sched_clock_offset) *
@@ -492,18 +508,19 @@ static int hv_cs_enable(struct clocksource *cs)
#endif
static struct clocksource hyperv_cs_tsc = {
- .name = "hyperv_clocksource_tsc_page",
- .rating = 500,
- .read = read_hv_clock_tsc_cs,
- .mask = CLOCKSOURCE_MASK(64),
- .flags = CLOCK_SOURCE_IS_CONTINUOUS,
- .suspend= suspend_hv_clock_tsc,
- .resume = resume_hv_clock_tsc,
+ .name = "hyperv_clocksource_tsc_page",
+ .rating = 500,
+ .read = read_hv_clock_tsc_cs,
+ .read_snapshot = read_hv_clock_tsc_cs_snapshot,
+ .mask = CLOCKSOURCE_MASK(64),
+ .flags = CLOCK_SOURCE_IS_CONTINUOUS,
+ .suspend = suspend_hv_clock_tsc,
+ .resume = resume_hv_clock_tsc,
#ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK
- .enable = hv_cs_enable,
- .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK,
+ .enable = hv_cs_enable,
+ .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK,
#else
- .vdso_clock_mode = VDSO_CLOCKMODE_NONE,
+ .vdso_clock_mode = VDSO_CLOCKMODE_NONE,
#endif
};
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 1501c7d..a166926 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -198,7 +198,9 @@ static struct clocksource gic_clocksource = {
.name = "GIC",
.read = gic_hpt_read,
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
.vdso_clock_mode = VDSO_CLOCKMODE_GIC,
+#endif
};
static void gic_clocksource_unstable(char *reason)
diff --git a/drivers/clocksource/mmio.c b/drivers/clocksource/mmio.c
index cd5fbf4..0fee8ed 100644
--- a/drivers/clocksource/mmio.c
+++ b/drivers/clocksource/mmio.c
@@ -21,21 +21,25 @@ u64 clocksource_mmio_readl_up(struct clocksource *c)
{
return (u64)readl_relaxed(to_mmio_clksrc(c)->reg);
}
+EXPORT_SYMBOL_GPL(clocksource_mmio_readl_up);
u64 clocksource_mmio_readl_down(struct clocksource *c)
{
return ~(u64)readl_relaxed(to_mmio_clksrc(c)->reg) & c->mask;
}
+EXPORT_SYMBOL_GPL(clocksource_mmio_readl_down);
u64 clocksource_mmio_readw_up(struct clocksource *c)
{
return (u64)readw_relaxed(to_mmio_clksrc(c)->reg);
}
+EXPORT_SYMBOL_GPL(clocksource_mmio_readw_up);
u64 clocksource_mmio_readw_down(struct clocksource *c)
{
return ~(u64)readw_relaxed(to_mmio_clksrc(c)->reg) & c->mask;
}
+EXPORT_SYMBOL_GPL(clocksource_mmio_readw_down);
/**
* clocksource_mmio_init - Initialize a simple mmio based clocksource
@@ -46,9 +50,9 @@ u64 clocksource_mmio_readw_down(struct clocksource *c)
* @bits: Number of valid bits
* @read: One of clocksource_mmio_read*() above
*/
-int __init clocksource_mmio_init(void __iomem *base, const char *name,
- unsigned long hz, int rating, unsigned bits,
- u64 (*read)(struct clocksource *))
+int clocksource_mmio_init(void __iomem *base, const char *name,
+ unsigned long hz, int rating, unsigned bits,
+ u64 (*read)(struct clocksource *))
{
struct clocksource_mmio *cs;
@@ -68,3 +72,4 @@ int __init clocksource_mmio_init(void __iomem *base, const char *name,
return clocksource_register_hz(&cs->clksrc, hz);
}
+EXPORT_SYMBOL_GPL(clocksource_mmio_init);
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index 420202b..ba63433 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -19,7 +19,7 @@
*
* Free the irq resource
*/
-static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
+static void timer_of_irq_exit(struct of_timer_irq *of_irq)
{
struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -41,8 +41,8 @@ static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
*
* Returns 0 on success, < 0 otherwise
*/
-static __init int timer_of_irq_init(struct device_node *np,
- struct of_timer_irq *of_irq)
+static int timer_of_irq_init(struct device_node *np,
+ struct of_timer_irq *of_irq)
{
int ret;
struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -82,7 +82,7 @@ static __init int timer_of_irq_init(struct device_node *np,
*
* Disables and releases the refcount on the clk
*/
-static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
+static void timer_of_clk_exit(struct of_timer_clk *of_clk)
{
of_clk->rate = 0;
clk_disable_unprepare(of_clk->clk);
@@ -98,8 +98,8 @@ static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
*
* Returns 0 on success, < 0 otherwise
*/
-static __init int timer_of_clk_init(struct device_node *np,
- struct of_timer_clk *of_clk)
+static int timer_of_clk_init(struct device_node *np,
+ struct of_timer_clk *of_clk)
{
int ret;
@@ -137,13 +137,13 @@ static __init int timer_of_clk_init(struct device_node *np,
goto out;
}
-static __init void timer_of_base_exit(struct of_timer_base *of_base)
+static void timer_of_base_exit(struct of_timer_base *of_base)
{
iounmap(of_base->base);
}
-static __init int timer_of_base_init(struct device_node *np,
- struct of_timer_base *of_base)
+static int timer_of_base_init(struct device_node *np,
+ struct of_timer_base *of_base)
{
of_base->base = of_base->name ?
of_io_request_and_map(np, of_base->index, of_base->name) :
@@ -156,7 +156,7 @@ static __init int timer_of_base_init(struct device_node *np,
return 0;
}
-int __init timer_of_init(struct device_node *np, struct timer_of *to)
+int timer_of_init(struct device_node *np, struct timer_of *to)
{
int ret = -EINVAL;
int flags = 0;
@@ -200,6 +200,7 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
timer_of_base_exit(&to->of_base);
return ret;
}
+EXPORT_SYMBOL_GPL(timer_of_init);
/**
* timer_of_cleanup - release timer_of resources
@@ -208,7 +209,7 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
* Release the resources that has been used in timer_of_init().
* This function should be called in init error cases
*/
-void __init timer_of_cleanup(struct timer_of *to)
+void timer_of_cleanup(struct timer_of *to)
{
if (to->flags & TIMER_OF_IRQ)
timer_of_irq_exit(&to->of_irq);
@@ -219,3 +220,4 @@ void __init timer_of_cleanup(struct timer_of *to)
if (to->flags & TIMER_OF_BASE)
timer_of_base_exit(&to->of_base);
}
+EXPORT_SYMBOL_GPL(timer_of_cleanup);
diff --git a/drivers/clocksource/timer-of.h b/drivers/clocksource/timer-of.h
index 01a2c6b..74a632b 100644
--- a/drivers/clocksource/timer-of.h
+++ b/drivers/clocksource/timer-of.h
@@ -65,9 +65,8 @@ static inline unsigned long timer_of_period(struct timer_of *to)
return to->of_clk.period;
}
-extern int __init timer_of_init(struct device_node *np,
- struct timer_of *to);
+int timer_of_init(struct device_node *np, struct timer_of *to);
-extern void __init timer_of_cleanup(struct timer_of *to);
+void timer_of_cleanup(struct timer_of *to);
#endif
diff --git a/drivers/clocksource/timer-rtl-otto.c b/drivers/clocksource/timer-rtl-otto.c
index 6113d2f..dd236a7 100644
--- a/drivers/clocksource/timer-rtl-otto.c
+++ b/drivers/clocksource/timer-rtl-otto.c
@@ -225,7 +225,7 @@ static int rttm_enable_clocksource(struct clocksource *cs)
return 0;
}
-struct rttm_cs rttm_cs = {
+static struct rttm_cs rttm_cs = {
.to = {
.flags = TIMER_OF_BASE | TIMER_OF_CLOCK,
},
diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c
index f827d3f..d7e01299 100644
--- a/drivers/clocksource/timer-sun5i.c
+++ b/drivers/clocksource/timer-sun5i.c
@@ -286,6 +286,9 @@ static int sun5i_timer_probe(struct platform_device *pdev)
}
rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
+ if (IS_ERR(rstc))
+ return dev_err_probe(dev, PTR_ERR(rstc),
+ "failed to get reset\n");
if (rstc)
reset_control_deassert(rstc);
diff --git a/drivers/comedi/drivers/comedi_test.c b/drivers/comedi/drivers/comedi_test.c
index 01aafce..1f430ff 100644
--- a/drivers/comedi/drivers/comedi_test.c
+++ b/drivers/comedi/drivers/comedi_test.c
@@ -274,6 +274,7 @@ static int waveform_ai_cmdtest(struct comedi_device *dev,
/* Step 2a : make sure trigger sources are unique */
err |= comedi_check_trigger_is_unique(cmd->convert_src);
+ err |= comedi_check_trigger_is_unique(cmd->scan_begin_src);
err |= comedi_check_trigger_is_unique(cmd->stop_src);
/* Step 2b : and mutually compatible */
@@ -324,10 +325,10 @@ static int waveform_ai_cmdtest(struct comedi_device *dev,
arg = min(arg,
rounddown(UINT_MAX, (unsigned int)NSEC_PER_USEC));
arg = NSEC_PER_USEC * DIV_ROUND_CLOSEST(arg, NSEC_PER_USEC);
- if (cmd->scan_begin_arg == TRIG_TIMER) {
+ if (cmd->scan_begin_src == TRIG_TIMER) {
/* limit convert_arg to keep scan_begin_arg in range */
limit = UINT_MAX / cmd->scan_end_arg;
- limit = rounddown(limit, (unsigned int)NSEC_PER_SEC);
+ limit = rounddown(limit, (unsigned int)NSEC_PER_USEC);
arg = min(arg, limit);
}
err |= comedi_check_trigger_arg_is(&cmd->convert_arg, arg);
diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c
index 50bd30b..0b1dac6 100644
--- a/drivers/counter/counter-core.c
+++ b/drivers/counter/counter-core.c
@@ -124,7 +124,8 @@ struct counter_device *counter_alloc(size_t sizeof_priv)
err_dev_set_name:
- counter_chrdev_remove(counter);
+ put_device(dev);
+ return NULL;
err_chrdev_add:
ida_free(&counter_ida, dev->id);
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
index 027e6ea..53025a0 100644
--- a/drivers/cpufreq/Kconfig.x86
+++ b/drivers/cpufreq/Kconfig.x86
@@ -70,18 +70,6 @@
For details, take a look at:
<file:Documentation/admin-guide/pm/amd-pstate.rst>.
-config X86_AMD_PSTATE_DYNAMIC_EPP
- bool "AMD Processor P-State dynamic EPP support"
- depends on X86_AMD_PSTATE
- default n
- help
- Allow the kernel to dynamically change the energy performance
- value from events like ACPI platform profile and AC adapter plug
- events.
-
- This feature can also be changed at runtime, this configuration
- option only sets the kernel default value behavior.
-
config X86_AMD_PSTATE_UT
tristate "selftest for AMD Processor P-State driver"
depends on X86 && ACPI_PROCESSOR
@@ -126,32 +114,6 @@
By enabling this option the acpi_cpufreq driver provides the old
entry in addition to the new boost ones, for compatibility reasons.
-config ELAN_CPUFREQ
- tristate "AMD Elan SC400 and SC410"
- depends on MELAN
- help
- This adds the CPUFreq driver for AMD Elan SC400 and SC410
- processors.
-
- You need to specify the processor maximum speed as boot
- parameter: elanfreq=maxspeed (in kHz) or as module
- parameter "max_freq".
-
- For details, take a look at <file:Documentation/cpu-freq/>.
-
- If in doubt, say N.
-
-config SC520_CPUFREQ
- tristate "AMD Elan SC520"
- depends on MELAN
- help
- This adds the CPUFreq driver for AMD Elan SC520 processor.
-
- For details, take a look at <file:Documentation/cpu-freq/>.
-
- If in doubt, say N.
-
-
config X86_POWERNOW_K6
tristate "AMD Mobile K6-2/K6-3 PowerNow!"
depends on X86_32
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 385c9fc..6c7a39b 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -40,8 +40,6 @@
obj-$(CONFIG_X86_POWERNOW_K7) += powernow-k7.o
obj-$(CONFIG_X86_LONGHAUL) += longhaul.o
obj-$(CONFIG_X86_E_POWERSAVER) += e_powersaver.o
-obj-$(CONFIG_ELAN_CPUFREQ) += elanfreq.o
-obj-$(CONFIG_SC520_CPUFREQ) += sc520_freq.o
obj-$(CONFIG_X86_LONGRUN) += longrun.o
obj-$(CONFIG_X86_GX_SUSPMOD) += gx-suspmod.o
obj-$(CONFIG_X86_SPEEDSTEP_ICH) += speedstep-ich.o
diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c
index aa8a464..735b29f 100644
--- a/drivers/cpufreq/amd-pstate-ut.c
+++ b/drivers/cpufreq/amd-pstate-ut.c
@@ -274,20 +274,21 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode)
static int amd_pstate_ut_epp(u32 index)
{
- struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL;
- char *buf __free(cleanup_page) = NULL;
static const char * const epp_strings[] = {
- "performance",
- "balance_performance",
- "balance_power",
"power",
+ "balance_power",
+ "balance_performance",
+ "performance",
};
- struct amd_cpudata *cpudata;
+ char *buf __free(cleanup_page) = NULL;
+ struct cpufreq_policy *policy = NULL;
enum amd_pstate_mode orig_mode;
+ struct amd_cpudata *cpudata;
+ unsigned long orig_policy;
bool orig_dynamic_epp;
int ret, cpu = 0;
- int i;
u16 epp;
+ int i;
policy = cpufreq_cpu_get(cpu);
if (!policy)
@@ -297,11 +298,9 @@ static int amd_pstate_ut_epp(u32 index)
orig_mode = amd_pstate_get_status();
orig_dynamic_epp = cpudata->dynamic_epp;
- /* disable dynamic EPP before running test */
- if (cpudata->dynamic_epp) {
- pr_debug("Dynamic EPP is enabled, disabling it\n");
- amd_pstate_clear_dynamic_epp(policy);
- }
+ /* Drop reference before potential driver change. */
+ cpufreq_cpu_put(policy);
+ policy = NULL;
buf = (char *)__get_free_page(GFP_KERNEL);
if (!buf)
@@ -311,6 +310,27 @@ static int amd_pstate_ut_epp(u32 index)
if (ret)
goto out;
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ down_write(&policy->rwsem);
+ cpudata = policy->driver_data;
+ orig_policy = cpudata->policy;
+ cpudata->policy = CPUFREQ_POLICY_POWERSAVE;
+
+ /*
+ * Disable dynamic EPP before running test. If "orig_dynamic_epp" is
+ * true, the driver will do a redundant switch at the end and there
+ * is no need for enabling it again at the end of the test.
+ */
+ if (cpudata->dynamic_epp) {
+ pr_debug("Dynamic EPP is enabled, disabling it\n");
+ amd_pstate_clear_dynamic_epp(policy);
+ }
+
for (epp = 0; epp <= U8_MAX; epp++) {
u8 val;
@@ -358,6 +378,12 @@ static int amd_pstate_ut_epp(u32 index)
ret = 0;
out:
+ if (policy) {
+ cpudata->policy = orig_policy;
+ up_write(&policy->rwsem);
+ cpufreq_cpu_put(policy);
+ }
+
if (orig_dynamic_epp) {
int ret2;
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index 453084c..62b5d99 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -87,11 +87,7 @@ static struct cpufreq_driver amd_pstate_driver;
static struct cpufreq_driver amd_pstate_epp_driver;
static int cppc_state = AMD_PSTATE_UNDEFINED;
static bool amd_pstate_prefcore = true;
-#ifdef CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP
-static bool dynamic_epp = CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP;
-#else
static bool dynamic_epp;
-#endif
static struct quirk_entry *quirks;
/*
@@ -1291,6 +1287,8 @@ static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy)
return ret;
cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu);
+ if (!cpudata->profile_name)
+ return -ENOMEM;
cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu),
cpudata->profile_name,
@@ -1427,7 +1425,7 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy,
if (ret)
epp = epp_values[ret];
else
- epp = amd_pstate_get_balanced_epp(policy);
+ epp = cpudata->epp_default_dc;
}
if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
@@ -1707,6 +1705,8 @@ static int amd_pstate_change_driver_mode(int mode)
{
int ret;
+ lockdep_assert_held(&amd_pstate_driver_lock);
+
ret = amd_pstate_unregister_driver(0);
if (ret)
return ret;
@@ -1821,8 +1821,16 @@ static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b,
if (ret)
return ret;
- if (dynamic_epp == enabled)
+ guard(mutex)(&amd_pstate_driver_lock);
+
+ if (cppc_state != AMD_PSTATE_ACTIVE) {
+ pr_debug("dynamic_epp can only be toggled in active mode\n");
return -EINVAL;
+ }
+
+ /* Nothing to do */
+ if (dynamic_epp == enabled)
+ return count;
/* reinitialize with desired dynamic EPP value */
dynamic_epp = enabled;
@@ -1942,7 +1950,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
if (dynamic_epp)
ret = amd_pstate_set_dynamic_epp(policy);
else
- ret = amd_pstate_set_epp(policy, amd_pstate_get_balanced_epp(policy));
+ ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc);
if (ret)
goto free_cpudata1;
@@ -1970,12 +1978,13 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
if (cpudata) {
union perf_cached perf = READ_ONCE(cpudata->perf);
+ if (cpudata->dynamic_epp)
+ amd_pstate_clear_dynamic_epp(policy);
+
/* Reset CPPC_REQ MSR to the BIOS value */
amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);
amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
- if (cpudata->dynamic_epp)
- amd_pstate_clear_dynamic_epp(policy);
kfree(cpudata);
policy->driver_data = NULL;
}
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 44eb1b7..dda0d34 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -130,38 +130,11 @@ struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
-static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
-{
- struct kernel_cpustat kcpustat;
- u64 cur_wall_time;
- u64 idle_time;
- u64 busy_time;
-
- cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
-
- kcpustat_cpu_fetch(&kcpustat, cpu);
-
- busy_time = kcpustat.cpustat[CPUTIME_USER];
- busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
- busy_time += kcpustat.cpustat[CPUTIME_IRQ];
- busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
- busy_time += kcpustat.cpustat[CPUTIME_STEAL];
- busy_time += kcpustat.cpustat[CPUTIME_NICE];
-
- idle_time = cur_wall_time - busy_time;
- if (wall)
- *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
-
- return div_u64(idle_time, NSEC_PER_USEC);
-}
-
u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
- if (idle_time == -1ULL)
- return get_cpu_idle_time_jiffy(cpu, wall);
- else if (!io_busy)
+ if (!io_busy)
idle_time += get_cpu_iowait_time_us(cpu, wall);
return idle_time;
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 86f35e4..3c4a1f9 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -105,7 +105,7 @@ void gov_update_cpu_data(struct dbs_data *dbs_data)
j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time,
dbs_data->io_is_busy);
if (dbs_data->ignore_nice_load)
- j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+ j_cdbs->prev_cpu_nice = kcpustat_field(CPUTIME_NICE, j);
}
}
}
@@ -165,7 +165,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
j_cdbs->prev_cpu_idle = cur_idle_time;
if (ignore_nice) {
- u64 cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+ u64 cur_nice = kcpustat_field(CPUTIME_NICE, j);
idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC);
j_cdbs->prev_cpu_nice = cur_nice;
@@ -539,7 +539,7 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *policy)
j_cdbs->prev_load = 0;
if (ignore_nice)
- j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);
+ j_cdbs->prev_cpu_nice = kcpustat_field(CPUTIME_NICE, j);
}
gov->start(policy);
diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c
deleted file mode 100644
index fc5a580..0000000
--- a/drivers/cpufreq/elanfreq.c
+++ /dev/null
@@ -1,226 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * elanfreq: cpufreq driver for the AMD ELAN family
- *
- * (c) Copyright 2002 Robert Schwebel <r.schwebel@pengutronix.de>
- *
- * Parts of this code are (c) Sven Geggus <sven@geggus.net>
- *
- * All Rights Reserved.
- *
- * 2002-02-13: - initial revision for 2.4.18-pre9 by Robert Schwebel
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-
-#include <linux/delay.h>
-#include <linux/cpufreq.h>
-
-#include <asm/cpu_device_id.h>
-#include <linux/timex.h>
-#include <linux/io.h>
-
-#define REG_CSCIR 0x22 /* Chip Setup and Control Index Register */
-#define REG_CSCDR 0x23 /* Chip Setup and Control Data Register */
-
-/* Module parameter */
-static int max_freq;
-
-struct s_elan_multiplier {
- int clock; /* frequency in kHz */
- int val40h; /* PMU Force Mode register */
- int val80h; /* CPU Clock Speed Register */
-};
-
-/*
- * It is important that the frequencies
- * are listed in ascending order here!
- */
-static struct s_elan_multiplier elan_multiplier[] = {
- {1000, 0x02, 0x18},
- {2000, 0x02, 0x10},
- {4000, 0x02, 0x08},
- {8000, 0x00, 0x00},
- {16000, 0x00, 0x02},
- {33000, 0x00, 0x04},
- {66000, 0x01, 0x04},
- {99000, 0x01, 0x05}
-};
-
-static struct cpufreq_frequency_table elanfreq_table[] = {
- {0, 0, 1000},
- {0, 1, 2000},
- {0, 2, 4000},
- {0, 3, 8000},
- {0, 4, 16000},
- {0, 5, 33000},
- {0, 6, 66000},
- {0, 7, 99000},
- {0, 0, CPUFREQ_TABLE_END},
-};
-
-
-/**
- * elanfreq_get_cpu_frequency: determine current cpu speed
- *
- * Finds out at which frequency the CPU of the Elan SOC runs
- * at the moment. Frequencies from 1 to 33 MHz are generated
- * the normal way, 66 and 99 MHz are called "Hyperspeed Mode"
- * and have the rest of the chip running with 33 MHz.
- */
-
-static unsigned int elanfreq_get_cpu_frequency(unsigned int cpu)
-{
- u8 clockspeed_reg; /* Clock Speed Register */
-
- local_irq_disable();
- outb_p(0x80, REG_CSCIR);
- clockspeed_reg = inb_p(REG_CSCDR);
- local_irq_enable();
-
- if ((clockspeed_reg & 0xE0) == 0xE0)
- return 0;
-
- /* Are we in CPU clock multiplied mode (66/99 MHz)? */
- if ((clockspeed_reg & 0xE0) == 0xC0) {
- if ((clockspeed_reg & 0x01) == 0)
- return 66000;
- else
- return 99000;
- }
-
- /* 33 MHz is not 32 MHz... */
- if ((clockspeed_reg & 0xE0) == 0xA0)
- return 33000;
-
- return (1<<((clockspeed_reg & 0xE0) >> 5)) * 1000;
-}
-
-
-static int elanfreq_target(struct cpufreq_policy *policy,
- unsigned int state)
-{
- /*
- * Access to the Elan's internal registers is indexed via
- * 0x22: Chip Setup & Control Register Index Register (CSCI)
- * 0x23: Chip Setup & Control Register Data Register (CSCD)
- *
- */
-
- /*
- * 0x40 is the Power Management Unit's Force Mode Register.
- * Bit 6 enables Hyperspeed Mode (66/100 MHz core frequency)
- */
-
- local_irq_disable();
- outb_p(0x40, REG_CSCIR); /* Disable hyperspeed mode */
- outb_p(0x00, REG_CSCDR);
- local_irq_enable(); /* wait till internal pipelines and */
- udelay(1000); /* buffers have cleaned up */
-
- local_irq_disable();
-
- /* now, set the CPU clock speed register (0x80) */
- outb_p(0x80, REG_CSCIR);
- outb_p(elan_multiplier[state].val80h, REG_CSCDR);
-
- /* now, the hyperspeed bit in PMU Force Mode Register (0x40) */
- outb_p(0x40, REG_CSCIR);
- outb_p(elan_multiplier[state].val40h, REG_CSCDR);
- udelay(10000);
- local_irq_enable();
-
- return 0;
-}
-/*
- * Module init and exit code
- */
-
-static int elanfreq_cpu_init(struct cpufreq_policy *policy)
-{
- struct cpuinfo_x86 *c = &cpu_data(0);
- struct cpufreq_frequency_table *pos;
-
- /* capability check */
- if ((c->x86_vendor != X86_VENDOR_AMD) ||
- (c->x86 != 4) || (c->x86_model != 10))
- return -ENODEV;
-
- /* max freq */
- if (!max_freq)
- max_freq = elanfreq_get_cpu_frequency(0);
-
- /* table init */
- cpufreq_for_each_entry(pos, elanfreq_table)
- if (pos->frequency > max_freq)
- pos->frequency = CPUFREQ_ENTRY_INVALID;
-
- policy->freq_table = elanfreq_table;
- return 0;
-}
-
-
-#ifndef MODULE
-/**
- * elanfreq_setup - elanfreq command line parameter parsing
- *
- * elanfreq command line parameter. Use:
- * elanfreq=66000
- * to set the maximum CPU frequency to 66 MHz. Note that in
- * case you do not give this boot parameter, the maximum
- * frequency will fall back to _current_ CPU frequency which
- * might be lower. If you build this as a module, use the
- * max_freq module parameter instead.
- */
-static int __init elanfreq_setup(char *str)
-{
- max_freq = simple_strtoul(str, &str, 0);
- pr_warn("You're using the deprecated elanfreq command line option. Use elanfreq.max_freq instead, please!\n");
- return 1;
-}
-__setup("elanfreq=", elanfreq_setup);
-#endif
-
-
-static struct cpufreq_driver elanfreq_driver = {
- .get = elanfreq_get_cpu_frequency,
- .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
- .verify = cpufreq_generic_frequency_table_verify,
- .target_index = elanfreq_target,
- .init = elanfreq_cpu_init,
- .name = "elanfreq",
-};
-
-static const struct x86_cpu_id elan_id[] = {
- X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 10, NULL),
- {}
-};
-MODULE_DEVICE_TABLE(x86cpu, elan_id);
-
-static int __init elanfreq_init(void)
-{
- if (!x86_match_cpu(elan_id))
- return -ENODEV;
- return cpufreq_register_driver(&elanfreq_driver);
-}
-
-
-static void __exit elanfreq_exit(void)
-{
- cpufreq_unregister_driver(&elanfreq_driver);
-}
-
-
-module_param(max_freq, int, 0444);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Robert Schwebel <r.schwebel@pengutronix.de>, "
- "Sven Geggus <sven@geggus.net>");
-MODULE_DESCRIPTION("cpufreq driver for AMD's Elan CPUs");
-
-module_init(elanfreq_init);
-module_exit(elanfreq_exit);
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 1292da5..1f093e3 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -2279,7 +2279,7 @@ static int hwp_get_cpu_scaling(int cpu)
* Return the hybrid scaling factor for P-cores and use the
* default core scaling for E-cores.
*/
- if (hybrid_get_cpu_type(cpu) == INTEL_CPU_TYPE_CORE)
+ if (hybrid_get_cpu_type(cpu) != INTEL_CPU_TYPE_ATOM)
return hybrid_scaling_factor;
return core_get_scaling();
@@ -3734,6 +3734,7 @@ static const struct x86_cpu_id intel_hybrid_scaling_factor[] = {
X86_MATCH_VFM(INTEL_RAPTORLAKE, HYBRID_SCALING_FACTOR_ADL),
X86_MATCH_VFM(INTEL_RAPTORLAKE_P, HYBRID_SCALING_FACTOR_ADL),
X86_MATCH_VFM(INTEL_RAPTORLAKE_S, HYBRID_SCALING_FACTOR_ADL),
+ X86_MATCH_VFM(INTEL_BARTLETTLAKE, HYBRID_SCALING_FACTOR_ADL),
X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL),
X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL),
{}
diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c
index 1caaec7..f3aaca0 100644
--- a/drivers/cpufreq/longrun.c
+++ b/drivers/cpufreq/longrun.c
@@ -14,6 +14,7 @@
#include <asm/msr.h>
#include <asm/processor.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
static struct cpufreq_driver longrun_driver;
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
index 6b7caf4..6a930d7 100644
--- a/drivers/cpufreq/powernow-k7.c
+++ b/drivers/cpufreq/powernow-k7.c
@@ -29,6 +29,7 @@
#include <asm/timer.h> /* Needed for recalibrate_cpu_khz() */
#include <asm/msr.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#ifdef CONFIG_X86_POWERNOW_K7_ACPI
#include <linux/acpi.h>
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index 4d77eef..2b791f1 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -39,6 +39,7 @@
#include <asm/msr.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <linux/acpi.h>
#include <linux/mutex.h>
diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c
deleted file mode 100644
index b360f03..0000000
--- a/drivers/cpufreq/sc520_freq.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * sc520_freq.c: cpufreq driver for the AMD Elan sc520
- *
- * Copyright (C) 2005 Sean Young <sean@mess.org>
- *
- * Based on elanfreq.c
- *
- * 2005-03-30: - initial revision
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-
-#include <linux/delay.h>
-#include <linux/cpufreq.h>
-#include <linux/timex.h>
-#include <linux/io.h>
-
-#include <asm/cpu_device_id.h>
-
-#define MMCR_BASE 0xfffef000 /* The default base address */
-#define OFFS_CPUCTL 0x2 /* CPU Control Register */
-
-static __u8 __iomem *cpuctl;
-
-static struct cpufreq_frequency_table sc520_freq_table[] = {
- {0, 0x01, 100000},
- {0, 0x02, 133000},
- {0, 0, CPUFREQ_TABLE_END},
-};
-
-static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu)
-{
- u8 clockspeed_reg = *cpuctl;
-
- switch (clockspeed_reg & 0x03) {
- default:
- pr_err("error: cpuctl register has unexpected value %02x\n",
- clockspeed_reg);
- fallthrough;
- case 0x01:
- return 100000;
- case 0x02:
- return 133000;
- }
-}
-
-static int sc520_freq_target(struct cpufreq_policy *policy, unsigned int state)
-{
-
- u8 clockspeed_reg;
-
- local_irq_disable();
-
- clockspeed_reg = *cpuctl & ~0x03;
- *cpuctl = clockspeed_reg | sc520_freq_table[state].driver_data;
-
- local_irq_enable();
-
- return 0;
-}
-
-/*
- * Module init and exit code
- */
-
-static int sc520_freq_cpu_init(struct cpufreq_policy *policy)
-{
- struct cpuinfo_x86 *c = &cpu_data(0);
-
- /* capability check */
- if (c->x86_vendor != X86_VENDOR_AMD ||
- c->x86 != 4 || c->x86_model != 9)
- return -ENODEV;
-
- /* cpuinfo and default policy values */
- policy->cpuinfo.transition_latency = 1000000; /* 1ms */
- policy->freq_table = sc520_freq_table;
-
- return 0;
-}
-
-
-static struct cpufreq_driver sc520_freq_driver = {
- .get = sc520_freq_get_cpu_frequency,
- .verify = cpufreq_generic_frequency_table_verify,
- .target_index = sc520_freq_target,
- .init = sc520_freq_cpu_init,
- .name = "sc520_freq",
-};
-
-static const struct x86_cpu_id sc520_ids[] = {
- X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 9, NULL),
- {}
-};
-MODULE_DEVICE_TABLE(x86cpu, sc520_ids);
-
-static int __init sc520_freq_init(void)
-{
- int err;
-
- if (!x86_match_cpu(sc520_ids))
- return -ENODEV;
-
- cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1);
- if (!cpuctl) {
- pr_err("sc520_freq: error: failed to remap memory\n");
- return -ENOMEM;
- }
-
- err = cpufreq_register_driver(&sc520_freq_driver);
- if (err)
- iounmap(cpuctl);
-
- return err;
-}
-
-
-static void __exit sc520_freq_exit(void)
-{
- cpufreq_unregister_driver(&sc520_freq_driver);
- iounmap(cpuctl);
-}
-
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Sean Young <sean@mess.org>");
-MODULE_DESCRIPTION("cpufreq driver for AMD's Elan sc520 CPU");
-
-module_init(sc520_freq_init);
-module_exit(sc520_freq_exit);
-
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index f8b42e9..973716c 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -15,6 +15,7 @@
#include <linux/init.h>
#include <linux/cpufreq.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include <asm/tsc.h>
#include "speedstep-lib.h"
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index d1e9e0a..78f98ae 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -1374,7 +1374,9 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
return -EOPNOTSUPP;
}
- snp_prepare();
+ rc = snp_prepare();
+ if (rc)
+ return rc;
/*
* Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 71f3754..d504c63 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -792,9 +792,13 @@ int dma_buf_fd(struct dma_buf *dmabuf, int flags)
if (!dmabuf || !dmabuf->file)
return -EINVAL;
- fd = FD_ADD(flags, dmabuf->file);
+ fd = get_unused_fd_flags(flags);
+ if (fd < 0)
+ return fd;
+
DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
+ fd_install(fd, dmabuf->file);
return fd;
}
EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index af7ce62..75e3ae0 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -829,12 +829,21 @@ int dpll_device_delete_ntf(struct dpll_device *dpll)
return dpll_device_event_send(DPLL_CMD_DEVICE_DELETE_NTF, dpll);
}
-static int
-__dpll_device_change_ntf(struct dpll_device *dpll)
+/**
+ * __dpll_device_change_ntf - notify that the dpll device has been changed
+ * @dpll: registered dpll pointer
+ *
+ * Context: caller must hold dpll_lock. Suitable for use inside device
+ * callbacks which are already invoked under dpll_lock.
+ * Return: 0 if succeeds, error code otherwise.
+ */
+int __dpll_device_change_ntf(struct dpll_device *dpll)
{
+ lockdep_assert_held(&dpll_lock);
dpll_device_notify(dpll, DPLL_DEVICE_CHANGED);
return dpll_device_event_send(DPLL_CMD_DEVICE_CHANGE_NTF, dpll);
}
+EXPORT_SYMBOL_GPL(__dpll_device_change_ntf);
/**
* dpll_device_change_ntf - notify that the dpll device has been changed
@@ -900,11 +909,21 @@ int dpll_pin_delete_ntf(struct dpll_pin *pin)
return dpll_pin_event_send(DPLL_CMD_PIN_DELETE_NTF, pin);
}
+/**
+ * __dpll_pin_change_ntf - notify that the pin has been changed
+ * @pin: registered pin pointer
+ *
+ * Context: caller must hold dpll_lock. Suitable for use inside pin
+ * callbacks which are already invoked under dpll_lock.
+ * Return: 0 if succeeds, error code otherwise.
+ */
int __dpll_pin_change_ntf(struct dpll_pin *pin)
{
+ lockdep_assert_held(&dpll_lock);
dpll_pin_notify(pin, DPLL_PIN_CHANGED);
return dpll_pin_event_send(DPLL_CMD_PIN_CHANGE_NTF, pin);
}
+EXPORT_SYMBOL_GPL(__dpll_pin_change_ntf);
/**
* dpll_pin_change_ntf - notify that the pin has been changed
diff --git a/drivers/dpll/dpll_netlink.h b/drivers/dpll/dpll_netlink.h
index dd28b56d..a9cfd55 100644
--- a/drivers/dpll/dpll_netlink.h
+++ b/drivers/dpll/dpll_netlink.h
@@ -11,5 +11,3 @@ int dpll_device_delete_ntf(struct dpll_device *dpll);
int dpll_pin_create_ntf(struct dpll_pin *pin);
int dpll_pin_delete_ntf(struct dpll_pin *pin);
-
-int __dpll_pin_change_ntf(struct dpll_pin *pin);
diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c
index 5f1e70f..0a133b0 100644
--- a/drivers/dpll/zl3073x/core.c
+++ b/drivers/dpll/zl3073x/core.c
@@ -762,18 +762,15 @@ zl3073x_dev_periodic_work(struct kthread_work *work)
dev_warn(zldev->dev, "Failed to update phase offsets: %pe\n",
ERR_PTR(rc));
- /* Update measured input reference frequencies if any DPLL has
- * frequency monitoring enabled.
+ /* Update measured input reference frequencies if frequency
+ * monitoring is enabled.
*/
- list_for_each_entry(zldpll, &zldev->dplls, list) {
- if (zldpll->freq_monitor) {
- rc = zl3073x_ref_freq_meas_update(zldev);
- if (rc)
- dev_warn(zldev->dev,
- "Failed to update measured frequencies: %pe\n",
- ERR_PTR(rc));
- break;
- }
+ if (zldev->freq_monitor) {
+ rc = zl3073x_ref_freq_meas_update(zldev);
+ if (rc)
+ dev_warn(zldev->dev,
+ "Failed to update measured frequencies: %pe\n",
+ ERR_PTR(rc));
}
/* Update references' fractional frequency offsets */
diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h
index 9944062..addba37 100644
--- a/drivers/dpll/zl3073x/core.h
+++ b/drivers/dpll/zl3073x/core.h
@@ -57,6 +57,7 @@ struct zl3073x_chip_info {
* @work: periodic work
* @clock_id: clock id of the device
* @phase_avg_factor: phase offset measurement averaging factor
+ * @freq_monitor: is frequency monitor enabled
*/
struct zl3073x_dev {
struct device *dev;
@@ -77,9 +78,10 @@ struct zl3073x_dev {
struct kthread_worker *kworker;
struct kthread_delayed_work work;
- /* Devlink parameters */
+ /* Per-chip parameters */
u64 clock_id;
u8 phase_avg_factor;
+ bool freq_monitor;
};
extern const struct regmap_config zl3073x_regmap_config;
diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c
index c95e93e..0bfcbae 100644
--- a/drivers/dpll/zl3073x/dpll.c
+++ b/drivers/dpll/zl3073x/dpll.c
@@ -1079,15 +1079,6 @@ zl3073x_dpll_phase_offset_avg_factor_get(const struct dpll_device *dpll,
return 0;
}
-static void
-zl3073x_dpll_change_work(struct work_struct *work)
-{
- struct zl3073x_dpll *zldpll;
-
- zldpll = container_of(work, struct zl3073x_dpll, change_work);
- dpll_device_change_ntf(zldpll->dpll_dev);
-}
-
static int
zl3073x_dpll_phase_offset_avg_factor_set(const struct dpll_device *dpll,
void *dpll_priv, u32 factor,
@@ -1113,8 +1104,10 @@ zl3073x_dpll_phase_offset_avg_factor_set(const struct dpll_device *dpll,
* we have to send a notification for other DPLL devices.
*/
list_for_each_entry(item, &zldpll->dev->dplls, list) {
- if (item != zldpll)
- schedule_work(&item->change_work);
+ struct dpll_device *dpll_dev = READ_ONCE(item->dpll_dev);
+
+ if (item != zldpll && dpll_dev)
+ __dpll_device_change_ntf(dpll_dev);
}
return 0;
@@ -1219,7 +1212,7 @@ zl3073x_dpll_freq_monitor_get(const struct dpll_device *dpll,
{
struct zl3073x_dpll *zldpll = dpll_priv;
- if (zldpll->freq_monitor)
+ if (zldpll->dev->freq_monitor)
*state = DPLL_FEATURE_STATE_ENABLE;
else
*state = DPLL_FEATURE_STATE_DISABLE;
@@ -1233,9 +1226,19 @@ zl3073x_dpll_freq_monitor_set(const struct dpll_device *dpll,
enum dpll_feature_state state,
struct netlink_ext_ack *extack)
{
- struct zl3073x_dpll *zldpll = dpll_priv;
+ struct zl3073x_dpll *item, *zldpll = dpll_priv;
- zldpll->freq_monitor = (state == DPLL_FEATURE_STATE_ENABLE);
+ zldpll->dev->freq_monitor = (state == DPLL_FEATURE_STATE_ENABLE);
+
+ /* The frequency monitoring is common for all DPLL channels so after
+ * change we have to send a notification for other DPLL devices.
+ */
+ list_for_each_entry(item, &zldpll->dev->dplls, list) {
+ struct dpll_device *dpll_dev = READ_ONCE(item->dpll_dev);
+
+ if (item != zldpll && dpll_dev)
+ __dpll_device_change_ntf(dpll_dev);
+ }
return 0;
}
@@ -1394,8 +1397,8 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index)
err_register:
dpll_pin_put(pin->dpll_pin, &pin->tracker);
- pin->dpll_pin = NULL;
err_pin_get:
+ pin->dpll_pin = NULL;
fwnode_handle_put(pin->fwnode);
pin->fwnode = NULL;
zl3073x_pin_props_put(props);
@@ -1563,8 +1566,10 @@ zl3073x_dpll_pins_register(struct zl3073x_dpll *zldpll)
}
rc = zl3073x_dpll_pin_register(pin, index);
- if (rc)
+ if (rc) {
+ zl3073x_dpll_pin_free(pin);
goto error;
+ }
list_add(&pin->list, &zldpll->pins);
}
@@ -1625,13 +1630,13 @@ zl3073x_dpll_device_register(struct zl3073x_dpll *zldpll)
static void
zl3073x_dpll_device_unregister(struct zl3073x_dpll *zldpll)
{
- WARN(!zldpll->dpll_dev, "DPLL device is not registered\n");
+ struct dpll_device *dpll_dev = READ_ONCE(zldpll->dpll_dev);
- cancel_work_sync(&zldpll->change_work);
+ WARN(!dpll_dev, "DPLL device is not registered\n");
- dpll_device_unregister(zldpll->dpll_dev, &zldpll->ops, zldpll);
- dpll_device_put(zldpll->dpll_dev, &zldpll->tracker);
- zldpll->dpll_dev = NULL;
+ WRITE_ONCE(zldpll->dpll_dev, NULL);
+ dpll_device_unregister(dpll_dev, &zldpll->ops, zldpll);
+ dpll_device_put(dpll_dev, &zldpll->tracker);
}
/**
@@ -1750,7 +1755,7 @@ zl3073x_dpll_pin_measured_freq_check(struct zl3073x_dpll_pin *pin)
u8 ref_id;
u32 freq;
- if (!zldpll->freq_monitor)
+ if (!zldpll->dev->freq_monitor)
return false;
ref_id = zl3073x_input_pin_ref_get(pin->id);
@@ -1783,10 +1788,8 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
struct zl3073x_dev *zldev = zldpll->dev;
enum dpll_lock_status lock_status;
struct device *dev = zldev->dev;
- const struct zl3073x_chan *chan;
struct zl3073x_dpll_pin *pin;
int rc;
- u8 mode;
zldpll->check_count++;
@@ -1805,15 +1808,6 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
dpll_device_change_ntf(zldpll->dpll_dev);
}
- /* Input pin monitoring does make sense only in automatic
- * or forced reference modes.
- */
- chan = zl3073x_chan_state_get(zldev, zldpll->id);
- mode = zl3073x_chan_mode_get(chan);
- if (mode != ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
- mode != ZL_DPLL_MODE_REFSEL_MODE_REFLOCK)
- return;
-
/* Update phase offset latch registers for this DPLL if the phase
* offset monitor feature is enabled.
*/
@@ -1924,7 +1918,6 @@ zl3073x_dpll_alloc(struct zl3073x_dev *zldev, u8 ch)
zldpll->dev = zldev;
zldpll->id = ch;
INIT_LIST_HEAD(&zldpll->pins);
- INIT_WORK(&zldpll->change_work, zl3073x_dpll_change_work);
return zldpll;
}
diff --git a/drivers/dpll/zl3073x/dpll.h b/drivers/dpll/zl3073x/dpll.h
index 434c32a..21adcc1 100644
--- a/drivers/dpll/zl3073x/dpll.h
+++ b/drivers/dpll/zl3073x/dpll.h
@@ -15,13 +15,11 @@
* @id: DPLL index
* @check_count: periodic check counter
* @phase_monitor: is phase offset monitor enabled
- * @freq_monitor: is frequency monitor enabled
* @ops: DPLL device operations for this instance
* @dpll_dev: pointer to registered DPLL device
* @tracker: tracking object for the acquired reference
* @lock_status: last saved DPLL lock status
* @pins: list of pins
- * @change_work: device change notification work
*/
struct zl3073x_dpll {
struct list_head list;
@@ -29,13 +27,11 @@ struct zl3073x_dpll {
u8 id;
u8 check_count;
bool phase_monitor;
- bool freq_monitor;
struct dpll_device_ops ops;
struct dpll_device *dpll_dev;
dpll_tracker tracker;
enum dpll_lock_status lock_status;
struct list_head pins;
- struct work_struct change_work;
};
struct zl3073x_dpll *zl3073x_dpll_alloc(struct zl3073x_dev *zldev, u8 ch);
diff --git a/drivers/edac/versalnet_edac.c b/drivers/edac/versalnet_edac.c
index ec13155..97ec05d 100644
--- a/drivers/edac/versalnet_edac.c
+++ b/drivers/edac/versalnet_edac.c
@@ -777,9 +777,9 @@ static int init_one_mc(struct mc_priv *priv, struct platform_device *pdev, int i
u32 num_chans, rank, dwidth, config;
struct edac_mc_layer layers[2];
struct mem_ctl_info *mci;
+ char name[MC_NAME_LEN];
struct device *dev;
enum dev_type dt;
- char *name;
int rc;
config = priv->adec[CONF + i * ADEC_NUM];
@@ -813,13 +813,9 @@ static int init_one_mc(struct mc_priv *priv, struct platform_device *pdev, int i
layers[1].is_virt_csrow = false;
rc = -ENOMEM;
- name = kzalloc(MC_NAME_LEN, GFP_KERNEL);
- if (!name)
- return rc;
-
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
- goto err_name_free;
+ return rc;
mci = edac_mc_alloc(i, ARRAY_SIZE(layers), layers, sizeof(struct mc_priv));
if (!mci) {
@@ -858,8 +854,6 @@ static int init_one_mc(struct mc_priv *priv, struct platform_device *pdev, int i
edac_mc_free(mci);
err_dev_free:
kfree(dev);
-err_name_free:
- kfree(name);
return rc;
}
diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c
index 9576862..601c341 100644
--- a/drivers/firmware/arm_ffa/bus.c
+++ b/drivers/firmware/arm_ffa/bus.c
@@ -26,6 +26,8 @@ static int ffa_device_match(struct device *dev, const struct device_driver *drv)
id_table = to_ffa_driver(drv)->id_table;
ffa_dev = to_ffa_dev(dev);
+ if (!id_table)
+ return 0;
while (!uuid_is_null(&id_table->uuid)) {
/*
@@ -123,7 +125,7 @@ int ffa_driver_register(struct ffa_driver *driver, struct module *owner,
{
int ret;
- if (!driver->probe)
+ if (!driver->probe || !driver->id_table)
return -EINVAL;
driver->driver.bus = &ffa_bus_type;
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index eb27828..b9f17fd 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -87,6 +87,7 @@ static inline int ffa_to_linux_errno(int errno)
struct ffa_pcpu_irq {
struct ffa_drv_info *info;
+ struct work_struct notif_pcpu_work;
};
struct ffa_drv_info {
@@ -100,13 +101,13 @@ struct ffa_drv_info {
bool mem_ops_native;
bool msg_direct_req2_supp;
bool bitmap_created;
+ bool bus_notifier_registered;
bool notif_enabled;
unsigned int sched_recv_irq;
unsigned int notif_pend_irq;
unsigned int cpuhp_state;
struct ffa_pcpu_irq __percpu *irq_pcpu;
struct workqueue_struct *notif_pcpu_wq;
- struct work_struct notif_pcpu_work;
struct work_struct sched_recv_irq_work;
struct xarray partition_info;
DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS));
@@ -322,6 +323,12 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
#define PART_INFO_ID_MASK GENMASK(15, 0)
#define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16)
#define PART_INFO_PROPS_MASK GENMASK(63, 32)
+#define FFA_PART_INFO_GET_REGS_FIRST_REG 3
+#define FFA_PART_INFO_GET_REGS_REGS_PER_DESC 3
+#define FFA_PART_INFO_GET_REGS_MAX_DESC \
+ (((sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) - \
+ FFA_PART_INFO_GET_REGS_FIRST_REG) / \
+ FFA_PART_INFO_GET_REGS_REGS_PER_DESC)
#define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x))))
#define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x))))
#define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x))))
@@ -329,15 +336,13 @@ static int
__ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
struct ffa_partition_info *buffer, int num_parts)
{
- u16 buf_sz, start_idx, cur_idx, count = 0, prev_idx = 0, tag = 0;
+ u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0;
struct ffa_partition_info *buf = buffer;
ffa_value_t partition_info;
do {
__le64 *regs;
- int idx;
-
- start_idx = prev_idx ? prev_idx + 1 : 0;
+ int idx, nr_desc, buf_idx;
invoke_ffa_fn((ffa_value_t){
.a0 = FFA_PARTITION_INFO_GET_REGS,
@@ -353,15 +358,28 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
count = PARTITION_COUNT(partition_info.a2);
if (!buffer || !num_parts) /* count only */
return count;
+ if (count > num_parts)
+ return -EINVAL;
cur_idx = CURRENT_INDEX(partition_info.a2);
+ if (cur_idx < start_idx || cur_idx >= count)
+ return -EINVAL;
+
+ nr_desc = cur_idx - start_idx + 1;
+ if (nr_desc > FFA_PART_INFO_GET_REGS_MAX_DESC)
+ return -EINVAL;
+
+ buf_idx = buf - buffer;
+ if (buf_idx + nr_desc > num_parts)
+ return -EINVAL;
+
tag = UUID_INFO_TAG(partition_info.a2);
buf_sz = PARTITION_INFO_SZ(partition_info.a2);
if (buf_sz > sizeof(*buffer))
buf_sz = sizeof(*buffer);
regs = (void *)&partition_info.a3;
- for (idx = 0; idx < cur_idx - start_idx + 1; idx++, buf++) {
+ for (idx = 0; idx < nr_desc; idx++, buf++) {
union {
uuid_t uuid;
u64 regs[2];
@@ -379,7 +397,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
uuid_copy(&buf->uuid, &uuid_regs.uuid);
regs += 3;
}
- prev_idx = cur_idx;
+ start_idx = cur_idx + 1;
} while (cur_idx < (count - 1));
@@ -1189,7 +1207,7 @@ static int
ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback,
void *cb_data, bool is_registration)
{
- struct ffa_dev_part_info *partition = NULL, *tmp;
+ struct ffa_dev_part_info *partition = NULL;
struct list_head *phead;
bool cb_valid;
@@ -1202,11 +1220,11 @@ ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback,
return -EINVAL;
}
- list_for_each_entry_safe(partition, tmp, phead, node)
+ list_for_each_entry(partition, phead, node)
if (partition->dev == dev)
break;
- if (!partition) {
+ if (&partition->node == phead) {
pr_err("%s: No such partition ID 0x%x\n", __func__, dev->vm_id);
return -EINVAL;
}
@@ -1445,20 +1463,25 @@ static int ffa_notify_send(struct ffa_device *dev, int notify_id,
static void handle_notif_callbacks(u64 bitmap, enum notify_type type)
{
+ ffa_notifier_cb cb;
+ void *cb_data;
int notify_id;
- struct notifier_cb_info *cb_info = NULL;
for (notify_id = 0; notify_id <= FFA_MAX_NOTIFICATIONS && bitmap;
notify_id++, bitmap >>= 1) {
if (!(bitmap & 1))
continue;
- read_lock(&drv_info->notify_lock);
- cb_info = notifier_hnode_get_by_type(notify_id, type);
- read_unlock(&drv_info->notify_lock);
+ scoped_guard(read_lock, &drv_info->notify_lock) {
+ struct notifier_cb_info *cb_info;
- if (cb_info && cb_info->cb)
- cb_info->cb(notify_id, cb_info->cb_data);
+ cb_info = notifier_hnode_get_by_type(notify_id, type);
+ cb = cb_info ? cb_info->cb : NULL;
+ cb_data = cb_info ? cb_info->cb_data : NULL;
+ }
+
+ if (cb)
+ cb(notify_id, cb_data);
}
}
@@ -1466,39 +1489,56 @@ static void handle_fwk_notif_callbacks(u32 bitmap)
{
void *buf;
uuid_t uuid;
+ void *fwk_cb_data;
int notify_id = 0, target;
+ ffa_fwk_notifier_cb fwk_cb;
struct ffa_indirect_msg_hdr *msg;
- struct notifier_cb_info *cb_info = NULL;
+ size_t min_offset = offsetof(struct ffa_indirect_msg_hdr, uuid);
/* Only one framework notification defined and supported for now */
if (!(bitmap & FRAMEWORK_NOTIFY_RX_BUFFER_FULL))
return;
- mutex_lock(&drv_info->rx_lock);
+ scoped_guard(mutex, &drv_info->rx_lock) {
+ u32 offset, size;
- msg = drv_info->rx_buffer;
- buf = kmemdup((void *)msg + msg->offset, msg->size, GFP_KERNEL);
- if (!buf) {
- mutex_unlock(&drv_info->rx_lock);
- return;
+ msg = drv_info->rx_buffer;
+ offset = msg->offset;
+ size = msg->size;
+
+ if (!size || (offset != min_offset && offset < sizeof(*msg)) ||
+ offset > drv_info->rxtx_bufsz ||
+ size > drv_info->rxtx_bufsz - offset) {
+ pr_err("invalid framework notification message\n");
+ ffa_rx_release();
+ return;
+ }
+
+ buf = kmemdup((void *)msg + offset, size, GFP_KERNEL);
+ if (!buf) {
+ ffa_rx_release();
+ return;
+ }
+
+ target = SENDER_ID(msg->send_recv_id);
+ if (offset >= sizeof(*msg))
+ uuid_copy(&uuid, &msg->uuid);
+ else
+ uuid_copy(&uuid, &uuid_null);
+ ffa_rx_release();
}
- target = SENDER_ID(msg->send_recv_id);
- if (msg->offset >= sizeof(*msg))
- uuid_copy(&uuid, &msg->uuid);
- else
- uuid_copy(&uuid, &uuid_null);
+ scoped_guard(read_lock, &drv_info->notify_lock) {
+ struct notifier_cb_info *cb_info;
- mutex_unlock(&drv_info->rx_lock);
+ cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target,
+ &uuid);
+ fwk_cb = cb_info ? cb_info->fwk_cb : NULL;
+ fwk_cb_data = cb_info ? cb_info->cb_data : NULL;
+ }
- ffa_rx_release();
-
- read_lock(&drv_info->notify_lock);
- cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target, &uuid);
- read_unlock(&drv_info->notify_lock);
-
- if (cb_info && cb_info->fwk_cb)
- cb_info->fwk_cb(notify_id, cb_info->cb_data, buf);
+ if (fwk_cb)
+ fwk_cb(notify_id, fwk_cb_data, buf);
kfree(buf);
}
@@ -1539,10 +1579,11 @@ ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data)
static void notif_pcpu_irq_work_fn(struct work_struct *work)
{
- struct ffa_drv_info *info = container_of(work, struct ffa_drv_info,
+ struct ffa_pcpu_irq *pcpu = container_of(work, struct ffa_pcpu_irq,
notif_pcpu_work);
+ struct ffa_drv_info *info = pcpu->info;
- ffa_self_notif_handle(smp_processor_id(), true, info);
+ notif_get_and_handle(info);
}
static const struct ffa_info_ops ffa_drv_info_ops = {
@@ -1629,6 +1670,15 @@ static struct notifier_block ffa_bus_nb = {
.notifier_call = ffa_bus_notifier,
};
+static void ffa_bus_notifier_unregister(void)
+{
+ if (!drv_info->bus_notifier_registered)
+ return;
+
+ bus_unregister_notifier(&ffa_bus_type, &ffa_bus_nb);
+ drv_info->bus_notifier_registered = false;
+}
+
static int ffa_xa_add_partition_info(struct ffa_device *dev)
{
struct ffa_dev_part_info *info;
@@ -1712,6 +1762,8 @@ static void ffa_partitions_cleanup(void)
struct list_head *phead;
unsigned long idx;
+ ffa_bus_notifier_unregister();
+
/* Clean up/free all registered devices */
ffa_devices_unregister();
@@ -1739,11 +1791,14 @@ static int ffa_setup_partitions(void)
ret = bus_register_notifier(&ffa_bus_type, &ffa_bus_nb);
if (ret)
pr_err("Failed to register FF-A bus notifiers\n");
+ else
+ drv_info->bus_notifier_registered = true;
}
count = ffa_partition_probe(&uuid_null, &pbuf);
if (count <= 0) {
pr_info("%s: No partitions found, error %d\n", __func__, count);
+ ffa_bus_notifier_unregister();
return -EINVAL;
}
@@ -1811,7 +1866,7 @@ static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data)
struct ffa_drv_info *info = pcpu->info;
queue_work_on(smp_processor_id(), info->notif_pcpu_wq,
- &info->notif_pcpu_work);
+ &pcpu->notif_pcpu_work);
return IRQ_HANDLED;
}
@@ -1928,8 +1983,11 @@ static int ffa_init_pcpu_irq(void)
if (!irq_pcpu)
return -ENOMEM;
- for_each_present_cpu(cpu)
+ for_each_present_cpu(cpu) {
per_cpu_ptr(irq_pcpu, cpu)->info = drv_info;
+ INIT_WORK(&per_cpu_ptr(irq_pcpu, cpu)->notif_pcpu_work,
+ notif_pcpu_irq_work_fn);
+ }
drv_info->irq_pcpu = irq_pcpu;
@@ -1958,7 +2016,6 @@ static int ffa_init_pcpu_irq(void)
}
INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn);
- INIT_WORK(&drv_info->notif_pcpu_work, notif_pcpu_irq_work_fn);
drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification");
if (!drv_info->notif_pcpu_wq)
return -EINVAL;
@@ -2063,11 +2120,12 @@ static int __init ffa_init(void)
rxtx_bufsz = SZ_4K;
}
+ rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz);
drv_info->rxtx_bufsz = rxtx_bufsz;
drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
if (!drv_info->rx_buffer) {
ret = -ENOMEM;
- goto free_pages;
+ goto free_drv_info;
}
drv_info->tx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
@@ -2078,7 +2136,7 @@ static int __init ffa_init(void)
ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
virt_to_phys(drv_info->rx_buffer),
- PAGE_ALIGN(rxtx_bufsz) / FFA_PAGE_SIZE);
+ rxtx_bufsz / FFA_PAGE_SIZE);
if (ret) {
pr_err("failed to register FFA RxTx buffers\n");
goto free_pages;
diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c
index a253b61..a5db353 100644
--- a/drivers/firmware/efi/efi-pstore.c
+++ b/drivers/firmware/efi/efi-pstore.c
@@ -60,8 +60,10 @@ static int efi_pstore_open(struct pstore_info *psi)
return err;
psi->data = kzalloc(record_size, GFP_KERNEL);
- if (!psi->data)
+ if (!psi->data) {
+ efivar_unlock();
return -ENOMEM;
+ }
return 0;
}
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index d04be38..318d1cc9 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -402,21 +402,11 @@ static void __init efi_debugfs_init(void)
static inline void efi_debugfs_init(void) {}
#endif
-/*
- * We register the efi subsystem with the firmware subsystem and the
- * efivars subsystem with the efi subsystem, if the system was booted with
- * EFI.
- */
-static int __init efisubsys_init(void)
+static int __init efipostcore_init(void)
{
- int error;
-
if (!efi_enabled(EFI_RUNTIME_SERVICES))
efi.runtime_supported_mask = 0;
- if (!efi_enabled(EFI_BOOT))
- return 0;
-
if (efi.runtime_supported_mask) {
/*
* Since we process only one efi_runtime_service() at a time, an
@@ -428,9 +418,23 @@ static int __init efisubsys_init(void)
pr_err("Creating efi_rts_wq failed, EFI runtime services disabled.\n");
clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
efi.runtime_supported_mask = 0;
- return 0;
}
}
+ return 0;
+}
+postcore_initcall(efipostcore_init);
+
+/*
+ * We register the efi subsystem with the firmware subsystem and the
+ * efivars subsystem with the efi subsystem, if the system was booted with
+ * EFI.
+ */
+static int __init efisubsys_init(void)
+{
+ int error;
+
+ if (!efi_enabled(EFI_BOOT))
+ return 0;
if (efi_rt_services_supported(EFI_RT_SUPPORTED_TIME_SERVICES))
platform_device_register_simple("rtc-efi", 0, NULL, 0);
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 983a438..cfedb30 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -66,7 +66,7 @@
lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \
file.o mem.o random.o randomalloc.o pci.o \
skip_spaces.o lib-cmdline.o lib-ctype.o \
- alignedmem.o relocate.o printk.o vsprintf.o
+ alignedmem.o printk.o vsprintf.o
# include the stub's libfdt dependencies from lib/ when needed
libfdt-deps := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c \
diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c
index 7aa2f9a..f27f2e1 100644
--- a/drivers/firmware/efi/libstub/efi-stub-helper.c
+++ b/drivers/firmware/efi/libstub/efi-stub-helper.c
@@ -79,6 +79,10 @@ efi_status_t efi_parse_options(char const *cmdline)
efi_noinitrd = true;
} else if (IS_ENABLED(CONFIG_X86_64) && !strcmp(param, "no5lvl")) {
efi_no5lvl = true;
+ } else if (IS_ENABLED(CONFIG_LOONGARCH) &&
+ IS_ENABLED(CONFIG_HIBERNATION) &&
+ !strcmp(param, "resume") && val) {
+ efi_nokaslr = true; /* LoongArch can't KASLR for hibernation */
} else if (IS_ENABLED(CONFIG_ARCH_HAS_MEM_ENCRYPT) &&
!strcmp(param, "mem_encrypt") && val) {
if (parse_option_str(val, "on"))
diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h
index 979a218..fd91fc1 100644
--- a/drivers/firmware/efi/libstub/efistub.h
+++ b/drivers/firmware/efi/libstub/efistub.h
@@ -1104,13 +1104,6 @@ efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr,
efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
unsigned long *addr, unsigned long min);
-efi_status_t efi_relocate_kernel(unsigned long *image_addr,
- unsigned long image_size,
- unsigned long alloc_size,
- unsigned long preferred_addr,
- unsigned long alignment,
- unsigned long min_addr);
-
efi_status_t efi_parse_options(char const *cmdline);
void efi_parse_option_graphics(char *option);
diff --git a/drivers/firmware/efi/libstub/loongarch-stub.c b/drivers/firmware/efi/libstub/loongarch-stub.c
index 736b6aa..c87ac70 100644
--- a/drivers/firmware/efi/libstub/loongarch-stub.c
+++ b/drivers/firmware/efi/libstub/loongarch-stub.c
@@ -14,6 +14,86 @@ extern int kernel_asize;
extern int kernel_fsize;
extern int kernel_entry;
+/**
+ * efi_relocate_kernel() - copy memory area
+ * @image_addr: pointer to address of memory area to copy
+ * @image_size: size of memory area to copy
+ * @alloc_size: minimum size of memory to allocate, must be greater or
+ * equal to image_size
+ * @preferred_addr: preferred target address
+ * @alignment: minimum alignment of the allocated memory area. It
+ * should be a power of two.
+ * @min_addr: minimum target address
+ *
+ * Copy a memory area to a newly allocated memory area aligned according
+ * to @alignment but at least EFI_ALLOC_ALIGN. If the preferred address
+ * is not available, the allocated address will not be below @min_addr.
+ * On exit, @image_addr is updated to the target copy address that was used.
+ *
+ * This function is used to copy the Linux kernel verbatim. It does not apply
+ * any relocation changes.
+ *
+ * Return: status code
+ */
+static
+efi_status_t efi_relocate_kernel(unsigned long *image_addr,
+ unsigned long image_size,
+ unsigned long alloc_size,
+ unsigned long preferred_addr,
+ unsigned long alignment,
+ unsigned long min_addr)
+{
+ unsigned long cur_image_addr;
+ unsigned long new_addr = 0;
+ efi_status_t status;
+ unsigned long nr_pages;
+ efi_physical_addr_t efi_addr = preferred_addr;
+
+ if (!image_addr || !image_size || !alloc_size)
+ return EFI_INVALID_PARAMETER;
+ if (alloc_size < image_size)
+ return EFI_INVALID_PARAMETER;
+
+ cur_image_addr = *image_addr;
+
+ /*
+ * The EFI firmware loader could have placed the kernel image
+ * anywhere in memory, but the kernel has restrictions on the
+ * max physical address it can run at. Some architectures
+ * also have a preferred address, so first try to relocate
+ * to the preferred address. If that fails, allocate as low
+ * as possible while respecting the required alignment.
+ */
+ nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
+ status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS,
+ EFI_LOADER_DATA, nr_pages, &efi_addr);
+ new_addr = efi_addr;
+ /*
+ * If preferred address allocation failed allocate as low as
+ * possible.
+ */
+ if (status != EFI_SUCCESS) {
+ status = efi_low_alloc_above(alloc_size, alignment, &new_addr,
+ min_addr);
+ }
+ if (status != EFI_SUCCESS) {
+ efi_err("Failed to allocate usable memory for kernel.\n");
+ return status;
+ }
+
+ /*
+ * We know source/dest won't overlap since both memory ranges
+ * have been allocated by UEFI, so we can safely use memcpy.
+ */
+ memcpy((void *)new_addr, (void *)cur_image_addr, image_size);
+ efi_cache_sync_image(new_addr, image_size);
+
+ /* Return the new address of the relocated image. */
+ *image_addr = new_addr;
+
+ return status;
+}
+
efi_status_t handle_kernel_image(unsigned long *image_addr,
unsigned long *image_size,
unsigned long *reserve_addr,
diff --git a/drivers/firmware/efi/libstub/loongarch.c b/drivers/firmware/efi/libstub/loongarch.c
index 9825f52..2b0c87d 100644
--- a/drivers/firmware/efi/libstub/loongarch.c
+++ b/drivers/firmware/efi/libstub/loongarch.c
@@ -18,6 +18,27 @@ efi_status_t check_platform_features(void)
return EFI_SUCCESS;
}
+void efi_cache_sync_image(unsigned long image_base, unsigned long alloc_size)
+{
+ asm volatile ("ibar 0" ::: "memory");
+}
+
+unsigned long efi_get_kimg_kaslr_address(void)
+{
+ unsigned int random_offset = 0;
+
+#ifdef CONFIG_RANDOMIZE_BASE
+ if (!efi_nokaslr) {
+ efi_get_random_bytes(sizeof(random_offset), (u8 *)&random_offset);
+ random_offset ^= (random_get_entropy() << 16);
+ random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1);
+ random_offset = ALIGN(random_offset + SZ_64K, SZ_64K);
+ }
+#endif
+
+ return PHYSADDR(VMLINUX_LOAD_ADDRESS) + random_offset;
+}
+
struct exit_boot_struct {
efi_memory_desc_t *runtime_map;
int runtime_entry_count;
diff --git a/drivers/firmware/efi/libstub/mem.c b/drivers/firmware/efi/libstub/mem.c
index 9c82259..59f3f83 100644
--- a/drivers/firmware/efi/libstub/mem.c
+++ b/drivers/firmware/efi/libstub/mem.c
@@ -124,3 +124,85 @@ void efi_free(unsigned long size, unsigned long addr)
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
efi_bs_call(free_pages, addr, nr_pages);
}
+
+/**
+ * efi_low_alloc_above() - allocate pages at or above given address
+ * @size: size of the memory area to allocate
+ * @align: minimum alignment of the allocated memory area. It should
+ * a power of two.
+ * @addr: on exit the address of the allocated memory
+ * @min: minimum address to used for the memory allocation
+ *
+ * Allocate at the lowest possible address that is not below @min as
+ * EFI_LOADER_DATA. The allocated pages are aligned according to @align but at
+ * least EFI_ALLOC_ALIGN. The first allocated page will not below the address
+ * given by @min.
+ *
+ * Return: status code
+ */
+efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
+ unsigned long *addr, unsigned long min)
+{
+ struct efi_boot_memmap *map __free(efi_pool) = NULL;
+ efi_status_t status;
+ unsigned long nr_pages;
+ int i;
+
+ status = efi_get_memory_map(&map, false);
+ if (status != EFI_SUCCESS)
+ return status;
+
+ /*
+ * Enforce minimum alignment that EFI or Linux requires when
+ * requesting a specific address. We are doing page-based (or
+ * larger) allocations, and both the address and size must meet
+ * alignment constraints.
+ */
+ if (align < EFI_ALLOC_ALIGN)
+ align = EFI_ALLOC_ALIGN;
+
+ size = round_up(size, EFI_ALLOC_ALIGN);
+ nr_pages = size / EFI_PAGE_SIZE;
+ for (i = 0; i < map->map_size / map->desc_size; i++) {
+ efi_memory_desc_t *desc;
+ unsigned long m = (unsigned long)map->map;
+ u64 start, end;
+
+ desc = efi_memdesc_ptr(m, map->desc_size, i);
+
+ if (desc->type != EFI_CONVENTIONAL_MEMORY)
+ continue;
+
+ if (desc->attribute & EFI_MEMORY_HOT_PLUGGABLE)
+ continue;
+
+ if (efi_soft_reserve_enabled() &&
+ (desc->attribute & EFI_MEMORY_SP))
+ continue;
+
+ if (desc->num_pages < nr_pages)
+ continue;
+
+ start = desc->phys_addr;
+ end = start + desc->num_pages * EFI_PAGE_SIZE;
+
+ if (start < min)
+ start = min;
+
+ start = round_up(start, align);
+ if ((start + size) > end)
+ continue;
+
+ status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS,
+ EFI_LOADER_DATA, nr_pages, &start);
+ if (status == EFI_SUCCESS) {
+ *addr = start;
+ break;
+ }
+ }
+
+ if (i == map->map_size / map->desc_size)
+ return EFI_NOT_FOUND;
+
+ return EFI_SUCCESS;
+}
diff --git a/drivers/firmware/efi/libstub/relocate.c b/drivers/firmware/efi/libstub/relocate.c
deleted file mode 100644
index d4264bf..0000000
--- a/drivers/firmware/efi/libstub/relocate.c
+++ /dev/null
@@ -1,166 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include <linux/efi.h>
-#include <asm/efi.h>
-
-#include "efistub.h"
-
-/**
- * efi_low_alloc_above() - allocate pages at or above given address
- * @size: size of the memory area to allocate
- * @align: minimum alignment of the allocated memory area. It should
- * a power of two.
- * @addr: on exit the address of the allocated memory
- * @min: minimum address to used for the memory allocation
- *
- * Allocate at the lowest possible address that is not below @min as
- * EFI_LOADER_DATA. The allocated pages are aligned according to @align but at
- * least EFI_ALLOC_ALIGN. The first allocated page will not below the address
- * given by @min.
- *
- * Return: status code
- */
-efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
- unsigned long *addr, unsigned long min)
-{
- struct efi_boot_memmap *map __free(efi_pool) = NULL;
- efi_status_t status;
- unsigned long nr_pages;
- int i;
-
- status = efi_get_memory_map(&map, false);
- if (status != EFI_SUCCESS)
- return status;
-
- /*
- * Enforce minimum alignment that EFI or Linux requires when
- * requesting a specific address. We are doing page-based (or
- * larger) allocations, and both the address and size must meet
- * alignment constraints.
- */
- if (align < EFI_ALLOC_ALIGN)
- align = EFI_ALLOC_ALIGN;
-
- size = round_up(size, EFI_ALLOC_ALIGN);
- nr_pages = size / EFI_PAGE_SIZE;
- for (i = 0; i < map->map_size / map->desc_size; i++) {
- efi_memory_desc_t *desc;
- unsigned long m = (unsigned long)map->map;
- u64 start, end;
-
- desc = efi_memdesc_ptr(m, map->desc_size, i);
-
- if (desc->type != EFI_CONVENTIONAL_MEMORY)
- continue;
-
- if (desc->attribute & EFI_MEMORY_HOT_PLUGGABLE)
- continue;
-
- if (efi_soft_reserve_enabled() &&
- (desc->attribute & EFI_MEMORY_SP))
- continue;
-
- if (desc->num_pages < nr_pages)
- continue;
-
- start = desc->phys_addr;
- end = start + desc->num_pages * EFI_PAGE_SIZE;
-
- if (start < min)
- start = min;
-
- start = round_up(start, align);
- if ((start + size) > end)
- continue;
-
- status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS,
- EFI_LOADER_DATA, nr_pages, &start);
- if (status == EFI_SUCCESS) {
- *addr = start;
- break;
- }
- }
-
- if (i == map->map_size / map->desc_size)
- return EFI_NOT_FOUND;
-
- return EFI_SUCCESS;
-}
-
-/**
- * efi_relocate_kernel() - copy memory area
- * @image_addr: pointer to address of memory area to copy
- * @image_size: size of memory area to copy
- * @alloc_size: minimum size of memory to allocate, must be greater or
- * equal to image_size
- * @preferred_addr: preferred target address
- * @alignment: minimum alignment of the allocated memory area. It
- * should be a power of two.
- * @min_addr: minimum target address
- *
- * Copy a memory area to a newly allocated memory area aligned according
- * to @alignment but at least EFI_ALLOC_ALIGN. If the preferred address
- * is not available, the allocated address will not be below @min_addr.
- * On exit, @image_addr is updated to the target copy address that was used.
- *
- * This function is used to copy the Linux kernel verbatim. It does not apply
- * any relocation changes.
- *
- * Return: status code
- */
-efi_status_t efi_relocate_kernel(unsigned long *image_addr,
- unsigned long image_size,
- unsigned long alloc_size,
- unsigned long preferred_addr,
- unsigned long alignment,
- unsigned long min_addr)
-{
- unsigned long cur_image_addr;
- unsigned long new_addr = 0;
- efi_status_t status;
- unsigned long nr_pages;
- efi_physical_addr_t efi_addr = preferred_addr;
-
- if (!image_addr || !image_size || !alloc_size)
- return EFI_INVALID_PARAMETER;
- if (alloc_size < image_size)
- return EFI_INVALID_PARAMETER;
-
- cur_image_addr = *image_addr;
-
- /*
- * The EFI firmware loader could have placed the kernel image
- * anywhere in memory, but the kernel has restrictions on the
- * max physical address it can run at. Some architectures
- * also have a preferred address, so first try to relocate
- * to the preferred address. If that fails, allocate as low
- * as possible while respecting the required alignment.
- */
- nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
- status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS,
- EFI_LOADER_DATA, nr_pages, &efi_addr);
- new_addr = efi_addr;
- /*
- * If preferred address allocation failed allocate as low as
- * possible.
- */
- if (status != EFI_SUCCESS) {
- status = efi_low_alloc_above(alloc_size, alignment, &new_addr,
- min_addr);
- }
- if (status != EFI_SUCCESS) {
- efi_err("Failed to allocate usable memory for kernel.\n");
- return status;
- }
-
- /*
- * We know source/dest won't overlap since both memory ranges
- * have been allocated by UEFI, so we can safely use memcpy.
- */
- memcpy((void *)new_addr, (void *)cur_image_addr, image_size);
-
- /* Return the new address of the relocated image. */
- *image_addr = new_addr;
-
- return status;
-}
diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c
index c00d0ae..c3da05c 100644
--- a/drivers/firmware/efi/libstub/x86-5lvl.c
+++ b/drivers/firmware/efi/libstub/x86-5lvl.c
@@ -2,6 +2,7 @@
#include <linux/efi.h>
#include <asm/boot.h>
+#include <asm/cpuid/api.h>
#include <asm/desc.h>
#include <asm/efi.h>
diff --git a/drivers/firmware/efi/sysfb_efi.c b/drivers/firmware/efi/sysfb_efi.c
index 4c3986d..685283b 100644
--- a/drivers/firmware/efi/sysfb_efi.c
+++ b/drivers/firmware/efi/sysfb_efi.c
@@ -311,11 +311,14 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
.callback = efifb_swap_width_height,
},
{
- /* Lenovo IdeaPad Duet 3 10IGL5 with 1200x1920 portrait screen */
+ /*
+ * Lenovo IdeaPad Duet 3 10IGL5 and 10IGL5-LTE with
+ * 1200x1920 portrait screen
+ */
.matches = {
DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
- DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
- "IdeaPad Duet 3 10IGL5"),
+ /* Non exact match to also match the LTE version */
+ DMI_MATCH(DMI_PRODUCT_VERSION, "IdeaPad Duet 3 10IGL5"),
},
.callback = efifb_swap_width_height,
},
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 38ca190..e73bae6 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -539,12 +539,22 @@ static int psci_system_suspend(unsigned long unused)
static int psci_system_suspend_enter(suspend_state_t state)
{
+ pm_set_resume_via_firmware();
+
return cpu_suspend(0, psci_system_suspend);
}
+static int psci_system_suspend_begin(suspend_state_t state)
+{
+ pm_set_suspend_via_firmware();
+
+ return 0;
+}
+
static const struct platform_suspend_ops psci_suspend_ops = {
.valid = suspend_valid_only_mem,
.enter = psci_system_suspend_enter,
+ .begin = psci_system_suspend_begin,
};
static void __init psci_init_system_reset2(void)
diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.c b/drivers/firmware/samsung/exynos-acpm-dvfs.c
index 06bdf62..fdea7aa 100644
--- a/drivers/firmware/samsung/exynos-acpm-dvfs.c
+++ b/drivers/firmware/samsung/exynos-acpm-dvfs.c
@@ -31,6 +31,9 @@ static void acpm_dvfs_set_xfer(struct acpm_xfer *xfer, u32 *cmd, size_t cmdlen,
if (response) {
xfer->rxcnt = cmdlen;
xfer->rxd = cmd;
+ } else {
+ xfer->rxcnt = 0;
+ xfer->rxd = NULL;
}
}
diff --git a/drivers/firmware/samsung/exynos-acpm.c b/drivers/firmware/samsung/exynos-acpm.c
index 16c46ed..19db367 100644
--- a/drivers/firmware/samsung/exynos-acpm.c
+++ b/drivers/firmware/samsung/exynos-acpm.c
@@ -7,11 +7,12 @@
#include <linux/bitfield.h>
#include <linux/bitmap.h>
-#include <linux/bits.h>
+#include <linux/bitops.h>
#include <linux/cleanup.h>
#include <linux/container_of.h>
#include <linux/delay.h>
#include <linux/device.h>
+#include <linux/find.h>
#include <linux/firmware/samsung/exynos-acpm-protocol.h>
#include <linux/io.h>
#include <linux/iopoll.h>
@@ -104,12 +105,15 @@ struct acpm_queue {
*
* @cmd: pointer to where the data shall be saved.
* @n_cmd: number of 32-bit commands.
- * @response: true if the client expects the RX data.
+ * @rxcnt: expected length of the response in 32-bit words.
+ * @completed: flag indicating if the firmware response has been fully
+ * processed.
*/
struct acpm_rx_data {
u32 *cmd;
size_t n_cmd;
- bool response;
+ size_t rxcnt;
+ bool completed;
};
#define ACPM_SEQNUM_MAX 64
@@ -199,31 +203,33 @@ static void acpm_get_saved_rx(struct acpm_chan *achan,
const struct acpm_rx_data *rx_data = &achan->rx_data[tx_seqnum - 1];
u32 rx_seqnum;
- if (!rx_data->response)
+ if (!rx_data->rxcnt)
return;
rx_seqnum = FIELD_GET(ACPM_PROTOCOL_SEQNUM, rx_data->cmd[0]);
- if (rx_seqnum == tx_seqnum) {
+ if (rx_seqnum == tx_seqnum)
memcpy(xfer->rxd, rx_data->cmd, xfer->rxcnt * sizeof(*xfer->rxd));
- clear_bit(rx_seqnum - 1, achan->bitmap_seqnum);
- }
}
/**
* acpm_get_rx() - get response from RX queue.
* @achan: ACPM channel info.
* @xfer: reference to the transfer to get response for.
+ * @native_match: pointer to a boolean set to true if the thread natively
+ * processed its own sequence number during this call.
*
* Return: 0 on success, -errno otherwise.
*/
-static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
+static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer,
+ bool *native_match)
{
u32 rx_front, rx_seqnum, tx_seqnum, seqnum;
const void __iomem *base, *addr;
struct acpm_rx_data *rx_data;
u32 i, val, mlen;
- bool rx_set = false;
+
+ *native_match = false;
guard(mutex)(&achan->rx_lock);
@@ -232,10 +238,8 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
tx_seqnum = FIELD_GET(ACPM_PROTOCOL_SEQNUM, xfer->txd[0]);
- if (i == rx_front) {
- acpm_get_saved_rx(achan, xfer, tx_seqnum);
+ if (i == rx_front)
return 0;
- }
base = achan->rx.base;
mlen = achan->mlen;
@@ -256,11 +260,16 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
seqnum = rx_seqnum - 1;
rx_data = &achan->rx_data[seqnum];
- if (rx_data->response) {
+ if (rx_data->rxcnt) {
if (rx_seqnum == tx_seqnum) {
__ioread32_copy(xfer->rxd, addr, xfer->rxcnt);
- rx_set = true;
- clear_bit(seqnum, achan->bitmap_seqnum);
+ /*
+ * Signal completion to the polling thread.
+ * Pairs with smp_load_acquire() in polling
+ * loop.
+ */
+ smp_store_release(&rx_data->completed, true);
+ *native_match = true;
} else {
/*
* The RX data corresponds to another request.
@@ -268,10 +277,23 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
* clear yet the bitmap. It will be cleared
* after the response is copied to the request.
*/
- __ioread32_copy(rx_data->cmd, addr, xfer->rxcnt);
+ __ioread32_copy(rx_data->cmd, addr,
+ rx_data->rxcnt);
+ /*
+ * Signal completion to the polling thread.
+ * Pairs with smp_load_acquire() in polling
+ * loop.
+ */
+ smp_store_release(&rx_data->completed, true);
}
} else {
- clear_bit(seqnum, achan->bitmap_seqnum);
+ /*
+ * Signal completion to the polling thread.
+ * Pairs with smp_load_acquire() in polling loop.
+ */
+ smp_store_release(&rx_data->completed, true);
+ if (rx_seqnum == tx_seqnum)
+ *native_match = true;
}
i = (i + 1) % achan->qlen;
@@ -280,13 +302,6 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
/* We saved all responses, mark RX empty. */
writel(rx_front, achan->rx.rear);
- /*
- * If the response was not in this iteration of the queue, check if the
- * RX data was previously saved.
- */
- if (!rx_set)
- acpm_get_saved_rx(achan, xfer, tx_seqnum);
-
return 0;
}
@@ -301,6 +316,7 @@ static int acpm_dequeue_by_polling(struct acpm_chan *achan,
const struct acpm_xfer *xfer)
{
struct device *dev = achan->acpm->dev;
+ bool native_match;
ktime_t timeout;
u32 seqnum;
int ret;
@@ -309,12 +325,25 @@ static int acpm_dequeue_by_polling(struct acpm_chan *achan,
timeout = ktime_add_us(ktime_get(), ACPM_POLL_TIMEOUT_US);
do {
- ret = acpm_get_rx(achan, xfer);
+ ret = acpm_get_rx(achan, xfer, &native_match);
if (ret)
return ret;
- if (!test_bit(seqnum - 1, achan->bitmap_seqnum))
+ /*
+ * Safely check if our specific transaction has been processed.
+ * smp_load_acquire prevents the CPU from speculatively
+ * executing subsequent instructions before the transaction is
+ * synchronized.
+ */
+ if (smp_load_acquire(&achan->rx_data[seqnum - 1].completed)) {
+ /* Retrieve payload if another thread cached it for us */
+ if (!native_match)
+ acpm_get_saved_rx(achan, xfer, seqnum);
+
+ /* Relinquish ownership of the sequence slot */
+ clear_bit_unlock(seqnum - 1, achan->bitmap_seqnum);
return 0;
+ }
/* Determined experimentally. */
udelay(20);
@@ -362,29 +391,48 @@ static int acpm_wait_for_queue_slots(struct acpm_chan *achan, u32 next_tx_front)
* TX queue.
* @achan: ACPM channel info.
* @xfer: reference to the transfer being prepared.
+ *
+ * Return: 0 on success, -errno otherwise.
*/
-static void acpm_prepare_xfer(struct acpm_chan *achan,
- const struct acpm_xfer *xfer)
+static int acpm_prepare_xfer(struct acpm_chan *achan,
+ const struct acpm_xfer *xfer)
{
struct acpm_rx_data *rx_data;
u32 *txd = (u32 *)xfer->txd;
+ unsigned long size = ACPM_SEQNUM_MAX - 1;
+ unsigned long bit = achan->seqnum;
- /* Prevent chan->seqnum from being re-used */
- do {
- if (++achan->seqnum == ACPM_SEQNUM_MAX)
- achan->seqnum = 1;
- } while (test_bit(achan->seqnum - 1, achan->bitmap_seqnum));
+ bit = find_next_zero_bit(achan->bitmap_seqnum, size, bit);
+ if (bit >= size) {
+ bit = find_first_zero_bit(achan->bitmap_seqnum, size);
+ if (bit >= size) {
+ dev_err_ratelimited(achan->acpm->dev,
+ "ACPM sequence number pool exhausted\n");
+ return -EBUSY;
+ }
+ }
+ /*
+ * Execute the atomic set to formally claim the bit and establish
+ * LKMM Acquire semantics against the RX thread's clear_bit_unlock().
+ * A loop is unnecessary because allocations are strictly serialized
+ * by tx_lock.
+ */
+ if (WARN_ON_ONCE(test_and_set_bit_lock(bit, achan->bitmap_seqnum)))
+ return -EIO;
+
+ /* Flag the index based on seqnum. (seqnum: 1~63, bitmap: 0~62) */
+ achan->seqnum = bit + 1;
txd[0] |= FIELD_PREP(ACPM_PROTOCOL_SEQNUM, achan->seqnum);
/* Clear data for upcoming responses */
- rx_data = &achan->rx_data[achan->seqnum - 1];
+ rx_data = &achan->rx_data[bit];
+ rx_data->completed = false;
memset(rx_data->cmd, 0, sizeof(*rx_data->cmd) * rx_data->n_cmd);
- if (xfer->rxd)
- rx_data->response = true;
+ /* zero means no response expected */
+ rx_data->rxcnt = xfer->rxcnt;
- /* Flag the index based on seqnum. (seqnum: 1~63, bitmap: 0~62) */
- set_bit(achan->seqnum - 1, achan->bitmap_seqnum);
+ return 0;
}
/**
@@ -444,7 +492,9 @@ int acpm_do_xfer(struct acpm_handle *handle, const struct acpm_xfer *xfer)
if (ret)
return ret;
- acpm_prepare_xfer(achan, xfer);
+ ret = acpm_prepare_xfer(achan, xfer);
+ if (ret)
+ return ret;
/* Write TX command. */
__iowrite32_copy(achan->tx.base + achan->mlen * tx_front,
@@ -526,10 +576,11 @@ static int acpm_achan_alloc_cmds(struct acpm_chan *achan)
/**
* acpm_free_mbox_chans() - free mailbox channels.
- * @acpm: pointer to driver data.
+ * @data: pointer to driver data.
*/
-static void acpm_free_mbox_chans(struct acpm_info *acpm)
+static void acpm_free_mbox_chans(void *data)
{
+ struct acpm_info *acpm = data;
int i;
for (i = 0; i < acpm->num_chans; i++)
@@ -557,6 +608,10 @@ static int acpm_channels_init(struct acpm_info *acpm)
if (!acpm->chans)
return -ENOMEM;
+ ret = devm_add_action_or_reset(dev, acpm_free_mbox_chans, acpm);
+ if (ret)
+ return dev_err_probe(dev, ret, "Failed to add mbox free action.\n");
+
chans_shmem = acpm->sram_base + readl(&shmem->chans);
for (i = 0; i < acpm->num_chans; i++) {
@@ -578,10 +633,8 @@ static int acpm_channels_init(struct acpm_info *acpm)
cl->dev = dev;
achan->chan = mbox_request_channel(cl, 0);
- if (IS_ERR(achan->chan)) {
- acpm_free_mbox_chans(acpm);
+ if (IS_ERR(achan->chan))
return PTR_ERR(achan->chan);
- }
}
return 0;
diff --git a/drivers/fwctl/pds/main.c b/drivers/fwctl/pds/main.c
index 08872ee..68fe254 100644
--- a/drivers/fwctl/pds/main.c
+++ b/drivers/fwctl/pds/main.c
@@ -362,6 +362,9 @@ static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
void *out = NULL;
int err;
+ if (in_len < sizeof(*rpc))
+ return ERR_PTR(-EINVAL);
+
err = pdsfc_validate_rpc(pdsfc, rpc, scope);
if (err)
return ERR_PTR(err);
diff --git a/drivers/gpio/gpio-adnp.c b/drivers/gpio/gpio-adnp.c
index e5ac2d2..fe5bcaa 100644
--- a/drivers/gpio/gpio-adnp.c
+++ b/drivers/gpio/gpio-adnp.c
@@ -237,7 +237,9 @@ static irqreturn_t adnp_irq(int irq, void *data)
unsigned long pending;
int err;
- scoped_guard(mutex, &adnp->i2c_lock) {
+ {
+ guard(mutex)(&adnp->i2c_lock);
+
err = adnp_read(adnp, GPIO_PLR(adnp) + i, &level);
if (err < 0)
continue;
diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c
index 5915209..bc6699a 100644
--- a/drivers/gpio/gpio-aggregator.c
+++ b/drivers/gpio/gpio-aggregator.c
@@ -968,9 +968,12 @@ static int gpio_aggregator_activate(struct gpio_aggregator *aggr)
}
wait_for_device_probe();
- if (!device_is_bound(&pdev->dev)) {
- ret = -ENXIO;
- goto err_unregister_pdev;
+
+ scoped_guard(device, &pdev->dev) {
+ if (!device_is_bound(&pdev->dev)) {
+ ret = -ENXIO;
+ goto err_unregister_pdev;
+ }
}
aggr->pdev = pdev;
@@ -979,8 +982,8 @@ static int gpio_aggregator_activate(struct gpio_aggregator *aggr)
err_unregister_pdev:
platform_device_unregister(pdev);
err_remove_lookup_table:
- kfree(aggr->lookups->dev_id);
gpiod_remove_lookup_table(aggr->lookups);
+ kfree(aggr->lookups->dev_id);
err_remove_swnode:
fwnode_remove_software_node(swnode);
err_remove_lookups:
@@ -991,11 +994,15 @@ static int gpio_aggregator_activate(struct gpio_aggregator *aggr)
static void gpio_aggregator_deactivate(struct gpio_aggregator *aggr)
{
+ struct fwnode_handle *swnode;
+
+ swnode = dev_fwnode(&aggr->pdev->dev);
platform_device_unregister(aggr->pdev);
aggr->pdev = NULL;
gpiod_remove_lookup_table(aggr->lookups);
kfree(aggr->lookups->dev_id);
kfree(aggr->lookups);
+ fwnode_remove_software_node(swnode);
}
static void gpio_aggregator_lockup_configfs(struct gpio_aggregator *aggr,
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index 647b6f4..12f11a6 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -469,7 +469,7 @@ static int mxc_gpio_probe(struct platform_device *pdev)
* the handler is needed only once, but doing it for every port
* is more robust and easier.
*/
- port->irq_high = -1;
+ port->irq_high = 0;
port->mx_irq_handler = mx2_gpio_irq_handler;
} else
port->mx_irq_handler = mx3_gpio_irq_handler;
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 52e96cc..b9c905a 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -1411,7 +1411,7 @@ static int pca953x_resume(struct device *dev)
ret = regulator_enable(chip->regulator);
if (ret) {
dev_err(dev, "Failed to enable regulator: %d\n", ret);
- return 0;
+ return ret;
}
}
diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c
index 44d7ebd..bc97d5d 100644
--- a/drivers/gpio/gpio-rockchip.c
+++ b/drivers/gpio/gpio-rockchip.c
@@ -638,10 +638,17 @@ static int rockchip_gpiolib_register(struct rockchip_pin_bank *bank)
return ret;
}
+static void rockchip_clk_put(void *data)
+{
+ struct clk *clk = data;
+
+ clk_put(clk);
+}
+
static int rockchip_get_bank_data(struct rockchip_pin_bank *bank)
{
struct resource res;
- int id = 0;
+ int id = 0, ret;
if (of_address_to_resource(bank->of_node, 0, &res)) {
dev_err(bank->dev, "cannot find IO resource for bank\n");
@@ -656,11 +663,10 @@ static int rockchip_get_bank_data(struct rockchip_pin_bank *bank)
if (!bank->irq)
return -EINVAL;
- bank->clk = of_clk_get(bank->of_node, 0);
+ bank->clk = devm_clk_get_enabled(bank->dev, NULL);
if (IS_ERR(bank->clk))
return PTR_ERR(bank->clk);
- clk_prepare_enable(bank->clk);
id = readl(bank->reg_base + gpio_regs_v2.version_id);
switch (id) {
@@ -672,9 +678,13 @@ static int rockchip_get_bank_data(struct rockchip_pin_bank *bank)
bank->db_clk = of_clk_get(bank->of_node, 1);
if (IS_ERR(bank->db_clk)) {
dev_err(bank->dev, "cannot find debounce clk\n");
- clk_disable_unprepare(bank->clk);
return -EINVAL;
}
+
+ ret = devm_add_action_or_reset(bank->dev, rockchip_clk_put,
+ bank->db_clk);
+ if (ret)
+ return ret;
break;
case GPIO_TYPE_V1:
bank->gpio_regs = &gpio_regs_v1;
@@ -751,7 +761,6 @@ static int rockchip_gpio_probe(struct platform_device *pdev)
ret = rockchip_gpiolib_register(bank);
if (ret) {
- clk_disable_unprepare(bank->clk);
mutex_unlock(&bank->deferred_lock);
return ret;
}
@@ -792,7 +801,9 @@ static void rockchip_gpio_remove(struct platform_device *pdev)
{
struct rockchip_pin_bank *bank = platform_get_drvdata(pdev);
- clk_disable_unprepare(bank->clk);
+ irq_set_chained_handler_and_data(bank->irq, NULL, NULL);
+ if (bank->domain)
+ irq_domain_remove(bank->domain);
gpiochip_remove(&bank->gpio_chip);
}
diff --git a/drivers/gpio/gpio-shared-proxy.c b/drivers/gpio/gpio-shared-proxy.c
index 29d7d2e..6941e4b 100644
--- a/drivers/gpio/gpio-shared-proxy.c
+++ b/drivers/gpio/gpio-shared-proxy.c
@@ -103,9 +103,18 @@ static void gpio_shared_proxy_free(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_shared_proxy_data *proxy = gpiochip_get_data(gc);
struct gpio_shared_desc *shared_desc = proxy->shared_desc;
+ int ret;
guard(gpio_shared_desc_lock)(shared_desc);
+ if (proxy->voted_high) {
+ ret = gpio_shared_proxy_set_unlocked(proxy,
+ shared_desc->can_sleep ? gpiod_set_value_cansleep : gpiod_set_value, 0);
+ if (ret)
+ dev_err(proxy->dev,
+ "Failed to unset the shared GPIO value on release: %d\n", ret);
+ }
+
proxy->shared_desc->usecnt--;
dev_dbg(proxy->dev, "Shared GPIO freed, number of users: %u\n",
diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
index e19701c..0da2c5a 100644
--- a/drivers/gpio/gpio-sim.c
+++ b/drivers/gpio/gpio-sim.c
@@ -901,7 +901,7 @@ static int gpio_sim_device_activate(struct gpio_sim_device *dev)
struct platform_device *pdev;
struct fwnode_handle *swnode;
struct gpio_sim_bank *bank;
- int ret;
+ int ret = 0;
lockdep_assert_held(&dev->lock);
@@ -945,9 +945,12 @@ static int gpio_sim_device_activate(struct gpio_sim_device *dev)
}
wait_for_device_probe();
- if (!device_is_bound(&pdev->dev)) {
- ret = -ENXIO;
- goto err_unregister_pdev;
+
+ scoped_guard(device, &pdev->dev) {
+ if (!device_is_bound(&pdev->dev)) {
+ ret = -ENXIO;
+ goto err_unregister_pdev;
+ }
}
dev->pdev = pdev;
diff --git a/drivers/gpio/gpio-virtuser.c b/drivers/gpio/gpio-virtuser.c
index fe0eac9..846f868 100644
--- a/drivers/gpio/gpio-virtuser.c
+++ b/drivers/gpio/gpio-virtuser.c
@@ -397,7 +397,7 @@ static ssize_t gpio_virtuser_direction_do_write(struct file *file,
char buf[32], *trimmed;
int ret, dir, val = 0;
- if (count >= sizeof(buf))
+ if (*ppos != 0 || count >= sizeof(buf))
return -EINVAL;
ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
@@ -622,7 +622,7 @@ static ssize_t gpio_virtuser_consumer_write(struct file *file,
char buf[GPIO_VIRTUSER_NAME_BUF_LEN + 2];
int ret;
- if (count >= sizeof(buf))
+ if (*ppos != 0 || count >= sizeof(buf))
return -EINVAL;
ret = simple_write_to_buffer(buf, GPIO_VIRTUSER_NAME_BUF_LEN, ppos,
@@ -1477,9 +1477,12 @@ gpio_virtuser_device_activate(struct gpio_virtuser_device *dev)
}
wait_for_device_probe();
- if (!device_is_bound(&pdev->dev)) {
- ret = -ENXIO;
- goto err_unregister_pdev;
+
+ scoped_guard(device, &pdev->dev) {
+ if (!device_is_bound(&pdev->dev)) {
+ ret = -ENXIO;
+ goto err_unregister_pdev;
+ }
}
dev->pdev = pdev;
diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c
index f36b7c0..82f27db 100644
--- a/drivers/gpio/gpiolib-cdev.c
+++ b/drivers/gpio/gpiolib-cdev.c
@@ -1184,6 +1184,7 @@ static int gpio_v2_line_flags_validate(u64 flags)
static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc,
unsigned int num_lines)
{
+ size_t unused_attrs;
unsigned int i;
u64 flags;
int ret;
@@ -1191,9 +1192,21 @@ static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc,
if (lc->num_attrs > GPIO_V2_LINE_NUM_ATTRS_MAX)
return -EINVAL;
+ unused_attrs = GPIO_V2_LINE_NUM_ATTRS_MAX - lc->num_attrs;
+
if (!mem_is_zero(lc->padding, sizeof(lc->padding)))
return -EINVAL;
+ for (i = 0; i < lc->num_attrs; i++) {
+ if (lc->attrs[i].attr.padding != 0)
+ return -EINVAL;
+ }
+
+ if (unused_attrs) {
+ if (!mem_is_zero(&lc->attrs[lc->num_attrs], unused_attrs * sizeof(*lc->attrs)))
+ return -EINVAL;
+ }
+
for (i = 0; i < num_lines; i++) {
flags = gpio_v2_line_config_flags(lc, i);
ret = gpio_v2_line_flags_validate(flags);
diff --git a/drivers/gpio/gpiolib-shared.c b/drivers/gpio/gpiolib-shared.c
index e02d6b9..de72776 100644
--- a/drivers/gpio/gpiolib-shared.c
+++ b/drivers/gpio/gpiolib-shared.c
@@ -53,7 +53,7 @@ struct gpio_shared_entry {
unsigned int offset;
/* Index in the property value array. */
size_t index;
- /* Synchronizes the modification of shared_desc. */
+ /* Synchronizes the modification of shared_desc and offset. */
struct mutex lock;
struct gpio_shared_desc *shared_desc;
struct kref ref;
@@ -598,16 +598,13 @@ void gpio_device_teardown_shared(struct gpio_device *gdev)
struct gpio_shared_ref *ref;
list_for_each_entry(entry, &gpio_shared_list, list) {
- guard(mutex)(&entry->lock);
-
if (!device_match_fwnode(&gdev->dev, entry->fwnode))
continue;
- gpiod_free_commit(&gdev->descs[entry->offset]);
+ scoped_guard(mutex, &entry->lock)
+ gpiod_free_commit(&gdev->descs[entry->offset]);
list_for_each_entry(ref, &entry->refs, list) {
- guard(mutex)(&ref->lock);
-
if (ref->lookup) {
gpiod_remove_lookup_table(ref->lookup);
kfree(ref->lookup->table[0].key);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 8bc591d..fd50da4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1190,7 +1190,6 @@ struct amdgpu_device {
bool apu_prefer_gtt;
bool userq_halt_for_enforce_isolation;
- struct work_struct userq_reset_work;
struct amdgpu_uid *uid_info;
struct amdgpu_uma_carveout_info uma_info;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index d9e283f..9783a3ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -36,6 +36,9 @@
#include "amdgpu_ras.h"
#include "amdgpu_umc.h"
#include "amdgpu_reset.h"
+#if IS_ENABLED(CONFIG_HSA_AMD)
+#include "kfd_priv.h"
+#endif
/* Total memory size in system memory and all GPU VRAM. Used to
* estimate worst case amount of memory to reserve for page tables
@@ -320,6 +323,28 @@ void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev)
(void)amdgpu_reset_domain_schedule(adev->reset_domain, &adev->kfd.reset_work);
}
+void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev)
+{
+#if IS_ENABLED(CONFIG_HSA_AMD)
+ struct kfd_dev *kfd = adev->kfd.dev;
+ unsigned int i;
+
+ if (!kfd)
+ return;
+
+ for (i = 0; i < kfd->num_nodes; i++) {
+ struct kfd_node *node = kfd->nodes[i];
+
+ kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_DOORBELL |
+ KFD_MMAP_GPU_ID(node->id),
+ kfd_doorbell_process_slice(kfd));
+ kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_MMIO |
+ KFD_MMAP_GPU_ID(node->id),
+ PAGE_SIZE);
+ }
+#endif
+}
+
int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size,
u32 domain, void **mem_obj, uint64_t *gpu_addr,
void **cpu_ptr, bool cp_mqd_gfx9)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index cdbab7f..2b4108f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -358,6 +358,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
uint64_t size, u32 alloc_flag, int8_t xcp_id);
void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
uint64_t size, u32 alloc_flag, int8_t xcp_id);
+void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev);
u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 737ef1e..feab90e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2839,8 +2839,12 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
* that checks whether the PSP is running. A solution for those issues
* in the APU is to trigger a GPU reset, but this should be done during
* the unload phase to avoid adding boot latency and screen flicker.
+ * GFX V11 has GC block as default off IP. Every time AMDGPU driver sends
+ * a request to PMFW to unload MP1, PMFW will put GC in reset and power down
+ * the voltage. Hence, skipping reset for APUs with GFX V11 or later.
*/
- if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu) {
+ if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu &&
+ amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 0, 0)) {
r = amdgpu_asic_reset(adev);
if (r)
dev_err(adev->dev, "asic reset on %s failed\n", __func__);
@@ -3783,7 +3787,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
}
INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
- INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
amdgpu_coredump_init(adev);
@@ -5474,7 +5477,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
if (!amdgpu_sriov_vf(adev))
cancel_work(&adev->reset_work);
#endif
- cancel_work(&adev->userq_reset_work);
+ amdgpu_userq_mgr_cancel_reset_work(adev);
if (adev->kfd.dev)
cancel_work(&adev->kfd.reset_work);
@@ -5832,6 +5835,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
/* We need to lock reset domain only once both for XGMI and single device */
amdgpu_device_recovery_get_reset_lock(adev, &device_list);
+ /* unmap all the mappings of doorbell and framebuffer to prevent user space from
+ * accessing them
+ */
+ unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
+ amdgpu_amdkfd_clear_kfd_mapping(adev);
+
amdgpu_device_halt_activities(adev, job, reset_context, &device_list,
hive, need_emergency_restart);
if (need_emergency_restart)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index fcad7daa..80efeca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -304,7 +304,7 @@ static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev,
adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset;
adev->discovery.size =
adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb << 10;
- if (!adev->discovery.offset || !adev->discovery.size)
+ if (!adev->discovery.size)
return -EINVAL;
} else {
goto out;
@@ -3090,10 +3090,8 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
case IP_VERSION(11, 5, 1):
case IP_VERSION(11, 5, 2):
case IP_VERSION(11, 5, 3):
- adev->family = AMDGPU_FAMILY_GC_11_5_0;
- break;
case IP_VERSION(11, 5, 4):
- adev->family = AMDGPU_FAMILY_GC_11_5_4;
+ adev->family = AMDGPU_FAMILY_GC_11_5_0;
break;
case IP_VERSION(12, 0, 0):
case IP_VERSION(12, 0, 1):
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index e47921e2..60debd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -3149,17 +3149,15 @@ static int __init amdgpu_init(void)
r = amdgpu_sync_init();
if (r)
- goto error_sync;
-
- r = amdgpu_userq_fence_slab_init();
- if (r)
- goto error_fence;
+ return r;
amdgpu_register_atpx_handler();
amdgpu_acpi_detect();
- /* Ignore KFD init failures. Normal when CONFIG_HSA_AMD is not set. */
- amdgpu_amdkfd_init();
+ /* Ignore KFD init failures when CONFIG_HSA_AMD is not set. */
+ r = amdgpu_amdkfd_init();
+ if (r && r != -ENOENT)
+ goto error_fini_sync;
if (amdgpu_pp_feature_mask & PP_OVERDRIVE_MASK) {
add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
@@ -3170,10 +3168,8 @@ static int __init amdgpu_init(void)
/* let modprobe override vga console setting */
return pci_register_driver(&amdgpu_kms_pci_driver);
-error_fence:
+error_fini_sync:
amdgpu_sync_fini();
-
-error_sync:
return r;
}
@@ -3184,7 +3180,6 @@ static void __exit amdgpu_exit(void)
amdgpu_unregister_atpx_handler();
amdgpu_acpi_release();
amdgpu_sync_fini();
- amdgpu_userq_fence_slab_fini();
mmu_notifier_synchronize();
amdgpu_xcp_drv_release();
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index bc772ca..b6f849d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -262,12 +262,19 @@ void amdgpu_gart_table_ram_free(struct amdgpu_device *adev)
*/
int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
{
+ int r;
+
if (adev->gart.bo != NULL)
return 0;
- return amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE,
- AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo,
- NULL, (void *)&adev->gart.ptr);
+ r = amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo,
+ NULL, (void *)&adev->gart.ptr);
+ if (r)
+ return r;
+
+ memset_io(adev->gart.ptr, adev->gart.gart_pte_flags, adev->gart.table_size);
+ return 0;
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 5376035..fe6d988 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -31,6 +31,7 @@
#include <linux/pci.h>
#include <linux/dma-buf.h>
#include <linux/dma-fence-unwrap.h>
+#include <linux/uaccess.h>
#include <drm/amdgpu_drm.h>
#include <drm/drm_drv.h>
@@ -508,6 +509,9 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
if (offset_in_page(args->addr | args->size))
return -EINVAL;
+ if (!access_ok((void __user *)(uintptr_t)args->addr, args->size))
+ return -EFAULT;
+
/* reject unknown flag values */
if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
@@ -821,7 +825,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
struct drm_syncobj *timeline_syncobj = NULL;
struct dma_fence_chain *timeline_chain = NULL;
struct drm_exec exec;
- uint64_t vm_size;
+ uint64_t vm_size, tmp;
int r = 0;
/* Validate virtual address range against reserved regions. */
@@ -845,7 +849,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
vm_size -= AMDGPU_VA_RESERVED_TOP;
- if (args->va_address + args->map_size > vm_size) {
+ if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) {
dev_dbg(dev->dev,
"va_address 0x%llx is in top reserved area 0x%llx\n",
args->va_address + args->map_size, vm_size);
@@ -1089,9 +1093,16 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
* If that number is larger than the size of the array, the ioctl must
* be retried.
*/
+ if (args->num_entries > INT_MAX / sizeof(*vm_entries)) {
+ r = -EINVAL;
+ goto out_exec;
+ }
+
vm_entries = kvcalloc(args->num_entries, sizeof(*vm_entries), GFP_KERNEL);
- if (!vm_entries)
- return -ENOMEM;
+ if (!vm_entries) {
+ r = -ENOMEM;
+ goto out_exec;
+ }
amdgpu_vm_bo_va_for_each_valid_mapping(bo_va, mapping) {
if (num_mappings < args->num_entries) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 285e217..3d9497d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -314,7 +314,10 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
mc->gart_start = max_mc_address - mc->gart_size + 1;
break;
case AMDGPU_GART_PLACEMENT_LOW:
- mc->gart_start = 0;
+ if (size_bf >= mc->gart_size)
+ mc->gart_start = 0;
+ else
+ mc->gart_start = ALIGN(mc->fb_end, four_gb);
break;
case AMDGPU_GART_PLACEMENT_BEST_FIT:
default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 620fddd..a5d26b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -199,11 +199,18 @@ int amdgpu_gtt_mgr_alloc_entries(struct amdgpu_gtt_mgr *mgr,
enum drm_mm_insert_mode mode)
{
struct amdgpu_device *adev = container_of(mgr, typeof(*adev), mman.gtt_mgr);
+ u32 alignment = 0;
int r;
+ /* Align to TLB L2 cache entry size to work around "V bit HW bug" */
+ if (adev->asic_type == CHIP_TAHITI) {
+ alignment = 32 * 1024 / AMDGPU_GPU_PAGE_SIZE;
+ num_pages = ALIGN(num_pages, alignment);
+ }
+
spin_lock(&mgr->lock);
r = drm_mm_insert_node_in_range(&mgr->mm, mm_node, num_pages,
- 0, GART_ENTRY_WITHOUT_BO_COLOR, 0,
+ alignment, GART_ENTRY_WITHOUT_BO_COLOR, 0,
adev->gmc.gart_size >> PAGE_SHIFT,
mode);
spin_unlock(&mgr->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
index f72990a..5bfa5a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
@@ -51,8 +51,6 @@
#include "amdgpu_amdkfd.h"
#include "amdgpu_hmm.h"
-#define MAX_WALK_BYTE (2UL << 30)
-
/**
* amdgpu_hmm_invalidate_gfx - callback to notify about mm change
*
@@ -78,6 +76,7 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni,
mmu_interval_set_seq(mni, cur_seq);
+ amdgpu_vm_bo_invalidate(bo, false);
r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP,
false, MAX_SCHEDULE_TIMEOUT);
mutex_unlock(&adev->notifier_lock);
@@ -170,11 +169,13 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
void *owner,
struct amdgpu_hmm_range *range)
{
- unsigned long end;
+ const u64 max_bytes = SZ_2G;
+
+ struct hmm_range *hmm_range = &range->hmm_range;
unsigned long timeout;
unsigned long *pfns;
- int r = 0;
- struct hmm_range *hmm_range = &range->hmm_range;
+ unsigned long end;
+ int r;
pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
if (unlikely(!pfns)) {
@@ -191,8 +192,9 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
end = start + npages * PAGE_SIZE;
hmm_range->dev_private_owner = owner;
+ hmm_range->notifier_seq = mmu_interval_read_begin(notifier);
do {
- hmm_range->end = min(hmm_range->start + MAX_WALK_BYTE, end);
+ hmm_range->end = min(hmm_range->start + max_bytes, end);
pr_debug("hmm range: start = 0x%lx, end = 0x%lx",
hmm_range->start, hmm_range->end);
@@ -200,7 +202,6 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
retry:
- hmm_range->notifier_seq = mmu_interval_read_begin(notifier);
r = hmm_range_fault(hmm_range);
if (unlikely(r)) {
if (r == -EBUSY && !time_after(jiffies, timeout))
@@ -210,7 +211,7 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
if (hmm_range->end == end)
break;
- hmm_range->hmm_pfns += MAX_WALK_BYTE >> PAGE_SHIFT;
+ hmm_range->hmm_pfns += max_bytes >> PAGE_SHIFT;
hmm_range->start = hmm_range->end;
} while (hmm_range->end < end);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 06efce3..71272f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -873,68 +873,59 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
? -EFAULT : 0;
}
case AMDGPU_INFO_READ_MMR_REG: {
- int ret = 0;
- unsigned int n, alloc_size;
- uint32_t *regs;
unsigned int se_num = (info->read_mmr_reg.instance >>
AMDGPU_INFO_MMR_SE_INDEX_SHIFT) &
AMDGPU_INFO_MMR_SE_INDEX_MASK;
unsigned int sh_num = (info->read_mmr_reg.instance >>
AMDGPU_INFO_MMR_SH_INDEX_SHIFT) &
AMDGPU_INFO_MMR_SH_INDEX_MASK;
-
- if (!down_read_trylock(&adev->reset_domain->sem))
- return -ENOENT;
+ unsigned int alloc_size;
+ uint32_t *regs;
+ int ret;
/* set full masks if the userspace set all bits
* in the bitfields
*/
- if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) {
+ if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK)
se_num = 0xffffffff;
- } else if (se_num >= AMDGPU_GFX_MAX_SE) {
- ret = -EINVAL;
- goto out;
- }
+ else if (se_num >= AMDGPU_GFX_MAX_SE)
+ return -EINVAL;
- if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) {
+ if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK)
sh_num = 0xffffffff;
- } else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) {
- ret = -EINVAL;
- goto out;
- }
+ else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE)
+ return -EINVAL;
- if (info->read_mmr_reg.count > 128) {
- ret = -EINVAL;
- goto out;
- }
+ if (info->read_mmr_reg.count > 128)
+ return -EINVAL;
- regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL);
- if (!regs) {
- ret = -ENOMEM;
- goto out;
- }
+ regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs),
+ GFP_KERNEL);
+ if (!regs)
+ return -ENOMEM;
+ down_read(&adev->reset_domain->sem);
alloc_size = info->read_mmr_reg.count * sizeof(*regs);
-
amdgpu_gfx_off_ctrl(adev, false);
+ ret = 0;
for (i = 0; i < info->read_mmr_reg.count; i++) {
if (amdgpu_asic_read_register(adev, se_num, sh_num,
info->read_mmr_reg.dword_offset + i,
®s[i])) {
DRM_DEBUG_KMS("unallowed offset %#x\n",
info->read_mmr_reg.dword_offset + i);
- kfree(regs);
- amdgpu_gfx_off_ctrl(adev, true);
ret = -EFAULT;
- goto out;
+ break;
}
}
amdgpu_gfx_off_ctrl(adev, true);
- n = copy_to_user(out, regs, min(size, alloc_size));
- kfree(regs);
- ret = (n ? -EFAULT : 0);
-out:
up_read(&adev->reset_domain->sem);
+
+ if (!ret) {
+ ret = copy_to_user(out, regs, min(size, alloc_size))
+ ? -EFAULT : 0;
+ }
+ kfree(regs);
return ret;
}
case AMDGPU_INFO_DEV_INFO: {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 912c9af..4d68732 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -96,7 +96,8 @@ struct amdgpu_bo_va {
* if non-zero, cannot unmap from GPU because user queues may still access it
*/
unsigned int queue_refcount;
- atomic_t userq_va_mapped;
+ /* Indicates if this buffer is mapped for any user queue. Once set, never reset. */
+ bool userq_va_mapped;
};
struct amdgpu_bo {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 6c644cf..fc9f3ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -2280,7 +2280,8 @@ void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
list_for_each_entry(obj, &con->head, node) {
if (amdgpu_ras_is_supported(adev, obj->head.block) &&
(obj->attr_inuse == 1)) {
- sprintf(fs_info.debugfs_name, "%s_err_inject",
+ snprintf(fs_info.debugfs_name, sizeof(fs_info.debugfs_name),
+ "%s_err_inject",
get_ras_block_str(&obj->head));
fs_info.head = obj->head;
amdgpu_ras_debugfs_create(adev, &fs_info, dir);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index cdf4909..0c57fe2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -1950,7 +1950,7 @@ void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev)
if (!control || amdgpu_bad_page_threshold == 0)
return;
- if (control->ras_num_bad_pages >= ras->bad_page_cnt_threshold) {
+ if (control->ras_num_bad_pages > ras->bad_page_cnt_threshold) {
if (amdgpu_dpm_send_rma_reason(adev))
dev_warn(adev->dev, "Unable to send out-of-band RMA CPER");
else
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 66e8a2f..d6bee5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -552,8 +552,9 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
size_t size, loff_t *pos)
{
struct amdgpu_ring *ring = file_inode(f)->i_private;
- uint32_t value, result, early[3];
+ u32 value, result, early[3] = { 0 };
uint64_t p;
+ u32 avail_dw, start_dw, read_dw;
loff_t i;
int r;
@@ -565,10 +566,10 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
result = 0;
- if (*pos < 12) {
- if (ring->funcs->type == AMDGPU_RING_TYPE_CPER)
- mutex_lock(&ring->adev->cper.ring_lock);
+ if (ring->funcs->type == AMDGPU_RING_TYPE_CPER)
+ mutex_lock(&ring->adev->cper.ring_lock);
+ if (*pos < 12) {
early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask;
early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask;
early[2] = ring->wptr & ring->buf_mask;
@@ -600,13 +601,24 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
*pos += 4;
}
} else {
+ early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask;
+ early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask;
+
p = early[0];
if (early[0] <= early[1])
- size = (early[1] - early[0]);
+ avail_dw = early[1] - early[0];
else
- size = ring->ring_size - (early[0] - early[1]);
+ avail_dw = ring->buf_mask + 1 - (early[0] - early[1]);
- while (size) {
+ start_dw = (*pos > 12) ? ((*pos - 12) >> 2) : 0;
+ if (start_dw >= avail_dw)
+ goto out;
+
+ p = (p + start_dw) & ring->ptr_mask;
+ avail_dw -= start_dw;
+ read_dw = min_t(u32, avail_dw, size >> 2);
+
+ while (read_dw) {
if (p == early[1])
goto out;
@@ -619,9 +631,10 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
buf += 4;
result += 4;
- size--;
+ read_dw--;
p++;
p &= ring->ptr_mask;
+ *pos += 4;
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
index a0b479d..f4be192 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
@@ -175,11 +175,14 @@ int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *va,
{
unsigned long bit_pos;
- bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem);
- if (bit_pos >= adev->seq64.num_sem)
- return -ENOSPC;
+ for (;;) {
+ bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem);
+ if (bit_pos >= adev->seq64.num_sem)
+ return -ENOSPC;
- __set_bit(bit_pos, adev->seq64.used);
+ if (!test_and_set_bit(bit_pos, adev->seq64.used))
+ break;
+ }
*va = bit_pos * sizeof(u64) + amdgpu_seq64_get_va_base(adev);
@@ -205,7 +208,7 @@ void amdgpu_seq64_free(struct amdgpu_device *adev, u64 va)
bit_pos = (va - amdgpu_seq64_get_va_base(adev)) / sizeof(u64);
if (bit_pos < adev->seq64.num_sem)
- __clear_bit(bit_pos, adev->seq64.used);
+ clear_bit(bit_pos, adev->seq64.used);
}
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0dc68fb..3d2e00e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -75,6 +75,9 @@ static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
unsigned int type,
uint64_t size_in_page)
{
+ if (!size_in_page)
+ return 0;
+
return ttm_range_man_init(&adev->mman.bdev, type,
false, size_in_page);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 0238c27..b8ed931 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -130,6 +130,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
adev->umc.ras->ras_block.hw_ops->query_ras_error_address &&
adev->umc.max_ras_err_cnt_per_query) {
+ kfree(err_data->err_addr);
err_data->err_addr =
kzalloc_objs(struct eeprom_table_record,
adev->umc.max_ras_err_cnt_per_query);
@@ -160,6 +161,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
if (adev->umc.ras &&
adev->umc.ras->ecc_info_query_ras_error_address &&
adev->umc.max_ras_err_cnt_per_query) {
+ kfree(err_data->err_addr);
err_data->err_addr =
kzalloc_objs(struct eeprom_table_record,
adev->umc.max_ras_err_cnt_per_query);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index d5abf78..cf19250 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
return false;
}
-static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
+static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
{
- if (amdgpu_device_should_recover_gpu(adev)) {
- amdgpu_reset_domain_schedule(adev->reset_domain,
- &adev->userq_reset_work);
- /* Wait for the reset job to complete */
- flush_work(&adev->userq_reset_work);
- }
-}
-
-static int
-amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
-{
+ struct amdgpu_userq_mgr *uq_mgr =
+ container_of(work, struct amdgpu_userq_mgr,
+ reset_work);
struct amdgpu_device *adev = uq_mgr->adev;
const int queue_types[] = {
AMDGPU_RING_TYPE_COMPUTE,
@@ -103,15 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
};
const int num_queue_types = ARRAY_SIZE(queue_types);
bool gpu_reset = false;
- int r = 0;
- int i;
-
- /* Warning if current process mutex is not held */
- WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
+ int i, r;
if (unlikely(adev->debug_disable_gpu_ring_reset)) {
dev_err(adev->dev, "userq reset disabled by debug mask\n");
- return 0;
+ return;
}
/*
@@ -119,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
* skip all reset detection logic
*/
if (!amdgpu_gpu_recovery)
- return 0;
+ return;
/*
* Iterate through all queue types to detect and reset problematic queues
@@ -127,9 +115,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
*/
for (i = 0; i < num_queue_types; i++) {
int ring_type = queue_types[i];
- const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type];
+ const struct amdgpu_userq_funcs *funcs =
+ adev->userq_funcs[ring_type];
- if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))
+ if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
+ AMDGPU_RESET_TYPE_PER_QUEUE))
continue;
if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
@@ -142,46 +132,43 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
}
}
- if (gpu_reset)
- amdgpu_userq_gpu_reset(adev);
+ if (gpu_reset) {
+ struct amdgpu_reset_context reset_context;
- return r;
+ memset(&reset_context, 0, sizeof(reset_context));
+
+ reset_context.method = AMD_RESET_METHOD_NONE;
+ reset_context.reset_req_dev = adev;
+ reset_context.src = AMDGPU_RESET_SRC_USERQ;
+ set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+ /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
+
+ amdgpu_device_gpu_recover(adev, NULL, &reset_context);
+ }
}
static void amdgpu_userq_hang_detect_work(struct work_struct *work)
{
- struct amdgpu_usermode_queue *queue = container_of(work,
- struct amdgpu_usermode_queue,
- hang_detect_work.work);
- struct dma_fence *fence;
- struct amdgpu_userq_mgr *uq_mgr;
+ struct amdgpu_usermode_queue *queue =
+ container_of(work, struct amdgpu_usermode_queue,
+ hang_detect_work.work);
- if (!queue->userq_mgr)
- return;
-
- uq_mgr = queue->userq_mgr;
- fence = READ_ONCE(queue->hang_detect_fence);
- /* Fence already signaled – no action needed */
- if (!fence || dma_fence_is_signaled(fence))
- return;
-
- mutex_lock(&uq_mgr->userq_mutex);
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
- mutex_unlock(&uq_mgr->userq_mutex);
+ /*
+ * Don't schedule the work here! Scheduling or queue work from one reset
+ * handler to another is illegal if you don't take extra precautions!
+ */
+ amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
}
/*
* Start hang detection for a user queue fence. A delayed work will be scheduled
- * to check if the fence is still pending after the timeout period.
-*/
+ * to reset the queues when the fence doesn't signal in time.
+ */
void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
{
struct amdgpu_device *adev;
unsigned long timeout_ms;
- if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)
- return;
-
adev = queue->userq_mgr->adev;
/* Determine timeout based on queue type */
switch (queue->queue_type) {
@@ -199,45 +186,44 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
break;
}
- /* Store the fence to monitor and schedule hang detection */
- WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);
- schedule_delayed_work(&queue->hang_detect_work,
- msecs_to_jiffies(timeout_ms));
+ queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
+ msecs_to_jiffies(timeout_ms));
}
-static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue)
+void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
{
- INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);
- queue->hang_detect_fence = NULL;
-}
+ struct xarray *xa = &adev->userq_doorbell_xa;
+ struct amdgpu_usermode_queue *queue;
+ unsigned long flags;
+ int r;
-static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
- struct amdgpu_bo_va_mapping *va_map, u64 addr)
-{
- struct amdgpu_userq_va_cursor *va_cursor;
- struct userq_va_list;
+ xa_lock_irqsave(xa, flags);
+ queue = xa_load(xa, doorbell);
+ if (queue) {
+ r = amdgpu_userq_fence_driver_process(queue->fence_drv);
+ /*
+ * We are in interrupt context here, this *can't* wait for
+ * reset work to finish.
+ */
+ if (r >= 0)
+ cancel_delayed_work(&queue->hang_detect_work);
- va_cursor = kzalloc_obj(*va_cursor);
- if (!va_cursor)
- return -ENOMEM;
-
- INIT_LIST_HEAD(&va_cursor->list);
- va_cursor->gpu_addr = addr;
- atomic_set(&va_map->bo_va->userq_va_mapped, 1);
- list_add(&va_cursor->list, &queue->userq_va_list);
-
- return 0;
+ /* Restart the timer when there are still fences pending */
+ if (r == 1)
+ amdgpu_userq_start_hang_detect_work(queue);
+ }
+ xa_unlock_irqrestore(xa, flags);
}
int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
struct amdgpu_usermode_queue *queue,
- u64 addr, u64 expected_size)
+ u64 addr, u64 expected_size,
+ u64 *va_out)
{
struct amdgpu_bo_va_mapping *va_map;
struct amdgpu_vm *vm = queue->vm;
u64 user_addr;
u64 size;
- int r = 0;
/* Caller must hold vm->root.bo reservation */
dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv);
@@ -246,20 +232,18 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
size = expected_size >> AMDGPU_GPU_PAGE_SHIFT;
va_map = amdgpu_vm_bo_lookup_mapping(vm, user_addr);
- if (!va_map) {
- r = -EINVAL;
- goto out_err;
- }
+ if (!va_map)
+ return -EINVAL;
+
/* Only validate the userq whether resident in the VM mapping range */
if (user_addr >= va_map->start &&
va_map->last - user_addr + 1 >= size) {
- amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr);
+ va_map->bo_va->userq_va_mapped = true;
+ *va_out = user_addr;
return 0;
}
- r = -EINVAL;
-out_err:
- return r;
+ return -EINVAL;
}
static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
@@ -270,7 +254,7 @@ static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
dma_resv_assert_held(vm->root.bo->tbo.base.resv);
mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
- if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped))
+ if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped)
r = true;
else
r = false;
@@ -280,14 +264,16 @@ static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
{
- struct amdgpu_userq_va_cursor *va_cursor, *tmp;
- int r = 0;
+ int i, r = 0;
- list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {
- r += amdgpu_userq_buffer_va_mapped(queue->vm, va_cursor->gpu_addr);
+ for (i = 0; i < ARRAY_SIZE(queue->userq_vas.va_array); i++) {
+ if (!queue->userq_vas.va_array[i])
+ continue;
+ r += amdgpu_userq_buffer_va_mapped(queue->vm,
+ queue->userq_vas.va_array[i]);
dev_dbg(queue->userq_mgr->adev->dev,
"validate the userq mapping:%p va:%llx r:%d\n",
- queue, va_cursor->gpu_addr, r);
+ queue, queue->userq_vas.va_array[i], r);
}
if (r != 0)
@@ -296,35 +282,7 @@ static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
return false;
}
-static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping,
- struct amdgpu_userq_va_cursor *va_cursor)
-{
- atomic_set(&mapping->bo_va->userq_va_mapped, 0);
- list_del(&va_cursor->list);
- kfree(va_cursor);
-}
-static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
- struct amdgpu_usermode_queue *queue)
-{
- struct amdgpu_userq_va_cursor *va_cursor, *tmp;
- struct amdgpu_bo_va_mapping *mapping;
-
- /* Caller must hold vm->root.bo reservation */
- dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv);
-
- list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {
- mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr);
- if (!mapping) {
- return -EINVAL;
- }
- dev_dbg(adev->dev, "delete the userq:%p va:%llx\n",
- queue, va_cursor->gpu_addr);
- amdgpu_userq_buffer_va_list_del(mapping, va_cursor);
- }
-
- return 0;
-}
static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
{
@@ -332,23 +290,18 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- bool found_hung_queue = false;
- int r = 0;
+ int r;
if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
r = userq_funcs->preempt(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- found_hung_queue = true;
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
}
}
-
- if (found_hung_queue)
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
- return r;
+ return 0;
}
static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
@@ -377,24 +330,21 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- bool found_hung_queue = false;
- int r = 0;
+ int r;
if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
- (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+ (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+
r = userq_funcs->unmap(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- found_hung_queue = true;
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
}
}
- if (found_hung_queue)
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
-
- return r;
+ return 0;
}
static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
@@ -403,19 +353,19 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
struct amdgpu_device *adev = uq_mgr->adev;
const struct amdgpu_userq_funcs *userq_funcs =
adev->userq_funcs[queue->queue_type];
- int r = 0;
+ int r;
if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
r = userq_funcs->map(queue);
if (r) {
queue->state = AMDGPU_USERQ_STATE_HUNG;
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
+ return r;
} else {
queue->state = AMDGPU_USERQ_STATE_MAPPED;
}
}
- return r;
+ return 0;
}
static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
@@ -432,18 +382,14 @@ static void amdgpu_userq_cleanup(struct amdgpu_usermode_queue *queue)
{
struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
struct amdgpu_device *adev = uq_mgr->adev;
- const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
/* Wait for mode-1 reset to complete */
down_read(&adev->reset_domain->sem);
- uq_funcs->mqd_destroy(queue);
/* Use interrupt-safe locking since IRQ handlers may access these XArrays */
xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);
amdgpu_userq_fence_driver_free(queue);
queue->fence_drv = NULL;
- queue->userq_mgr = NULL;
- list_del(&queue->userq_va_list);
up_read(&adev->reset_domain->sem);
}
@@ -482,74 +428,15 @@ amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
dma_fence_put(ev_fence);
}
-int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
- struct amdgpu_userq_obj *userq_obj,
- int size)
-{
- struct amdgpu_device *adev = uq_mgr->adev;
- struct amdgpu_bo_param bp;
- int r;
- memset(&bp, 0, sizeof(bp));
- bp.byte_align = PAGE_SIZE;
- bp.domain = AMDGPU_GEM_DOMAIN_GTT;
- bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
- AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
- bp.type = ttm_bo_type_kernel;
- bp.size = size;
- bp.resv = NULL;
- bp.bo_ptr_size = sizeof(struct amdgpu_bo);
- r = amdgpu_bo_create(adev, &bp, &userq_obj->obj);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to allocate BO for userqueue (%d)", r);
- return r;
- }
-
- r = amdgpu_bo_reserve(userq_obj->obj, true);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to reserve BO to map (%d)", r);
- goto free_obj;
- }
-
- r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r);
- goto unresv;
- }
-
- r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r);
- goto unresv;
- }
-
- userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj);
- amdgpu_bo_unreserve(userq_obj->obj);
- memset(userq_obj->cpu_ptr, 0, size);
- return 0;
-
-unresv:
- amdgpu_bo_unreserve(userq_obj->obj);
-
-free_obj:
- amdgpu_bo_unref(&userq_obj->obj);
- return r;
-}
-
-void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
- struct amdgpu_userq_obj *userq_obj)
-{
- amdgpu_bo_kunmap(userq_obj->obj);
- amdgpu_bo_unref(&userq_obj->obj);
-}
-
-uint64_t
+static int
amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
struct amdgpu_db_info *db_info,
- struct drm_file *filp)
+ struct drm_file *filp,
+ u64 *index)
{
- uint64_t index;
+ u64 doorbell_index;
struct drm_gem_object *gobj;
struct amdgpu_userq_obj *db_obj = db_info->db_obj;
int r, db_size;
@@ -596,12 +483,13 @@ amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
goto unpin_bo;
}
- index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj,
- db_info->doorbell_offset, db_size);
+ doorbell_index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj,
+ db_info->doorbell_offset, db_size);
drm_dbg_driver(adev_to_drm(uq_mgr->adev),
- "[Usermode queues] doorbell index=%lld\n", index);
+ "[Usermode queues] doorbell index=%lld\n", doorbell_index);
amdgpu_bo_unreserve(db_obj->obj);
- return index;
+ *index = doorbell_index;
+ return 0;
unpin_bo:
amdgpu_bo_unpin(db_obj->obj);
@@ -616,9 +504,7 @@ static int
amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue)
{
struct amdgpu_device *adev = uq_mgr->adev;
- struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
- struct amdgpu_vm *vm = &fpriv->vm;
-
+ const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
int r = 0;
cancel_delayed_work_sync(&uq_mgr->resume_work);
@@ -626,33 +512,21 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
/* Cancel any pending hang detection work and cleanup */
cancel_delayed_work_sync(&queue->hang_detect_work);
- r = amdgpu_bo_reserve(vm->root.bo, false);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to reserve root bo during userqueue destroy\n");
- return r;
- }
- amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
- amdgpu_bo_unreserve(vm->root.bo);
-
mutex_lock(&uq_mgr->userq_mutex);
- queue->hang_detect_fence = NULL;
amdgpu_userq_wait_for_last_fence(queue);
#if defined(CONFIG_DEBUG_FS)
debugfs_remove_recursive(queue->debugfs_queue);
#endif
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
r = amdgpu_userq_unmap_helper(queue);
- /*TODO: It requires a reset for userq hw unmap error*/
- if (r) {
- drm_warn(adev_to_drm(uq_mgr->adev), "trying to destroy a HW mapping userq\n");
- queue->state = AMDGPU_USERQ_STATE_HUNG;
- }
-
atomic_dec(&uq_mgr->userq_count[queue->queue_type]);
amdgpu_userq_cleanup(queue);
mutex_unlock(&uq_mgr->userq_mutex);
+ cancel_delayed_work_sync(&queue->hang_detect_work);
+ uq_funcs->mqd_destroy(queue);
+ queue->userq_mgr = NULL;
+
amdgpu_bo_reserve(queue->db_obj.obj, true);
amdgpu_bo_unpin(queue->db_obj.obj);
amdgpu_bo_unreserve(queue->db_obj.obj);
@@ -724,14 +598,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
const struct amdgpu_userq_funcs *uq_funcs;
struct amdgpu_usermode_queue *queue;
struct amdgpu_db_info db_info;
- bool skip_map_queue;
- u32 qid;
uint64_t index;
- int r = 0;
- int priority =
- (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
- AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
+ int priority;
+ u32 qid;
+ int r;
+ priority =
+ (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
+ >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
r = amdgpu_userq_priority_permit(filp, priority);
if (r)
return r;
@@ -744,128 +618,121 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
uq_funcs = adev->userq_funcs[args->in.ip_type];
if (!uq_funcs) {
- drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",
- args->in.ip_type);
r = -EINVAL;
goto err_pm_runtime;
}
queue = kzalloc_obj(struct amdgpu_usermode_queue);
if (!queue) {
- drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
r = -ENOMEM;
goto err_pm_runtime;
}
- INIT_LIST_HEAD(&queue->userq_va_list);
+ kref_init(&queue->refcount);
queue->doorbell_handle = args->in.doorbell_handle;
queue->queue_type = args->in.ip_type;
queue->vm = &fpriv->vm;
queue->priority = priority;
-
- db_info.queue_type = queue->queue_type;
- db_info.doorbell_handle = queue->doorbell_handle;
- db_info.db_obj = &queue->db_obj;
- db_info.doorbell_offset = args->in.doorbell_offset;
-
queue->userq_mgr = uq_mgr;
+ INIT_DELAYED_WORK(&queue->hang_detect_work,
+ amdgpu_userq_hang_detect_work);
- /* Validate the userq virtual address.*/
- r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+ r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
if (r)
goto free_queue;
- if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) ||
- amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
- amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
+ xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
+ mutex_init(&queue->fence_drv_lock);
+ /* Make sure the queue can actually run with those virtual addresses. */
+ r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+ if (r)
+ goto free_fence_drv;
+
+ if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
+ args->in.queue_size,
+ &queue->userq_vas.va.queue_rb) ||
+ amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
+ AMDGPU_GPU_PAGE_SIZE,
+ &queue->userq_vas.va.rptr) ||
+ amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
+ AMDGPU_GPU_PAGE_SIZE,
+ &queue->userq_vas.va.wptr)) {
r = -EINVAL;
amdgpu_bo_unreserve(fpriv->vm.root.bo);
- goto clean_mapping;
+ goto free_fence_drv;
}
amdgpu_bo_unreserve(fpriv->vm.root.bo);
/* Convert relative doorbell offset into absolute doorbell index */
- index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
- if (index == (uint64_t)-EINVAL) {
+ db_info.queue_type = queue->queue_type;
+ db_info.doorbell_handle = queue->doorbell_handle;
+ db_info.db_obj = &queue->db_obj;
+ db_info.doorbell_offset = args->in.doorbell_offset;
+ r = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp, &index);
+ if (r) {
drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
- r = -EINVAL;
- goto clean_mapping;
+ goto free_fence_drv;
}
queue->doorbell_index = index;
- xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
- r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
- if (r) {
- drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
- goto clean_mapping;
- }
-
r = uq_funcs->mqd_create(queue, &args->in);
if (r) {
drm_file_err(uq_mgr->file, "Failed to create Queue\n");
- goto clean_fence_driver;
+ goto clean_doorbell_bo;
}
+ /* Update VM owner at userq submit-time for page-fault attribution. */
+ amdgpu_vm_set_task_info(&fpriv->vm);
+
+ r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
+ GFP_KERNEL));
+ if (r)
+ goto clean_mqd;
+
amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
/* don't map the queue if scheduling is halted */
- if (adev->userq_halt_for_enforce_isolation &&
- ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
- (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
- skip_map_queue = true;
- else
- skip_map_queue = false;
- if (!skip_map_queue) {
+ if (!adev->userq_halt_for_enforce_isolation ||
+ ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
+ (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
r = amdgpu_userq_map_helper(queue);
if (r) {
drm_file_err(uq_mgr->file, "Failed to map Queue\n");
- goto clean_mqd;
+ mutex_unlock(&uq_mgr->userq_mutex);
+ goto erase_doorbell;
}
}
- /* drop this refcount during queue destroy */
- kref_init(&queue->refcount);
-
- /* Wait for mode-1 reset to complete */
- down_read(&adev->reset_domain->sem);
-
- r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
- XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
- if (r) {
- if (!skip_map_queue)
- amdgpu_userq_unmap_helper(queue);
- r = -ENOMEM;
- goto clean_reset_domain;
- }
-
- r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
- if (r) {
- xa_erase(&uq_mgr->userq_xa, qid);
- if (!skip_map_queue)
- amdgpu_userq_unmap_helper(queue);
- goto clean_reset_domain;
- }
- up_read(&adev->reset_domain->sem);
-
- amdgpu_debugfs_userq_init(filp, queue, qid);
- amdgpu_userq_init_hang_detect_work(queue);
-
- args->out.queue_id = qid;
atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
mutex_unlock(&uq_mgr->userq_mutex);
+
+ r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
+ XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
+ GFP_KERNEL);
+ if (r) {
+ /*
+ * This drops the last reference which should take care of
+ * all cleanup.
+ */
+ amdgpu_userq_put(queue);
+ return r;
+ }
+
+ amdgpu_debugfs_userq_init(filp, queue, qid);
+ args->out.queue_id = qid;
return 0;
-clean_reset_domain:
- up_read(&adev->reset_domain->sem);
+erase_doorbell:
+ xa_erase_irq(&adev->userq_doorbell_xa, index);
clean_mqd:
- mutex_unlock(&uq_mgr->userq_mutex);
uq_funcs->mqd_destroy(queue);
-clean_fence_driver:
+clean_doorbell_bo:
+ amdgpu_bo_reserve(queue->db_obj.obj, true);
+ amdgpu_bo_unpin(queue->db_obj.obj);
+ amdgpu_bo_unreserve(queue->db_obj.obj);
+ amdgpu_bo_unref(&queue->db_obj.obj);
+free_fence_drv:
amdgpu_userq_fence_driver_free(queue);
-clean_mapping:
- amdgpu_bo_reserve(fpriv->vm.root.bo, true);
- amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
- amdgpu_bo_unreserve(fpriv->vm.root.bo);
free_queue:
kfree(queue);
err_pm_runtime:
@@ -1187,7 +1054,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
bo = range->bo;
ret = amdgpu_ttm_tt_get_user_pages(bo, range);
if (ret)
- goto unlock_all;
+ goto free_ranges;
}
invalidated = true;
@@ -1214,6 +1081,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
unlock_all:
drm_exec_fini(&exec);
+free_ranges:
xa_for_each(&xa, tmp_key, range) {
if (!range)
continue;
@@ -1254,7 +1122,6 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
unsigned long queue_id;
int ret = 0, r;
- amdgpu_userq_detect_and_reset_queues(uq_mgr);
/* Try to unmap all the queues in this process ctx */
xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
r = amdgpu_userq_preempt_helper(queue);
@@ -1262,29 +1129,16 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
ret = r;
}
- if (ret)
+ if (ret) {
drm_file_err(uq_mgr->file,
"Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
+ amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
+ &uq_mgr->reset_work);
+ flush_work(&uq_mgr->reset_work);
+ }
return ret;
}
-void amdgpu_userq_reset_work(struct work_struct *work)
-{
- struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
- userq_reset_work);
- struct amdgpu_reset_context reset_context;
-
- memset(&reset_context, 0, sizeof(reset_context));
-
- reset_context.method = AMD_RESET_METHOD_NONE;
- reset_context.reset_req_dev = adev;
- reset_context.src = AMDGPU_RESET_SRC_USERQ;
- set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
- /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
-
- amdgpu_device_gpu_recover(adev, NULL, &reset_context);
-}
-
static void
amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
{
@@ -1318,9 +1172,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
userq_mgr->file = file_priv;
INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
+ INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
return 0;
}
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
+{
+ struct xarray *xa = &adev->userq_doorbell_xa;
+ struct amdgpu_usermode_queue *queue;
+ unsigned long flags, queue_id;
+
+ xa_lock_irqsave(xa, flags);
+ xa_for_each(xa, queue_id, queue) {
+ cancel_delayed_work(&queue->hang_detect_work);
+ cancel_work(&queue->userq_mgr->reset_work);
+ }
+ xa_unlock_irqrestore(xa, flags);
+}
+
void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
{
cancel_delayed_work_sync(&userq_mgr->resume_work);
@@ -1346,6 +1215,14 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
}
xa_destroy(&userq_mgr->userq_xa);
+
+ /*
+ * Drain any in-flight reset_work. By this point all queues are freed
+ * and userq_count is 0, so if reset_work starts now it exits early.
+ * We still need to wait in case it was already executing gpu_recover.
+ */
+ cancel_work_sync(&userq_mgr->reset_work);
+
mutex_destroy(&userq_mgr->userq_mutex);
}
@@ -1364,7 +1241,6 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
uqm = queue->userq_mgr;
cancel_delayed_work_sync(&uqm->resume_work);
guard(mutex)(&uqm->userq_mutex);
- amdgpu_userq_detect_and_reset_queues(uqm);
if (adev->in_s0ix)
r = amdgpu_userq_preempt_helper(queue);
else
@@ -1423,7 +1299,6 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
(queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
(queue->xcp_id == idx)) {
- amdgpu_userq_detect_and_reset_queues(uqm);
r = amdgpu_userq_preempt_helper(queue);
if (r)
ret = r;
@@ -1496,23 +1371,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
{
const struct amdgpu_userq_funcs *userq_funcs;
struct amdgpu_usermode_queue *queue;
- struct amdgpu_userq_mgr *uqm;
unsigned long queue_id;
+ /* TODO: We probably need a new lock for the queue state */
xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
- uqm = queue->userq_mgr;
- cancel_delayed_work_sync(&uqm->resume_work);
- if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
- amdgpu_userq_wait_for_last_fence(queue);
- userq_funcs = adev->userq_funcs[queue->queue_type];
- userq_funcs->unmap(queue);
- /* just mark all queues as hung at this point.
- * if unmap succeeds, we could map again
- * in amdgpu_userq_post_reset() if vram is not lost
- */
- queue->state = AMDGPU_USERQ_STATE_HUNG;
- amdgpu_userq_fence_driver_force_completion(queue);
- }
+ if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
+ continue;
+
+ userq_funcs = adev->userq_funcs[queue->queue_type];
+ userq_funcs->unmap(queue);
+ /* just mark all queues as hung at this point.
+ * if unmap succeeds, we could map again
+ * in amdgpu_userq_post_reset() if vram is not lost
+ */
+ queue->state = AMDGPU_USERQ_STATE_HUNG;
+ amdgpu_userq_fence_driver_force_completion(queue);
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 675fe63..28cfc66 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -48,11 +48,6 @@ struct amdgpu_userq_obj {
struct amdgpu_bo *obj;
};
-struct amdgpu_userq_va_cursor {
- u64 gpu_addr;
- struct list_head list;
-};
-
struct amdgpu_usermode_queue {
int queue_type;
enum amdgpu_userq_state state;
@@ -66,17 +61,44 @@ struct amdgpu_usermode_queue {
struct amdgpu_userq_obj db_obj;
struct amdgpu_userq_obj fw_obj;
struct amdgpu_userq_obj wptr_obj;
+
+ /**
+ * @fence_drv_lock: Protecting @fence_drv_xa.
+ */
+ struct mutex fence_drv_lock;
+
+ /**
+ * @fence_drv_xa:
+ *
+ * References to the external fence drivers returned by wait_ioctl.
+ * Dropped on the next signaled dma_fence or queue destruction.
+ */
struct xarray fence_drv_xa;
struct amdgpu_userq_fence_driver *fence_drv;
struct dma_fence *last_fence;
u32 xcp_id;
int priority;
struct dentry *debugfs_queue;
- struct delayed_work hang_detect_work;
- struct dma_fence *hang_detect_fence;
+
+ /**
+ * @hang_detect_work:
+ *
+ * Delayed work which runs when userq_fences time out.
+ */
+ struct delayed_work hang_detect_work;
struct kref refcount;
- struct list_head userq_va_list;
+ union {
+ struct {
+ u64 queue_rb;
+ u64 wptr;
+ u64 rptr;
+ u64 eop;
+ u64 shadow;
+ u64 csa;
+ } va;
+ u64 va_array[6];
+ } userq_vas;
};
struct amdgpu_userq_funcs {
@@ -105,6 +127,13 @@ struct amdgpu_userq_mgr {
struct amdgpu_device *adev;
struct delayed_work resume_work;
struct drm_file *file;
+
+ /**
+ * @reset_work:
+ *
+ * Reset work which is used when eviction fails.
+ */
+ struct work_struct reset_work;
atomic_t userq_count[AMDGPU_RING_TYPE_MAX];
};
@@ -123,25 +152,15 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
struct amdgpu_device *adev);
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
-int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
- struct amdgpu_userq_obj *userq_obj,
- int size);
-
-void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
- struct amdgpu_userq_obj *userq_obj);
-
void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr);
void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
struct amdgpu_eviction_fence_mgr *evf_mgr);
-uint64_t amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
- struct amdgpu_db_info *db_info,
- struct drm_file *filp);
-
u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev);
bool amdgpu_userq_enabled(struct drm_device *dev);
@@ -156,10 +175,12 @@ void amdgpu_userq_reset_work(struct work_struct *work);
void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
+void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
struct amdgpu_usermode_queue *queue,
- u64 addr, u64 expected_size);
+ u64 addr, u64 expected_size, u64 *va_out);
+
void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping,
uint64_t saddr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index da39ac8..a41fb72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -32,29 +32,9 @@
#include "amdgpu.h"
#include "amdgpu_userq_fence.h"
-static const struct dma_fence_ops amdgpu_userq_fence_ops;
-static struct kmem_cache *amdgpu_userq_fence_slab;
-
#define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
-int amdgpu_userq_fence_slab_init(void)
-{
- amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
- sizeof(struct amdgpu_userq_fence),
- 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!amdgpu_userq_fence_slab)
- return -ENOMEM;
-
- return 0;
-}
-
-void amdgpu_userq_fence_slab_fini(void)
-{
- rcu_barrier();
- kmem_cache_destroy(amdgpu_userq_fence_slab);
-}
+static const struct dma_fence_ops amdgpu_userq_fence_ops;
static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
{
@@ -141,6 +121,7 @@ amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
userq->last_fence = NULL;
amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
xa_destroy(&userq->fence_drv_xa);
+ mutex_destroy(&userq->fence_drv_lock);
/* Drop the queue's ownership reference to fence_drv explicitly */
amdgpu_userq_fence_driver_put(userq->fence_drv);
}
@@ -154,7 +135,14 @@ amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
userq_fence->fence_drv_array_count = 0;
}
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
+/*
+ * Returns:
+ * -ENOENT when no fences were processes
+ * 1 when more fences are pending
+ * 0 when no fences are pending any more
+ */
+int
+amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
{
struct amdgpu_userq_fence *userq_fence, *tmp;
LIST_HEAD(to_be_signaled);
@@ -162,9 +150,6 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
unsigned long flags;
u64 rptr;
- if (!fence_drv)
- return;
-
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
rptr = amdgpu_userq_fence_read(fence_drv);
@@ -177,6 +162,9 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
&userq_fence->link);
spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
+ if (list_empty(&to_be_signaled))
+ return -ENOENT;
+
list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
fence = &userq_fence->base;
list_del_init(&userq_fence->link);
@@ -188,6 +176,8 @@ void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_d
dma_fence_put(fence);
}
+ /* That doesn't need to be accurate so no locking */
+ return list_empty(&fence_drv->fences) ? 0 : 1;
}
void amdgpu_userq_fence_driver_destroy(struct kref *ref)
@@ -229,80 +219,84 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
}
-static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
+static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
+ struct amdgpu_userq_fence **pfence)
{
- *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
- return *userq_fence ? 0 : -ENOMEM;
+ struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
+ struct amdgpu_userq_fence *userq_fence;
+ void *entry;
+
+ userq_fence = kmalloc(sizeof(*userq_fence), GFP_KERNEL);
+ if (!userq_fence)
+ return -ENOMEM;
+
+ /*
+ * Get the next unused entry, since we fill from the start this can be
+ * used as size to allocate the array.
+ */
+ mutex_lock(&userq->fence_drv_lock);
+ XA_STATE(xas, &userq->fence_drv_xa, 0);
+
+ rcu_read_lock();
+ do {
+ entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
+ } while (xas_retry(&xas, entry));
+ rcu_read_unlock();
+
+ userq_fence->fence_drv_array = kvmalloc_array(xas.xa_index,
+ sizeof(fence_drv),
+ GFP_KERNEL);
+ if (!userq_fence->fence_drv_array) {
+ mutex_unlock(&userq->fence_drv_lock);
+ kfree(userq_fence);
+ return -ENOMEM;
+ }
+
+ userq_fence->fence_drv_array_count = xas.xa_index;
+ xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
+ 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
+ xa_destroy(&userq->fence_drv_xa);
+
+ mutex_unlock(&userq->fence_drv_lock);
+
+ amdgpu_userq_fence_driver_get(fence_drv);
+ userq_fence->fence_drv = fence_drv;
+
+ *pfence = userq_fence;
+ return 0;
}
-static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
- struct amdgpu_userq_fence *userq_fence,
- u64 seq, struct dma_fence **f)
+static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
+ struct amdgpu_userq_fence *fence,
+ u64 seq)
{
- struct amdgpu_userq_fence_driver *fence_drv;
- struct dma_fence *fence;
+ struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
unsigned long flags;
bool signaled = false;
- fence_drv = userq->fence_drv;
- if (!fence_drv)
- return -EINVAL;
-
- spin_lock_init(&userq_fence->lock);
- INIT_LIST_HEAD(&userq_fence->link);
- fence = &userq_fence->base;
- userq_fence->fence_drv = fence_drv;
-
- dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
+ spin_lock_init(&fence->lock);
+ dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
fence_drv->context, seq);
- amdgpu_userq_fence_driver_get(fence_drv);
- dma_fence_get(fence);
+ /* Make sure the fence is visible to the hang detect worker */
+ dma_fence_put(userq->last_fence);
+ userq->last_fence = dma_fence_get(&fence->base);
- if (!xa_empty(&userq->fence_drv_xa)) {
- struct amdgpu_userq_fence_driver *stored_fence_drv;
- unsigned long index, count = 0;
- int i = 0;
-
- xa_lock(&userq->fence_drv_xa);
- xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
- count++;
-
- userq_fence->fence_drv_array =
- kvmalloc_objs(struct amdgpu_userq_fence_driver *, count,
- GFP_ATOMIC);
-
- if (userq_fence->fence_drv_array) {
- xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
- userq_fence->fence_drv_array[i] = stored_fence_drv;
- __xa_erase(&userq->fence_drv_xa, index);
- i++;
- }
- }
-
- userq_fence->fence_drv_array_count = i;
- xa_unlock(&userq->fence_drv_xa);
- } else {
- userq_fence->fence_drv_array = NULL;
- userq_fence->fence_drv_array_count = 0;
- }
-
- /* Check if hardware has already processed the job */
+ /* Check if hardware has already processed the fence */
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
- if (!dma_fence_is_signaled(fence)) {
- list_add_tail(&userq_fence->link, &fence_drv->fences);
+ if (!dma_fence_is_signaled(&fence->base)) {
+ dma_fence_get(&fence->base);
+ list_add_tail(&fence->link, &fence_drv->fences);
} else {
+ INIT_LIST_HEAD(&fence->link);
signaled = true;
- dma_fence_put(fence);
}
spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
if (signaled)
- amdgpu_userq_fence_put_fence_drv_array(userq_fence);
-
- *f = fence;
-
- return 0;
+ amdgpu_userq_fence_put_fence_drv_array(fence);
+ else
+ amdgpu_userq_start_hang_detect_work(userq);
}
static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
@@ -342,7 +336,7 @@ static void amdgpu_userq_fence_free(struct rcu_head *rcu)
amdgpu_userq_fence_driver_put(fence_drv);
kvfree(userq_fence->fence_drv_array);
- kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
+ kfree(userq_fence);
}
static void amdgpu_userq_fence_release(struct dma_fence *f)
@@ -376,56 +370,48 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
{
struct amdgpu_bo_va_mapping *mapping;
struct amdgpu_bo *bo;
+ struct drm_exec exec;
u64 addr, *ptr;
- int r;
-
- r = amdgpu_bo_reserve(queue->vm->root.bo, false);
- if (r)
- return r;
+ int ret;
addr = queue->userq_prop->wptr_gpu_addr;
addr &= AMDGPU_GMC_HOLE_MASK;
- mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
- if (!mapping) {
- amdgpu_bo_unreserve(queue->vm->root.bo);
- DRM_ERROR("Failed to lookup amdgpu_bo_va_mapping\n");
- return -EINVAL;
+ drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
+ drm_exec_until_all_locked(&exec) {
+ ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(ret))
+ goto lock_error;
+
+ mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
+ if (!mapping) {
+ ret = -EINVAL;
+ goto lock_error;
+ }
+
+ ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(ret))
+ goto lock_error;
}
- bo = amdgpu_bo_ref(mapping->bo_va->base.bo);
- amdgpu_bo_unreserve(queue->vm->root.bo);
- r = amdgpu_bo_reserve(bo, true);
- if (r) {
- amdgpu_bo_unref(&bo);
- DRM_ERROR("Failed to reserve userqueue wptr bo");
- return r;
- }
-
- r = amdgpu_bo_kmap(bo, (void **)&ptr);
- if (r) {
+ bo = mapping->bo_va->base.bo;
+ ret = amdgpu_bo_kmap(bo, (void **)&ptr);
+ if (ret) {
DRM_ERROR("Failed mapping the userqueue wptr bo");
- goto map_error;
+ goto lock_error;
}
*wptr = le64_to_cpu(*ptr);
amdgpu_bo_kunmap(bo);
- amdgpu_bo_unreserve(bo);
- amdgpu_bo_unref(&bo);
-
+ drm_exec_fini(&exec);
return 0;
-map_error:
- amdgpu_bo_unreserve(bo);
- amdgpu_bo_unref(&bo);
-
- return r;
-}
-
-static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
-{
- dma_fence_put(fence);
+lock_error:
+ drm_exec_fini(&exec);
+ return ret;
}
static void
@@ -471,13 +457,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
const unsigned int num_read_bo_handles = args->num_bo_read_handles;
struct amdgpu_fpriv *fpriv = filp->driver_priv;
struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
+
struct drm_gem_object **gobj_write, **gobj_read;
u32 *syncobj_handles, num_syncobj_handles;
- struct amdgpu_userq_fence *userq_fence;
- struct amdgpu_usermode_queue *queue = NULL;
- struct drm_syncobj **syncobj = NULL;
- struct dma_fence *fence;
+ struct amdgpu_usermode_queue *queue;
+ struct amdgpu_userq_fence *fence;
+ struct drm_syncobj **syncobj;
struct drm_exec exec;
+ void __user *ptr;
int r, i, entry;
u64 wptr;
@@ -489,13 +476,14 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
return -EINVAL;
num_syncobj_handles = args->num_syncobj_handles;
- syncobj_handles = memdup_array_user(u64_to_user_ptr(args->syncobj_handles),
- num_syncobj_handles, sizeof(u32));
+ ptr = u64_to_user_ptr(args->syncobj_handles);
+ syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
+ sizeof(u32));
if (IS_ERR(syncobj_handles))
return PTR_ERR(syncobj_handles);
- /* Array of pointers to the looked up syncobjs */
- syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
+ syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
+ GFP_KERNEL);
if (!syncobj) {
r = -ENOMEM;
goto free_syncobj_handles;
@@ -509,21 +497,17 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
}
}
- r = drm_gem_objects_lookup(filp,
- u64_to_user_ptr(args->bo_read_handles),
- num_read_bo_handles,
- &gobj_read);
+ ptr = u64_to_user_ptr(args->bo_read_handles);
+ r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
if (r)
goto free_syncobj;
- r = drm_gem_objects_lookup(filp,
- u64_to_user_ptr(args->bo_write_handles),
- num_write_bo_handles,
+ ptr = u64_to_user_ptr(args->bo_write_handles);
+ r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
&gobj_write);
if (r)
goto put_gobj_read;
- /* Retrieve the user queue */
queue = amdgpu_userq_get(userq_mgr, args->queue_id);
if (!queue) {
r = -ENOENT;
@@ -532,73 +516,61 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
if (r)
- goto put_gobj_write;
+ goto put_queue;
- r = amdgpu_userq_fence_alloc(&userq_fence);
+ r = amdgpu_userq_fence_alloc(queue, &fence);
if (r)
- goto put_gobj_write;
+ goto put_queue;
/* We are here means UQ is active, make sure the eviction fence is valid */
amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
- /* Create a new fence */
- r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
- if (r) {
- mutex_unlock(&userq_mgr->userq_mutex);
- kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
- goto put_gobj_write;
- }
+ /* Create the new fence */
+ amdgpu_userq_fence_init(queue, fence, wptr);
- dma_fence_put(queue->last_fence);
- queue->last_fence = dma_fence_get(fence);
- amdgpu_userq_start_hang_detect_work(queue);
mutex_unlock(&userq_mgr->userq_mutex);
+ /*
+ * This needs to come after the fence is created since
+ * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
+ * locks.
+ */
drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
(num_read_bo_handles + num_write_bo_handles));
- /* Lock all BOs with retry handling */
drm_exec_until_all_locked(&exec) {
- r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
+ r = drm_exec_prepare_array(&exec, gobj_read,
+ num_read_bo_handles, 1);
drm_exec_retry_on_contention(&exec);
- if (r) {
- amdgpu_userq_fence_cleanup(fence);
+ if (r)
goto exec_fini;
- }
- r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
+ r = drm_exec_prepare_array(&exec, gobj_write,
+ num_write_bo_handles, 1);
drm_exec_retry_on_contention(&exec);
- if (r) {
- amdgpu_userq_fence_cleanup(fence);
+ if (r)
goto exec_fini;
- }
}
- for (i = 0; i < num_read_bo_handles; i++) {
- if (!gobj_read || !gobj_read[i]->resv)
- continue;
-
- dma_resv_add_fence(gobj_read[i]->resv, fence,
+ /* And publish the new fence in the BOs and syncobj */
+ for (i = 0; i < num_read_bo_handles; i++)
+ dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
DMA_RESV_USAGE_READ);
- }
- for (i = 0; i < num_write_bo_handles; i++) {
- if (!gobj_write || !gobj_write[i]->resv)
- continue;
-
- dma_resv_add_fence(gobj_write[i]->resv, fence,
+ for (i = 0; i < num_write_bo_handles; i++)
+ dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
DMA_RESV_USAGE_WRITE);
- }
- /* Add the created fence to syncobj/BO's */
for (i = 0; i < num_syncobj_handles; i++)
- drm_syncobj_replace_fence(syncobj[i], fence);
-
- /* drop the reference acquired in fence creation function */
- dma_fence_put(fence);
+ drm_syncobj_replace_fence(syncobj[i], &fence->base);
exec_fini:
+ /* drop the reference acquired in fence creation function */
+ dma_fence_put(&fence->base);
+
drm_exec_fini(&exec);
+put_queue:
+ amdgpu_userq_put(queue);
put_gobj_write:
for (i = 0; i < num_write_bo_handles; i++)
drm_gem_object_put(gobj_write[i]);
@@ -609,15 +581,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
kvfree(gobj_read);
free_syncobj:
while (entry-- > 0)
- if (syncobj[entry])
- drm_syncobj_put(syncobj[entry]);
+ drm_syncobj_put(syncobj[entry]);
kfree(syncobj);
free_syncobj_handles:
kfree(syncobj_handles);
- if (queue)
- amdgpu_userq_put(queue);
-
return r;
}
@@ -892,8 +860,10 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
* Otherwise, we would gather those references until we don't
* have any more space left and crash.
*/
+ mutex_lock(&waitq->fence_drv_lock);
r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
xa_limit_32b, GFP_KERNEL);
+ mutex_unlock(&waitq->fence_drv_lock);
if (r)
goto put_waitq;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
index d56246a..0bd5161 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
@@ -58,15 +58,12 @@ struct amdgpu_userq_fence_driver {
char timeline_name[TASK_COMM_LEN];
};
-int amdgpu_userq_fence_slab_init(void);
-void amdgpu_userq_fence_slab_fini(void);
-
void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
struct amdgpu_userq_fence_driver **fence_drv_req);
void amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq);
-void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
+int amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv);
void amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq);
void amdgpu_userq_fence_driver_destroy(struct kref *ref);
int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 115a7b2..c9f88ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1631,6 +1631,7 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
{
struct amdgpu_bo_va *bo_va;
struct dma_resv *resv;
+ struct amdgpu_bo *bo;
bool clear, unlock;
int r;
@@ -1650,11 +1651,13 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
while (!list_empty(&vm->invalidated)) {
bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
base.vm_status);
- resv = bo_va->base.bo->tbo.base.resv;
+ bo = bo_va->base.bo;
+ resv = bo->tbo.base.resv;
spin_unlock(&vm->status_lock);
/* Try to reserve the BO to avoid clearing its ptes */
- if (!adev->debug_vm && dma_resv_trylock(resv)) {
+ if (!adev->debug_vm && !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
+ dma_resv_trylock(resv)) {
clear = false;
unlock = true;
/* The caller is already holding the reservation lock */
@@ -2002,7 +2005,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
* during user requests GEM unmap IOCTL except for forcing the unmap
* from user space.
*/
- if (unlikely(atomic_read(&bo_va->userq_va_mapped) > 0))
+ if (unlikely(bo_va->userq_va_mapped))
amdgpu_userq_gem_va_unmap_validate(adev, mapping, saddr);
list_del(&mapping->list);
@@ -3023,11 +3026,22 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
is_compute_context = vm->is_compute_context;
- if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid,
- node_id, addr >> PAGE_SHIFT, ts, write_fault)) {
+ if (is_compute_context) {
+ /* Unreserve root since svm_range_restore_pages might try to reserve it. */
+ /* TODO: rework svm_range_restore_pages so that this isn't necessary. */
amdgpu_bo_unreserve(root);
+
+ if (!svm_range_restore_pages(adev, pasid, vmid,
+ node_id, addr >> PAGE_SHIFT, ts, write_fault)) {
+ amdgpu_bo_unref(&root);
+ return true;
+ }
amdgpu_bo_unref(&root);
- return true;
+
+ /* Re-acquire the VM lock, could be that the VM was freed in between. */
+ vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid);
+ if (!vm)
+ return false;
}
addr /= AMDGPU_GPU_PAGE_SIZE;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
index fd88138..f27f917 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
@@ -562,6 +562,11 @@ static void vpe_ring_emit_fence(struct amdgpu_ring *ring, uint64_t addr,
amdgpu_ring_write(ring, 0);
}
+ /* WA: Force sync after TRAP to avoid VPE1 fail to power off */
+ if (ring->adev->vpe.collaborate_mode) {
+ amdgpu_ring_write(ring, VPE_CMD_HEADER(VPE_CMD_OPCODE_COLLAB_SYNC, 0));
+ amdgpu_ring_write(ring, 0xabcd);
+ }
}
static void vpe_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
@@ -968,7 +973,7 @@ static const struct amdgpu_ring_funcs vpe_ring_funcs = {
.emit_frame_size =
5 + /* vpe_ring_init_cond_exec */
6 + /* vpe_ring_emit_pipeline_sync */
- 10 + 10 + 10 + /* vpe_ring_emit_fence */
+ 12 + 12 + 12 + /* vpe_ring_emit_fence */
/* vpe_ring_emit_vm_flush */
SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 8c82e90..d40ab1e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -6523,15 +6523,7 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
DRM_DEBUG("IH: CP EOP\n");
if (adev->enable_mes && doorbell_offset) {
- struct amdgpu_usermode_queue *queue;
- struct xarray *xa = &adev->userq_doorbell_xa;
- unsigned long flags;
-
- xa_lock_irqsave(xa, flags);
- queue = xa_load(xa, doorbell_offset);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
- xa_unlock_irqrestore(xa, flags);
+ amdgpu_userq_process_fence_irq(adev, doorbell_offset);
} else {
me_id = (entry->ring_id & 0x0c) >> 2;
pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 65c3382..c35372e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -602,6 +602,13 @@ static int gfx_v12_0_init_microcode(struct amdgpu_device *adev)
"amdgpu/%s_pfp.bin", ucode_prefix);
if (err)
goto out;
+
+ adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
+ (union amdgpu_firmware_header *)
+ adev->gfx.pfp_fw->data, 2, 0);
+ if (adev->gfx.rs64_enable)
+ dev_dbg(adev->dev, "CP RS64 enable\n");
+
amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
@@ -4854,15 +4861,7 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev,
DRM_DEBUG("IH: CP EOP\n");
if (adev->enable_mes && doorbell_offset) {
- struct xarray *xa = &adev->userq_doorbell_xa;
- struct amdgpu_usermode_queue *queue;
- unsigned long flags;
-
- xa_lock_irqsave(xa, flags);
- queue = xa_load(xa, doorbell_offset);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
- xa_unlock_irqrestore(xa, flags);
+ amdgpu_userq_process_fence_irq(adev, doorbell_offset);
} else {
me_id = (entry->ring_id & 0x0c) >> 2;
pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index 68fd3c0..68db1bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -3643,16 +3643,7 @@ static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
DRM_DEBUG("IH: CP EOP\n");
if (adev->enable_mes && doorbell_offset) {
- struct xarray *xa = &adev->userq_doorbell_xa;
- struct amdgpu_usermode_queue *queue;
- unsigned long flags;
-
- xa_lock_irqsave(xa, flags);
- queue = xa_load(xa, doorbell_offset);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
-
- xa_unlock_irqrestore(xa, flags);
+ amdgpu_userq_process_fence_irq(adev, doorbell_offset);
} else {
me_id = (entry->ring_id & 0x0c) >> 2;
pipe_id = (entry->ring_id & 0x03) >> 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 73223d9..ac90d8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1571,6 +1571,71 @@ static void gfx_v6_0_setup_spi(struct amdgpu_device *adev)
mutex_unlock(&adev->grbm_idx_mutex);
}
+/**
+ * gfx_v6_0_setup_tcc() - setup which TCCs are used
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Verify whether the current GPU has any TCCs disabled,
+ * which can happen when the GPU is harvested and some
+ * memory channels are disabled, reducing the memory bus width.
+ * For example, on the Radeon HD 7870 XT (Tahiti LE).
+ *
+ * If some TCCs are disabled, we need to make sure that
+ * the disabled TCCs are not used, and the remaining TCCs
+ * are used optimally.
+ *
+ * TCP_CHAN_STEER_LO/HI control which TCC is used by TCP channels.
+ * TCP_ADDR_CONFIG.NUM_TCC_BANKS controls how many channels are used.
+ *
+ * For optimal performance:
+ * - Rely on the CHAN_STEER from the golden registers table,
+ * only skip disabled TCCs but keep the mapping order.
+ * - Limit NUM_TCC_BANKS to number of active TCCs to avoid thrashing,
+ * which performs better than using the same TCC twice.
+ */
+static void gfx_v6_0_setup_tcc(struct amdgpu_device *adev)
+{
+ u32 i, tcc, tcp_addr_config, num_active_tcc = 0;
+ u64 chan_steer, patched_chan_steer = 0;
+ const u32 num_max_tcc = adev->gfx.config.max_texture_channel_caches;
+ const u32 dis_tcc_mask =
+ amdgpu_gfx_create_bitmask(num_max_tcc) &
+ (REG_GET_FIELD(RREG32(mmCGTS_TCC_DISABLE),
+ CGTS_TCC_DISABLE, TCC_DISABLE) |
+ REG_GET_FIELD(RREG32(mmCGTS_USER_TCC_DISABLE),
+ CGTS_USER_TCC_DISABLE, TCC_DISABLE));
+
+ /* When no TCC is disabled, the golden registers table already has optimal TCC setup */
+ if (!dis_tcc_mask)
+ return;
+
+ /* Each 4-bit nibble contains the index of a TCC used by all TCPs */
+ chan_steer = RREG32(mmTCP_CHAN_STEER_LO) | ((u64)RREG32(mmTCP_CHAN_STEER_HI) << 32ull);
+
+ /* Patch the TCP to TCC mapping to skip disabled TCCs */
+ for (i = 0; i < num_max_tcc; ++i) {
+ tcc = (chan_steer >> (u64)(4 * i)) & 0xf;
+
+ if (!((1 << tcc) & dis_tcc_mask)) {
+ /* Copy enabled TCC indices to the patched register value. */
+ patched_chan_steer |= (u64)tcc << (u64)(4 * num_active_tcc);
+ ++num_active_tcc;
+ }
+ }
+
+ WARN_ON(num_active_tcc != num_max_tcc - hweight32(dis_tcc_mask));
+
+ /* Patch number of TCCs used by TCPs */
+ tcp_addr_config = REG_SET_FIELD(RREG32(mmTCP_ADDR_CONFIG),
+ TCP_ADDR_CONFIG, NUM_TCC_BANKS,
+ num_active_tcc - 1);
+
+ WREG32(mmTCP_ADDR_CONFIG, tcp_addr_config);
+ WREG32(mmTCP_CHAN_STEER_HI, upper_32_bits(patched_chan_steer));
+ WREG32(mmTCP_CHAN_STEER_LO, lower_32_bits(patched_chan_steer));
+}
+
static void gfx_v6_0_config_init(struct amdgpu_device *adev)
{
adev->gfx.config.double_offchip_lds_buf = 0;
@@ -1729,6 +1794,7 @@ static void gfx_v6_0_constants_init(struct amdgpu_device *adev)
gfx_v6_0_tiling_mode_table_init(adev);
gfx_v6_0_setup_rb(adev);
+ gfx_v6_0_setup_tcc(adev);
gfx_v6_0_setup_spi(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 95be105..86c7c2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -5660,9 +5660,6 @@ static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
{
struct amdgpu_device *adev = ring->adev;
- /* we only allocate 32bit for each seq wb address */
- BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
-
/* write fence seq to the "addr" */
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
index 9fe8d10..cffb1e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
@@ -802,6 +802,7 @@ static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v2_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v2_0_dec_ring_get_rptr,
.get_wptr = jpeg_v2_0_dec_ring_get_wptr,
.set_wptr = jpeg_v2_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
index 20983f1..13a6e24 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
@@ -693,6 +693,7 @@ static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v2_5_dec_ring_get_rptr,
.get_wptr = jpeg_v2_5_dec_ring_get_wptr,
.set_wptr = jpeg_v2_5_dec_ring_set_wptr,
@@ -724,6 +725,7 @@ static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = {
static const struct amdgpu_ring_funcs jpeg_v2_6_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v2_5_dec_ring_get_rptr,
.get_wptr = jpeg_v2_5_dec_ring_get_wptr,
.set_wptr = jpeg_v2_5_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
index 98f5e06..d0445df 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
@@ -594,6 +594,7 @@ static const struct amd_ip_funcs jpeg_v3_0_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v3_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v3_0_dec_ring_get_rptr,
.get_wptr = jpeg_v3_0_dec_ring_get_wptr,
.set_wptr = jpeg_v3_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
index 0bd8382..6fd4238 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
@@ -759,6 +759,7 @@ static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v4_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v4_0_dec_ring_get_rptr,
.get_wptr = jpeg_v4_0_dec_ring_get_wptr,
.set_wptr = jpeg_v4_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
index 82abe181..0c74658 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
@@ -1219,6 +1219,7 @@ static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v4_0_3_dec_ring_get_rptr,
.get_wptr = jpeg_v4_0_3_dec_ring_get_wptr,
.set_wptr = jpeg_v4_0_3_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
index 54fd9c8..a43582b 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
@@ -804,6 +804,7 @@ static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v4_0_5_dec_ring_get_rptr,
.get_wptr = jpeg_v4_0_5_dec_ring_get_wptr,
.set_wptr = jpeg_v4_0_5_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
index 46bf15d..72a4b2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
@@ -680,6 +680,7 @@ static const struct amd_ip_funcs jpeg_v5_0_0_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v5_0_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v5_0_0_dec_ring_get_rptr,
.get_wptr = jpeg_v5_0_0_dec_ring_get_wptr,
.set_wptr = jpeg_v5_0_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
index edecbfe..2503167 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
@@ -884,6 +884,7 @@ static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v5_0_1_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v5_0_1_dec_ring_get_rptr,
.get_wptr = jpeg_v5_0_1_dec_ring_get_wptr,
.set_wptr = jpeg_v5_0_1_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
index 285c459..7a4ecea 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
@@ -703,6 +703,7 @@ static const struct amd_ip_funcs jpeg_v5_0_2_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v5_0_2_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v5_0_2_dec_ring_get_rptr,
.get_wptr = jpeg_v5_0_2_dec_ring_get_wptr,
.set_wptr = jpeg_v5_0_2_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
index 1821dced..e754681 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
@@ -661,6 +661,7 @@ static const struct amd_ip_funcs jpeg_v5_3_0_ip_funcs = {
static const struct amdgpu_ring_funcs jpeg_v5_3_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_JPEG,
.align_mask = 0xf,
+ .no_user_fence = true,
.get_rptr = jpeg_v5_3_0_dec_ring_get_rptr,
.get_wptr = jpeg_v5_3_0_dec_ring_get_wptr,
.set_wptr = jpeg_v5_3_0_dec_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 2fc39a6..98aa00e 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -31,89 +31,70 @@
#define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
static int
-mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo)
-{
- int ret;
-
- ret = amdgpu_bo_reserve(bo, true);
- if (ret) {
- DRM_ERROR("Failed to reserve bo. ret %d\n", ret);
- goto err_reserve_bo_failed;
- }
-
- ret = amdgpu_ttm_alloc_gart(&bo->tbo);
- if (ret) {
- DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
- goto err_map_bo_gart_failed;
- }
-
- amdgpu_bo_unreserve(bo);
- bo = amdgpu_bo_ref(bo);
-
- return 0;
-
-err_map_bo_gart_failed:
- amdgpu_bo_unreserve(bo);
-err_reserve_bo_failed:
- return ret;
-}
-
-static int
mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
struct amdgpu_userq_mgr *uq_mgr,
struct amdgpu_usermode_queue *queue,
uint64_t wptr)
{
struct amdgpu_bo_va_mapping *wptr_mapping;
- struct amdgpu_vm *wptr_vm;
struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
+ struct amdgpu_bo *obj;
+ struct amdgpu_vm *vm = queue->vm;
+ struct drm_exec exec;
int ret;
- wptr_vm = queue->vm;
- ret = amdgpu_bo_reserve(wptr_vm->root.bo, false);
- if (ret)
- return ret;
-
wptr &= AMDGPU_GMC_HOLE_MASK;
- wptr_mapping = amdgpu_vm_bo_lookup_mapping(wptr_vm, wptr >> PAGE_SHIFT);
- amdgpu_bo_unreserve(wptr_vm->root.bo);
- if (!wptr_mapping) {
- DRM_ERROR("Failed to lookup wptr bo\n");
- return -EINVAL;
+
+ drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
+ drm_exec_until_all_locked(&exec) {
+ ret = amdgpu_vm_lock_pd(vm, &exec, 1);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(ret))
+ goto fail_lock;
+
+ wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
+ if (!wptr_mapping) {
+ ret = -EINVAL;
+ goto fail_lock;
+ }
+
+ obj = wptr_mapping->bo_va->base.bo;
+ ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
+ drm_exec_retry_on_contention(&exec);
+ if (unlikely(ret))
+ goto fail_lock;
}
- wptr_obj->obj = wptr_mapping->bo_va->base.bo;
+ wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
- DRM_ERROR("Requested GART mapping for wptr bo larger than one page\n");
- return -EINVAL;
- }
-
- ret = mes_userq_map_gtt_bo_to_gart(wptr_obj->obj);
- if (ret) {
- DRM_ERROR("Failed to map wptr bo to GART\n");
- return ret;
- }
-
- ret = amdgpu_bo_reserve(wptr_obj->obj, true);
- if (ret) {
- DRM_ERROR("Failed to reserve wptr bo\n");
- return ret;
+ ret = -EINVAL;
+ goto fail_map;
}
/* TODO use eviction fence instead of pinning. */
ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
if (ret) {
- drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin wptr bo\n");
- goto unresv_bo;
+ DRM_ERROR("Failed to pin wptr bo. ret %d\n", ret);
+ goto fail_map;
+ }
+
+ ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
+ if (ret) {
+ DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
+ goto fail_alloc_gart;
}
queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
- amdgpu_bo_unreserve(wptr_obj->obj);
+ drm_exec_fini(&exec);
return 0;
-unresv_bo:
- amdgpu_bo_unreserve(wptr_obj->obj);
+fail_alloc_gart:
+ amdgpu_bo_unpin(wptr_obj->obj);
+fail_map:
+ amdgpu_bo_unref(&wptr_obj->obj);
+fail_lock:
+ drm_exec_fini(&exec);
return ret;
}
@@ -211,12 +192,16 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
* for the same.
*/
size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ;
- r = amdgpu_userq_create_object(uq_mgr, ctx, size);
+ r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
+ AMDGPU_GEM_DOMAIN_GTT,
+ &ctx->obj, &ctx->gpu_addr,
+ &ctx->cpu_ptr);
if (r) {
DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
return r;
}
+ memset(ctx->cpu_ptr, 0, size);
return 0;
}
@@ -289,13 +274,19 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
return -ENOMEM;
}
- r = amdgpu_userq_create_object(uq_mgr, &queue->mqd,
- AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
+ r = amdgpu_bo_create_kernel(adev,
+ AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size),
+ 0, AMDGPU_GEM_DOMAIN_GTT,
+ &queue->mqd.obj, &queue->mqd.gpu_addr,
+ &queue->mqd.cpu_ptr);
if (r) {
DRM_ERROR("Failed to create MQD object for userqueue\n");
goto free_props;
}
+ memset(queue->mqd.cpu_ptr, 0,
+ AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
+
/* Initialize the MQD BO with user given values */
userq_props->wptr_gpu_addr = mqd_user->wptr_va;
userq_props->rptr_gpu_addr = mqd_user->rptr_va;
@@ -327,8 +318,9 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
kfree(compute_mqd);
goto free_mqd;
}
- r = amdgpu_userq_input_va_validate(adev, queue, compute_mqd->eop_va,
- 2048);
+ r = amdgpu_userq_input_va_validate(adev, queue,
+ compute_mqd->eop_va, 2048,
+ &queue->userq_vas.va.eop);
amdgpu_bo_unreserve(queue->vm->root.bo);
if (r) {
kfree(compute_mqd);
@@ -377,7 +369,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
goto free_mqd;
}
r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
- shadow_info.shadow_size);
+ shadow_info.shadow_size,
+ &queue->userq_vas.va.shadow);
if (r) {
amdgpu_bo_unreserve(queue->vm->root.bo);
kfree(mqd_gfx_v11);
@@ -385,7 +378,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
}
r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
- shadow_info.csa_size);
+ shadow_info.csa_size,
+ &queue->userq_vas.va.csa);
amdgpu_bo_unreserve(queue->vm->root.bo);
if (r) {
kfree(mqd_gfx_v11);
@@ -415,7 +409,8 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
goto free_mqd;
}
r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
- 32);
+ 32,
+ &queue->userq_vas.va.csa);
amdgpu_bo_unreserve(queue->vm->root.bo);
if (r) {
kfree(mqd_sdma_v11);
@@ -451,10 +446,12 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
return 0;
free_ctx:
- amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);
+ amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
+ &queue->fw_obj.cpu_ptr);
free_mqd:
- amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);
+ amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
+ &queue->mqd.cpu_ptr);
free_props:
kfree(userq_props);
@@ -464,11 +461,12 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
{
- struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
- amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);
+ amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
+ &queue->fw_obj.cpu_ptr);
kfree(queue->userq_prop);
- amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);
+ amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
+ &queue->mqd.cpu_ptr);
}
static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 44f0f23..e64f2f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -889,7 +889,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
/* write the fence */
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
/* zero in first two bits */
- BUG_ON(addr & 0x3);
+ WARN_ON(addr & 0x3);
amdgpu_ring_write(ring, lower_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(addr));
amdgpu_ring_write(ring, lower_32_bits(seq));
@@ -899,7 +899,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
addr += 4;
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
/* zero in first two bits */
- BUG_ON(addr & 0x3);
+ WARN_ON(addr & 0x3);
amdgpu_ring_write(ring, lower_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(seq));
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 0f530bb..8ca46e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1662,17 +1662,8 @@ static int sdma_v6_0_process_fence_irq(struct amdgpu_device *adev,
u32 doorbell_offset = entry->src_data[0];
if (adev->enable_mes && doorbell_offset) {
- struct amdgpu_usermode_queue *queue;
- struct xarray *xa = &adev->userq_doorbell_xa;
- unsigned long flags;
-
doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
-
- xa_lock_irqsave(xa, flags);
- queue = xa_load(xa, doorbell_offset);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
- xa_unlock_irqrestore(xa, flags);
+ amdgpu_userq_process_fence_irq(adev, doorbell_offset);
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 9ed817b..37191e2 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1594,17 +1594,8 @@ static int sdma_v7_0_process_fence_irq(struct amdgpu_device *adev,
u32 doorbell_offset = entry->src_data[0];
if (adev->enable_mes && doorbell_offset) {
- struct xarray *xa = &adev->userq_doorbell_xa;
- struct amdgpu_usermode_queue *queue;
- unsigned long flags;
-
doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
-
- xa_lock_irqsave(xa, flags);
- queue = xa_load(xa, doorbell_offset);
- if (queue)
- amdgpu_userq_fence_driver_process(queue->fence_drv);
- xa_unlock_irqrestore(xa, flags);
+ amdgpu_userq_process_fence_irq(adev, doorbell_offset);
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
index fea576a..efb3fde 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
@@ -242,6 +242,10 @@ static void uvd_v3_1_mc_resume(struct amdgpu_device *adev)
uint64_t addr;
uint32_t size;
+ /* When the keyselect is already set, don't perturb it. */
+ if (RREG32(mmUVD_FW_START))
+ return;
+
/* program the VCPU memory controller bits 0-27 */
addr = (adev->uvd.inst->gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3;
size = AMDGPU_UVD_FIRMWARE_SIZE(adev) >> 3;
@@ -284,6 +288,12 @@ static int uvd_v3_1_fw_validate(struct amdgpu_device *adev)
int i;
uint32_t keysel = adev->uvd.keyselect;
+ if (RREG32(mmUVD_FW_START) & UVD_FW_STATUS__PASS_MASK) {
+ dev_dbg(adev->dev, "UVD keyselect already set: 0x%x (on CPU: 0x%x)\n",
+ RREG32(mmUVD_FW_START), adev->uvd.keyselect);
+ return 0;
+ }
+
WREG32(mmUVD_FW_START, keysel);
for (i = 0; i < 10; ++i) {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
index 5b7b46d..93253db 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
@@ -42,9 +42,10 @@
#include "oss/oss_1_0_d.h"
#include "oss/oss_1_0_sh_mask.h"
+#define VCE_V1_0_ALIGNMENT (32 * 1024)
#define VCE_V1_0_FW_SIZE (256 * 1024)
#define VCE_V1_0_STACK_SIZE (64 * 1024)
-#define VCE_V1_0_DATA_SIZE (7808 * (AMDGPU_MAX_VCE_HANDLES + 1))
+#define VCE_V1_0_DATA_SIZE (ALIGN(7808 * (AMDGPU_MAX_VCE_HANDLES + 1), VCE_V1_0_ALIGNMENT))
#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02
static void vce_v1_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -177,7 +178,7 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev)
}
/**
- * vce_v1_0_load_fw_signature - load firmware signature into VCPU BO
+ * vce_v1_0_load_fw() - load firmware signature into VCPU BO
*
* @adev: amdgpu_device pointer
*
@@ -185,21 +186,26 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev)
* This function finds the signature appropriate for the current
* ASIC and writes that into the VCPU BO.
*/
-static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev)
+static int vce_v1_0_load_fw(struct amdgpu_device *adev)
{
const struct common_firmware_header *hdr;
struct vce_v1_0_fw_signature *sign;
- unsigned int ucode_offset;
+ u32 ucode_offset;
+ u32 ucode_size;
uint32_t chip_id;
u32 *cpu_addr;
int i;
hdr = (const struct common_firmware_header *)adev->vce.fw->data;
ucode_offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
+ ucode_size = hdr->ucode_size_bytes - sizeof(struct vce_v1_0_fw_signature *);
cpu_addr = adev->vce.cpu_addr;
sign = (void *)adev->vce.fw->data + ucode_offset;
+ if (ucode_size > VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET)
+ return -EINVAL;
+
switch (adev->asic_type) {
case CHIP_TAHITI:
chip_id = 0x01000014;
@@ -226,12 +232,14 @@ static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev)
return -EINVAL;
}
+ memset_io(&cpu_addr[0], 0, amdgpu_bo_size(adev->vce.vcpu_bo));
+
cpu_addr += (256 - 64) / 4;
memcpy_toio(&cpu_addr[0], &sign->val[i].nonce[0], 16);
cpu_addr[4] = cpu_to_le32(le32_to_cpu(sign->length) + 64);
memset_io(&cpu_addr[5], 0, 44);
- memcpy_toio(&cpu_addr[16], &sign[1], hdr->ucode_size_bytes - sizeof(*sign));
+ memcpy_toio(&cpu_addr[16], &sign[1], ucode_size);
cpu_addr += (le32_to_cpu(sign->length) + 64) / 4;
memcpy_toio(&cpu_addr[0], &sign->val[i].sigval[0], 16);
@@ -312,18 +320,23 @@ static int vce_v1_0_mc_resume(struct amdgpu_device *adev)
WREG32(mmVCE_VCPU_SCRATCH7, AMDGPU_MAX_VCE_HANDLES);
offset = adev->vce.gpu_addr + AMDGPU_VCE_FIRMWARE_OFFSET;
- size = VCE_V1_0_FW_SIZE;
- WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
+ size = VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
+ WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset);
WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
offset += size;
size = VCE_V1_0_STACK_SIZE;
- WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset & 0x7fffffff);
+ WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT));
+ WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT));
+ WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset);
WREG32(mmVCE_VCPU_CACHE_SIZE1, size);
offset += size;
size = VCE_V1_0_DATA_SIZE;
- WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset & 0x7fffffff);
+ WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT));
+ WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT));
+ WARN_ON((offset + size - adev->vce.gpu_addr) > amdgpu_bo_size(adev->vce.vcpu_bo));
+ WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset);
WREG32(mmVCE_VCPU_CACHE_SIZE2, size);
WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);
@@ -527,22 +540,31 @@ static int vce_v1_0_early_init(struct amdgpu_ip_block *ip_block)
* To accomodate that, we put GART to the LOW address range
* and reserve some GART pages where we map the VCPU BO,
* so that it gets a 32-bit address.
+ *
+ * The BAR address is zero and we can't change it
+ * due to the firmware validation mechanism.
+ * It seems that it fails to initialize if the address is >= 128 MiB.
*/
static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev)
{
u64 bo_size = amdgpu_bo_size(adev->vce.vcpu_bo);
- u64 max_vcpu_bo_addr = 0xffffffff - bo_size;
+ u64 max_vcpu_bo_addr = 0x07ffffff - bo_size;
u64 num_pages = ALIGN(bo_size, AMDGPU_GPU_PAGE_SIZE) / AMDGPU_GPU_PAGE_SIZE;
u64 pa = amdgpu_gmc_vram_pa(adev, adev->vce.vcpu_bo);
u64 flags = AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | AMDGPU_PTE_VALID;
u64 vce_gart_start_offs;
int r;
- r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr,
- &adev->vce.gart_node, num_pages,
- DRM_MM_INSERT_LOW);
- if (r)
- return r;
+ if (adev->gmc.vram_start < adev->gmc.gart_start)
+ return amdgpu_bo_gpu_offset(adev->vce.vcpu_bo) <= max_vcpu_bo_addr ? 0 : -EINVAL;
+
+ if (!drm_mm_node_allocated(&adev->vce.gart_node)) {
+ r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr,
+ &adev->vce.gart_node, num_pages,
+ DRM_MM_INSERT_LOW);
+ if (r)
+ return r;
+ }
vce_gart_start_offs = amdgpu_gtt_node_to_byte_offset(&adev->vce.gart_node);
@@ -553,8 +575,6 @@ static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev)
amdgpu_gart_map_vram_range(adev, pa, adev->vce.gart_node.start,
num_pages, flags, adev->gart.ptr);
adev->vce.gpu_addr = adev->gmc.gart_start + vce_gart_start_offs;
- if (adev->vce.gpu_addr > max_vcpu_bo_addr)
- return -EINVAL;
return 0;
}
@@ -574,10 +594,7 @@ static int vce_v1_0_sw_init(struct amdgpu_ip_block *ip_block)
if (r)
return r;
- r = amdgpu_vce_resume(adev);
- if (r)
- return r;
- r = vce_v1_0_load_fw_signature(adev);
+ r = vce_v1_0_load_fw(adev);
if (r)
return r;
r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev);
@@ -696,10 +713,7 @@ static int vce_v1_0_resume(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
int r;
- r = amdgpu_vce_resume(adev);
- if (r)
- return r;
- r = vce_v1_0_load_fw_signature(adev);
+ r = vce_v1_0_load_fw(adev);
if (r)
return r;
r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index db149ed..3a6fc86 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -37,9 +37,14 @@
#include "oss/oss_2_0_d.h"
#include "oss/oss_2_0_sh_mask.h"
+
+/* Use 24K to be safe. The FW supposedly only requires 23744 bytes. */
+#define VCE_V2_0_DATA_ENTRY_SIZE (24 * 1024)
+
#define VCE_V2_0_FW_SIZE (256 * 1024)
#define VCE_V2_0_STACK_SIZE (64 * 1024)
-#define VCE_V2_0_DATA_SIZE (23552 * AMDGPU_MAX_VCE_HANDLES)
+#define VCE_V2_0_DATA_SIZE (VCE_V2_0_DATA_ENTRY_SIZE * (AMDGPU_MAX_VCE_HANDLES + 1))
+
#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02
static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -183,7 +188,7 @@ static void vce_v2_0_mc_resume(struct amdgpu_device *adev)
WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8));
offset = AMDGPU_VCE_FIRMWARE_OFFSET;
- size = VCE_V2_0_FW_SIZE;
+ size = VCE_V2_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 03d79e4..c69f7d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -574,7 +574,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
} else
WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8));
offset = AMDGPU_VCE_FIRMWARE_OFFSET;
- size = VCE_V3_0_FW_SIZE;
+ size = VCE_V3_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
index e35fae9..0442bfc 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -2113,6 +2113,7 @@ static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_DEC,
.align_mask = 0xf,
+ .no_user_fence = true,
.secure_submission_supported = true,
.get_rptr = vcn_v2_0_dec_ring_get_rptr,
.get_wptr = vcn_v2_0_dec_ring_get_wptr,
@@ -2145,6 +2146,7 @@ static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v2_0_enc_ring_get_rptr,
.get_wptr = vcn_v2_0_enc_ring_get_wptr,
.set_wptr = vcn_v2_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index 006a154..8b8184f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -1778,6 +1778,7 @@ static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_DEC,
.align_mask = 0xf,
+ .no_user_fence = true,
.secure_submission_supported = true,
.get_rptr = vcn_v2_5_dec_ring_get_rptr,
.get_wptr = vcn_v2_5_dec_ring_get_wptr,
@@ -1879,6 +1880,7 @@ static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v2_5_enc_ring_get_rptr,
.get_wptr = vcn_v2_5_enc_ring_get_wptr,
.set_wptr = vcn_v2_5_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 6fb4fcd..81bba3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -1856,6 +1856,7 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_DEC,
.align_mask = 0x3f,
.nop = VCN_DEC_SW_CMD_NO_OP,
+ .no_user_fence = true,
.secure_submission_supported = true,
.get_rptr = vcn_v3_0_dec_ring_get_rptr,
.get_wptr = vcn_v3_0_dec_ring_get_wptr,
@@ -1972,6 +1973,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
uint32_t offset, size, *create;
+ uint64_t buf_end;
if (msg[0] != RDECODE_MESSAGE_CREATE)
continue;
@@ -1979,7 +1981,8 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
offset = msg[1];
size = msg[2];
- if (size < 4 || offset + size > end - addr) {
+ if (size < 4 || check_add_overflow(offset, size, &buf_end) ||
+ buf_end > end - addr) {
DRM_ERROR("VCN message buffer exceeds BO bounds!\n");
r = -EINVAL;
goto out;
@@ -2036,6 +2039,7 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_DEC,
.align_mask = 0xf,
+ .no_user_fence = true,
.secure_submission_supported = true,
.get_rptr = vcn_v3_0_dec_ring_get_rptr,
.get_wptr = vcn_v3_0_dec_ring_get_wptr,
@@ -2138,6 +2142,7 @@ static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v3_0_enc_ring_get_rptr,
.get_wptr = vcn_v3_0_enc_ring_get_wptr,
.set_wptr = vcn_v3_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 5dec926..ff7269b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -1889,6 +1889,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
uint32_t offset, size, *create;
+ uint64_t buf_end;
if (msg[0] != RDECODE_MESSAGE_CREATE)
continue;
@@ -1896,7 +1897,8 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
offset = msg[1];
size = msg[2];
- if (size < 4 || offset + size > end - addr) {
+ if (size < 4 || check_add_overflow(offset, size, &buf_end) ||
+ buf_end > end - addr) {
DRM_ERROR("VCN message buffer exceeds BO bounds!\n");
r = -EINVAL;
goto out;
@@ -1994,6 +1996,7 @@ static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.extra_bytes = sizeof(struct amdgpu_vcn_rb_metadata),
.get_rptr = vcn_v4_0_unified_ring_get_rptr,
.get_wptr = vcn_v4_0_unified_ring_get_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index ff3013b..10e8fc2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -1775,6 +1775,7 @@ static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v4_0_3_unified_ring_get_rptr,
.get_wptr = vcn_v4_0_3_unified_ring_get_wptr,
.set_wptr = vcn_v4_0_3_unified_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
index 1f6a229..1571cc5 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
@@ -1483,6 +1483,7 @@ static struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v4_0_5_unified_ring_get_rptr,
.get_wptr = vcn_v4_0_5_unified_ring_get_wptr,
.set_wptr = vcn_v4_0_5_unified_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
index 6109124..d5f49fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
@@ -1207,6 +1207,7 @@ static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v5_0_0_unified_ring_get_rptr,
.get_wptr = vcn_v5_0_0_unified_ring_get_wptr,
.set_wptr = vcn_v5_0_0_unified_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
index c28c6af..54fbf8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
@@ -1419,6 +1419,7 @@ static const struct amdgpu_ring_funcs vcn_v5_0_1_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v5_0_1_unified_ring_get_rptr,
.get_wptr = vcn_v5_0_1_unified_ring_get_wptr,
.set_wptr = vcn_v5_0_1_unified_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
index c3d3cc0..bbc172d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
@@ -994,6 +994,7 @@ static const struct amdgpu_ring_funcs vcn_v5_0_2_unified_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_ENC,
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
+ .no_user_fence = true,
.get_rptr = vcn_v5_0_2_unified_ring_get_rptr,
.get_wptr = vcn_v5_0_2_unified_ring_get_wptr,
.set_wptr = vcn_v5_0_2_unified_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 55ea5145..8785f78 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -25,6 +25,7 @@
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/file.h>
+#include <linux/overflow.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
@@ -66,6 +67,21 @@ static const struct class kfd_class = {
.name = kfd_dev_name,
};
+/*
+ * Cache the address space of the chardev on first open so that the reset
+ * path can drop all userspace mappings of doorbell and MMIO ranges via
+ * unmap_mapping_range().
+ */
+static struct address_space *kfd_dev_mapping;
+
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen)
+{
+ struct address_space *mapping = READ_ONCE(kfd_dev_mapping);
+
+ if (mapping)
+ unmap_mapping_range(mapping, holebegin, holelen, 1);
+}
+
static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
{
struct kfd_process_device *pdd;
@@ -132,6 +148,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
if (iminor(inode) != 0)
return -ENODEV;
+ /*
+ * /dev/kfd is a single chardev so all opens share one inode. Cache
+ * its address_space on the first open for use by the reset path.
+ */
+ if (!READ_ONCE(kfd_dev_mapping))
+ cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping);
+
is_32bit_user_mode = in_compat_syscall();
if (is_32bit_user_mode) {
@@ -1359,7 +1382,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
if (WARN_ON_ONCE(!peer_pdd))
continue;
- kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(peer_pdd);
}
kfree(devices_arr);
@@ -1454,7 +1477,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
if (WARN_ON_ONCE(!peer_pdd))
continue;
if (flush_tlb)
- kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
+ kfd_flush_tlb(peer_pdd);
/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv);
@@ -1695,6 +1718,16 @@ static int kfd_ioctl_smi_events(struct file *filep,
return kfd_smi_event_open(pdd->dev, &args->anon_fd);
}
+static int kfd_ioctl_svm_validate(void *kdata, unsigned int usize)
+{
+ struct kfd_ioctl_svm_args *args = kdata;
+ size_t expected = struct_size(args, attrs, args->nattr);
+
+ if (expected == SIZE_MAX || usize < expected)
+ return -EINVAL;
+ return 0;
+}
+
#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
static int kfd_ioctl_set_xnack_mode(struct file *filep,
@@ -2267,6 +2300,11 @@ static int criu_restore_devices(struct kfd_process *p,
ret = -EINVAL;
goto exit;
}
+
+ if (pdd->drm_file) {
+ ret = -EINVAL;
+ goto exit;
+ }
pdd->user_gpu_id = device_buckets[i].user_gpu_id;
drm_file = fget(device_buckets[i].drm_fd);
@@ -2277,11 +2315,6 @@ static int criu_restore_devices(struct kfd_process *p,
goto exit;
}
- if (pdd->drm_file) {
- ret = -EINVAL;
- goto exit;
- }
-
/* create the vm using render nodes for kfd pdd */
if (kfd_process_device_init_vm(pdd, drm_file)) {
pr_err("could not init vm for given pdd\n");
@@ -3209,7 +3242,11 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v
#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
- .cmd_drv = 0, .name = #ioctl}
+ .validate = NULL, .cmd_drv = 0, .name = #ioctl}
+
+#define AMDKFD_IOCTL_DEF_V(ioctl, _func, _validate, _flags) \
+ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
+ .validate = _validate, .cmd_drv = 0, .name = #ioctl}
/** Ioctl table */
static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
@@ -3306,7 +3343,8 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
kfd_ioctl_smi_events, 0),
- AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
+ AMDKFD_IOCTL_DEF_V(AMDKFD_IOC_SVM, kfd_ioctl_svm,
+ kfd_ioctl_svm_validate, 0),
AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
kfd_ioctl_set_xnack_mode, 0),
@@ -3431,6 +3469,12 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
memset(kdata, 0, usize);
}
+ if (ioctl->validate) {
+ retcode = ioctl->validate(kdata, usize);
+ if (retcode)
+ goto err_i1;
+ }
+
retcode = func(filep, process, kdata);
if (cmd & IOC_OUT)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 8ff97bf..b7f8f7f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -1737,37 +1737,6 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
return false;
}
-/* check if there is kfd process still uses adev */
-static bool kgd2kfd_check_device_idle(struct amdgpu_device *adev)
-{
- struct kfd_process *p;
- struct hlist_node *p_temp;
- unsigned int temp;
- struct kfd_node *dev;
-
- mutex_lock(&kfd_processes_mutex);
-
- if (hash_empty(kfd_processes_table)) {
- mutex_unlock(&kfd_processes_mutex);
- return true;
- }
-
- /* check if there is device still use adev */
- hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
- for (int i = 0; i < p->n_pdds; i++) {
- dev = p->pdds[i]->dev;
- if (dev->adev == adev) {
- mutex_unlock(&kfd_processes_mutex);
- return false;
- }
- }
- }
-
- mutex_unlock(&kfd_processes_mutex);
-
- return true;
-}
-
/** kgd2kfd_teardown_processes - gracefully tear down existing
* kfd processes that use adev
*
@@ -1800,7 +1769,7 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
mutex_unlock(&kfd_processes_mutex);
/* wait all kfd processes use adev terminate */
- while (!kgd2kfd_check_device_idle(adev))
+ while (!!atomic_read(&adev->kfd.dev->kfd_processes_count))
cond_resched();
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ab3b2e7..0d7296c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -475,6 +475,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
} else {
/* For CP queues on SOC15 */
if (restore_id) {
+ if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
+ return -EINVAL;
+
/* make sure that ID is free */
if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
return -EINVAL;
@@ -572,7 +575,7 @@ static int allocate_vmid(struct device_queue_manager *dqm,
qpd->vmid,
qpd->page_table_base);
/* invalidate the VM context after pasid and vmid mapping is set up */
- kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(qpd_to_pdd(qpd));
if (dqm->dev->kfd2kgd->set_scratch_backing_va)
dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
@@ -610,7 +613,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
if (flush_texture_cache_nocpsch(q->device, qpd))
dev_err(dev, "Failed to flush TC\n");
- kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(qpd_to_pdd(qpd));
/* Release the vmid mapping */
set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
@@ -1284,7 +1287,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
dqm->dev->adev,
qpd->vmid,
qpd->page_table_base);
- kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(pdd);
}
/* Take a safe reference to the mm_struct, which may otherwise
@@ -1587,6 +1590,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
}
if (restore_sdma_id) {
+ if (*restore_sdma_id >= get_num_sdma_queues(dqm))
+ return -EINVAL;
+
/* Re-use existing sdma_id */
if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
dev_err(dev, "SDMA queue already in use\n");
@@ -1613,6 +1619,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
return -ENOMEM;
}
if (restore_sdma_id) {
+ if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm))
+ return -EINVAL;
+
/* Re-use existing sdma_id */
if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
dev_err(dev, "SDMA queue already in use\n");
@@ -3299,12 +3308,14 @@ static void copy_context_work_handler(struct work_struct *work)
static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
{
- size_t array_size = num_queues * sizeof(uint32_t);
-
if (!usr_queue_id_array)
return NULL;
- return memdup_user(usr_queue_id_array, array_size);
+ if (num_queues > KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
+ return ERR_PTR(-EINVAL);
+
+ return memdup_user(usr_queue_id_array,
+ array_size(num_queues, sizeof(uint32_t)));
}
int resume_queues(struct kfd_process *p,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index e8f97de..f6d9d81 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -364,11 +364,15 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
{
struct v9_mqd *m;
struct kfd_context_save_area_header header;
+ u32 cntl_stack_size;
+ u32 cntl_stack_offset;
/* Control stack is located one page after MQD. */
void *mqd_ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE);
m = get_mqd(mqd);
+ cntl_stack_size = min_t(u32, m->cp_hqd_cntl_stack_size, q->ctl_stack_size);
+ cntl_stack_offset = min_t(u32, m->cp_hqd_cntl_stack_offset, cntl_stack_size);
*ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
m->cp_hqd_cntl_stack_offset;
@@ -384,9 +388,10 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state)))
return -EFAULT;
- if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset,
- mqd_ctl_stack + m->cp_hqd_cntl_stack_offset,
- *ctl_stack_used_size))
+ *ctl_stack_used_size = cntl_stack_size - cntl_stack_offset;
+
+ if (copy_to_user(ctl_stack + cntl_stack_offset, mqd_ctl_stack + cntl_stack_offset,
+ *ctl_stack_used_size))
return -EFAULT;
return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 6e333bfa..d5b0778 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -395,6 +395,7 @@ enum kfd_mempool {
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen);
/**
* enum kfd_unmap_queues_filter - Enum for queue filters.
@@ -1047,10 +1048,13 @@ extern struct srcu_struct kfd_processes_srcu;
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
void *data);
+typedef int amdkfd_ioctl_validate_t(void *kdata, unsigned int usize);
+
struct amdkfd_ioctl_desc {
unsigned int cmd;
int flags;
amdkfd_ioctl_t *func;
+ amdkfd_ioctl_validate_t *validate;
unsigned int cmd_drv;
const char *name;
};
@@ -1551,13 +1555,13 @@ void kfd_signal_reset_event(struct kfd_node *dev);
void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
void kfd_signal_process_terminate_event(struct kfd_process *p);
-static inline void kfd_flush_tlb(struct kfd_process_device *pdd,
- enum TLB_FLUSH_TYPE type)
+static inline void kfd_flush_tlb(struct kfd_process_device *pdd)
{
struct amdgpu_device *adev = pdd->dev->adev;
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
- amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask);
+ amdgpu_vm_flush_compute_tlb(adev, vm, TLB_FLUSH_HEAVYWEIGHT,
+ pdd->dev->xcc_mask);
}
static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index b120fdb..3841943 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1366,6 +1366,12 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
pr_debug("CPU[0x%llx 0x%llx] -> GPU[0x%llx 0x%llx]\n", start, last,
gpu_start, gpu_end);
+
+ if (!amdgpu_vm_ready(vm)) {
+ pr_debug("VM not ready, canceling unmap\n");
+ return -EINVAL;
+ }
+
return amdgpu_vm_update_range(adev, vm, false, true, true, false, NULL, gpu_start,
gpu_end, init_pte_value, 0, 0, NULL, NULL,
fence);
@@ -1418,7 +1424,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
if (r)
break;
}
- kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT);
+ kfd_flush_tlb(pdd);
}
return r;
@@ -1443,6 +1449,11 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,
last_start, last_start + npages - 1, readonly);
+ if (!amdgpu_vm_ready(vm)) {
+ pr_debug("VM not ready, canceling map\n");
+ return -EINVAL;
+ }
+
for (i = offset; i < offset + npages; i++) {
uint64_t gpu_start;
uint64_t gpu_end;
@@ -1560,7 +1571,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
}
}
- kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
+ kfd_flush_tlb(pdd);
}
return r;
@@ -3721,6 +3732,9 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
svms = &p->svms;
+ if (!process_info)
+ return -EINVAL;
+
mutex_lock(&process_info->lock);
svm_range_list_lock_and_flush_work(svms, mm);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index e96a12f..5fc5d56 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -1903,7 +1903,11 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
goto error;
}
- init_data.asic_id.chip_family = adev->family;
+ /* special handling for early revisions of GC 11.5.4 */
+ if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4))
+ init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4;
+ else
+ init_data.asic_id.chip_family = adev->family;
init_data.asic_id.pci_revision_id = adev->pdev->revision;
init_data.asic_id.hw_internal_rev = adev->external_rev_id;
@@ -9404,9 +9408,21 @@ static void manage_dm_interrupts(struct amdgpu_device *adev,
if (acrtc_state) {
timing = &acrtc_state->stream->timing;
- if (amdgpu_ip_version(adev, DCE_HWIP, 0) <
- IP_VERSION(3, 5, 0) ||
- !(adev->flags & AMD_IS_APU)) {
+ if (amdgpu_ip_version(adev, DCE_HWIP, 0) >=
+ IP_VERSION(3, 2, 0) &&
+ !(adev->flags & AMD_IS_APU)) {
+ /*
+ * DGPUs NV3x and newer that support idle optimizations
+ * experience intermittent flip-done timeouts on cursor
+ * updates. Restore 5s offdelay behavior for now.
+ *
+ * Discussion on the issue:
+ * https://lore.kernel.org/amd-gfx/20260217191632.1243826-1-sysdadmin@m1k.cloud/
+ */
+ config.offdelay_ms = 5000;
+ config.disable_immediate = false;
+ } else if (amdgpu_ip_version(adev, DCE_HWIP, 0) <
+ IP_VERSION(3, 5, 0)) {
/*
* Older HW and DGPU have issues with instant off;
* use a 2 frame offdelay.
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index 3b8ae77..a3cb054 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -1032,6 +1032,45 @@ dm_helpers_read_acpi_edid(struct amdgpu_dm_connector *aconnector)
return drm_edid_read_custom(connector, dm_helpers_probe_acpi_edid, connector);
}
+static const struct drm_edid *
+dm_helpers_read_vbios_hardcoded_edid(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
+{
+ struct dc_bios *bios = link->ctx->dc_bios;
+ struct embedded_panel_info info;
+ const struct drm_edid *edid;
+ enum bp_result r;
+
+ if (!dc_is_embedded_signal(link->connector_signal) ||
+ !bios->funcs->get_embedded_panel_info)
+ return NULL;
+
+ memset(&info, 0, sizeof(info));
+ r = bios->funcs->get_embedded_panel_info(bios, &info);
+
+ if (r != BP_RESULT_OK) {
+ dm_error("Error when reading embedded panel info: %u\n", r);
+ return NULL;
+ }
+
+ if (!info.fake_edid || !info.fake_edid_size) {
+ dm_error("Embedded panel info doesn't contain an EDID\n");
+ return NULL;
+ }
+
+ edid = drm_edid_alloc(info.fake_edid, info.fake_edid_size);
+
+ if (!drm_edid_valid(edid)) {
+ dm_error("EDID from embedded panel info is invalid\n");
+ drm_edid_free(edid);
+ return NULL;
+ }
+
+ aconnector->base.display_info.width_mm = info.panel_width_mm;
+ aconnector->base.display_info.height_mm = info.panel_height_mm;
+
+ return edid;
+}
+
void populate_hdmi_info_from_connector(struct drm_hdmi_info *hdmi, struct dc_edid_caps *edid_caps)
{
edid_caps->scdc_present = hdmi->scdc.supported;
@@ -1052,6 +1091,9 @@ enum dc_edid_status dm_helpers_read_local_edid(
if (link->aux_mode)
ddc = &aconnector->dm_dp_aux.aux.ddc;
+ else if (link->ddc_hw_inst == GPIO_DDC_LINE_UNKNOWN &&
+ dc_is_embedded_signal(link->connector_signal))
+ ddc = NULL;
else
ddc = &aconnector->i2c->base;
@@ -1065,6 +1107,8 @@ enum dc_edid_status dm_helpers_read_local_edid(
drm_edid = dm_helpers_read_acpi_edid(aconnector);
if (drm_edid)
drm_info(connector->dev, "Using ACPI provided EDID for %s\n", connector->name);
+ else if (!ddc)
+ drm_edid = dm_helpers_read_vbios_hardcoded_edid(link, aconnector);
else
drm_edid = drm_edid_read_ddc(connector, ddc);
drm_edid_connector_update(connector, drm_edid);
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index dd36207..c307f42 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -794,11 +794,13 @@ static enum bp_result bios_parser_external_encoder_control(
static enum bp_result bios_parser_dac_load_detection(
struct dc_bios *dcb,
- enum engine_id engine_id)
+ enum engine_id engine_id,
+ struct graphics_object_id ext_enc_id)
{
struct bios_parser *bp = BP_FROM_DCB(dcb);
struct dc_context *ctx = dcb->ctx;
struct bp_load_detection_parameters bp_params = {0};
+ struct bp_external_encoder_control ext_cntl = {0};
enum bp_result bp_result = BP_RESULT_UNSUPPORTED;
uint32_t bios_0_scratch;
uint32_t device_id_mask = 0;
@@ -824,6 +826,13 @@ static enum bp_result bios_parser_dac_load_detection(
bp_params.engine_id = engine_id;
bp_result = bp->cmd_tbl.dac_load_detection(bp, &bp_params);
+ } else if (ext_enc_id.id) {
+ if (!bp->cmd_tbl.external_encoder_control)
+ return BP_RESULT_UNSUPPORTED;
+
+ ext_cntl.action = EXTERNAL_ENCODER_CONTROL_DAC_LOAD_DETECT;
+ ext_cntl.encoder_id = ext_enc_id;
+ bp_result = bp->cmd_tbl.external_encoder_control(bp, &ext_cntl);
}
if (bp_result != BP_RESULT_OK)
@@ -1304,6 +1313,60 @@ static enum bp_result bios_parser_get_embedded_panel_info(
return BP_RESULT_FAILURE;
}
+static enum bp_result get_embedded_panel_extra_info(
+ struct bios_parser *bp,
+ struct embedded_panel_info *info,
+ const uint32_t table_offset)
+{
+ uint8_t *record = bios_get_image(&bp->base, table_offset, 1);
+ ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record;
+ ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record;
+
+ while (*record != ATOM_RECORD_END_TYPE) {
+ switch (*record) {
+ case LCD_MODE_PATCH_RECORD_MODE_TYPE:
+ record += sizeof(ATOM_PATCH_RECORD_MODE);
+ break;
+ case LCD_RTS_RECORD_TYPE:
+ record += sizeof(ATOM_LCD_RTS_RECORD);
+ break;
+ case LCD_CAP_RECORD_TYPE:
+ record += sizeof(ATOM_LCD_MODE_CONTROL_CAP);
+ break;
+ case LCD_FAKE_EDID_PATCH_RECORD_TYPE:
+ fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record;
+ if (fake_edid_record->ucFakeEDIDLength) {
+ if (fake_edid_record->ucFakeEDIDLength == 128)
+ info->fake_edid_size =
+ fake_edid_record->ucFakeEDIDLength;
+ else
+ info->fake_edid_size =
+ fake_edid_record->ucFakeEDIDLength * 128;
+
+ info->fake_edid = fake_edid_record->ucFakeEDIDString;
+
+ record += struct_size(fake_edid_record,
+ ucFakeEDIDString,
+ info->fake_edid_size);
+ } else {
+ /* empty fake edid record must be 3 bytes long */
+ record += sizeof(ATOM_FAKE_EDID_PATCH_RECORD) + 1;
+ }
+ break;
+ case LCD_PANEL_RESOLUTION_RECORD_TYPE:
+ panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record;
+ info->panel_width_mm = panel_res_record->usHSize;
+ info->panel_height_mm = panel_res_record->usVSize;
+ record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD);
+ break;
+ default:
+ return BP_RESULT_BADBIOSTABLE;
+ }
+ }
+
+ return BP_RESULT_OK;
+}
+
static enum bp_result get_embedded_panel_info_v1_2(
struct bios_parser *bp,
struct embedded_panel_info *info)
@@ -1420,6 +1483,10 @@ static enum bp_result get_embedded_panel_info_v1_2(
if (ATOM_PANEL_MISC_API_ENABLED & lvds->ucLVDS_Misc)
info->lcd_timing.misc_info.API_ENABLED = true;
+ if (lvds->usExtInfoTableOffset)
+ return get_embedded_panel_extra_info(bp, info,
+ le16_to_cpu(lvds->usExtInfoTableOffset) + DATA_TABLES(LCD_Info));
+
return BP_RESULT_OK;
}
@@ -1545,6 +1612,10 @@ static enum bp_result get_embedded_panel_info_v1_3(
(uint32_t) (ATOM_PANEL_MISC_V13_GREY_LEVEL &
lvds->ucLCD_Misc) >> ATOM_PANEL_MISC_V13_GREY_LEVEL_SHIFT;
+ if (lvds->usExtInfoTableOffset)
+ return get_embedded_panel_extra_info(bp, info,
+ le16_to_cpu(lvds->usExtInfoTableOffset) + DATA_TABLES(LCD_Info));
+
return BP_RESULT_OK;
}
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index a1c08e1..c51c4b2 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -493,6 +493,10 @@ static enum bp_result get_gpio_i2c_info(
- sizeof(struct atom_common_table_header))
/ sizeof(struct atom_gpio_pin_assignment);
+ if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut),
+ le16_to_cpu(header->table_header.structuresize)))
+ return BP_RESULT_BADBIOSTABLE;
+
pin = (struct atom_gpio_pin_assignment *) header->gpio_pin;
for (table_index = 0; table_index < count; table_index++) {
@@ -681,6 +685,11 @@ static enum bp_result bios_parser_get_gpio_pin_info(
count = (le16_to_cpu(header->table_header.structuresize)
- sizeof(struct atom_common_table_header))
/ sizeof(struct atom_gpio_pin_assignment);
+
+ if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut),
+ le16_to_cpu(header->table_header.structuresize)))
+ return BP_RESULT_BADBIOSTABLE;
+
for (i = 0; i < count; ++i) {
if (header->gpio_pin[i].gpio_id != gpio_id)
continue;
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
index 8d2cf95..e00dc05 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
@@ -37,10 +37,13 @@ uint8_t *bios_get_image(struct dc_bios *bp,
uint32_t offset,
uint32_t size)
{
- if (bp->bios && offset + size < bp->bios_size)
- return bp->bios + offset;
- else
+ if (!bp->bios)
return NULL;
+
+ if (offset > bp->bios_size || size > bp->bios_size - offset)
+ return NULL;
+
+ return bp->bios + offset;
}
#include "reg_helper.h"
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 419f894..b3530fb 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -6071,7 +6071,11 @@ bool dc_process_dmub_aux_transfer_async(struct dc *dc,
uint8_t action;
union dmub_rb_cmd cmd = {0};
- ASSERT(payload->length <= 16);
+ if (link_index >= dc->link_count || !dc->links[link_index])
+ return false;
+
+ if (payload->length > sizeof(cmd.dp_aux_access.aux_control.dpaux.data))
+ return false;
cmd.dp_aux_access.header.type = DMUB_CMD__DP_AUX_ACCESS;
cmd.dp_aux_access.header.payload_bytes = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 7f55ba0..37714d4 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -1682,7 +1682,7 @@ struct dc_scratch_space {
struct dc_link_training_overrides preferred_training_settings;
struct dp_audio_test_data audio_test_data;
- uint8_t ddc_hw_inst;
+ enum gpio_ddc_line ddc_hw_inst;
uint8_t hpd_src;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
index 6f96c5c..526f716 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_bios_types.h
@@ -102,7 +102,8 @@ struct dc_vbios_funcs {
struct bp_external_encoder_control *cntl);
enum bp_result (*dac_load_detection)(
struct dc_bios *bios,
- enum engine_id engine_id);
+ enum engine_id engine_id,
+ struct graphics_object_id ext_enc_id);
enum bp_result (*transmitter_control)(
struct dc_bios *bios,
struct bp_transmitter_control *cntl);
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
index c4d4eea..1f23dfc 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
@@ -105,15 +105,26 @@ static void dccg21_update_dpp_dto(struct dccg *dccg, int dpp_inst, int req_dppcl
* dccg2_init() unconditionally overwrites MICROSECOND_TIME_BASE_DIV to
* 0x00120264, destroying the marker before it can be read.
*
- * Guard the call: if the S0i3 marker is present, skip dccg2_init() so the
+ * Guard the call: if the S0i3 marker is present, skip init so the
* WA can function correctly. bios_golden_init() will handle init in that case.
+ *
+ * DCN21 uses 48MHz refclk, not 100MHz, so we must explicitly set the correct
+ * values (48MHz is taken from rn_clk_mgr_construct()).
*/
static void dccg21_init(struct dccg *dccg)
{
+ struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
if (dccg2_is_s0i3_golden_init_wa_done(dccg))
return;
- dccg2_init(dccg);
+ /* 48MHz refclk from rn_clk_mgr_construct() */
+ REG_WRITE(MICROSECOND_TIME_BASE_DIV, 0x00120230);
+ REG_WRITE(MILLISECOND_TIME_BASE_DIV, 0x0010bb80);
+ REG_WRITE(DISPCLK_FREQ_CHANGE_CNTL, 0x0e01003c);
+
+ if (REG(REFCLK_CNTL))
+ REG_WRITE(REFCLK_CNTL, 0);
}
static const struct dccg_funcs dccg21_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index 5f40ae9..e15fd14 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -1102,7 +1102,9 @@ void dce110_link_encoder_hw_init(
ASSERT(result == BP_RESULT_OK);
}
- aux_initialize(enc110);
+
+ if (enc110->aux_regs)
+ aux_initialize(enc110);
/* reinitialize HPD.
* hpd_initialize() will pass DIG_FE id to HW context.
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
index 6f2a0d5..62fe5c3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.c
@@ -40,8 +40,8 @@
#define FN(reg_name, field_name) \
mcif_wb30->mcif_wb_shift->field_name, mcif_wb30->mcif_wb_mask->field_name
-#define MCIF_ADDR(addr) (((unsigned long long)addr & 0xffffffffff) + 0xFE) >> 8
-#define MCIF_ADDR_HIGH(addr) (unsigned long long)addr >> 40
+#define MCIF_ADDR(addr) ((uint32_t)((((unsigned long long)(addr) & 0xffffffffffULL) + 0xFEULL) >> 8))
+#define MCIF_ADDR_HIGH(addr) ((uint32_t)(((unsigned long long)(addr)) >> 40))
/* wbif programming guide:
* 1. set up wbif parameter:
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
index a2c4635..95f8b7c 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
@@ -646,6 +646,9 @@ enum gpio_result dal_ddc_change_mode(
enum gpio_ddc_line dal_ddc_get_line(
const struct ddc *ddc)
{
+ if (!ddc)
+ return GPIO_DDC_LINE_UNKNOWN;
+
return (enum gpio_ddc_line)dal_gpio_get_enum(ddc->pin_data);
}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 5273ca0..f0abbb7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -665,16 +665,45 @@ void dce110_update_info_frame(struct pipe_ctx *pipe_ctx)
}
static void
-dce110_dac_encoder_control(struct pipe_ctx *pipe_ctx, bool enable)
+dce110_external_encoder_control(enum bp_external_encoder_control_action action,
+ struct dc_link *link,
+ struct dc_crtc_timing *timing)
{
- struct dc_link *link = pipe_ctx->stream->link;
+ struct dc *dc = link->ctx->dc;
struct dc_bios *bios = link->ctx->dc_bios;
- struct bp_encoder_control encoder_control = {0};
+ const struct dc_link_settings *link_settings = &link->cur_link_settings;
+ enum bp_result bp_result = BP_RESULT_OK;
+ struct bp_external_encoder_control ext_cntl = {
+ .action = action,
+ .connector_obj_id = link->link_enc->connector,
+ .encoder_id = link->ext_enc_id,
+ .lanes_number = link_settings->lane_count,
+ .link_rate = link_settings->link_rate,
- encoder_control.action = enable ? ENCODER_CONTROL_ENABLE : ENCODER_CONTROL_DISABLE;
- encoder_control.engine_id = link->link_enc->analog_engine;
- encoder_control.pixel_clock = pipe_ctx->stream->timing.pix_clk_100hz / 10;
- bios->funcs->encoder_control(bios, &encoder_control);
+ /* Use signal type of the real link encoder, ie. DP */
+ .signal = link->connector_signal,
+
+ /* We don't know the timing yet when executing the SETUP action,
+ * so use a reasonably high default value. It seems that ENABLE
+ * can change the actual pixel clock but doesn't work with higher
+ * pixel clocks than what SETUP was called with.
+ */
+ .pixel_clock = timing ? timing->pix_clk_100hz / 10 : 300000,
+ .color_depth = timing ? timing->display_color_depth : COLOR_DEPTH_888,
+ };
+ DC_LOGGER_INIT(dc->ctx);
+
+ bp_result = bios->funcs->external_encoder_control(bios, &ext_cntl);
+
+ if (bp_result != BP_RESULT_OK)
+ DC_LOG_ERROR("Failed to execute external encoder action: 0x%x\n", action);
+}
+
+static void
+dce110_prepare_ddc(struct dc_link *link)
+{
+ if (link->ext_enc_id.id)
+ dce110_external_encoder_control(EXTERNAL_ENCODER_CONTROL_DDC_SETUP, link, NULL);
}
static bool
@@ -684,7 +713,8 @@ dce110_dac_load_detect(struct dc_link *link)
struct link_encoder *link_enc = link->link_enc;
enum bp_result bp_result;
- bp_result = bios->funcs->dac_load_detection(bios, link_enc->analog_engine);
+ bp_result = bios->funcs->dac_load_detection(
+ bios, link_enc->analog_engine, link->ext_enc_id);
return bp_result == BP_RESULT_OK;
}
@@ -700,7 +730,6 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
uint32_t early_control = 0;
struct timing_generator *tg = pipe_ctx->stream_res.tg;
- link_hwss->setup_stream_attribute(pipe_ctx);
link_hwss->setup_stream_encoder(pipe_ctx);
dc->hwss.update_info_frame(pipe_ctx);
@@ -719,8 +748,8 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
tg->funcs->set_early_control(tg, early_control);
- if (dc_is_rgb_signal(pipe_ctx->stream->signal))
- dce110_dac_encoder_control(pipe_ctx, true);
+ if (link->ext_enc_id.id)
+ dce110_external_encoder_control(EXTERNAL_ENCODER_CONTROL_ENABLE, link, timing);
}
static enum bp_result link_transmitter_control(
@@ -1219,8 +1248,8 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
link_enc->transmitter - TRANSMITTER_UNIPHY_A);
}
- if (dc_is_rgb_signal(pipe_ctx->stream->signal))
- dce110_dac_encoder_control(pipe_ctx, false);
+ if (link->ext_enc_id.id)
+ dce110_external_encoder_control(EXTERNAL_ENCODER_CONTROL_DISABLE, link, NULL);
}
void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
@@ -1603,22 +1632,6 @@ static enum dc_status dce110_enable_stream_timing(
return DC_OK;
}
-static void
-dce110_select_crtc_source(struct pipe_ctx *pipe_ctx)
-{
- struct dc_link *link = pipe_ctx->stream->link;
- struct dc_bios *bios = link->ctx->dc_bios;
- struct bp_crtc_source_select crtc_source_select = {0};
- enum engine_id engine_id = link->link_enc->preferred_engine;
-
- if (dc_is_rgb_signal(pipe_ctx->stream->signal))
- engine_id = link->link_enc->analog_engine;
- crtc_source_select.controller_id = CONTROLLER_ID_D0 + pipe_ctx->stream_res.tg->inst;
- crtc_source_select.color_depth = pipe_ctx->stream->timing.display_color_depth;
- crtc_source_select.engine_id = engine_id;
- crtc_source_select.sink_signal = pipe_ctx->stream->signal;
- bios->funcs->select_crtc_source(bios, &crtc_source_select);
-}
enum dc_status dce110_apply_single_controller_ctx_to_hw(
struct pipe_ctx *pipe_ctx,
@@ -1639,10 +1652,6 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
hws->funcs.disable_stream_gating(dc, pipe_ctx);
}
- if (pipe_ctx->stream->signal == SIGNAL_TYPE_RGB) {
- dce110_select_crtc_source(pipe_ctx);
- }
-
if (pipe_ctx->stream_res.audio != NULL) {
struct audio_output audio_output = {0};
@@ -1722,8 +1731,7 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
pipe_ctx->stream_res.tg->funcs->set_static_screen_control(
pipe_ctx->stream_res.tg, event_triggers, 2);
- if (!dc_is_virtual_signal(pipe_ctx->stream->signal) &&
- !dc_is_rgb_signal(pipe_ctx->stream->signal))
+ if (!dc_is_virtual_signal(pipe_ctx->stream->signal))
pipe_ctx->stream_res.stream_enc->funcs->dig_connect_to_otg(
pipe_ctx->stream_res.stream_enc,
pipe_ctx->stream_res.tg->inst);
@@ -3376,6 +3384,15 @@ void dce110_enable_tmds_link_output(struct dc_link *link,
link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
}
+static void dce110_enable_analog_link_output(
+ struct dc_link *link,
+ uint32_t pix_clk_100hz)
+{
+ link->link_enc->funcs->enable_analog_output(
+ link->link_enc,
+ pix_clk_100hz);
+}
+
void dce110_enable_dp_link_output(
struct dc_link *link,
const struct link_resource *link_res,
@@ -3423,6 +3440,11 @@ void dce110_enable_dp_link_output(
}
}
+ if (link->ext_enc_id.id) {
+ dce110_external_encoder_control(EXTERNAL_ENCODER_CONTROL_INIT, link, NULL);
+ dce110_external_encoder_control(EXTERNAL_ENCODER_CONTROL_SETUP, link, NULL);
+ }
+
if (dc->link_srv->dp_get_encoding_format(link_settings) == DP_8b_10b_ENCODING) {
if (dc->clk_mgr->funcs->notify_link_rate_change)
dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link);
@@ -3513,8 +3535,10 @@ static const struct hw_sequencer_funcs dce110_funcs = {
.enable_lvds_link_output = dce110_enable_lvds_link_output,
.enable_tmds_link_output = dce110_enable_tmds_link_output,
.enable_dp_link_output = dce110_enable_dp_link_output,
+ .enable_analog_link_output = dce110_enable_analog_link_output,
.disable_link_output = dce110_disable_link_output,
.dac_load_detect = dce110_dac_load_detect,
+ .prepare_ddc = dce110_prepare_ddc,
};
static const struct hwseq_private_funcs dce110_private_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index 7e7682d7..ae4c4ad 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -568,7 +568,9 @@ static bool construct_phy(struct dc_link *link,
goto ddc_create_fail;
}
- if (!link->ddc->ddc_pin) {
+ /* Embedded display connectors such as LVDS may not have DDC. */
+ if (!link->ddc->ddc_pin &&
+ !dc_is_embedded_signal(link->connector_signal)) {
DC_ERROR("Failed to get I2C info for connector!\n");
goto ddc_create_fail;
}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
index 6a25dcf..d2d56a1 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c
@@ -753,7 +753,8 @@ static struct link_encoder *dce60_link_encoder_create(
enc_init_data,
&link_enc_feature,
&link_enc_regs[link_regs_id],
- &link_enc_aux_regs[enc_init_data->channel - 1],
+ enc_init_data->channel == CHANNEL_ID_UNKNOWN ?
+ NULL : &link_enc_aux_regs[enc_init_data->channel - 1],
enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ?
NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]);
return &enc110->base;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
index 33be49b..6c00497 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
@@ -760,7 +760,8 @@ static struct link_encoder *dce80_link_encoder_create(
enc_init_data,
&link_enc_feature,
&link_enc_regs[link_regs_id],
- &link_enc_aux_regs[enc_init_data->channel - 1],
+ enc_init_data->channel == CHANNEL_ID_UNKNOWN ?
+ NULL : &link_enc_aux_regs[enc_init_data->channel - 1],
enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ?
NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]);
return &enc110->base;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index 82f81b5..3751f7a 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -92,9 +92,14 @@
#include "dml/dcn32/dcn32_fpu.h"
#include "dc_state_priv.h"
+#include "dc_fpu.h"
#include "dml2_0/dml2_wrapper.h"
+#if !defined(DC_RUN_WITH_PREEMPTION_ENABLED)
+#define DC_RUN_WITH_PREEMPTION_ENABLED(code) code
+#endif
+
#define DC_LOGGER_INIT(logger)
enum dcn32_clk_src_array_id {
@@ -1684,7 +1689,8 @@ static void dcn32_enable_phantom_plane(struct dc *dc,
if (curr_pipe->top_pipe && curr_pipe->top_pipe->plane_state == curr_pipe->plane_state)
phantom_plane = prev_phantom_plane;
else
- phantom_plane = dc_state_create_phantom_plane(dc, context, curr_pipe->plane_state);
+ DC_RUN_WITH_PREEMPTION_ENABLED(phantom_plane =
+ dc_state_create_phantom_plane(dc, context, curr_pipe->plane_state));
if (!phantom_plane)
continue;
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
index 38a77fa..a0f03fb 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
@@ -153,6 +153,10 @@ struct embedded_panel_info {
uint32_t drr_enabled;
uint32_t min_drr_refresh_rate;
bool realtek_eDPToLVDS;
+ uint16_t panel_width_mm;
+ uint16_t panel_height_mm;
+ uint16_t fake_edid_size;
+ const uint8_t *fake_edid;
};
struct dc_firmware_info {
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
index 3694246..c3aff5d 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
@@ -3076,6 +3076,10 @@ static bool si_dpm_vblank_too_short(void *handle)
/* we never hit the non-gddr5 limit so disable it */
u32 switch_limit = adev->gmc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 0;
+ /* Disregard vblank time when there are no displays connected */
+ if (!adev->pm.pm_display_cfg.num_display)
+ return false;
+
/* Consider zero vblank time too short and disable MCLK switching.
* Note that the vblank time is set to maximum when no displays are attached,
* so we'll still enable MCLK switching in that case.
diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c
index 731355b..3650e7b 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c
@@ -1333,12 +1333,13 @@ static int ci_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
dev_id = adev->pdev->device;
- if ((dpm_table->mclk_table.count >= 2)
- && ((dev_id == 0x67B0) || (dev_id == 0x67B1))) {
- smu_data->smc_state_table.MemoryLevel[1].MinVddci =
- smu_data->smc_state_table.MemoryLevel[0].MinVddci;
- smu_data->smc_state_table.MemoryLevel[1].MinMvdd =
- smu_data->smc_state_table.MemoryLevel[0].MinMvdd;
+ if ((dpm_table->mclk_table.count >= 2) &&
+ ((dev_id == 0x67B0) || (dev_id == 0x67B1)) &&
+ (adev->pdev->revision == 0)) {
+ smu_data->smc_state_table.MemoryLevel[1].MinVddc =
+ smu_data->smc_state_table.MemoryLevel[0].MinVddc;
+ smu_data->smc_state_table.MemoryLevel[1].MinVddcPhases =
+ smu_data->smc_state_table.MemoryLevel[0].MinVddcPhases;
}
smu_data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F;
CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.MemoryLevel[0].ActivityLevel);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
index 7f386ff..9d8b122 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
@@ -425,6 +425,7 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu)
dpm_table->dpm_levels[0].enabled = true;
dpm_table->dpm_levels[1].value = pptable->GfxclkFmax;
dpm_table->dpm_levels[1].enabled = true;
+ dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
} else {
dpm_table->count = 1;
dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index cd0a23f..0df8c05 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -1129,6 +1129,7 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
/* gfxclk dpm table setup */
dpm_table = &dpm_context->dpm_tables.gfx_table;
dpm_table->clk_type = SMU_GFXCLK;
+ dpm_table->flags = SMU_DPM_TABLE_FINE_GRAINED;
if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
/* In the case of gfxclk, only fine-grained dpm is honored.
* Get min/max values from FW.
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
index c3cb368..940b431 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
@@ -435,10 +435,12 @@ int smu_v15_0_fini_smc_tables(struct smu_context *smu)
smu_table->watermarks_table = NULL;
smu_table->metrics_time = 0;
+ kfree(smu_dpm->dpm_policies);
kfree(smu_dpm->dpm_context);
kfree(smu_dpm->golden_dpm_context);
kfree(smu_dpm->dpm_current_power_state);
kfree(smu_dpm->dpm_request_power_state);
+ smu_dpm->dpm_policies = NULL;
smu_dpm->dpm_context = NULL;
smu_dpm->golden_dpm_context = NULL;
smu_dpm->dpm_context_size = 0;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 3d49e58..90c7127 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -1370,7 +1370,7 @@ int smu_cmn_print_dpm_clk_levels(struct smu_context *smu,
level_index = 1;
}
- if (!is_fine_grained) {
+ if (!is_fine_grained || count == 1) {
for (i = 0; i < count; i++) {
freq_match = !is_deep_sleep &&
smu_cmn_freqs_match(
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index 814713c..553a1df 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -758,7 +758,9 @@ static int chipone_i2c_probe(struct i2c_client *client)
dev_set_drvdata(dev, icn);
i2c_set_clientdata(client, icn);
- drm_bridge_add(&icn->bridge);
+ ret = devm_drm_bridge_add(dev, &icn->bridge);
+ if (ret)
+ return ret;
return chipone_dsi_host_attach(icn);
}
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
index 441fd32..d64e328 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
@@ -222,52 +222,58 @@ static const struct drm_bridge_funcs imx8qxp_pxl2dpi_bridge_funcs = {
imx8qxp_pxl2dpi_bridge_atomic_get_output_bus_fmts,
};
-static struct device_node *
+static int
imx8qxp_pxl2dpi_get_available_ep_from_port(struct imx8qxp_pxl2dpi *p2d,
- u32 port_id)
+ u32 port_id,
+ struct device_node **ep)
{
- struct device_node *port, *ep;
+ struct device_node *port;
+ int ret = 0;
int ep_cnt;
+ *ep = NULL;
+
port = of_graph_get_port_by_id(p2d->dev->of_node, port_id);
if (!port) {
DRM_DEV_ERROR(p2d->dev, "failed to get port@%u\n", port_id);
- return ERR_PTR(-ENODEV);
+ return -ENODEV;
}
ep_cnt = of_get_available_child_count(port);
if (ep_cnt == 0) {
DRM_DEV_ERROR(p2d->dev, "no available endpoints of port@%u\n",
port_id);
- ep = ERR_PTR(-ENODEV);
+ ret = -ENODEV;
goto out;
} else if (ep_cnt > 1) {
DRM_DEV_ERROR(p2d->dev,
"invalid available endpoints of port@%u\n",
port_id);
- ep = ERR_PTR(-EINVAL);
+ ret = -EINVAL;
goto out;
}
- ep = of_get_next_available_child(port, NULL);
- if (!ep) {
+ *ep = of_get_next_available_child(port, NULL);
+ if (!*ep) {
DRM_DEV_ERROR(p2d->dev,
"failed to get available endpoint of port@%u\n",
port_id);
- ep = ERR_PTR(-ENODEV);
+ ret = -ENODEV;
goto out;
}
out:
of_node_put(port);
- return ep;
+ return ret;
}
static int imx8qxp_pxl2dpi_find_next_bridge(struct imx8qxp_pxl2dpi *p2d)
{
- struct device_node *ep __free(device_node) =
- imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 1);
- if (IS_ERR(ep))
- return PTR_ERR(ep);
+ struct device_node *ep __free(device_node) = NULL;
+ int ret;
+
+ ret = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 1, &ep);
+ if (ret)
+ return ret;
struct device_node *remote __free(device_node) = of_graph_get_remote_port_parent(ep);
if (!remote || !of_device_is_available(remote)) {
@@ -291,9 +297,9 @@ static int imx8qxp_pxl2dpi_set_pixel_link_sel(struct imx8qxp_pxl2dpi *p2d)
struct of_endpoint endpoint;
int ret;
- ep = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 0);
- if (IS_ERR(ep))
- return PTR_ERR(ep);
+ ret = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 0, &ep);
+ if (ret)
+ return ret;
ret = of_graph_parse_endpoint(ep, &endpoint);
if (ret) {
diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
index 9246e9c..ed21f09 100644
--- a/drivers/gpu/drm/bridge/ite-it66121.c
+++ b/drivers/gpu/drm/bridge/ite-it66121.c
@@ -1559,6 +1559,11 @@ static int it66121_probe(struct i2c_client *client)
return ret;
}
+ ctx->gpio_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+ if (IS_ERR(ctx->gpio_reset))
+ return dev_err_probe(dev, PTR_ERR(ctx->gpio_reset),
+ "Failed to get reset GPIO\n");
+
it66121_hw_reset(ctx);
ctx->regmap = devm_regmap_init_i2c(client, &it66121_regmap_config);
diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
index c9e6505..2d02cc6 100644
--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
@@ -251,7 +251,6 @@ static void ge_b850v3_lvds_remove(void)
goto out;
drm_bridge_remove(&ge_b850v3_lvds_ptr->bridge);
-
ge_b850v3_lvds_ptr = NULL;
out:
mutex_unlock(&ge_b850v3_lvds_dev_mutex);
@@ -261,6 +260,7 @@ static int ge_b850v3_register(void)
{
struct i2c_client *stdp4028_i2c = ge_b850v3_lvds_ptr->stdp4028_i2c;
struct device *dev = &stdp4028_i2c->dev;
+ int ret;
/* drm bridge initialization */
ge_b850v3_lvds_ptr->bridge.ops = DRM_BRIDGE_OP_DETECT |
@@ -277,11 +277,15 @@ static int ge_b850v3_register(void)
if (!stdp4028_i2c->irq)
return 0;
- return devm_request_threaded_irq(&stdp4028_i2c->dev,
- stdp4028_i2c->irq, NULL,
- ge_b850v3_lvds_irq_handler,
- IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
- "ge-b850v3-lvds-dp", ge_b850v3_lvds_ptr);
+ ret = devm_request_threaded_irq(&stdp4028_i2c->dev,
+ stdp4028_i2c->irq, NULL,
+ ge_b850v3_lvds_irq_handler,
+ IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+ "ge-b850v3-lvds-dp", ge_b850v3_lvds_ptr);
+ if (ret)
+ drm_bridge_remove(&ge_b850v3_lvds_ptr->bridge);
+
+ return ret;
}
static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c)
diff --git a/drivers/gpu/drm/bridge/tda998x_drv.c b/drivers/gpu/drm/bridge/tda998x_drv.c
index d9b3881..6c427bc 100644
--- a/drivers/gpu/drm/bridge/tda998x_drv.c
+++ b/drivers/gpu/drm/bridge/tda998x_drv.c
@@ -1293,7 +1293,7 @@ static const struct drm_edid *tda998x_edid_read(struct tda998x_priv *priv,
* can't handle signals gracefully.
*/
if (tda998x_edid_delay_wait(priv))
- return 0;
+ return NULL;
if (priv->rev == TDA19988)
reg_clear(priv, REG_TX4, TX4_PD_RAM);
@@ -1762,7 +1762,7 @@ static const struct drm_bridge_funcs tda998x_bridge_funcs = {
static int tda998x_get_audio_ports(struct tda998x_priv *priv,
struct device_node *np)
{
- const u32 *port_data;
+ const __be32 *port_data;
u32 size;
int i;
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
index c598b99..e7db4e4 100644
--- a/drivers/gpu/drm/drm_color_mgmt.c
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -831,7 +831,7 @@ static void fill_palette_332(struct drm_crtc *crtc, u16 r, u16 g, u16 b,
}
/**
- * drm_crtc_fill_palette_332 - Programs a default palette for R332-like formats
+ * drm_crtc_fill_palette_332 - Programs a default palette for RGB332-like formats
* @crtc: The displaying CRTC
* @set_palette: Callback for programming the hardware gamma LUT
*
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 985c283..6756754 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -697,6 +697,7 @@ static void drm_dev_init_release(struct drm_device *dev, void *res)
mutex_destroy(&dev->master_mutex);
mutex_destroy(&dev->clientlist_mutex);
mutex_destroy(&dev->filelist_mutex);
+ mutex_destroy(&dev->gem_lru_mutex);
}
static int drm_dev_init(struct drm_device *dev,
@@ -738,6 +739,7 @@ static int drm_dev_init(struct drm_device *dev,
INIT_LIST_HEAD(&dev->vblank_event_list);
spin_lock_init(&dev->event_lock);
+ mutex_init(&dev->gem_lru_mutex);
mutex_init(&dev->filelist_mutex);
mutex_init(&dev->clientlist_mutex);
mutex_init(&dev->master_mutex);
diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c
index e2b62e5..cc99681 100644
--- a/drivers/gpu/drm/drm_dumb_buffers.c
+++ b/drivers/gpu/drm/drm_dumb_buffers.c
@@ -70,8 +70,11 @@ static int drm_mode_align_dumb(struct drm_mode_create_dumb *args,
if (!pitch)
return -EINVAL;
- if (hw_pitch_align)
+ if (hw_pitch_align) {
pitch = roundup(pitch, hw_pitch_align);
+ if (pitch < hw_pitch_align)
+ return -EINVAL;
+ }
if (!hw_size_align)
hw_size_align = PAGE_SIZE;
@@ -80,7 +83,7 @@ static int drm_mode_align_dumb(struct drm_mode_create_dumb *args,
if (check_mul_overflow(args->height, pitch, &size))
return -EINVAL;
- size = ALIGN(size, hw_size_align);
+ size = roundup(size, hw_size_align);
if (!size)
return -EINVAL;
@@ -199,6 +202,13 @@ int drm_mode_create_dumb(struct drm_device *dev,
if (!args->width || !args->height || !args->bpp)
return -EINVAL;
+ /* Reject unreasonable inputs early. Dumb buffers are for software
+ * rendering; nothing legitimate needs more than 8192x8192 at 32bpp.
+ * This prevents overflows in downstream alignment helpers.
+ */
+ if (args->width >= 8192 || args->height >= 8192 || args->bpp > 32)
+ return -EINVAL;
+
/* overflow checks for 32bit size calculations */
if (args->bpp > U32_MAX - 8)
return -EINVAL;
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index a80a335..1541fc8 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -490,7 +490,7 @@ static void drm_fb_helper_memory_range_to_clip(struct fb_info *info, off_t off,
* the number of horizontal pixels that need an update.
*/
off_t bit_off = (off % line_length) * 8;
- off_t bit_end = (end % line_length) * 8;
+ off_t bit_end = bit_off + len * 8;
x1 = bit_off / info->var.bits_per_pixel;
x2 = DIV_ROUND_UP(bit_end, info->var.bits_per_pixel);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index d642426..e12cdf9 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -1019,7 +1019,7 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_gem_change_handle *args = data;
- struct drm_gem_object *obj;
+ struct drm_gem_object *obj, *idrobj;
int handle, ret;
if (!drm_core_check_feature(dev, DRIVER_GEM))
@@ -1042,12 +1042,31 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
mutex_lock(&file_priv->prime.lock);
spin_lock(&file_priv->table_lock);
+
+ /* When create_tail allocs an obj idr, it needs to first alloc as NULL,
+ * then later replace with the correct object. This is not necessary
+ * here, because the only operations that could race are drm_prime
+ * bookkeeping, and we hold the prime lock.
+ */
ret = idr_alloc(&file_priv->object_idr, obj, handle, handle + 1,
GFP_NOWAIT);
- spin_unlock(&file_priv->table_lock);
- if (ret < 0)
- goto out_unlock;
+ if (ret < 0) {
+ spin_unlock(&file_priv->table_lock);
+ goto out_unlock;
+ }
+
+ idrobj = idr_replace(&file_priv->object_idr, NULL, handle);
+ if (idrobj != obj) {
+ idr_replace(&file_priv->object_idr, idrobj, handle);
+ idr_remove(&file_priv->object_idr, args->new_handle);
+ spin_unlock(&file_priv->table_lock);
+ ret = -ENOENT;
+ goto out_unlock;
+ }
+
+ idr_replace(&file_priv->object_idr, NULL, args->handle);
+ spin_unlock(&file_priv->table_lock);
if (obj->dma_buf) {
ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf,
@@ -1055,6 +1074,7 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
if (ret < 0) {
spin_lock(&file_priv->table_lock);
idr_remove(&file_priv->object_idr, handle);
+ idr_replace(&file_priv->object_idr, obj, args->handle);
spin_unlock(&file_priv->table_lock);
goto out_unlock;
}
@@ -1066,7 +1086,9 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
spin_lock(&file_priv->table_lock);
idr_remove(&file_priv->object_idr, args->handle);
+ idrobj = idr_replace(&file_priv->object_idr, obj, handle);
spin_unlock(&file_priv->table_lock);
+ WARN_ON(idrobj != NULL);
out_unlock:
mutex_unlock(&file_priv->prime.lock);
@@ -1541,12 +1563,10 @@ EXPORT_SYMBOL(drm_gem_unlock_reservations);
* drm_gem_lru_init - initialize a LRU
*
* @lru: The LRU to initialize
- * @lock: The lock protecting the LRU
*/
void
-drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
+drm_gem_lru_init(struct drm_gem_lru *lru)
{
- lru->lock = lock;
lru->count = 0;
INIT_LIST_HEAD(&lru->list);
}
@@ -1571,14 +1591,10 @@ drm_gem_lru_remove_locked(struct drm_gem_object *obj)
void
drm_gem_lru_remove(struct drm_gem_object *obj)
{
- struct drm_gem_lru *lru = obj->lru;
-
- if (!lru)
- return;
-
- mutex_lock(lru->lock);
- drm_gem_lru_remove_locked(obj);
- mutex_unlock(lru->lock);
+ mutex_lock(&obj->dev->gem_lru_mutex);
+ if (obj->lru)
+ drm_gem_lru_remove_locked(obj);
+ mutex_unlock(&obj->dev->gem_lru_mutex);
}
EXPORT_SYMBOL(drm_gem_lru_remove);
@@ -1593,7 +1609,7 @@ EXPORT_SYMBOL(drm_gem_lru_remove);
void
drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
{
- lockdep_assert_held_once(lru->lock);
+ lockdep_assert_held_once(&obj->dev->gem_lru_mutex);
if (obj->lru)
drm_gem_lru_remove_locked(obj);
@@ -1617,9 +1633,9 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
void
drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
{
- mutex_lock(lru->lock);
+ mutex_lock(&obj->dev->gem_lru_mutex);
drm_gem_lru_move_tail_locked(lru, obj);
- mutex_unlock(lru->lock);
+ mutex_unlock(&obj->dev->gem_lru_mutex);
}
EXPORT_SYMBOL(drm_gem_lru_move_tail);
@@ -1633,6 +1649,7 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail);
* of the shrink callback to check for this (ie. dma_resv_test_signaled())
* or if necessary block until the buffer becomes idle.
*
+ * @dev: DRM device the LRU belongs to
* @lru: The LRU to scan
* @nr_to_scan: The number of pages to try to reclaim
* @remaining: The number of pages left to reclaim, should be initialized by caller
@@ -1640,7 +1657,8 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail);
* @ticket: Optional ww_acquire_ctx context to use for locking
*/
unsigned long
-drm_gem_lru_scan(struct drm_gem_lru *lru,
+drm_gem_lru_scan(struct drm_device *dev,
+ struct drm_gem_lru *lru,
unsigned int nr_to_scan,
unsigned long *remaining,
bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
@@ -1650,9 +1668,9 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
struct drm_gem_object *obj;
unsigned freed = 0;
- drm_gem_lru_init(&still_in_lru, lru->lock);
+ drm_gem_lru_init(&still_in_lru);
- mutex_lock(lru->lock);
+ mutex_lock(&dev->gem_lru_mutex);
while (freed < nr_to_scan) {
obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
@@ -1675,7 +1693,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
* rest of the loop body, to reduce contention with other
* code paths that need the LRU lock
*/
- mutex_unlock(lru->lock);
+ mutex_unlock(&dev->gem_lru_mutex);
if (ticket)
ww_acquire_init(ticket, &reservation_ww_class);
@@ -1709,7 +1727,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
tail:
drm_gem_object_put(obj);
- mutex_lock(lru->lock);
+ mutex_lock(&dev->gem_lru_mutex);
}
/*
@@ -1721,7 +1739,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
list_splice_tail(&still_in_lru.list, &lru->list);
lru->count += still_in_lru.count;
- mutex_unlock(lru->lock);
+ mutex_unlock(&dev->gem_lru_mutex);
return freed;
}
diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
index 9166c353..88808e9 100644
--- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c
+++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
@@ -172,8 +172,8 @@ int drm_gem_fb_init_with_funcs(struct drm_device *dev,
}
for (i = 0; i < info->num_planes; i++) {
- unsigned int width = mode_cmd->width / (i ? info->hsub : 1);
- unsigned int height = mode_cmd->height / (i ? info->vsub : 1);
+ unsigned int width = drm_format_info_plane_width(info, mode_cmd->width, i);
+ unsigned int height = drm_format_info_plane_height(info, mode_cmd->height, i);
unsigned int min_size;
objs[i] = drm_gem_object_lookup(file, mode_cmd->handles[i]);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index df4232d7..3cc50d6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -116,16 +116,18 @@ int etnaviv_sched_push_job(struct etnaviv_gem_submit *submit)
*/
mutex_lock(&gpu->sched_lock);
+ ret = xa_alloc_cyclic(&gpu->user_fences, &submit->out_fence_id,
+ NULL, xa_limit_32b, &gpu->next_user_fence,
+ GFP_KERNEL);
+ if (ret < 0)
+ goto out_unlock;
+
drm_sched_job_arm(&submit->sched_job);
submit->out_fence = dma_fence_get(&submit->sched_job.s_fence->finished);
- ret = xa_alloc_cyclic(&gpu->user_fences, &submit->out_fence_id,
- submit->out_fence, xa_limit_32b,
- &gpu->next_user_fence, GFP_KERNEL);
- if (ret < 0) {
- drm_sched_job_cleanup(&submit->sched_job);
- goto out_unlock;
- }
+
+ xa_store(&gpu->user_fences, submit->out_fence_id,
+ submit->out_fence, GFP_KERNEL);
/* the scheduler holds on to the job now */
kref_get(&submit->refcount);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index 29a8366..e68c954 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -423,7 +423,9 @@ static int exynos_mic_probe(struct platform_device *pdev)
mic->bridge.of_node = dev->of_node;
- drm_bridge_add(&mic->bridge);
+ ret = devm_drm_bridge_add(dev, &mic->bridge);
+ if (ret)
+ goto err;
pm_runtime_enable(dev);
@@ -443,12 +445,8 @@ static int exynos_mic_probe(struct platform_device *pdev)
static void exynos_mic_remove(struct platform_device *pdev)
{
- struct exynos_mic *mic = platform_get_drvdata(pdev);
-
component_del(&pdev->dev, &exynos_mic_component_ops);
pm_runtime_disable(&pdev->dev);
-
- drm_bridge_remove(&mic->bridge);
}
static const struct of_device_id exynos_mic_of_match[] = {
diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c
index 6b6b44e..4fbc22a59 100644
--- a/drivers/gpu/drm/gma500/mmu.c
+++ b/drivers/gpu/drm/gma500/mmu.c
@@ -7,6 +7,8 @@
#include <linux/highmem.h>
#include <linux/vmalloc.h>
+#include <asm/cpuid/api.h>
+
#include "mmu.h"
#include "psb_drv.h"
#include "psb_reg.h"
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 58d7e19..403d21c 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -580,6 +580,7 @@ static int oaktrail_hdmi_get_modes(struct drm_connector *connector)
} else {
edid = (struct edid *)raw_edid;
/* FIXME ? edid = drm_get_edid(connector, i2c_adap); */
+ i2c_put_adapter(i2c_adap);
}
if (edid) {
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index 884d324..e194d0cc 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -293,7 +293,7 @@ void oaktrail_lvds_init(struct drm_device *dev,
{
struct gma_encoder *gma_encoder;
struct gma_connector *gma_connector;
- struct gma_i2c_chan *ddc_bus;
+ struct gma_i2c_chan *ddc_bus = NULL;
struct drm_connector *connector;
struct drm_encoder *encoder;
struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
@@ -367,6 +367,8 @@ void oaktrail_lvds_init(struct drm_device *dev,
if (edid == NULL && dev_priv->lpc_gpio_base) {
ddc_bus = oaktrail_lvds_i2c_init(dev);
if (!IS_ERR(ddc_bus)) {
+ if (i2c_adap)
+ i2c_put_adapter(i2c_adap);
i2c_adap = &ddc_bus->base;
edid = drm_get_edid(connector, i2c_adap);
}
@@ -421,7 +423,10 @@ void oaktrail_lvds_init(struct drm_device *dev,
err_unlock:
mutex_unlock(&dev->mode_config.mutex);
- gma_i2c_destroy(to_gma_i2c_chan(connector->ddc));
+ if (!IS_ERR_OR_NULL(ddc_bus))
+ gma_i2c_destroy(ddc_bus);
+ else if (i2c_adap)
+ i2c_put_adapter(i2c_adap);
drm_encoder_cleanup(encoder);
err_connector_cleanup:
drm_connector_cleanup(connector);
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
index 051ecc5..4e6f703 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
@@ -391,8 +391,11 @@ static int hyperv_get_supported_resolution(struct hv_device *hdev)
return -ETIMEDOUT;
}
- if (msg->resolution_resp.resolution_count == 0) {
- drm_err(dev, "No supported resolutions\n");
+ if (msg->resolution_resp.resolution_count == 0 ||
+ msg->resolution_resp.resolution_count >
+ SYNTHVID_MAX_RESOLUTION_COUNT) {
+ drm_err(dev, "Invalid resolution count: %d\n",
+ msg->resolution_resp.resolution_count);
return -ENODEV;
}
@@ -417,30 +420,92 @@ static int hyperv_get_supported_resolution(struct hv_device *hdev)
return 0;
}
-static void hyperv_receive_sub(struct hv_device *hdev)
+static void hyperv_receive_sub(struct hv_device *hdev, u32 bytes_recvd)
{
struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
struct synthvid_msg *msg;
+ size_t hdr_size;
+ size_t need;
if (!hv)
return;
- msg = (struct synthvid_msg *)hv->recv_buf;
-
- /* Complete the wait event */
- if (msg->vid_hdr.type == SYNTHVID_VERSION_RESPONSE ||
- msg->vid_hdr.type == SYNTHVID_RESOLUTION_RESPONSE ||
- msg->vid_hdr.type == SYNTHVID_VRAM_LOCATION_ACK) {
- memcpy(hv->init_buf, msg, VMBUS_MAX_PACKET_SIZE);
- complete(&hv->wait);
+ hdr_size = sizeof(struct pipe_msg_hdr) +
+ sizeof(struct synthvid_msg_hdr);
+ if (bytes_recvd < hdr_size) {
+ drm_err_ratelimited(&hv->dev,
+ "synthvid packet too small for header: %u\n",
+ bytes_recvd);
return;
}
- if (msg->vid_hdr.type == SYNTHVID_FEATURE_CHANGE) {
+ msg = (struct synthvid_msg *)hv->recv_buf;
+ need = hdr_size;
+
+ switch (msg->vid_hdr.type) {
+ case SYNTHVID_VERSION_RESPONSE:
+ need += sizeof(struct synthvid_version_resp);
+ break;
+ case SYNTHVID_RESOLUTION_RESPONSE:
+ /*
+ * The resolution response is variable length: the host
+ * fills resolution_count entries, not the full
+ * SYNTHVID_MAX_RESOLUTION_COUNT array. Require the fixed
+ * prefix first so resolution_count can be read, then
+ * demand exactly the count-sized array.
+ */
+ need += offsetof(struct synthvid_supported_resolution_resp,
+ supported_resolution);
+ if (bytes_recvd < need)
+ break;
+ if (msg->resolution_resp.resolution_count >
+ SYNTHVID_MAX_RESOLUTION_COUNT) {
+ drm_err_ratelimited(&hv->dev,
+ "synthvid resolution count too large: %u\n",
+ msg->resolution_resp.resolution_count);
+ return;
+ }
+ need += msg->resolution_resp.resolution_count *
+ sizeof(struct hvd_screen_info);
+ break;
+ case SYNTHVID_VRAM_LOCATION_ACK:
+ need += sizeof(struct synthvid_vram_location_ack);
+ break;
+ case SYNTHVID_FEATURE_CHANGE:
+ /*
+ * Not a completion-driving message: validate its own payload
+ * and consume it here rather than falling through to the
+ * memcpy/complete shared by the wait-event responses.
+ */
+ if (bytes_recvd < need +
+ sizeof(struct synthvid_feature_change)) {
+ drm_err_ratelimited(&hv->dev,
+ "synthvid feature change packet too small: %u\n",
+ bytes_recvd);
+ return;
+ }
hv->dirt_needed = msg->feature_chg.is_dirt_needed;
if (hv->dirt_needed)
hyperv_hide_hw_ptr(hv->hdev);
+ return;
+ default:
+ return;
}
+
+ /*
+ * Shared completion path for the wait-event responses
+ * (VERSION_RESPONSE, RESOLUTION_RESPONSE, VRAM_LOCATION_ACK):
+ * require the type-specific payload before handing the buffer to
+ * the waiter.
+ */
+ if (bytes_recvd < need) {
+ drm_err_ratelimited(&hv->dev,
+ "synthvid packet too small for type %u: %u < %zu\n",
+ msg->vid_hdr.type, bytes_recvd, need);
+ return;
+ }
+ memcpy(hv->init_buf, msg, bytes_recvd);
+ complete(&hv->wait);
}
static void hyperv_receive(void *ctx)
@@ -461,9 +526,21 @@ static void hyperv_receive(void *ctx)
ret = vmbus_recvpacket(hdev->channel, recv_buf,
VMBUS_MAX_PACKET_SIZE,
&bytes_recvd, &req_id);
- if (bytes_recvd > 0 &&
- recv_buf->pipe_hdr.type == PIPE_MSG_DATA)
- hyperv_receive_sub(hdev);
+ if (ret) {
+ /*
+ * A nonzero return (e.g. -ENOBUFS for an oversized
+ * packet) is itself a malformed message: bytes_recvd
+ * then reports the required length rather than a copied
+ * payload, so it must not be forwarded to the
+ * sub-handler. Channel recovery is not attempted.
+ */
+ drm_err_ratelimited(&hv->dev,
+ "vmbus_recvpacket failed: %d (need %u)\n",
+ ret, bytes_recvd);
+ } else if (bytes_recvd > 0 &&
+ recv_buf->pipe_hdr.type == PIPE_MSG_DATA) {
+ hyperv_receive_sub(hdev, bytes_recvd);
+ }
} while (bytes_recvd > 0 && ret == 0);
}
@@ -508,9 +585,13 @@ int hyperv_connect_vsp(struct hv_device *hdev)
ret = hyperv_get_supported_resolution(hdev);
if (ret)
drm_err(dev, "Failed to get supported resolution from host, use default\n");
- } else {
+ }
+
+ if (!hv->screen_width_max) {
hv->screen_width_max = SYNTHVID_WIDTH_WIN8;
hv->screen_height_max = SYNTHVID_HEIGHT_WIN8;
+ hv->preferred_width = SYNTHVID_WIDTH_WIN8;
+ hv->preferred_height = SYNTHVID_HEIGHT_WIN8;
}
hv->mmio_megabytes = hdev->channel->offermsg.offer.mmio_megabytes;
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index e795065..6d1cffc 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -3976,7 +3976,7 @@ xelpd_program_plane_pre_csc_lut(struct intel_dsb *dsb,
intel_de_write_dsb(display, dsb,
PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0),
(1 << 24));
- } while (i++ > 130);
+ } while (i++ < 130);
} else {
for (i = 0; i < lut_size; i++) {
u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1);
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index d9baca2..78afcd4 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -497,6 +497,7 @@ struct intel_display {
u8 vblank_enabled;
int vblank_enable_count;
+ bool vblank_status_last_notified;
struct work_struct vblank_notify_work;
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index 70c1bba..aedf392 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -1773,8 +1773,12 @@ static void intel_display_vblank_notify_work(struct work_struct *work)
struct intel_display *display =
container_of(work, typeof(*display), irq.vblank_notify_work);
int vblank_enable_count = READ_ONCE(display->irq.vblank_enable_count);
+ bool vblank_status = !!vblank_enable_count;
- intel_psr_notify_vblank_enable_disable(display, vblank_enable_count);
+ if (display->irq.vblank_status_last_notified != vblank_status) {
+ intel_psr_notify_vblank_enable_disable(display, vblank_status);
+ display->irq.vblank_status_last_notified = vblank_status;
+ }
}
int bdw_enable_vblank(struct drm_crtc *_crtc)
@@ -1787,10 +1791,10 @@ int bdw_enable_vblank(struct drm_crtc *_crtc)
if (gen11_dsi_configure_te(crtc, true))
return 0;
+ spin_lock_irqsave(&display->irq.lock, irqflags);
if (crtc->vblank_psr_notify && display->irq.vblank_enable_count++ == 0)
schedule_work(&display->irq.vblank_notify_work);
- spin_lock_irqsave(&display->irq.lock, irqflags);
bdw_enable_pipe_irq(display, pipe, GEN8_PIPE_VBLANK);
spin_unlock_irqrestore(&display->irq.lock, irqflags);
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index f6cd0a0..2e6a857 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -584,6 +584,7 @@ struct intel_connector {
struct {
u8 dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
+ u8 intel_wa_dpcd;
bool support;
bool su_support;
@@ -1789,6 +1790,8 @@ struct intel_psr {
u8 active_non_psr_pipes;
const char *no_psr_reason;
+
+ struct ref_tracker *vblank_wakeref;
};
struct intel_dp {
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 4955bd8..6ef2a00 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -3119,8 +3119,13 @@ static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc
drm_WARN_ON(display->drm,
vsc->bpc == 6 && vsc->pixelformat != DP_PIXELFORMAT_RGB);
- /* all YCbCr are always limited range */
- vsc->dynamic_range = DP_DYNAMIC_RANGE_CTA;
+ /* All YCbCr formats are always limited range. */
+ if (vsc->pixelformat == DP_PIXELFORMAT_RGB)
+ vsc->dynamic_range = crtc_state->limited_color_range ?
+ DP_DYNAMIC_RANGE_CTA : DP_DYNAMIC_RANGE_VESA;
+ else
+ vsc->dynamic_range = DP_DYNAMIC_RANGE_CTA;
+
vsc->content_type = DP_CONTENT_TYPE_NOT_DEFINED;
}
@@ -5298,7 +5303,7 @@ int intel_dp_as_sdp_unpack(struct drm_dp_as_sdp *as_sdp,
as_sdp->length = sdp->sdp_header.HB3 & DP_ADAPTIVE_SYNC_SDP_LENGTH;
as_sdp->mode = sdp->db[0] & DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE;
as_sdp->vtotal = (sdp->db[2] << 8) | sdp->db[1];
- as_sdp->target_rr = (u64)sdp->db[3] | ((u64)sdp->db[4] & 0x3);
+ as_sdp->target_rr = ((sdp->db[4] & 0x3) << 8) | sdp->db[3];
as_sdp->target_rr_divider = sdp->db[4] & 0x20 ? true : false;
return 0;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index b20ec3e..9c9b641 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -12,6 +12,7 @@
#include "intel_dp.h"
#include "intel_dp_aux.h"
#include "intel_dp_aux_regs.h"
+#include "intel_parent.h"
#include "intel_pps.h"
#include "intel_quirks.h"
#include "intel_tc.h"
@@ -60,18 +61,29 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp)
struct intel_display *display = to_intel_display(intel_dp);
i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
const unsigned int timeout_ms = 10;
+ bool done = true;
u32 status;
- bool done;
+ int ret;
+ if (intel_parent_irq_enabled(display)) {
#define C (((status = intel_de_read_notrace(display, ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
- done = wait_event_timeout(display->gmbus.wait_queue, C,
- msecs_to_jiffies_timeout(timeout_ms));
+ done = wait_event_timeout(display->gmbus.wait_queue, C,
+ msecs_to_jiffies_timeout(timeout_ms));
+
+#undef C
+ } else {
+ ret = intel_de_wait_ms(display, ch_ctl,
+ DP_AUX_CH_CTL_SEND_BUSY, 0,
+ timeout_ms, &status);
+
+ if (ret == -ETIMEDOUT)
+ done = false;
+ }
if (!done)
drm_err(display->drm,
"%s: did not complete or timeout within %ums (status 0x%08x)\n",
intel_dp->aux.name, timeout_ms, status);
-#undef C
return status;
}
diff --git a/drivers/gpu/drm/i915/display/intel_dpcd.h b/drivers/gpu/drm/i915/display/intel_dpcd.h
new file mode 100644
index 0000000..4aea532
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dpcd.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_DPCD_H__
+#define __INTEL_DPCD_H__
+
+#define INTEL_DPCD_INTEL_WA_REGISTER_CAPS 0x3f0
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_EARLYSCANLINE_SDP_SUPPORT_MASK REG_GENMASK(1, 0)
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_FALL_BACK_TO_PSR1 0
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITH_EARLY_SCANLINE 1
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITHOUT_EARLY_SCANLINE 2
+
+#endif /* __INTEL_DPCD_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c
index 5390ceb..82f445c 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_plane.c
@@ -373,7 +373,7 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
bool changed = false;
int i = 0;
- iter_colorop = plane_state->uapi.color_pipeline;
+ iter_colorop = from_plane_state->uapi.color_pipeline;
while (iter_colorop) {
for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 53c10ae..598fe76 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -43,6 +43,7 @@
#include "intel_display_wa.h"
#include "intel_dmc.h"
#include "intel_dp.h"
+#include "intel_dpcd.h"
#include "intel_dp_aux.h"
#include "intel_dsb.h"
#include "intel_frontbuffer.h"
@@ -716,8 +717,14 @@ static void _psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *co
connector->dp.psr_caps.su_support ? "" : "not ");
}
- if (connector->dp.psr_caps.su_support)
+ if (connector->dp.psr_caps.su_support) {
+ ret = drm_dp_dpcd_read_byte(&intel_dp->aux,
+ INTEL_DPCD_INTEL_WA_REGISTER_CAPS,
+ &connector->dp.psr_caps.intel_wa_dpcd);
+ if (ret < 0)
+ return;
_psr_compute_su_granularity(intel_dp, connector);
+ }
}
void intel_psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
@@ -1358,9 +1365,35 @@ static bool psr2_granularity_check(struct intel_crtc_state *crtc_state,
return true;
}
-static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,
- struct intel_crtc_state *crtc_state)
+static bool apply_scanline_indication_wa(struct intel_crtc_state *crtc_state,
+ struct intel_connector *connector)
{
+ struct intel_dp *intel_dp = intel_attached_dp(connector);
+ u8 early_scanline_support = connector->dp.psr_caps.intel_wa_dpcd &
+ INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_EARLYSCANLINE_SDP_SUPPORT_MASK;
+
+ if (intel_dp->edp_dpcd[0] >= DP_EDP_15)
+ return true;
+
+ switch (early_scanline_support) {
+ case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_FALL_BACK_TO_PSR1:
+ crtc_state->req_psr2_sdp_prior_scanline = false;
+ return false;
+ case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITH_EARLY_SCANLINE:
+ return true;
+ case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITHOUT_EARLY_SCANLINE:
+ crtc_state->req_psr2_sdp_prior_scanline = false;
+ return true;
+ default:
+ MISSING_CASE(early_scanline_support);
+ return false;
+ }
+}
+
+static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_crtc_state *crtc_state,
+ struct intel_connector *connector)
+{
+ struct intel_dp *intel_dp = intel_attached_dp(connector);
struct intel_display *display = to_intel_display(intel_dp);
const struct drm_display_mode *adjusted_mode = &crtc_state->uapi.adjusted_mode;
u32 hblank_total, hblank_ns, req_ns;
@@ -1379,7 +1412,8 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d
return false;
crtc_state->req_psr2_sdp_prior_scanline = true;
- return true;
+
+ return apply_scanline_indication_wa(crtc_state, connector);
}
static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
@@ -1660,7 +1694,7 @@ static bool intel_sel_update_config_valid(struct intel_crtc_state *crtc_state,
conn_state))
goto unsupported;
- if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
+ if (!_compute_psr2_sdp_prior_scanline_indication(crtc_state, connector)) {
drm_dbg_kms(display->drm,
"Selective update not enabled, SDP indication do not fit in hblank\n");
goto unsupported;
@@ -4122,27 +4156,22 @@ void intel_psr_notify_vblank_enable_disable(struct intel_display *display,
struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
mutex_lock(&intel_dp->psr.lock);
- if (intel_dp->psr.panel_replay_enabled) {
- mutex_unlock(&intel_dp->psr.lock);
- break;
+ if (CAN_PANEL_REPLAY(intel_dp)) {
+ if (enable)
+ intel_dp->psr.vblank_wakeref =
+ intel_display_power_get(display,
+ POWER_DOMAIN_DC_OFF);
+ else
+ intel_display_power_put(display, POWER_DOMAIN_DC_OFF,
+ intel_dp->psr.vblank_wakeref);
}
- if (intel_dp->psr.enabled && intel_dp->psr.pkg_c_latency_used)
+ if (intel_dp->psr.enabled && !intel_dp->psr.panel_replay_enabled &&
+ intel_dp->psr.pkg_c_latency_used)
intel_psr_apply_underrun_on_idle_wa_locked(intel_dp);
mutex_unlock(&intel_dp->psr.lock);
- return;
}
-
- /*
- * NOTE: intel_display_power_set_target_dc_state is used
- * only by PSR * code for DC3CO handling. DC3CO target
- * state is currently disabled in * PSR code. If DC3CO
- * is taken into use we need take that into account here
- * as well.
- */
- intel_display_power_set_target_dc_state(display, enable ? DC_STATE_DISABLE :
- DC_STATE_EN_UPTO_DC6);
}
static void
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
index de70517..df3fcc2 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
@@ -419,8 +419,6 @@ void i915_ttm_free_cached_io_rsgt(struct drm_i915_gem_object *obj)
int i915_ttm_purge(struct drm_i915_gem_object *obj)
{
struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
- struct i915_ttm_tt *i915_tt =
- container_of(bo->ttm, typeof(*i915_tt), ttm);
struct ttm_operation_ctx ctx = {
.interruptible = true,
.no_wait_gpu = false,
@@ -435,16 +433,22 @@ int i915_ttm_purge(struct drm_i915_gem_object *obj)
if (ret)
return ret;
- if (bo->ttm && i915_tt->filp) {
- /*
- * The below fput(which eventually calls shmem_truncate) might
- * be delayed by worker, so when directly called to purge the
- * pages(like by the shrinker) we should try to be more
- * aggressive and release the pages immediately.
- */
- shmem_truncate_range(file_inode(i915_tt->filp),
- 0, (loff_t)-1);
- fput(fetch_and_zero(&i915_tt->filp));
+ if (bo->ttm) {
+ struct i915_ttm_tt *i915_tt =
+ container_of(bo->ttm, typeof(*i915_tt), ttm);
+
+ if (i915_tt->filp) {
+ /*
+ * The below fput(which eventually calls shmem_truncate)
+ * might be delayed by worker, so when directly called
+ * to purge the pages(like by the shrinker) we should
+ * try to be more aggressive and release the pages
+ * immediately.
+ */
+ shmem_truncate_range(file_inode(i915_tt->filp),
+ 0, (loff_t)-1);
+ fput(fetch_and_zero(&i915_tt->filp));
+ }
}
obj->write_domain = 0;
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 984d005..adff482 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -132,7 +132,8 @@ void __i915_request_reset(struct i915_request *rq, bool guilty)
rcu_read_lock(); /* protect the GEM context */
if (guilty) {
i915_request_set_error_once(rq, -EIO);
- __i915_request_skip(rq);
+ if (!i915_request_signaled(rq))
+ __i915_request_skip(rq);
banned = mark_guilty(rq);
} else {
i915_request_set_error_once(rq, -EAGAIN);
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index 385a634..d9be7a5 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -750,9 +750,8 @@ static bool has_auxccs(struct drm_device *drm)
{
struct drm_i915_private *i915 = to_i915(drm);
- return IS_GRAPHICS_VER(i915, 9, 12) ||
- IS_ALDERLAKE_P(i915) ||
- IS_METEORLAKE(i915);
+ return IS_GRAPHICS_VER(i915, 9, 12) &&
+ !HAS_FLAT_CCS(i915);
}
static bool has_fenced_regions(struct drm_device *drm)
diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c
index e154cb3..6193811 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_trace.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c
@@ -558,6 +558,6 @@ pvr_fw_trace_debugfs_init(struct pvr_device *pvr_dev, struct dentry *dir)
&pvr_fw_trace_fops);
}
- debugfs_create_file("trace_mask", 0600, dir, fw_trace,
+ debugfs_create_file("trace_mask", 0600, dir, pvr_dev,
&pvr_fw_trace_mask_fops);
}
diff --git a/drivers/gpu/drm/loongson/lsdc_drv.c b/drivers/gpu/drm/loongson/lsdc_drv.c
index 1ece1ea..3440507 100644
--- a/drivers/gpu/drm/loongson/lsdc_drv.c
+++ b/drivers/gpu/drm/loongson/lsdc_drv.c
@@ -293,7 +293,7 @@ static int lsdc_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
vga_client_register(pdev, lsdc_vga_set_decode);
- drm_kms_helper_poll_init(ddev);
+ drmm_kms_helper_poll_init(ddev);
if (loongson_vblank) {
ret = drm_vblank_init(ddev, descp->num_of_crtc);
diff --git a/drivers/gpu/drm/mediatek/mtk_cec.c b/drivers/gpu/drm/mediatek/mtk_cec.c
index c7be530..b8ccd6e 100644
--- a/drivers/gpu/drm/mediatek/mtk_cec.c
+++ b/drivers/gpu/drm/mediatek/mtk_cec.c
@@ -240,7 +240,7 @@ static const struct of_device_id mtk_cec_of_ids[] = {
};
MODULE_DEVICE_TABLE(of, mtk_cec_of_ids);
-struct platform_driver mtk_cec_driver = {
+static struct platform_driver mtk_cec_driver = {
.probe = mtk_cec_probe,
.remove = mtk_cec_remove,
.driver = {
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
index 6358e1a..2acbdb0 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
@@ -328,7 +328,7 @@ static const struct of_device_id mtk_hdmi_ddc_match[] = {
};
MODULE_DEVICE_TABLE(of, mtk_hdmi_ddc_match);
-struct platform_driver mtk_hdmi_ddc_driver = {
+static struct platform_driver mtk_hdmi_ddc_driver = {
.probe = mtk_hdmi_ddc_probe,
.remove = mtk_hdmi_ddc_remove,
.driver = {
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c
index d937219..31e81a6d 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c
@@ -389,7 +389,7 @@ static const struct of_device_id mtk_hdmi_ddc_v2_match[] = {
};
MODULE_DEVICE_TABLE(of, mtk_hdmi_ddc_v2_match);
-struct platform_driver mtk_hdmi_ddc_v2_driver = {
+static struct platform_driver mtk_hdmi_ddc_v2_driver = {
.probe = mtk_hdmi_ddc_v2_probe,
.driver = {
.name = "mediatek-hdmi-ddc-v2",
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
index b5c7383..a8eb6fd 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
@@ -50,7 +50,7 @@ enum mtk_hdmi_v2_clk_id {
MTK_HDMI_V2_CLK_COUNT,
};
-const char *const mtk_hdmi_v2_clk_names[MTK_HDMI_V2_CLK_COUNT] = {
+static const char *const mtk_hdmi_v2_clk_names[MTK_HDMI_V2_CLK_COUNT] = {
[MTK_HDMI_V2_CLK_HDMI_APB_SEL] = "bus",
[MTK_HDMI_V2_CLK_HDCP_SEL] = "hdcp",
[MTK_HDMI_V2_CLK_HDCP_24M_SEL] = "hdcp24m",
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index d5aba07..7a3e3c2 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -2621,7 +2621,6 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
struct platform_device *pdev = priv->gpu_pdev;
struct adreno_platform_config *config = pdev->dev.platform_data;
const struct adreno_info *info = config->info;
- struct device_node *node;
struct a6xx_gpu *a6xx_gpu;
struct adreno_gpu *adreno_gpu;
struct msm_gpu *gpu;
@@ -2643,7 +2642,8 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
adreno_gpu->registers = NULL;
/* Check if there is a GMU phandle and set it up */
- node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
+ struct device_node *node __free(device_node) =
+ of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
/* FIXME: How do we gracefully handle this? */
BUG_ON(!node);
@@ -2690,7 +2690,6 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
ret = a6xx_gmu_wrapper_init(a6xx_gpu, node);
else
ret = a6xx_gmu_init(a6xx_gpu, node);
- of_node_put(node);
if (ret) {
a6xx_destroy(&(a6xx_gpu->base.base));
return ERR_PTR(ret);
@@ -2740,6 +2739,7 @@ const struct adreno_gpu_funcs a6xx_gpu_funcs = {
.create_private_vm = a6xx_create_private_vm,
.get_rptr = a6xx_get_rptr,
.progress = a6xx_progress,
+ .sysprof_setup = a6xx_gmu_sysprof_setup,
},
.init = a6xx_gpu_init,
.get_timestamp = a6xx_gmu_get_timestamp,
@@ -2808,6 +2808,7 @@ const struct adreno_gpu_funcs a7xx_gpu_funcs = {
.create_private_vm = a6xx_create_private_vm,
.get_rptr = a6xx_get_rptr,
.progress = a6xx_progress,
+ .sysprof_setup = a6xx_gmu_sysprof_setup,
},
.init = a6xx_gpu_init,
.get_timestamp = a6xx_gmu_get_timestamp,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
index 487c273..186a73c 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
@@ -289,6 +289,8 @@ static int a8xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
(gmu->nr_gpu_freqs * num_gx_votes * sizeof(gmu->gx_arc_votes[0])) +
(gmu->nr_gmu_freqs * num_cx_votes * sizeof(gmu->cx_arc_votes[0]));
tbl = kzalloc(size, GFP_KERNEL);
+ if (!tbl)
+ return -ENOMEM;
tbl->type = HFI_TABLE_GPU_PERF;
/* First fill GX votes */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 4edfe80..fc38331 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -17,7 +17,7 @@ MODULE_PARM_DESC(snapshot_debugbus, "Include debugbus sections in GPU devcoredum
module_param_named(snapshot_debugbus, snapshot_debugbus, bool, 0600);
int enable_preemption = -1;
-MODULE_PARM_DESC(enable_preemption, "Enable preemption (A7xx only) (1=on , 0=disable, -1=auto (default))");
+MODULE_PARM_DESC(enable_preemption, "Enable preemption (A7xx+ only) (1=on , 0=disable, -1=auto (default))");
module_param(enable_preemption, int, 0600);
bool disable_acd;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 66f80f2..03f96a1 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -376,7 +376,7 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
*value = adreno_gpu->info->gmem;
return 0;
case MSM_PARAM_GMEM_BASE:
- if (adreno_gpu->info->family >= ADRENO_6XX_GEN4)
+ if (adreno_gpu->info->family >= ADRENO_6XX_GEN3)
*value = 0;
else
*value = 0x100000;
@@ -424,15 +424,21 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
*value = vm->mm_range;
return 0;
case MSM_PARAM_HIGHEST_BANK_BIT:
+ if (!adreno_gpu->ubwc_config)
+ return UERR(ENOENT, drm, "no UBWC on this platform");
*value = adreno_gpu->ubwc_config->highest_bank_bit;
return 0;
case MSM_PARAM_RAYTRACING:
*value = adreno_gpu->has_ray_tracing;
return 0;
case MSM_PARAM_UBWC_SWIZZLE:
+ if (!adreno_gpu->ubwc_config)
+ return UERR(ENOENT, drm, "no UBWC on this platform");
*value = adreno_gpu->ubwc_config->ubwc_swizzle;
return 0;
case MSM_PARAM_MACROTILE_MODE:
+ if (!adreno_gpu->ubwc_config)
+ return UERR(ENOENT, drm, "no UBWC on this platform");
*value = adreno_gpu->ubwc_config->macrotile_mode;
return 0;
case MSM_PARAM_UCHE_TRAP_BASE:
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
index b7b06e4..06da1583 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
@@ -480,7 +480,7 @@ const struct dpu_mdss_cfg dpu_kaanapali_cfg = {
.wb_count = ARRAY_SIZE(kaanapali_wb),
.wb = kaanapali_wb,
.cwb_count = ARRAY_SIZE(kaanapali_cwb),
- .cwb = sm8650_cwb,
+ .cwb = kaanapali_cwb,
.intf_count = ARRAY_SIZE(kaanapali_intf),
.intf = kaanapali_intf,
.vbif = &sm8650_vbif,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
index 6e8883d..590922c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
@@ -61,7 +61,7 @@ static int _dpu_format_populate_plane_sizes_ubwc(
bool meta = MSM_FORMAT_IS_UBWC(fmt);
if (MSM_FORMAT_IS_YUV(fmt)) {
- unsigned int stride, sclines;
+ unsigned int stride, y_sclines, uv_sclines;
unsigned int y_tile_width, y_tile_height;
unsigned int y_meta_stride, y_meta_scanlines;
unsigned int uv_meta_stride, uv_meta_scanlines;
@@ -77,23 +77,25 @@ static int _dpu_format_populate_plane_sizes_ubwc(
y_tile_width = 32;
}
- sclines = round_up(fb->height, 16);
+ y_sclines = round_up(fb->height, 16);
+ uv_sclines = round_up((fb->height+1)>>1, 16);
y_tile_height = 4;
} else {
stride = round_up(fb->width, 128);
y_tile_width = 32;
- sclines = round_up(fb->height, 32);
+ y_sclines = round_up(fb->height, 32);
+ uv_sclines = round_up((fb->height+1)>>1, 32);
y_tile_height = 8;
}
layout->plane_pitch[0] = stride;
layout->plane_size[0] = round_up(layout->plane_pitch[0] *
- sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+ y_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
layout->plane_pitch[1] = stride;
layout->plane_size[1] = round_up(layout->plane_pitch[1] *
- sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+ uv_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
if (!meta)
return 0;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
index 7545c02..6f2370c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
@@ -5,6 +5,7 @@
#include <drm/drm_edid.h>
#include <drm/drm_framebuffer.h>
+#include <drm/drm_managed.h>
#include "dpu_writeback.h"
@@ -125,7 +126,7 @@ int dpu_writeback_init(struct drm_device *dev, struct drm_encoder *enc,
struct dpu_wb_connector *dpu_wb_conn;
int rc = 0;
- dpu_wb_conn = devm_kzalloc(dev->dev, sizeof(*dpu_wb_conn), GFP_KERNEL);
+ dpu_wb_conn = drmm_kzalloc(dev, sizeof(*dpu_wb_conn), GFP_KERNEL);
if (!dpu_wb_conn)
return -ENOMEM;
diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
index 427d3ee..e603ab3 100644
--- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
+++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
@@ -5,11 +5,11 @@
#define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
-#include <generated/utsrelease.h>
+#include <linux/utsname.h>
#include "msm_disp_snapshot.h"
-static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *base_addr)
+static void msm_disp_state_dump_regs(u32 **reg, u32 len, void __iomem *base_addr)
{
u32 len_padded;
u32 num_rows;
@@ -19,11 +19,11 @@ static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *b
void __iomem *end_addr;
int i;
- len_padded = aligned_len * REG_DUMP_ALIGN;
- num_rows = aligned_len / REG_DUMP_ALIGN;
+ len_padded = round_up(len, REG_DUMP_ALIGN);
+ num_rows = DIV_ROUND_UP(len, REG_DUMP_ALIGN);
addr = base_addr;
- end_addr = base_addr + aligned_len;
+ end_addr = base_addr + len;
*reg = kvzalloc(len_padded, GFP_KERNEL);
if (!*reg)
@@ -48,8 +48,8 @@ static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *b
static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
void __iomem *base_addr, struct drm_printer *p)
{
+ void __iomem *addr, *end_addr;
int i;
- void __iomem *addr;
u32 num_rows;
if (!dump_addr) {
@@ -58,6 +58,7 @@ static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
}
addr = base_addr;
+ end_addr = base_addr + len;
num_rows = len / REG_DUMP_ALIGN;
for (i = 0; i < num_rows; i++) {
@@ -67,6 +68,17 @@ static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
dump_addr[i * 4 + 2], dump_addr[i * 4 + 3]);
addr += REG_DUMP_ALIGN;
}
+
+ if (addr != end_addr) {
+ drm_printf(p, "0x%lx : %08x",
+ (unsigned long)(addr - base_addr),
+ dump_addr[i * 4]);
+ if (addr + 0x4 < end_addr)
+ drm_printf(p, " %08x", dump_addr[i * 4 + 1]);
+ if (addr + 0x8 < end_addr)
+ drm_printf(p, " %08x", dump_addr[i * 4 + 2]);
+ drm_printf(p, "\n");
+ }
}
void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p)
@@ -79,7 +91,7 @@ void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p)
}
drm_printf(p, "---\n");
- drm_printf(p, "kernel: " UTS_RELEASE "\n");
+ drm_printf(p, "kernel: %s\n", init_utsname()->release);
drm_printf(p, "module: " KBUILD_MODNAME "\n");
drm_printf(p, "dpu devcoredump\n");
drm_printf(p, "time: %ptSp\n", &state->time);
@@ -185,7 +197,7 @@ void msm_disp_snapshot_add_block(struct msm_disp_state *disp_state, u32 len,
va_end(va);
INIT_LIST_HEAD(&new_blk->node);
- new_blk->size = ALIGN(len, REG_DUMP_ALIGN);
+ new_blk->size = len;
new_blk->base_addr = base_addr;
msm_disp_state_dump_regs(&new_blk->state, new_blk->size, base_addr);
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 565d425..982abaa 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -2033,6 +2033,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
/* fixup base address by io offset */
msm_host->ctrl_base += cfg->io_offset;
+ msm_host->ctrl_size -= cfg->io_offset;
ret = devm_regulator_bulk_get_const(&pdev->dev, cfg->num_regulators,
cfg->regulator_data,
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 195f40e..cc2bcd1 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -128,11 +128,10 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv,
/*
* Initialize the LRUs:
*/
- mutex_init(&priv->lru.lock);
- drm_gem_lru_init(&priv->lru.unbacked, &priv->lru.lock);
- drm_gem_lru_init(&priv->lru.pinned, &priv->lru.lock);
- drm_gem_lru_init(&priv->lru.willneed, &priv->lru.lock);
- drm_gem_lru_init(&priv->lru.dontneed, &priv->lru.lock);
+ drm_gem_lru_init(&priv->lru.unbacked);
+ drm_gem_lru_init(&priv->lru.pinned);
+ drm_gem_lru_init(&priv->lru.willneed);
+ drm_gem_lru_init(&priv->lru.dontneed);
/* Initialize stall-on-fault */
spin_lock_init(&priv->fault_stall_lock);
@@ -140,7 +139,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv,
/* Teach lockdep about lock ordering wrt. shrinker: */
fs_reclaim_acquire(GFP_KERNEL);
- might_lock(&priv->lru.lock);
+ might_lock(&ddev->gem_lru_mutex);
fs_reclaim_release(GFP_KERNEL);
if (priv->kms_init) {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 6d847d5..617b3c4 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -150,13 +150,6 @@ struct msm_drm_private {
* DONTNEED state (ie. can be purged)
*/
struct drm_gem_lru dontneed;
-
- /**
- * lock:
- *
- * Protects manipulation of all of the LRUs.
- */
- struct mutex lock;
} lru;
struct notifier_block vmap_notifier;
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 2cb3ab0..efd3d3c 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -177,11 +177,11 @@ static void update_lru_locked(struct drm_gem_object *obj)
static void update_lru(struct drm_gem_object *obj)
{
- struct msm_drm_private *priv = obj->dev->dev_private;
+ struct drm_device *dev = obj->dev;
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
update_lru_locked(obj);
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
}
static struct page **get_pages(struct drm_gem_object *obj)
@@ -292,11 +292,11 @@ void msm_gem_pin_obj_locked(struct drm_gem_object *obj)
static void pin_obj_locked(struct drm_gem_object *obj)
{
- struct msm_drm_private *priv = obj->dev->dev_private;
+ struct drm_device *dev = obj->dev;
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
msm_gem_pin_obj_locked(obj);
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
}
struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
@@ -487,16 +487,16 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct drm_gpuva *vma)
void msm_gem_unpin_locked(struct drm_gem_object *obj)
{
- struct msm_drm_private *priv = obj->dev->dev_private;
+ struct drm_device *dev = obj->dev;
struct msm_gem_object *msm_obj = to_msm_bo(obj);
msm_gem_assert_locked(obj);
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
msm_obj->pin_count--;
GEM_WARN_ON(msm_obj->pin_count < 0);
update_lru_locked(obj);
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
}
/* Special unpin path for use in fence-signaling path, avoiding the need
@@ -507,10 +507,10 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj)
*/
void msm_gem_unpin_active(struct drm_gem_object *obj)
{
- struct msm_drm_private *priv = obj->dev->dev_private;
+ struct drm_device *dev = obj->dev;
struct msm_gem_object *msm_obj = to_msm_bo(obj);
- GEM_WARN_ON(!mutex_is_locked(&priv->lru.lock));
+ GEM_WARN_ON(!mutex_is_locked(&dev->gem_lru_mutex));
msm_obj->pin_count--;
GEM_WARN_ON(msm_obj->pin_count < 0);
@@ -797,12 +797,12 @@ void msm_gem_put_vaddr(struct drm_gem_object *obj)
*/
int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
{
- struct msm_drm_private *priv = obj->dev->dev_private;
+ struct drm_device *dev = obj->dev;
struct msm_gem_object *msm_obj = to_msm_bo(obj);
msm_gem_lock(obj);
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
if (msm_obj->madv != __MSM_MADV_PURGED)
msm_obj->madv = madv;
@@ -814,7 +814,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
*/
update_lru_locked(obj);
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
msm_gem_unlock(obj);
@@ -824,7 +824,6 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
void msm_gem_purge(struct drm_gem_object *obj)
{
struct drm_device *dev = obj->dev;
- struct msm_drm_private *priv = obj->dev->dev_private;
struct msm_gem_object *msm_obj = to_msm_bo(obj);
msm_gem_assert_locked(obj);
@@ -839,10 +838,10 @@ void msm_gem_purge(struct drm_gem_object *obj)
put_pages(obj);
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
/* A one-way transition: */
msm_obj->madv = __MSM_MADV_PURGED;
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
drm_gem_free_mmap_offset(obj);
diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c
index 31fa51a4..9d2788f 100644
--- a/drivers/gpu/drm/msm/msm_gem_shrinker.c
+++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c
@@ -43,8 +43,7 @@ msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
}
static bool
-with_vm_locks(struct ww_acquire_ctx *ticket,
- void (*fn)(struct drm_gem_object *obj),
+with_vm_locks(void (*fn)(struct drm_gem_object *obj),
struct drm_gem_object *obj)
{
/*
@@ -52,7 +51,7 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
* success paths
*/
struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
- int ret = 0;
+ bool locked = true;
drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
@@ -60,23 +59,14 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
if (resv == obj->resv)
continue;
- ret = dma_resv_lock(resv, ticket);
-
/*
- * Since we already skip the case when the VM and obj
- * share a resv (ie. _NO_SHARE objs), we don't expect
- * to hit a double-locking scenario... which the lock
- * unwinding cannot really cope with.
+ * dma_resv_lock can't be used due to acquiring 'ticket' before the
+ * fs_reclaim lock, which is held in shrinker context
*/
- WARN_ON(ret == -EALREADY);
-
- /*
- * Don't bother with slow-lock / backoff / retry sequence,
- * if we can't get the lock just give up and move on to
- * the next object.
- */
- if (ret)
+ if (!dma_resv_trylock(resv)) {
+ locked = false;
goto out_unlock;
+ }
/*
* Hold a ref to prevent the vm_bo from being freed
@@ -108,11 +98,11 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
}
}
- return ret == 0;
+ return locked;
}
static bool
-purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+purge(struct drm_gem_object *obj, struct ww_acquire_ctx *unused)
{
if (!is_purgeable(to_msm_bo(obj)))
return false;
@@ -120,11 +110,11 @@ purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
if (msm_gem_active(obj))
return false;
- return with_vm_locks(ticket, msm_gem_purge, obj);
+ return with_vm_locks(msm_gem_purge, obj);
}
static bool
-evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+evict(struct drm_gem_object *obj, struct ww_acquire_ctx *unused)
{
if (is_unevictable(to_msm_bo(obj)))
return false;
@@ -132,7 +122,7 @@ evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
if (msm_gem_active(obj))
return false;
- return with_vm_locks(ticket, msm_gem_evict, obj);
+ return with_vm_locks(msm_gem_evict, obj);
}
static bool
@@ -164,7 +154,6 @@ static unsigned long
msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
{
struct msm_drm_private *priv = shrinker->private_data;
- struct ww_acquire_ctx ticket;
struct {
struct drm_gem_lru *lru;
bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket);
@@ -185,11 +174,14 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
if (!stages[i].cond)
continue;
+ /*
+ * 'ticket' not needed on trylock paths
+ */
stages[i].freed =
- drm_gem_lru_scan(stages[i].lru, nr,
+ drm_gem_lru_scan(priv->dev, stages[i].lru, nr,
&stages[i].remaining,
stages[i].shrink,
- &ticket);
+ NULL);
nr -= stages[i].freed;
freed += stages[i].freed;
remaining += stages[i].remaining;
@@ -255,7 +247,7 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
unsigned long remaining = 0;
for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
- unmapped += drm_gem_lru_scan(lrus[idx],
+ unmapped += drm_gem_lru_scan(priv->dev, lrus[idx],
vmap_shrink_limit - unmapped,
&remaining,
vmap_shrink,
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 26ea8a2..3c6bc90 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -352,7 +352,7 @@ static int submit_fence_sync(struct msm_gem_submit *submit)
static int submit_pin_objects(struct msm_gem_submit *submit)
{
- struct msm_drm_private *priv = submit->dev->dev_private;
+ struct drm_device *dev = submit->dev;
int i, ret = 0;
for (i = 0; i < submit->nr_bos; i++) {
@@ -381,11 +381,11 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
* get_pages() which could trigger reclaim.. and if we held the LRU lock
* could trigger deadlock with the shrinker).
*/
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
for (i = 0; i < submit->nr_bos; i++) {
msm_gem_pin_obj_locked(submit->bos[i].obj);
}
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
submit->bos_pinned = true;
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 1a952b1..c4cfe03 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -702,7 +702,7 @@ static struct dma_fence *
msm_vma_job_run(struct drm_sched_job *_job)
{
struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
- struct msm_drm_private *priv = job->vm->drm->dev_private;
+ struct drm_device *dev = job->vm->drm;
struct msm_gem_vm *vm = to_msm_vm(job->vm);
struct drm_gem_object *obj;
int ret = vm->unusable ? -EINVAL : 0;
@@ -745,13 +745,13 @@ msm_vma_job_run(struct drm_sched_job *_job)
if (ret)
msm_gem_vm_unusable(job->vm);
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
job_foreach_bo (obj, job) {
msm_gem_unpin_active(obj);
}
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
/* VM_BIND ops are synchronous, so no fence to wait on: */
return NULL;
@@ -1305,7 +1305,7 @@ vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
return PTR_ERR(pages);
}
- struct msm_drm_private *priv = job->vm->drm->dev_private;
+ struct drm_device *dev = job->vm->drm;
/*
* A second loop while holding the LRU lock (a) avoids acquiring/dropping
@@ -1314,10 +1314,10 @@ vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
* get_pages() which could trigger reclaim.. and if we held the LRU lock
* could trigger deadlock with the shrinker).
*/
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
job_foreach_bo (obj, job)
msm_gem_pin_obj_locked(obj);
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
job->bos_pinned = true;
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 930e54d..3f3925b 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -13,11 +13,11 @@
#include "msm_gpu_trace.h"
//#include "adreno/adreno_gpu.h"
-#include <generated/utsrelease.h>
#include <linux/string_helpers.h>
#include <linux/devcoredump.h>
#include <linux/sched/task.h>
#include <linux/sched/mm.h>
+#include <linux/utsname.h>
/*
* Power Management:
@@ -196,7 +196,7 @@ static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset,
p = drm_coredump_printer(&iter);
drm_printf(&p, "---\n");
- drm_printf(&p, "kernel: " UTS_RELEASE "\n");
+ drm_printf(&p, "kernel: %s\n", init_utsname()->release);
drm_printf(&p, "module: " KBUILD_MODNAME "\n");
drm_printf(&p, "time: %ptSp\n", &state->time);
if (state->comm)
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index 7d449e5..058c71c 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -677,7 +677,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
int prot)
{
struct msm_iommu *iommu = to_msm_iommu(mmu);
- size_t ret;
+ ssize_t ret;
WARN_ON(off != 0);
@@ -686,7 +686,8 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
iova |= GENMASK_ULL(63, 49);
ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);
- WARN_ON(!ret);
+ if (ret < 0)
+ return ret;
return (ret == len) ? 0 : -EINVAL;
}
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 30ddb53..2d6b930 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -16,13 +16,13 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
struct msm_gem_submit *submit = to_msm_submit(job);
struct msm_fence_context *fctx = submit->ring->fctx;
struct msm_gpu *gpu = submit->gpu;
- struct msm_drm_private *priv = gpu->dev->dev_private;
+ struct drm_device *dev = gpu->dev;
unsigned nr_cmds = submit->nr_cmds;
int i;
msm_fence_init(submit->hw_fence, fctx);
- mutex_lock(&priv->lru.lock);
+ mutex_lock(&dev->gem_lru_mutex);
for (i = 0; i < submit->nr_bos; i++) {
struct drm_gem_object *obj = submit->bos[i].obj;
@@ -32,7 +32,7 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
submit->bos_pinned = false;
- mutex_unlock(&priv->lru.lock);
+ mutex_unlock(&dev->gem_lru_mutex);
/* TODO move submit path over to using a per-ring lock.. */
mutex_lock(&gpu->lock);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 72848ed..b101e14 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -2513,6 +2513,7 @@ static const struct nvkm_device_chip
nv170_chipset = {
.name = "GA100",
.bar = { 0x00000001, tu102_bar_new },
+ .bios = { 0x00000001, nvkm_bios_new },
.devinit = { 0x00000001, ga100_devinit_new },
.fault = { 0x00000001, tu102_fault_new },
.fb = { 0x00000001, ga100_fb_new },
@@ -2529,7 +2530,6 @@ nv170_chipset = {
.vfn = { 0x00000001, ga100_vfn_new },
.ce = { 0x000003ff, ga100_ce_new },
.fifo = { 0x00000001, ga100_fifo_new },
- .sec2 = { 0x00000001, tu102_sec2_new },
};
static const struct nvkm_device_chip
@@ -3341,7 +3341,6 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
case 0x166: device->chip = &nv166_chipset; break;
case 0x167: device->chip = &nv167_chipset; break;
case 0x168: device->chip = &nv168_chipset; break;
- case 0x170: device->chip = &nv170_chipset; break;
case 0x172: device->chip = &nv172_chipset; break;
case 0x173: device->chip = &nv173_chipset; break;
case 0x174: device->chip = &nv174_chipset; break;
@@ -3361,6 +3360,14 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
case 0x1b6: device->chip = &nv1b6_chipset; break;
case 0x1b7: device->chip = &nv1b7_chipset; break;
default:
+ if (nvkm_boolopt(device->cfgopt, "NvEnableUnsupportedChipsets", false)) {
+ switch (device->chipset) {
+ case 0x170: device->chip = &nv170_chipset; break;
+ default:
+ break;
+ }
+ }
+
if (!device->chip) {
nvdev_error(device, "unknown chipset (%08x)\n", boot0);
ret = -ENODEV;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
index fdd820e..27a13ae 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga100.c
@@ -41,11 +41,15 @@ ga100_gsp_flcn = {
static const struct nvkm_gsp_func
ga100_gsp = {
.flcn = &ga100_gsp_flcn,
+ .fwsec = &tu102_gsp_fwsec,
.sig_section = ".fwsignature_ga100",
.booter.ctor = tu102_gsp_booter_ctor,
+ .fwsec_sb.ctor = tu102_gsp_fwsec_sb_ctor,
+ .fwsec_sb.dtor = tu102_gsp_fwsec_sb_dtor,
+
.dtor = r535_gsp_dtor,
.oneinit = tu102_gsp_oneinit,
.init = tu102_gsp_init,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
index dd82c76..19cb269 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/tu102.c
@@ -318,13 +318,8 @@ tu102_gsp_oneinit(struct nvkm_gsp *gsp)
if (ret)
return ret;
- /*
- * Calculate FB layout. FRTS is a memory region created by the FWSEC-FRTS firmware.
- * FWSEC comes from VBIOS. So on systems with no VBIOS (e.g. GA100), the FRTS does
- * not exist. Therefore, use the existence of VBIOS to determine whether to reserve
- * an FRTS region.
- */
- gsp->fb.wpr2.frts.size = device->bios ? 0x100000 : 0;
+ /* Calculate FB layout. */
+ gsp->fb.wpr2.frts.size = 0x100000;
gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
@@ -348,12 +343,9 @@ tu102_gsp_oneinit(struct nvkm_gsp *gsp)
if (ret)
return ret;
- /* Only boot FWSEC-FRTS if it actually exists */
- if (gsp->fb.wpr2.frts.size) {
- ret = nvkm_gsp_fwsec_frts(gsp);
- if (WARN_ON(ret))
- return ret;
- }
+ ret = nvkm_gsp_fwsec_frts(gsp);
+ if (WARN_ON(ret))
+ return ret;
/* Reset GSP into RISC-V mode. */
ret = gsp->func->reset(gsp);
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index d6863b2..d592f4f 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -208,6 +208,7 @@
depends on OF
depends on DRM_MIPI_DSI
depends on BACKLIGHT_CLASS_DEVICE
+ select DRM_DISPLAY_DSC_HELPER
select DRM_KMS_HELPER
help
Say Y here if you want to enable support for Himax HX83121A-based
diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
index d5fe105..658ce64 100644
--- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
+++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
@@ -1324,6 +1324,8 @@ static int boe_panel_disable(struct drm_panel *panel)
mipi_dsi_dcs_set_display_off_multi(&ctx);
mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
+ boe->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
mipi_dsi_msleep(&ctx, 150);
return ctx.accum_err;
diff --git a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
index 4f8d6d8..dbdb7e3 100644
--- a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
+++ b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
@@ -98,9 +98,7 @@ static int feiyang_enable(struct drm_panel *panel)
/* T12 (video & logic signal rise + backlight rise) T12 >= 200ms */
msleep(200);
- mipi_dsi_dcs_set_display_on(ctx->dsi);
-
- return 0;
+ return mipi_dsi_dcs_set_display_on(ctx->dsi);
}
static int feiyang_disable(struct drm_panel *panel)
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83102.c b/drivers/gpu/drm/panel/panel-himax-hx83102.c
index 8b2a68e..a5e5c9e 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83102.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83102.c
@@ -937,6 +937,8 @@ static int hx83102_disable(struct drm_panel *panel)
mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+ dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
mipi_dsi_msleep(&dsi_ctx, 150);
return dsi_ctx.accum_err;
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83121a.c b/drivers/gpu/drm/panel/panel-himax-hx83121a.c
index ebe643b..bed79aa 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83121a.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83121a.c
@@ -596,8 +596,8 @@ static int himax_probe(struct mipi_dsi_device *dsi)
ctx = devm_drm_panel_alloc(dev, struct himax, panel, &himax_panel_funcs,
DRM_MODE_CONNECTOR_DSI);
- if (!ctx)
- return -ENOMEM;
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
ret = devm_regulator_bulk_get_const(&dsi->dev,
ARRAY_SIZE(himax_supplies),
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 711f510..074c099 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -390,6 +390,8 @@ panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
true, timeout);
if (!ret)
ret = timeout ? -ETIMEDOUT : -EBUSY;
+ else if (ret > 0)
+ ret = 0;
drm_gem_object_put(gem_obj);
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index 2bbb116..1e6a239 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -118,12 +118,13 @@ qxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
/* Complete initialization. */
ret = drm_dev_register(&qdev->ddev, ent->driver_data);
if (ret)
- goto modeset_cleanup;
+ goto poll_fini;
drm_client_setup(&qdev->ddev, NULL);
return 0;
-modeset_cleanup:
+poll_fini:
+ drm_kms_helper_poll_fini(&qdev->ddev);
qxl_modeset_fini(qdev);
unload:
qxl_device_fini(qdev);
@@ -154,6 +155,7 @@ qxl_pci_remove(struct pci_dev *pdev)
{
struct drm_device *dev = pci_get_drvdata(pdev);
+ drm_kms_helper_poll_fini(dev);
drm_dev_unregister(dev);
drm_atomic_helper_shutdown(dev);
if (pci_is_vga(pdev) && pdev->revision < 5)
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index 22321eb..703848f 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -2461,7 +2461,8 @@ static void ci_register_patching_mc_arb(struct radeon_device *rdev,
if (patch &&
((rdev->pdev->device == 0x67B0) ||
- (rdev->pdev->device == 0x67B1))) {
+ (rdev->pdev->device == 0x67B1)) &&
+ (rdev->pdev->revision == 0)) {
if ((memory_clock > 100000) && (memory_clock <= 125000)) {
tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff;
*dram_timimg2 &= ~0x00ff0000;
@@ -3304,7 +3305,8 @@ static int ci_populate_all_memory_levels(struct radeon_device *rdev)
pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
if ((dpm_table->mclk_table.count >= 2) &&
- ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1))) {
+ ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1)) &&
+ (rdev->pdev->revision == 0)) {
pi->smc_state_table.MemoryLevel[1].MinVddc =
pi->smc_state_table.MemoryLevel[0].MinVddc;
pi->smc_state_table.MemoryLevel[1].MinVddcPhases =
@@ -4493,7 +4495,8 @@ static int ci_register_patching_mc_seq(struct radeon_device *rdev,
if (patch &&
((rdev->pdev->device == 0x67B0) ||
- (rdev->pdev->device == 0x67B1))) {
+ (rdev->pdev->device == 0x67B1)) &&
+ (rdev->pdev->revision == 0)) {
for (i = 0; i < table->last; i++) {
if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 3142ef4..9196f85 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -312,8 +312,10 @@ static int evergreen_surface_check(struct radeon_cs_parser *p,
case ARRAY_2D_TILED_THIN1:
return evergreen_surface_check_2d(p, surf, prefix);
default:
- dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
- __func__, __LINE__, prefix, surf->mode);
+ if (prefix) {
+ dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
+ __func__, __LINE__, prefix, surf->mode);
+ }
return -EINVAL;
}
return -EINVAL;
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index b7397827..360a88c 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -741,6 +741,7 @@ static int sti_hda_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct sti_hda *hda;
struct resource *res;
+ int ret;
DRM_INFO("%s\n", __func__);
@@ -779,7 +780,9 @@ static int sti_hda_probe(struct platform_device *pdev)
return PTR_ERR(hda->clk_hddac);
}
- drm_bridge_add(&hda->bridge);
+ ret = devm_drm_bridge_add(dev, &hda->bridge);
+ if (ret)
+ return ret;
platform_set_drvdata(pdev, hda);
@@ -788,10 +791,7 @@ static int sti_hda_probe(struct platform_device *pdev)
static void sti_hda_remove(struct platform_device *pdev)
{
- struct sti_hda *hda = platform_get_drvdata(pdev);
-
component_del(&pdev->dev, &sti_hda_ops);
- drm_bridge_remove(&hda->bridge);
}
static const struct of_device_id hda_of_match[] = {
diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c
index d38ba70..247cf13 100644
--- a/drivers/gpu/drm/sysfb/ofdrm.c
+++ b/drivers/gpu/drm/sysfb/ofdrm.c
@@ -350,6 +350,7 @@ static void ofdrm_pci_release(void *data)
struct pci_dev *pcidev = data;
pci_disable_device(pcidev);
+ pci_dev_put(pcidev);
}
static int ofdrm_device_init_pci(struct ofdrm_device *odev)
@@ -375,6 +376,7 @@ static int ofdrm_device_init_pci(struct ofdrm_device *odev)
if (ret) {
drm_err(dev, "pci_enable_device(%s) failed: %d\n",
dev_name(&pcidev->dev), ret);
+ pci_dev_put(pcidev);
return ret;
}
ret = devm_add_action_or_reset(&pdev->dev, ofdrm_pci_release, pcidev);
diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c
index 3bae91d..278bb23 100644
--- a/drivers/gpu/drm/tiny/appletbdrm.c
+++ b/drivers/gpu/drm/tiny/appletbdrm.c
@@ -353,7 +353,7 @@ static int appletbdrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
frames_size +
sizeof(struct appletbdrm_fb_request_footer), 16);
- appletbdrm_state->request = kzalloc(request_size, GFP_KERNEL);
+ appletbdrm_state->request = kvzalloc(request_size, GFP_KERNEL);
if (!appletbdrm_state->request)
return -ENOMEM;
@@ -543,7 +543,7 @@ static void appletbdrm_primary_plane_destroy_state(struct drm_plane *plane,
{
struct appletbdrm_plane_state *appletbdrm_state = to_appletbdrm_plane_state(state);
- kfree(appletbdrm_state->request);
+ kvfree(appletbdrm_state->request);
kfree(appletbdrm_state->response);
__drm_gem_destroy_shadow_plane_state(&appletbdrm_state->base);
diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c
index 222e4ae..5d8dc5e 100644
--- a/drivers/gpu/drm/tiny/bochs.c
+++ b/drivers/gpu/drm/tiny/bochs.c
@@ -761,25 +761,21 @@ static int bochs_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent
ret = pcim_enable_device(pdev);
if (ret)
- goto err_free_dev;
+ return ret;
pci_set_drvdata(pdev, dev);
ret = bochs_load(bochs);
if (ret)
- goto err_free_dev;
+ return ret;
ret = drm_dev_register(dev, 0);
if (ret)
- goto err_free_dev;
+ return ret;
drm_client_setup(dev, NULL);
return ret;
-
-err_free_dev:
- drm_dev_put(dev);
- return ret;
}
static void bochs_pci_remove(struct pci_dev *pdev)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index d85f0a3..bcd76f6 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -739,7 +739,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo,
may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM);
ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL);
if (ret) {
- if (ret != -ENOSPC && ret != -EAGAIN) {
+ if (ret != -ENOSPC) {
dmem_cgroup_pool_state_put(limit_pool);
return ret;
}
@@ -1177,17 +1177,13 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
bdev->funcs->swap_notify(bo);
if (ttm_tt_is_populated(tt)) {
- spin_lock(&bdev->lru_lock);
- ttm_resource_del_bulk_move(bo->resource, bo);
- spin_unlock(&bdev->lru_lock);
-
ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags);
-
- spin_lock(&bdev->lru_lock);
- if (ret)
- ttm_resource_add_bulk_move(bo->resource, bo);
- ttm_resource_move_to_lru_tail(bo->resource);
- spin_unlock(&bdev->lru_lock);
+ if (!ret) {
+ spin_lock(&bdev->lru_lock);
+ ttm_resource_del_bulk_move_unevictable(bo->resource, bo);
+ ttm_resource_move_to_lru_tail(bo->resource);
+ spin_unlock(&bdev->lru_lock);
+ }
}
out:
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index f83b7d5..3e3c201 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -1112,19 +1112,14 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
if (lret < 0)
return lret;
- if (bo->bulk_move) {
- spin_lock(&bdev->lru_lock);
- ttm_resource_del_bulk_move(bo->resource, bo);
- spin_unlock(&bdev->lru_lock);
- }
-
lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags)
{.purge = flags.purge,
.writeback = flags.writeback});
- if (lret <= 0 && bo->bulk_move) {
+ if (lret > 0) {
spin_lock(&bdev->lru_lock);
- ttm_resource_add_bulk_move(bo->resource, bo);
+ ttm_resource_del_bulk_move_unevictable(bo->resource, bo);
+ ttm_resource_move_to_lru_tail(bo->resource);
spin_unlock(&bdev->lru_lock);
}
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 26a3689..278bbe7 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -206,6 +206,14 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
return NULL;
}
+static void __free_pages_gpu_account(struct page *p, unsigned int order,
+ bool reclaim)
+{
+ mod_lruvec_page_state(p, reclaim ? NR_GPU_RECLAIM : NR_GPU_ACTIVE,
+ -(1 << order));
+ __free_pages(p, order);
+}
+
/* Reset the caching and pages of size 1 << order */
static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,
unsigned int order, struct page *p, bool reclaim)
@@ -223,9 +231,7 @@ static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,
#endif
if (!pool || !ttm_pool_uses_dma_alloc(pool)) {
- mod_lruvec_page_state(p, reclaim ? NR_GPU_RECLAIM : NR_GPU_ACTIVE,
- -(1 << order));
- __free_pages(p, order);
+ __free_pages_gpu_account(p, order, reclaim);
return;
}
@@ -606,7 +612,7 @@ static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore,
*/
ttm_pool_split_for_swap(restore->pool, p);
copy_highpage(restore->alloced_page + i, p);
- __free_pages(p, 0);
+ __free_pages_gpu_account(p, 0, false);
}
restore->restored_pages++;
@@ -1068,7 +1074,7 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
if (flags->purge) {
shrunken += num_pages;
page->private = 0;
- __free_pages(page, order);
+ __free_pages_gpu_account(page, order, false);
memset(tt->pages + i, 0,
num_pages * sizeof(*tt->pages));
}
@@ -1109,7 +1115,7 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
}
handle = shandle;
tt->pages[i] = ttm_backup_handle_to_page_ptr(handle);
- put_page(page);
+ __free_pages_gpu_account(page, 0, false);
shrunken++;
}
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 9f36631..154d673 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -292,6 +292,19 @@ void ttm_resource_del_bulk_move(struct ttm_resource *res,
ttm_lru_bulk_move_del(bo->bulk_move, res);
}
+/*
+ * Remove a resource from its bulk_move, bypassing the unevictable check.
+ * Use only when the resource is known to still be tracked in the range despite
+ * the BO having just become unevictable; asserts that this is the case.
+ */
+void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res,
+ struct ttm_buffer_object *bo)
+{
+ WARN_ON_ONCE(!ttm_resource_unevictable(res, bo));
+ if (bo->bulk_move)
+ ttm_lru_bulk_move_del(bo->bulk_move, res);
+}
+
/* Move a resource to the LRU or bulk tail */
void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
{
@@ -385,8 +398,11 @@ int ttm_resource_alloc(struct ttm_buffer_object *bo,
if (man->cg) {
ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool);
- if (ret)
+ if (ret) {
+ if (ret == -EAGAIN)
+ ret = -ENOSPC;
return ret;
+ }
}
ret = man->func->alloc(man, bo, place, res_ptr);
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index 08a0e94..17950fe 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -285,13 +285,12 @@ static struct urb *udl_get_urb_locked(struct udl_device *udl, long timeout)
return unode->urb;
}
-#define GET_URB_TIMEOUT HZ
struct urb *udl_get_urb(struct udl_device *udl)
{
struct urb *urb;
spin_lock_irq(&udl->urbs.lock);
- urb = udl_get_urb_locked(udl, GET_URB_TIMEOUT);
+ urb = udl_get_urb_locked(udl, HZ * 2);
spin_unlock_irq(&udl->urbs.lock);
return urb;
}
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 231e829..1ca073a 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -21,6 +21,7 @@
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_print.h>
#include <drm/drm_probe_helper.h>
#include <drm/drm_vblank.h>
@@ -342,8 +343,10 @@ static void udl_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atom
return;
urb = udl_get_urb(udl);
- if (!urb)
+ if (!urb) {
+ drm_err_ratelimited(dev, "get urb failed when enabling crtc\n");
goto out;
+ }
buf = (char *)urb->transfer_buffer;
buf = udl_vidreg_lock(buf);
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 1855ef5..94bf628 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -126,20 +126,6 @@ v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
}
static void
-v3d_cpu_job_free(struct drm_sched_job *sched_job)
-{
- struct v3d_cpu_job *job = to_cpu_job(sched_job);
-
- v3d_timestamp_query_info_free(&job->timestamp_query,
- job->timestamp_query.count);
-
- v3d_performance_query_info_free(&job->performance_query,
- job->performance_query.count);
-
- v3d_job_cleanup(&job->base);
-}
-
-static void
v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
{
struct v3d_perfmon *perfmon = v3d->global_perfmon;
@@ -830,7 +816,7 @@ static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
static const struct drm_sched_backend_ops v3d_cpu_sched_ops = {
.run_job = v3d_cpu_job_run,
- .free_job = v3d_cpu_job_free
+ .free_job = v3d_sched_job_free
};
static int
diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index ee4512d..3ddd53b 100644
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -123,6 +123,24 @@ v3d_render_job_free(struct kref *ref)
v3d_job_free(ref);
}
+static void
+v3d_cpu_job_free(struct kref *ref)
+{
+ struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job,
+ base.refcount);
+
+ v3d_timestamp_query_info_free(&job->timestamp_query,
+ job->timestamp_query.count);
+
+ v3d_performance_query_info_free(&job->performance_query,
+ job->performance_query.count);
+
+ if (job->indirect_csd.indirect)
+ drm_gem_object_put(job->indirect_csd.indirect);
+
+ v3d_job_free(ref);
+}
+
void v3d_job_cleanup(struct v3d_job *job)
{
if (!job)
@@ -1302,7 +1320,7 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type);
ret = v3d_job_init(v3d, file_priv, &cpu_job->base,
- v3d_job_free, 0, &se, V3D_CPU);
+ v3d_cpu_job_free, 0, &se, V3D_CPU);
if (ret) {
v3d_job_deallocate((void *)&cpu_job);
goto fail;
@@ -1385,8 +1403,6 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
v3d_job_cleanup((void *)csd_job);
v3d_job_cleanup(clean_job);
v3d_put_multisync_post_deps(&se);
- kvfree(cpu_job->timestamp_query.queries);
- kvfree(cpu_job->performance_query.queries);
return ret;
}
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index f17660a7..2f35319 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -317,6 +317,7 @@ virtio_gpu_array_from_handles(struct drm_file *drm_file, u32 *handles, u32 nents
void virtio_gpu_array_add_obj(struct virtio_gpu_object_array *objs,
struct drm_gem_object *obj);
int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs);
+int virtio_gpu_lock_one_resv_uninterruptible(struct virtio_gpu_object_array *objs);
void virtio_gpu_array_unlock_resv(struct virtio_gpu_object_array *objs);
void virtio_gpu_array_add_fence(struct virtio_gpu_object_array *objs,
struct dma_fence *fence);
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index f22dc5c..435d37d 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -238,6 +238,23 @@ int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs)
return ret;
}
+int virtio_gpu_lock_one_resv_uninterruptible(struct virtio_gpu_object_array *objs)
+{
+ int ret;
+
+ if (objs->nents != 1)
+ return -EINVAL;
+
+ dma_resv_lock(objs->objs[0]->resv, NULL);
+
+ ret = dma_resv_reserve_fences(objs->objs[0]->resv, 1);
+ if (ret) {
+ virtio_gpu_array_unlock_resv(objs);
+ return ret;
+ }
+ return 0;
+}
+
void virtio_gpu_array_unlock_resv(struct virtio_gpu_object_array *objs)
{
if (objs->nents == 1) {
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index a126d1b..6523524 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -215,7 +215,10 @@ static void virtio_gpu_resource_flush(struct drm_plane *plane,
if (!objs)
return;
virtio_gpu_array_add_obj(objs, vgfb->base.obj[0]);
- virtio_gpu_array_lock_resv(objs);
+ if (virtio_gpu_lock_one_resv_uninterruptible(objs)) {
+ virtio_gpu_array_put_free(objs);
+ return;
+ }
virtio_gpu_cmd_resource_flush(vgdev, bo->hw_res_handle, x, y,
width, height, objs,
vgplane_st->fence);
@@ -459,7 +462,10 @@ static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
if (!objs)
return;
virtio_gpu_array_add_obj(objs, vgfb->base.obj[0]);
- virtio_gpu_array_lock_resv(objs);
+ if (virtio_gpu_lock_one_resv_uninterruptible(objs)) {
+ virtio_gpu_array_put_free(objs);
+ return;
+ }
virtio_gpu_cmd_transfer_to_host_2d
(vgdev, 0,
plane->state->crtc_w,
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index 49de1c2..03242e8 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -88,6 +88,7 @@
xe_irq.o \
xe_late_bind_fw.o \
xe_lrc.o \
+ xe_mem_pool.o \
xe_migrate.o \
xe_mmio.o \
xe_mmio_gem.o \
diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
index 29c72aa..33494b8 100644
--- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
+++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
@@ -37,9 +37,17 @@ static bool intel_hdcp_gsc_check_status(struct drm_device *drm)
struct xe_device *xe = to_xe_device(drm);
struct xe_tile *tile = xe_device_get_root_tile(xe);
struct xe_gt *gt = tile->media_gt;
- struct xe_gsc *gsc = >->uc.gsc;
+ struct xe_gsc *gsc;
- if (!gsc || !xe_uc_fw_is_available(&gsc->fw)) {
+ if (!gt) {
+ drm_dbg_kms(&xe->drm,
+ "not checking GSC status for HDCP2.x: media GT not present or disabled\n");
+ return false;
+ }
+
+ gsc = >->uc.gsc;
+
+ if (!xe_uc_fw_is_available(&gsc->fw)) {
drm_dbg_kms(&xe->drm,
"GSC Components not ready for HDCP2.x\n");
return false;
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 4ebaa08..353fe0b 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -152,10 +152,11 @@
#define XEHPG_INSTDONE_GEOM_SVGUNIT XE_REG_MCR(0x666c)
-#define CACHE_MODE_1 XE_REG(0x7004, XE_REG_OPTION_MASKED)
+#define CACHE_MODE_1 XE_REG_MCR(0x7004, XE_REG_OPTION_MASKED)
#define MSAA_OPTIMIZATION_REDUC_DISABLE REG_BIT(11)
#define COMMON_SLICE_CHICKEN1 XE_REG(0x7010, XE_REG_OPTION_MASKED)
+#define XEHP_COMMON_SLICE_CHICKEN1 XE_REG_MCR(0x7010, XE_REG_OPTION_MASKED)
#define DISABLE_BOTTOM_CLIP_RECTANGLE_TEST REG_BIT(14)
#define HIZ_CHICKEN XE_REG(0x7018, XE_REG_OPTION_MASKED)
@@ -178,6 +179,7 @@
#define XEHPG_SC_INSTDONE_EXTRA2 XE_REG_MCR(0x7108)
#define COMMON_SLICE_CHICKEN4 XE_REG(0x7300, XE_REG_OPTION_MASKED)
+#define XEHP_COMMON_SLICE_CHICKEN4 XE_REG_MCR(0x7300, XE_REG_OPTION_MASKED)
#define SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE REG_BIT(12)
#define DISABLE_TDC_LOAD_BALANCING_CALC REG_BIT(6)
#define HW_FILTERING REG_BIT(5)
@@ -583,7 +585,7 @@
#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32)
#define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32)
-#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0)
+#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0, XE_REG_OPTION_MASKED)
#define CPSS_AWARE_DIS REG_BIT(3)
#define SARB_CHICKEN1 XE_REG_MCR(0xe90c)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index a7c2dc7..6b51885 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -897,10 +897,10 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo,
new_state == XE_MADV_PURGEABLE_PURGED);
/* Once purged, always purged - cannot transition out */
- xe_assert(xe, !(bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED &&
+ xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED &&
new_state != XE_MADV_PURGEABLE_PURGED));
- bo->madv_purgeable = new_state;
+ bo->purgeable.state = new_state;
xe_bo_set_purgeable_shrinker(bo, new_state);
}
@@ -2322,8 +2322,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
}
/* XE_BO_FLAG_GGTTx requires XE_BO_FLAG_GGTT also be set */
- if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT))
+ if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) {
+ xe_bo_free(bo);
return ERR_PTR(-EINVAL);
+ }
if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&
!(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&
@@ -2342,8 +2344,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
alignment = SZ_4K >> PAGE_SHIFT;
}
- if (type == ttm_bo_type_device && aligned_size != size)
+ if (type == ttm_bo_type_device && aligned_size != size) {
+ xe_bo_free(bo);
return ERR_PTR(-EINVAL);
+ }
if (!bo) {
bo = xe_bo_alloc();
@@ -2364,7 +2368,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
INIT_LIST_HEAD(&bo->vram_userfault_link);
/* Initialize purge advisory state */
- bo->madv_purgeable = XE_MADV_PURGEABLE_WILLNEED;
+ bo->purgeable.state = XE_MADV_PURGEABLE_WILLNEED;
drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 68dea7d..6340317 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -251,7 +251,7 @@ static inline bool xe_bo_is_protected(const struct xe_bo *bo)
static inline bool xe_bo_is_purged(struct xe_bo *bo)
{
xe_bo_assert_held(bo);
- return bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED;
+ return bo->purgeable.state == XE_MADV_PURGEABLE_PURGED;
}
/**
@@ -268,11 +268,95 @@ static inline bool xe_bo_is_purged(struct xe_bo *bo)
static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo)
{
xe_bo_assert_held(bo);
- return bo->madv_purgeable == XE_MADV_PURGEABLE_DONTNEED;
+ return bo->purgeable.state == XE_MADV_PURGEABLE_DONTNEED;
}
void xe_bo_set_purgeable_state(struct xe_bo *bo, enum xe_madv_purgeable_state new_state);
+/**
+ * xe_bo_willneed_get_locked() - Acquire a WILLNEED holder on a BO
+ * @bo: Buffer object
+ *
+ * Increments willneed_count and, on a 0->1 transition, promotes the BO
+ * from DONTNEED to WILLNEED. PURGED is terminal and is never modified.
+ *
+ * Caller must hold the BO's dma-resv lock.
+ */
+static inline void xe_bo_willneed_get_locked(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ /* Imported BOs are owned externally; do not track purgeability. */
+ if (drm_gem_is_imported(&bo->ttm.base))
+ return;
+
+ if (bo->purgeable.willneed_count++ == 0 && xe_bo_madv_is_dontneed(bo))
+ xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_WILLNEED);
+}
+
+/**
+ * xe_bo_willneed_put_locked() - Release a WILLNEED holder on a BO
+ * @bo: Buffer object
+ *
+ * Decrements willneed_count and, on a 1->0 transition, marks the BO
+ * DONTNEED only if it still has VMAs (implying all active VMAs are
+ * DONTNEED). If the last VMA is being removed, preserve the current BO
+ * state to match the previous VMA-walk semantics.
+ *
+ * PURGED is terminal and the BO state is never modified.
+ *
+ * Caller must hold the BO's dma-resv lock.
+ */
+static inline void xe_bo_willneed_put_locked(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ if (drm_gem_is_imported(&bo->ttm.base))
+ return;
+
+ xe_assert(xe_bo_device(bo), bo->purgeable.willneed_count > 0);
+ if (--bo->purgeable.willneed_count == 0 && bo->purgeable.vma_count > 0 &&
+ !xe_bo_is_purged(bo))
+ xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED);
+}
+
+/**
+ * xe_bo_vma_count_inc_locked() - Account a new VMA on a BO
+ * @bo: Buffer object
+ *
+ * Increments vma_count.
+ *
+ * Caller must hold the BO's dma-resv lock.
+ */
+static inline void xe_bo_vma_count_inc_locked(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ if (drm_gem_is_imported(&bo->ttm.base))
+ return;
+
+ bo->purgeable.vma_count++;
+}
+
+/**
+ * xe_bo_vma_count_dec_locked() - Account a VMA removal on a BO
+ * @bo: Buffer object
+ *
+ * Decrements vma_count.
+ *
+ * Caller must hold the BO's dma-resv lock.
+ */
+static inline void xe_bo_vma_count_dec_locked(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ if (drm_gem_is_imported(&bo->ttm.base))
+ return;
+
+ xe_assert(xe_bo_device(bo), bo->purgeable.vma_count > 0);
+ bo->purgeable.vma_count--;
+}
+
static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo)
{
if (likely(bo)) {
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index ff8317b..077e35b 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -18,6 +18,7 @@
#include "xe_ggtt_types.h"
struct xe_device;
+struct xe_mem_pool_node;
struct xe_vm;
#define XE_BO_MAX_PLACEMENTS 3
@@ -88,7 +89,7 @@ struct xe_bo {
bool ccs_cleared;
/** @bb_ccs: BB instructions of CCS read/write. Valid only for VF */
- struct xe_bb *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT];
+ struct xe_mem_pool_node *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT];
/**
* @cpu_caching: CPU caching mode. Currently only used for userspace
@@ -110,10 +111,32 @@ struct xe_bo {
u64 min_align;
/**
- * @madv_purgeable: user space advise on BO purgeability, protected
- * by BO's dma-resv lock.
+ * @purgeable: Purgeability state and accounting.
+ *
+ * All fields are protected by the BO's dma-resv lock.
*/
- u32 madv_purgeable;
+ struct {
+ /**
+ * @purgeable.state: BO purgeability state
+ * (WILLNEED/DONTNEED/PURGED).
+ */
+ u32 state;
+
+ /**
+ * @purgeable.vma_count: Number of VMAs currently mapping this BO.
+ */
+ u32 vma_count;
+
+ /**
+ * @purgeable.willneed_count: Number of active WILLNEED holders.
+ *
+ * Counts WILLNEED VMAs plus active dma-buf exports for
+ * non-imported BOs. The BO flips to DONTNEED on a 1->0
+ * transition only when VMAs still exist; if the last VMA is
+ * removed, the previous BO state is preserved.
+ */
+ u32 willneed_count;
+ } purgeable;
};
#endif
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 7f9602b..8a920e5 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -193,6 +193,18 @@ static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
return 0;
}
+static void xe_dma_buf_release(struct dma_buf *dmabuf)
+{
+ struct drm_gem_object *obj = dmabuf->priv;
+ struct xe_bo *bo = gem_to_xe_bo(obj);
+
+ xe_bo_lock(bo, false);
+ xe_bo_willneed_put_locked(bo);
+ xe_bo_unlock(bo);
+
+ drm_gem_dmabuf_release(dmabuf);
+}
+
static const struct dma_buf_ops xe_dmabuf_ops = {
.attach = xe_dma_buf_attach,
.detach = xe_dma_buf_detach,
@@ -200,7 +212,7 @@ static const struct dma_buf_ops xe_dmabuf_ops = {
.unpin = xe_dma_buf_unpin,
.map_dma_buf = xe_dma_buf_map,
.unmap_dma_buf = xe_dma_buf_unmap,
- .release = drm_gem_dmabuf_release,
+ .release = xe_dma_buf_release,
.begin_cpu_access = xe_dma_buf_begin_cpu_access,
.mmap = drm_gem_dmabuf_mmap,
.vmap = drm_gem_dmabuf_vmap,
@@ -241,26 +253,33 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
ret = -EINVAL;
goto out_unlock;
}
+
+ xe_bo_willneed_get_locked(bo);
xe_bo_unlock(bo);
ret = ttm_bo_setup_export(&bo->ttm, &ctx);
if (ret)
- return ERR_PTR(ret);
+ goto out_put;
buf = drm_gem_prime_export(obj, flags);
- if (!IS_ERR(buf))
- buf->ops = &xe_dmabuf_ops;
+ if (IS_ERR(buf)) {
+ ret = PTR_ERR(buf);
+ goto out_put;
+ }
+ buf->ops = &xe_dmabuf_ops;
return buf;
+out_put:
+ xe_bo_lock(bo, false);
+ xe_bo_willneed_put_locked(bo);
out_unlock:
xe_bo_unlock(bo);
return ERR_PTR(ret);
}
static struct drm_gem_object *
-xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
- struct dma_buf *dma_buf)
+xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
{
struct dma_resv *resv = dma_buf->resv;
struct xe_device *xe = to_xe_device(dev);
@@ -281,7 +300,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
if (ret)
break;
- bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
+ bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size,
0, /* Will require 1way or 2way for vm_bind */
ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec);
drm_exec_retry_on_contention(&exec);
@@ -332,7 +351,6 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
const struct dma_buf_attach_ops *attach_ops;
struct dma_buf_attachment *attach;
struct drm_gem_object *obj;
- struct xe_bo *bo;
if (dma_buf->ops == &xe_dmabuf_ops) {
obj = dma_buf->priv;
@@ -348,13 +366,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
}
/*
- * Don't publish the bo until we have a valid attachment, and a
- * valid attachment needs the bo address. So pre-create a bo before
- * creating the attachment and publish.
+ * This needs to happen before the attach, since it will create a new
+ * attachment for this, and add it to the list of attachments, at which
+ * point it is globally visible, and at any point the export side can
+ * call into on invalidate_mappings callback, which require a working
+ * object.
*/
- bo = xe_bo_alloc();
- if (IS_ERR(bo))
- return ERR_CAST(bo);
+ obj = xe_dma_buf_create_obj(dev, dma_buf);
+ if (IS_ERR(obj))
+ return obj;
attach_ops = &xe_dma_buf_attach_ops;
#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
@@ -362,26 +382,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
attach_ops = test->attach_ops;
#endif
- attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
+ attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj);
if (IS_ERR(attach)) {
- obj = ERR_CAST(attach);
- goto out_err;
+ xe_bo_put(gem_to_xe_bo(obj));
+ return ERR_CAST(attach);
}
- /* Errors here will take care of freeing the bo. */
- obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
- if (IS_ERR(obj))
- return obj;
-
-
get_dma_buf(dma_buf);
obj->import_attach = attach;
return obj;
-
-out_err:
- xe_bo_free(bo);
-
- return obj;
}
#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c
index c34408c..dddcdd0 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.c
+++ b/drivers/gpu/drm/xe/xe_eu_stall.c
@@ -869,14 +869,14 @@ static int xe_eu_stall_stream_close(struct inode *inode, struct file *file)
struct xe_eu_stall_data_stream *stream = file->private_data;
struct xe_gt *gt = stream->gt;
- drm_dev_put(>->tile->xe->drm);
-
mutex_lock(>->eu_stall->stream_lock);
xe_eu_stall_disable_locked(stream);
xe_eu_stall_data_buf_destroy(stream);
xe_eu_stall_stream_free(stream);
mutex_unlock(>->eu_stall->stream_lock);
+ drm_dev_put(>->tile->xe->drm);
+
return 0;
}
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index b287d0e..071b8c4 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -1405,7 +1405,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
if (q->vm && q->hwe->hw_engine_group) {
err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q);
if (err)
- goto put_exec_queue;
+ goto kill_exec_queue;
}
}
@@ -1416,12 +1416,15 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
/* user id alloc must always be last in ioctl to prevent UAF */
err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
if (err)
- goto kill_exec_queue;
+ goto del_hw_engine_group;
args->exec_queue_id = id;
return 0;
+del_hw_engine_group:
+ if (q->vm && q->hwe && q->hwe->hw_engine_group)
+ xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
kill_exec_queue:
xe_exec_queue_kill(q);
delete_queue_group:
@@ -1760,7 +1763,7 @@ void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
unsigned int type)
{
- xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
+ xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
dma_fence_put(q->tlb_inval[type].last_fence);
diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c
index e5c234f..aab59dc 100644
--- a/drivers/gpu/drm/xe/xe_gsc.c
+++ b/drivers/gpu/drm/xe/xe_gsc.c
@@ -166,7 +166,7 @@ static int query_compatibility_version(struct xe_gsc *gsc)
&rd_offset);
if (err) {
xe_gt_err(gt, "HuC: invalid GSC reply for version query (err=%d)\n", err);
- return err;
+ goto out_bo;
}
compat->major = version_query_rd(xe, &bo->vmap, rd_offset, proj_major);
@@ -482,8 +482,7 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
EXEC_QUEUE_FLAG_PERMANENT, 0);
if (IS_ERR(q)) {
xe_gt_err(gt, "Failed to create queue for GSC submission\n");
- err = PTR_ERR(q);
- goto out_bo;
+ return PTR_ERR(q);
}
wq = alloc_ordered_workqueue("gsc-ordered-wq", 0);
@@ -506,8 +505,6 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
out_q:
xe_exec_queue_put(q);
-out_bo:
- xe_bo_unpin_map_no_vm(bo);
return err;
}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
index 87a164e..01fe03b 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
@@ -385,10 +385,10 @@ static int pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid, void *buf
if (xe_gt_is_media_type(gt))
for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++)
- regs[n] = xe_mmio_read32(>->mmio, MED_VF_SW_FLAG(n));
+ regs[n] = xe_mmio_read32(&mmio, MED_VF_SW_FLAG(n));
else
for (n = 0; n < VF_SW_FLAG_COUNT; n++)
- regs[n] = xe_mmio_read32(>->mmio, VF_SW_FLAG(n));
+ regs[n] = xe_mmio_read32(&mmio, VF_SW_FLAG(n));
return 0;
}
@@ -407,10 +407,10 @@ static int pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid,
if (xe_gt_is_media_type(gt))
for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++)
- xe_mmio_write32(>->mmio, MED_VF_SW_FLAG(n), regs[n]);
+ xe_mmio_write32(&mmio, MED_VF_SW_FLAG(n), regs[n]);
else
for (n = 0; n < VF_SW_FLAG_COUNT; n++)
- xe_mmio_write32(>->mmio, VF_SW_FLAG(n), regs[n]);
+ xe_mmio_write32(&mmio, VF_SW_FLAG(n), regs[n]);
return 0;
}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
index 7d532bd..a85ba44 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
@@ -114,8 +114,10 @@ int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32
* VFs with no events are not printed.
*
* This function can only be called on PF.
+ *
+ * Return: always 0
*/
-void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p)
{
unsigned int n, total_vfs = xe_gt_sriov_pf_get_totalvfs(gt);
const struct xe_gt_sriov_monitor *data;
@@ -144,4 +146,6 @@ void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p
#undef __format
#undef __value
}
+
+ return 0;
}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
index 7ca9351..0b8f088 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
@@ -13,7 +13,7 @@ struct drm_printer;
struct xe_gt;
void xe_gt_sriov_pf_monitor_flr(struct xe_gt *gt, u32 vfid);
-void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p);
#ifdef CONFIG_PCI_IOV
int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
index 8989c8e..0cd9d77 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
@@ -1137,13 +1137,15 @@ void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val)
}
/**
- * xe_gt_sriov_vf_print_config - Print VF self config.
+ * xe_gt_sriov_vf_print_config() - Print VF self config.
* @gt: the &xe_gt
* @p: the &drm_printer
*
* This function is for VF use only.
+ *
+ * Return: always 0.
*/
-void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
{
struct xe_gt_sriov_vf_selfconfig *config = >->sriov.vf.self_config;
struct xe_device *xe = gt_to_xe(gt);
@@ -1170,16 +1172,20 @@ void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
drm_printf(p, "GuC contexts:\t%u\n", config->num_ctxs);
drm_printf(p, "GuC doorbells:\t%u\n", config->num_dbs);
+
+ return 0;
}
/**
- * xe_gt_sriov_vf_print_runtime - Print VF's runtime regs received from PF.
+ * xe_gt_sriov_vf_print_runtime() - Print VF's runtime regs received from PF.
* @gt: the &xe_gt
* @p: the &drm_printer
*
* This function is for VF use only.
+ *
+ * Return: always 0.
*/
-void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
{
struct vf_runtime_reg *vf_regs = gt->sriov.vf.runtime.regs;
unsigned int size = gt->sriov.vf.runtime.num_regs;
@@ -1188,16 +1194,20 @@ void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
for (; size--; vf_regs++)
drm_printf(p, "%#x = %#x\n", vf_regs->offset, vf_regs->value);
+
+ return 0;
}
/**
- * xe_gt_sriov_vf_print_version - Print VF ABI versions.
+ * xe_gt_sriov_vf_print_version() - Print VF ABI versions.
* @gt: the &xe_gt
* @p: the &drm_printer
*
* This function is for VF use only.
+ *
+ * Return: always 0.
*/
-void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
{
struct xe_device *xe = gt_to_xe(gt);
struct xe_uc_fw_version *guc_version = >->sriov.vf.guc_version;
@@ -1227,6 +1237,8 @@ void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
GUC_RELAY_VERSION_LATEST_MAJOR, GUC_RELAY_VERSION_LATEST_MINOR);
drm_printf(p, "\thandshake:\t%u.%u\n",
pf_version->major, pf_version->minor);
+
+ return 0;
}
static bool vf_post_migration_shutdown(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
index a6f7127..79878f2 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
@@ -35,9 +35,9 @@ bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt);
u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg);
void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);
-void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p);
-void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p);
-void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p);
int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt);
int xe_vf_migration_fixups_complete_count(struct xe_gt *gt);
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index 8b55cf2..fffb5d6 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -145,6 +145,13 @@ struct xe_gt {
/** @info.has_indirect_ring_state: GT has indirect ring state support */
u8 has_indirect_ring_state:1;
/**
+ * @info.has_xe2_blt_instructions: GT supports Xe2-style MEM_SET
+ * and MEM_COPY blitter functionality. Note that despite the
+ * name, some Xe1 platforms may also support this "Xe2-style"
+ * feature.
+ */
+ u8 has_xe2_blt_instructions:1;
+ /**
* @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse
* registers the geometry XeCore mask spans.
*/
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index 81b5f01..5760251 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -512,12 +512,9 @@ static void guc_golden_lrc_init(struct xe_guc_ads *ads)
* that starts after the execlists LRC registers. This is
* required to allow the GuC to restore just the engine state
* when a watchdog reset occurs.
- * We calculate the engine state size by removing the size of
- * what comes before it in the context image (which is identical
- * on all engines).
*/
ads_blob_write(ads, ads.eng_state_size[guc_class],
- real_size - xe_lrc_skip_size(xe));
+ xe_lrc_engine_state_size(gt, class));
ads_blob_write(ads, ads.golden_context_lrca[guc_class],
addr_ggtt);
@@ -770,6 +767,11 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
}
}
+ if (XE_GT_WA(hwe->gt, 16023105232))
+ guc_mmio_regset_write_one(ads, regset_map,
+ RING_IDLEDLY(hwe->mmio_base),
+ count++);
+
return count;
}
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index a145234..912182d 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -261,22 +261,10 @@ static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
static void guc_submit_fini(void *arg)
{
struct xe_guc *guc = arg;
-
- /* Forcefully kill any remaining exec queues */
- xe_guc_ct_stop(&guc->ct);
- guc_submit_reset_prepare(guc);
- xe_guc_softreset(guc);
- xe_guc_submit_stop(guc);
- xe_uc_fw_sanitize(&guc->fw);
- xe_guc_submit_pause_abort(guc);
-}
-
-static void guc_submit_wedged_fini(void *arg)
-{
- struct xe_guc *guc = arg;
struct xe_exec_queue *q;
unsigned long index;
+ /* Drop any wedged queue refs */
mutex_lock(&guc->submission_state.lock);
xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
if (exec_queue_wedged(q)) {
@@ -286,6 +274,14 @@ static void guc_submit_wedged_fini(void *arg)
}
}
mutex_unlock(&guc->submission_state.lock);
+
+ /* Forcefully kill any remaining exec queues */
+ xe_guc_ct_stop(&guc->ct);
+ guc_submit_reset_prepare(guc);
+ xe_guc_softreset(guc);
+ xe_guc_submit_stop(guc);
+ xe_uc_fw_sanitize(&guc->fw);
+ xe_guc_submit_pause_abort(guc);
}
static const struct xe_exec_queue_ops guc_exec_queue_ops;
@@ -1320,10 +1316,8 @@ static void disable_scheduling_deregister(struct xe_guc *guc,
void xe_guc_submit_wedge(struct xe_guc *guc)
{
struct xe_device *xe = guc_to_xe(guc);
- struct xe_gt *gt = guc_to_gt(guc);
struct xe_exec_queue *q;
unsigned long index;
- int err;
xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
@@ -1335,15 +1329,6 @@ void xe_guc_submit_wedge(struct xe_guc *guc)
return;
if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
- err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
- guc_submit_wedged_fini, guc);
- if (err) {
- xe_gt_err(gt, "Failed to register clean-up on wedged.mode=%s; "
- "Although device is wedged.\n",
- xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET));
- return;
- }
-
mutex_lock(&guc->submission_state.lock);
xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
if (xe_exec_queue_get_unless_zero(q))
@@ -1688,6 +1673,14 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q)
struct xe_guc_exec_queue *ge = q->guc;
struct xe_guc *guc = exec_queue_to_guc(q);
+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
+ struct xe_exec_queue_group *group = q->multi_queue.group;
+
+ mutex_lock(&group->list_lock);
+ list_del(&q->multi_queue.link);
+ mutex_unlock(&group->list_lock);
+ }
+
release_guc_id(guc, q);
xe_sched_entity_fini(&ge->entity);
xe_sched_fini(&ge->sched);
@@ -1709,14 +1702,6 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w)
guard(xe_pm_runtime)(guc_to_xe(guc));
trace_xe_exec_queue_destroy(q);
- if (xe_exec_queue_is_multi_queue_secondary(q)) {
- struct xe_exec_queue_group *group = q->multi_queue.group;
-
- mutex_lock(&group->list_lock);
- list_del(&q->multi_queue.link);
- mutex_unlock(&group->list_lock);
- }
-
/* Confirm no work left behind accessing device structures */
cancel_delayed_work_sync(&ge->sched.base.work_tdr);
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index 9d12a0d..4af9f0d 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -746,9 +746,16 @@ size_t xe_lrc_reg_size(struct xe_device *xe)
return 80 * sizeof(u32);
}
-size_t xe_lrc_skip_size(struct xe_device *xe)
+/**
+ * xe_lrc_engine_state_size() - Get size of the engine state within LRC
+ * @gt: the &xe_gt struct instance
+ * @class: Hardware engine class
+ *
+ * Returns: Size of the engine state
+ */
+size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class)
{
- return LRC_PPHWSP_SIZE + xe_lrc_reg_size(xe);
+ return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt));
}
static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc)
@@ -1214,7 +1221,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc,
if (xe_gt_WARN_ON(lrc->gt, max_len < 3))
return -ENOSPC;
- *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
+ *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1);
*cmd++ = CS_DEBUG_MODE2(0).addr;
*cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index e7c975f..5440663 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -130,7 +130,7 @@ u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc);
struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc);
size_t xe_lrc_reg_size(struct xe_device *xe);
-size_t xe_lrc_skip_size(struct xe_device *xe);
+size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class);
void xe_lrc_dump_default(struct drm_printer *p,
struct xe_gt *gt,
diff --git a/drivers/gpu/drm/xe/xe_mem_pool.c b/drivers/gpu/drm/xe/xe_mem_pool.c
new file mode 100644
index 0000000..d5e24d6
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mem_pool.c
@@ -0,0 +1,403 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/kernel.h>
+
+#include <drm/drm_managed.h>
+
+#include "instructions/xe_mi_commands.h"
+#include "xe_bo.h"
+#include "xe_device_types.h"
+#include "xe_map.h"
+#include "xe_mem_pool.h"
+#include "xe_mem_pool_types.h"
+#include "xe_tile_printk.h"
+
+/**
+ * struct xe_mem_pool - DRM MM pool for sub-allocating memory from a BO on an
+ * XE tile.
+ *
+ * The XE memory pool is a DRM MM manager that provides sub-allocation of memory
+ * from a backing buffer object (BO) on a specific XE tile. It is designed to
+ * manage memory for GPU workloads, allowing for efficient allocation and
+ * deallocation of memory regions within the BO.
+ *
+ * The memory pool maintains a primary BO that is pinned in the GGTT and mapped
+ * into the CPU address space for direct access. Optionally, it can also maintain
+ * a shadow BO that can be used for atomic updates to the primary BO's contents.
+ *
+ * The API provided by the memory pool allows clients to allocate and free memory
+ * regions, retrieve GPU and CPU addresses, and synchronize data between the
+ * primary and shadow BOs as needed.
+ */
+struct xe_mem_pool {
+ /** @base: Range allocator over [0, @size) in bytes */
+ struct drm_mm base;
+ /** @bo: Active pool BO (GGTT-pinned, CPU-mapped). */
+ struct xe_bo *bo;
+ /** @shadow: Shadow BO for atomic command updates. */
+ struct xe_bo *shadow;
+ /** @swap_guard: Timeline guard updating @bo and @shadow */
+ struct mutex swap_guard;
+ /** @cpu_addr: CPU virtual address of the active BO. */
+ void *cpu_addr;
+ /** @is_iomem: Indicates if the BO mapping is I/O memory. */
+ bool is_iomem;
+};
+
+static struct xe_mem_pool *node_to_pool(struct xe_mem_pool_node *node)
+{
+ return container_of(node->sa_node.mm, struct xe_mem_pool, base);
+}
+
+static struct xe_tile *pool_to_tile(struct xe_mem_pool *pool)
+{
+ return pool->bo->tile;
+}
+
+static void fini_pool_action(struct drm_device *drm, void *arg)
+{
+ struct xe_mem_pool *pool = arg;
+
+ if (pool->is_iomem)
+ kvfree(pool->cpu_addr);
+
+ drm_mm_takedown(&pool->base);
+}
+
+static int pool_shadow_init(struct xe_mem_pool *pool)
+{
+ struct xe_tile *tile = pool->bo->tile;
+ struct xe_device *xe = tile_to_xe(tile);
+ struct xe_bo *shadow;
+ int ret;
+
+ xe_assert(xe, !pool->shadow);
+
+ ret = drmm_mutex_init(&xe->drm, &pool->swap_guard);
+ if (ret)
+ return ret;
+
+ if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
+ fs_reclaim_acquire(GFP_KERNEL);
+ might_lock(&pool->swap_guard);
+ fs_reclaim_release(GFP_KERNEL);
+ }
+ shadow = xe_managed_bo_create_pin_map(xe, tile,
+ xe_bo_size(pool->bo),
+ XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+ XE_BO_FLAG_GGTT |
+ XE_BO_FLAG_GGTT_INVALIDATE |
+ XE_BO_FLAG_PINNED_NORESTORE);
+ if (IS_ERR(shadow))
+ return PTR_ERR(shadow);
+
+ pool->shadow = shadow;
+
+ return 0;
+}
+
+/**
+ * xe_mem_pool_init() - Initialize memory pool.
+ * @tile: the &xe_tile where allocate.
+ * @size: number of bytes to allocate.
+ * @guard: the size of the guard region at the end of the BO that is not
+ * sub-allocated, in bytes.
+ * @flags: flags to use to create shadow pool.
+ *
+ * Initializes a memory pool for sub-allocating memory from a backing BO on the
+ * specified XE tile. The backing BO is pinned in the GGTT and mapped into
+ * the CPU address space for direct access. Optionally, a shadow BO can also be
+ * initialized for atomic updates to the primary BO's contents.
+ *
+ * Returns: a pointer to the &xe_mem_pool, or an error pointer on failure.
+ */
+struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size,
+ u32 guard, int flags)
+{
+ struct xe_device *xe = tile_to_xe(tile);
+ struct xe_mem_pool *pool;
+ struct xe_bo *bo;
+ u32 managed_size;
+ int ret;
+
+ xe_tile_assert(tile, size > guard);
+ managed_size = size - guard;
+
+ pool = drmm_kzalloc(&xe->drm, sizeof(*pool), GFP_KERNEL);
+ if (!pool)
+ return ERR_PTR(-ENOMEM);
+
+ bo = xe_managed_bo_create_pin_map(xe, tile, size,
+ XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+ XE_BO_FLAG_GGTT |
+ XE_BO_FLAG_GGTT_INVALIDATE |
+ XE_BO_FLAG_PINNED_NORESTORE);
+ if (IS_ERR(bo)) {
+ xe_tile_err(tile, "Failed to prepare %uKiB BO for mem pool (%pe)\n",
+ size / SZ_1K, bo);
+ return ERR_CAST(bo);
+ }
+ pool->bo = bo;
+ pool->is_iomem = bo->vmap.is_iomem;
+
+ if (pool->is_iomem) {
+ pool->cpu_addr = kvzalloc(size, GFP_KERNEL);
+ if (!pool->cpu_addr)
+ return ERR_PTR(-ENOMEM);
+ } else {
+ pool->cpu_addr = bo->vmap.vaddr;
+ }
+
+ if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY) {
+ ret = pool_shadow_init(pool);
+
+ if (ret)
+ goto out_err;
+ }
+
+ drm_mm_init(&pool->base, 0, managed_size);
+ ret = drmm_add_action_or_reset(&xe->drm, fini_pool_action, pool);
+ if (ret)
+ return ERR_PTR(ret);
+
+ return pool;
+
+out_err:
+ if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY)
+ xe_tile_err(tile,
+ "Failed to initialize shadow BO for mem pool (%d)\n", ret);
+ if (bo->vmap.is_iomem)
+ kvfree(pool->cpu_addr);
+ return ERR_PTR(ret);
+}
+
+/**
+ * xe_mem_pool_sync() - Copy the entire contents of the main pool to shadow pool.
+ * @pool: the memory pool containing the primary and shadow BOs.
+ *
+ * Copies the entire contents of the primary pool to the shadow pool. This must
+ * be done after xe_mem_pool_init() with the XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY
+ * flag to ensure that the shadow pool has the same initial contents as the primary
+ * pool. After this initial synchronization, clients can choose to synchronize the
+ * shadow pool with the primary pool on a node basis using
+ * xe_mem_pool_sync_shadow_locked() as needed.
+ *
+ * Return: None.
+ */
+void xe_mem_pool_sync(struct xe_mem_pool *pool)
+{
+ struct xe_tile *tile = pool_to_tile(pool);
+ struct xe_device *xe = tile_to_xe(tile);
+
+ xe_tile_assert(tile, pool->shadow);
+
+ xe_map_memcpy_to(xe, &pool->shadow->vmap, 0,
+ pool->cpu_addr, xe_bo_size(pool->bo));
+}
+
+/**
+ * xe_mem_pool_swap_shadow_locked() - Swap the primary BO with the shadow BO.
+ * @pool: the memory pool containing the primary and shadow BOs.
+ *
+ * Swaps the primary buffer object with the shadow buffer object in the mem
+ * pool. This allows for atomic updates to the contents of the primary BO
+ * by first writing to the shadow BO and then swapping it with the primary BO.
+ * Swap_guard must be held to ensure synchronization with any concurrent swap
+ * operations.
+ *
+ * Return: None.
+ */
+void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool)
+{
+ struct xe_tile *tile = pool_to_tile(pool);
+
+ xe_tile_assert(tile, pool->shadow);
+ lockdep_assert_held(&pool->swap_guard);
+
+ swap(pool->bo, pool->shadow);
+ if (!pool->bo->vmap.is_iomem)
+ pool->cpu_addr = pool->bo->vmap.vaddr;
+}
+
+/**
+ * xe_mem_pool_sync_shadow_locked() - Copy node from primary pool to shadow pool.
+ * @node: the node allocated in the memory pool.
+ *
+ * Copies the specified batch buffer from the primary pool to the shadow pool.
+ * Swap_guard must be held to ensure synchronization with any concurrent swap
+ * operations.
+ *
+ * Return: None.
+ */
+void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node)
+{
+ struct xe_mem_pool *pool = node_to_pool(node);
+ struct xe_tile *tile = pool_to_tile(pool);
+ struct xe_device *xe = tile_to_xe(tile);
+ struct drm_mm_node *sa_node = &node->sa_node;
+
+ xe_tile_assert(tile, pool->shadow);
+ lockdep_assert_held(&pool->swap_guard);
+
+ xe_map_memcpy_to(xe, &pool->shadow->vmap,
+ sa_node->start,
+ pool->cpu_addr + sa_node->start,
+ sa_node->size);
+}
+
+/**
+ * xe_mem_pool_gpu_addr() - Retrieve GPU address of memory pool.
+ * @pool: the memory pool
+ *
+ * Returns: GGTT address of the memory pool.
+ */
+u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool)
+{
+ return xe_bo_ggtt_addr(pool->bo);
+}
+
+/**
+ * xe_mem_pool_cpu_addr() - Retrieve CPU address of manager pool.
+ * @pool: the memory pool
+ *
+ * Returns: CPU virtual address of memory pool.
+ */
+void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool)
+{
+ return pool->cpu_addr;
+}
+
+/**
+ * xe_mem_pool_bo_swap_guard() - Retrieve the mutex used to guard swap
+ * operations on a memory pool.
+ * @pool: the memory pool
+ *
+ * Returns: Swap guard mutex or NULL if shadow pool is not created.
+ */
+struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool)
+{
+ if (!pool->shadow)
+ return NULL;
+
+ return &pool->swap_guard;
+}
+
+/**
+ * xe_mem_pool_bo_flush_write() - Copy the data from the sub-allocation
+ * to the GPU memory.
+ * @node: the node allocated in the memory pool to flush.
+ */
+void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node)
+{
+ struct xe_mem_pool *pool = node_to_pool(node);
+ struct xe_tile *tile = pool_to_tile(pool);
+ struct xe_device *xe = tile_to_xe(tile);
+ struct drm_mm_node *sa_node = &node->sa_node;
+
+ if (!pool->bo->vmap.is_iomem)
+ return;
+
+ xe_map_memcpy_to(xe, &pool->bo->vmap, sa_node->start,
+ pool->cpu_addr + sa_node->start,
+ sa_node->size);
+}
+
+/**
+ * xe_mem_pool_bo_sync_read() - Copy the data from GPU memory to the
+ * sub-allocation.
+ * @node: the node allocated in the memory pool to read back.
+ */
+void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node)
+{
+ struct xe_mem_pool *pool = node_to_pool(node);
+ struct xe_tile *tile = pool_to_tile(pool);
+ struct xe_device *xe = tile_to_xe(tile);
+ struct drm_mm_node *sa_node = &node->sa_node;
+
+ if (!pool->bo->vmap.is_iomem)
+ return;
+
+ xe_map_memcpy_from(xe, pool->cpu_addr + sa_node->start,
+ &pool->bo->vmap, sa_node->start, sa_node->size);
+}
+
+/**
+ * xe_mem_pool_alloc_node() - Allocate a new node for use with xe_mem_pool.
+ *
+ * Returns: node structure or an ERR_PTR(-ENOMEM).
+ */
+struct xe_mem_pool_node *xe_mem_pool_alloc_node(void)
+{
+ struct xe_mem_pool_node *node = kzalloc_obj(*node);
+
+ if (!node)
+ return ERR_PTR(-ENOMEM);
+
+ return node;
+}
+
+/**
+ * xe_mem_pool_insert_node() - Insert a node into the memory pool.
+ * @pool: the memory pool to insert into
+ * @node: the node to insert
+ * @size: the size of the node to be allocated in bytes.
+ *
+ * Inserts a node into the specified memory pool using drm_mm for
+ * allocation.
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+int xe_mem_pool_insert_node(struct xe_mem_pool *pool,
+ struct xe_mem_pool_node *node, u32 size)
+{
+ if (!pool)
+ return -EINVAL;
+
+ return drm_mm_insert_node(&pool->base, &node->sa_node, size);
+}
+
+/**
+ * xe_mem_pool_free_node() - Free a node allocated from the memory pool.
+ * @node: the node to free
+ *
+ * Returns: None.
+ */
+void xe_mem_pool_free_node(struct xe_mem_pool_node *node)
+{
+ if (!node)
+ return;
+
+ drm_mm_remove_node(&node->sa_node);
+ kfree(node);
+}
+
+/**
+ * xe_mem_pool_node_cpu_addr() - Retrieve CPU address of the node.
+ * @node: the node allocated in the memory pool
+ *
+ * Returns: CPU virtual address of the node.
+ */
+void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node)
+{
+ struct xe_mem_pool *pool = node_to_pool(node);
+
+ return xe_mem_pool_cpu_addr(pool) + node->sa_node.start;
+}
+
+/**
+ * xe_mem_pool_dump() - Dump the state of the DRM MM manager for debugging.
+ * @pool: the memory pool info be dumped.
+ * @p: The DRM printer to use for output.
+ *
+ * Only the drm managed region is dumped, not the state of the BOs or any other
+ * pool information.
+ *
+ * Returns: None.
+ */
+void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p)
+{
+ drm_mm_print(&pool->base, p);
+}
diff --git a/drivers/gpu/drm/xe/xe_mem_pool.h b/drivers/gpu/drm/xe/xe_mem_pool.h
new file mode 100644
index 0000000..89cd255
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mem_pool.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+#ifndef _XE_MEM_POOL_H_
+#define _XE_MEM_POOL_H_
+
+#include <linux/sizes.h>
+#include <linux/types.h>
+
+#include <drm/drm_mm.h>
+#include "xe_mem_pool_types.h"
+
+struct drm_printer;
+struct xe_mem_pool;
+struct xe_tile;
+
+struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size,
+ u32 guard, int flags);
+void xe_mem_pool_sync(struct xe_mem_pool *pool);
+void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool);
+void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node);
+u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool);
+void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool);
+struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool);
+void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node);
+void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node);
+struct xe_mem_pool_node *xe_mem_pool_alloc_node(void);
+int xe_mem_pool_insert_node(struct xe_mem_pool *pool,
+ struct xe_mem_pool_node *node, u32 size);
+void xe_mem_pool_free_node(struct xe_mem_pool_node *node);
+void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node);
+void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_mem_pool_types.h b/drivers/gpu/drm/xe/xe_mem_pool_types.h
new file mode 100644
index 0000000..d5e926c
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mem_pool_types.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_MEM_POOL_TYPES_H_
+#define _XE_MEM_POOL_TYPES_H_
+
+#include <drm/drm_mm.h>
+
+#define XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY BIT(0)
+
+/**
+ * struct xe_mem_pool_node - Sub-range allocations from mem pool.
+ */
+struct xe_mem_pool_node {
+ /** @sa_node: drm_mm_node for this allocation. */
+ struct drm_mm_node sa_node;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
index 811e071..579af47 100644
--- a/drivers/gpu/drm/xe/xe_memirq.c
+++ b/drivers/gpu/drm/xe/xe_memirq.c
@@ -427,13 +427,25 @@ static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
return __memirq_received(memirq, vector, offset, name, true);
}
+static void memirq_assume_received(struct xe_memirq *memirq, const char *source,
+ u16 offset, const char *status)
+{
+ memirq_debug(memirq, "ASSUME %s %s(%u)\n", source, status, offset);
+}
+
static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status,
struct xe_hw_engine *hwe)
{
memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);
- if (memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name))
- xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT);
+ /*
+ * The programming note says to assume that GT_MI_USER_INTERRUPT is always
+ * set. Check and clear related status byte just for a debug.
+ */
+ if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) &&
+ !memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name))
+ memirq_assume_received(memirq, hwe->name, ilog2(GT_MI_USER_INTERRUPT), "USER");
+ xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT);
}
static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status,
@@ -443,8 +455,14 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat
memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);
- if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
- xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
+ /*
+ * The programming note says to assume that GUC_INTR_GUC2HOST is always
+ * set. Check and clear related status byte just for a debug.
+ */
+ if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) &&
+ !memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
+ memirq_assume_received(memirq, name, ilog2(GUC_INTR_GUC2HOST), "GUC2HOST");
+ xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
/*
* This is a software interrupt that must be cleared after it's consumed
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index fc918b4..a22413f 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -29,6 +29,7 @@
#include "xe_hw_engine.h"
#include "xe_lrc.h"
#include "xe_map.h"
+#include "xe_mem_pool.h"
#include "xe_mocs.h"
#include "xe_printk.h"
#include "xe_pt.h"
@@ -1166,11 +1167,12 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
u32 batch_size, batch_size_allocated;
struct xe_device *xe = gt_to_xe(gt);
struct xe_res_cursor src_it, ccs_it;
+ struct xe_mem_pool *bb_pool;
struct xe_sriov_vf_ccs_ctx *ctx;
- struct xe_sa_manager *bb_pool;
u64 size = xe_bo_size(src_bo);
- struct xe_bb *bb = NULL;
+ struct xe_mem_pool_node *bb;
u64 src_L0, src_L0_ofs;
+ struct xe_bb xe_bb_tmp;
u32 src_L0_pt;
int err;
@@ -1208,18 +1210,18 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
size -= src_L0;
}
- bb = xe_bb_alloc(gt);
+ bb = xe_mem_pool_alloc_node();
if (IS_ERR(bb))
return PTR_ERR(bb);
bb_pool = ctx->mem.ccs_bb_pool;
- scoped_guard(mutex, xe_sa_bo_swap_guard(bb_pool)) {
- xe_sa_bo_swap_shadow(bb_pool);
+ scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) {
+ xe_mem_pool_swap_shadow_locked(bb_pool);
- err = xe_bb_init(bb, bb_pool, batch_size);
+ err = xe_mem_pool_insert_node(bb_pool, bb, batch_size * sizeof(u32));
if (err) {
xe_gt_err(gt, "BB allocation failed.\n");
- xe_bb_free(bb, NULL);
+ kfree(bb);
return err;
}
@@ -1227,6 +1229,7 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
size = xe_bo_size(src_bo);
batch_size = 0;
+ xe_bb_tmp = (struct xe_bb){ .cs = xe_mem_pool_node_cpu_addr(bb), .len = 0 };
/*
* Emit PTE and copy commands here.
* The CCS copy command can only support limited size. If the size to be
@@ -1255,24 +1258,27 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
batch_size += EMIT_COPY_CCS_DW;
- emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src);
+ emit_pte(m, &xe_bb_tmp, src_L0_pt, false, true, &src_it, src_L0, src);
- emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
+ emit_pte(m, &xe_bb_tmp, ccs_pt, false, false, &ccs_it, ccs_size, src);
- bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
- flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt,
+ xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len,
+ flush_flags);
+ flush_flags = xe_migrate_ccs_copy(m, &xe_bb_tmp, src_L0_ofs, src_is_pltt,
src_L0_ofs, dst_is_pltt,
src_L0, ccs_ofs, true);
- bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags);
+ xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len,
+ flush_flags);
size -= src_L0;
}
- xe_assert(xe, (batch_size_allocated == bb->len));
+ xe_assert(xe, (batch_size_allocated == xe_bb_tmp.len));
+ xe_assert(xe, bb->sa_node.size == xe_bb_tmp.len * sizeof(u32));
src_bo->bb_ccs[read_write] = bb;
xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
- xe_sa_bo_sync_shadow(bb->bo);
+ xe_mem_pool_sync_shadow_locked(bb);
}
return 0;
@@ -1297,10 +1303,10 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
enum xe_sriov_vf_ccs_rw_ctxs read_write)
{
- struct xe_bb *bb = src_bo->bb_ccs[read_write];
+ struct xe_mem_pool_node *bb = src_bo->bb_ccs[read_write];
struct xe_device *xe = xe_bo_device(src_bo);
+ struct xe_mem_pool *bb_pool;
struct xe_sriov_vf_ccs_ctx *ctx;
- struct xe_sa_manager *bb_pool;
u32 *cs;
xe_assert(xe, IS_SRIOV_VF(xe));
@@ -1308,17 +1314,17 @@ void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
ctx = &xe->sriov.vf.ccs.contexts[read_write];
bb_pool = ctx->mem.ccs_bb_pool;
- guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
- xe_sa_bo_swap_shadow(bb_pool);
+ scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) {
+ xe_mem_pool_swap_shadow_locked(bb_pool);
- cs = xe_sa_bo_cpu_addr(bb->bo);
- memset(cs, MI_NOOP, bb->len * sizeof(u32));
- xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
+ cs = xe_mem_pool_node_cpu_addr(bb);
+ memset(cs, MI_NOOP, bb->sa_node.size);
+ xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
- xe_sa_bo_sync_shadow(bb->bo);
-
- xe_bb_free(bb, NULL);
- src_bo->bb_ccs[read_write] = NULL;
+ xe_mem_pool_sync_shadow_locked(bb);
+ xe_mem_pool_free_node(bb);
+ src_bo->bb_ccs[read_write] = NULL;
+ }
}
/**
@@ -1518,23 +1524,9 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
bb->len += len;
}
-static bool has_service_copy_support(struct xe_gt *gt)
-{
- /*
- * What we care about is whether the architecture was designed with
- * service copy functionality (specifically the new MEM_SET / MEM_COPY
- * instructions) so check the architectural engine list rather than the
- * actual list since these instructions are usable on BCS0 even if
- * all of the actual service copy engines (BCS1-BCS8) have been fused
- * off.
- */
- return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8,
- XE_HW_ENGINE_BCS1);
-}
-
static u32 emit_clear_cmd_len(struct xe_gt *gt)
{
- if (has_service_copy_support(gt))
+ if (gt->info.has_xe2_blt_instructions)
return PVC_MEM_SET_CMD_LEN_DW;
else
return XY_FAST_COLOR_BLT_DW;
@@ -1543,7 +1535,7 @@ static u32 emit_clear_cmd_len(struct xe_gt *gt)
static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
u32 size, u32 pitch, bool is_vram)
{
- if (has_service_copy_support(gt))
+ if (gt->info.has_xe2_blt_instructions)
emit_clear_link_copy(gt, bb, src_ofs, size, pitch);
else
emit_clear_main_copy(gt, bb, src_ofs, size, pitch,
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index 6337e67..d908f4e 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -2032,8 +2032,10 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f
if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
return -ENOENT;
- if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1))
- return -EOPNOTSUPP;
+ if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) {
+ ret = -EOPNOTSUPP;
+ goto err_exec_q;
+ }
}
/*
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 01673d2..c2ecd27e 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -118,6 +118,7 @@ static const struct xe_graphics_desc graphics_xe2 = {
static const struct xe_graphics_desc graphics_xe3p_lpg = {
XE2_GFX_FEATURES,
+ .has_indirect_ring_state = 1,
.multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE),
.num_geometry_xecore_fuse_regs = 3,
.num_compute_xecore_fuse_regs = 3,
@@ -851,6 +852,15 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile,
gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs;
/*
+ * Even if the service copy engines wind up being fused off, their
+ * presence in the IP descriptor indicates that the platform supports
+ * Xe2-style MEM_SET and MEM_COPY functionality.
+ */
+ if (graphics_desc->hw_engine_mask & GENMASK(XE_HW_ENGINE_BCS8,
+ XE_HW_ENGINE_BCS1))
+ gt->info.has_xe2_blt_instructions = true;
+
+ /*
* Before media version 13, the media IP was part of the primary GT
* so we need to add the media engines to the primary GT's engine list.
*/
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c
index 80577e4..8cc3131 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.c
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c
@@ -226,7 +226,7 @@ void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent,
}
range_start = reg & REG_GENMASK(25, range_bit);
- range_end = range_start | REG_GENMASK(range_bit, 0);
+ range_end = range_start | REG_GENMASK(range_bit - 1, 0);
switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) {
case RING_FORCE_TO_NONPRIV_ACCESS_RW:
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c
index 6c4b164..150a241 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c
@@ -149,10 +149,11 @@ pf_migration_consume(struct xe_device *xe, unsigned int vfid)
for_each_gt(gt, xe, gt_id) {
data = xe_gt_sriov_pf_migration_save_consume(gt, vfid);
- if (data && PTR_ERR(data) != EAGAIN)
+ if (!data)
+ continue;
+ if (!IS_ERR(data) || PTR_ERR(data) != -EAGAIN)
return data;
- if (PTR_ERR(data) == -EAGAIN)
- more_data = true;
+ more_data = true;
}
if (!more_data)
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
index db023fb..09b99fb 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
@@ -14,9 +14,9 @@
#include "xe_guc.h"
#include "xe_guc_submit.h"
#include "xe_lrc.h"
+#include "xe_mem_pool.h"
#include "xe_migrate.h"
#include "xe_pm.h"
-#include "xe_sa.h"
#include "xe_sriov_printk.h"
#include "xe_sriov_vf.h"
#include "xe_sriov_vf_ccs.h"
@@ -141,43 +141,47 @@ static u64 get_ccs_bb_pool_size(struct xe_device *xe)
static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx)
{
+ struct xe_mem_pool *pool;
struct xe_device *xe = tile_to_xe(tile);
- struct xe_sa_manager *sa_manager;
+ u32 *pool_cpu_addr, *last_dw_addr;
u64 bb_pool_size;
- int offset, err;
+ int err;
bb_pool_size = get_ccs_bb_pool_size(xe);
xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
- sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16,
- XE_SA_BO_MANAGER_FLAG_SHADOW);
-
- if (IS_ERR(sa_manager)) {
- xe_sriov_err(xe, "Suballocator init failed with error: %pe\n",
- sa_manager);
- err = PTR_ERR(sa_manager);
+ pool = xe_mem_pool_init(tile, bb_pool_size, sizeof(u32),
+ XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY);
+ if (IS_ERR(pool)) {
+ xe_sriov_err(xe, "xe_mem_pool_init failed with error: %pe\n",
+ pool);
+ err = PTR_ERR(pool);
return err;
}
- offset = 0;
- xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP,
- bb_pool_size);
- xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP,
- bb_pool_size);
+ pool_cpu_addr = xe_mem_pool_cpu_addr(pool);
+ memset(pool_cpu_addr, 0, bb_pool_size);
- offset = bb_pool_size - sizeof(u32);
- xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END);
- xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END);
+ last_dw_addr = pool_cpu_addr + (bb_pool_size / sizeof(u32)) - 1;
+ *last_dw_addr = MI_BATCH_BUFFER_END;
- ctx->mem.ccs_bb_pool = sa_manager;
+ /**
+ * Sync the main copy and shadow copy so that the shadow copy is
+ * replica of main copy. We sync only BBs after init part. So, we
+ * need to make sure the main pool and shadow copy are in sync after
+ * this point. This is needed as GuC may read the BB commands from
+ * shadow copy.
+ */
+ xe_mem_pool_sync(pool);
+ ctx->mem.ccs_bb_pool = pool;
return 0;
}
static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx)
{
- u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
+ u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool);
struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
u32 dw[10], i = 0;
@@ -388,7 +392,7 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe)
#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32))
void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
{
- u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
+ u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool);
struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
struct xe_device *xe = gt_to_xe(ctx->mig_q->gt);
@@ -412,8 +416,8 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo)
struct xe_device *xe = xe_bo_device(bo);
enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
struct xe_sriov_vf_ccs_ctx *ctx;
+ struct xe_mem_pool_node *bb;
struct xe_tile *tile;
- struct xe_bb *bb;
int err = 0;
xe_assert(xe, IS_VF_CCS_READY(xe));
@@ -445,7 +449,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo)
{
struct xe_device *xe = xe_bo_device(bo);
enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
- struct xe_bb *bb;
+ struct xe_mem_pool_node *bb;
xe_assert(xe, IS_VF_CCS_READY(xe));
@@ -471,8 +475,8 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo)
*/
void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
{
- struct xe_sa_manager *bb_pool;
enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
+ struct xe_mem_pool *bb_pool;
if (!IS_VF_CCS_READY(xe))
return;
@@ -485,7 +489,7 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read");
drm_printf(p, "-------------------------\n");
- drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool));
+ xe_mem_pool_dump(bb_pool, p);
drm_puts(p, "\n");
}
}
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h
index 22c4999..6fc8f97 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h
@@ -17,9 +17,6 @@ enum xe_sriov_vf_ccs_rw_ctxs {
XE_SRIOV_VF_CCS_CTX_COUNT
};
-struct xe_migrate;
-struct xe_sa_manager;
-
/**
* struct xe_sriov_vf_ccs_ctx - VF CCS migration context data.
*/
@@ -33,7 +30,7 @@ struct xe_sriov_vf_ccs_ctx {
/** @mem: memory data */
struct {
/** @mem.ccs_bb_pool: Pool from which batch buffers are allocated. */
- struct xe_sa_manager *ccs_bb_pool;
+ struct xe_mem_pool *ccs_bb_pool;
} mem;
};
diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h
index 33932fd..0048100 100644
--- a/drivers/gpu/drm/xe/xe_tile_types.h
+++ b/drivers/gpu/drm/xe/xe_tile_types.h
@@ -106,8 +106,6 @@ struct xe_tile {
struct xe_lmtt lmtt;
} pf;
struct {
- /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */
- struct xe_ggtt_node *ggtt_balloon[2];
/** @sriov.vf.self_config: VF configuration data */
struct xe_tile_sriov_vf_selfconfig self_config;
} vf;
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index f8de6a4..fcb6698 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -97,7 +97,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
{ XE_RTP_NAME("Tuning: Set STLB Bank Hash Mode to 4KB"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED),
IS_INTEGRATED),
- XE_RTP_ACTIONS(FIELD_SET(XEHP_GAMSTLB_CTRL, BANK_HASH_MODE,
+ XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE,
BANK_HASH_4KB_MODE))
},
};
@@ -129,7 +129,7 @@ static const struct xe_rtp_entry_sr engine_tunings[] = {
static const struct xe_rtp_entry_sr lrc_tunings[] = {
{ XE_RTP_NAME("Tuning: Windower HW Filtering"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, HW_FILTERING))
+ XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, HW_FILTERING))
},
/* DG2 */
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 56e2db5..ab6cc1f 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1120,6 +1120,25 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
xe_bo_assert_held(bo);
+ /*
+ * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This
+ * gates new vm_bind ioctls (user supplies WILLNEED) while
+ * still allowing partial-unbind / remap splits whose new VMAs
+ * inherit the parent's DONTNEED attr. It must also run before
+ * xe_bo_willneed_get_locked() below so a 0->1 holder bump
+ * cannot silently promote DONTNEED back to WILLNEED.
+ */
+ if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) {
+ if (xe_bo_madv_is_dontneed(bo)) {
+ xe_vma_free(vma);
+ return ERR_PTR(-EBUSY);
+ }
+ if (xe_bo_is_purged(bo)) {
+ xe_vma_free(vma);
+ return ERR_PTR(-EINVAL);
+ }
+ }
+
vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base);
if (IS_ERR(vm_bo)) {
xe_vma_free(vma);
@@ -1131,6 +1150,10 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
vma->gpuva.gem.offset = bo_offset_or_userptr;
drm_gpuva_link(&vma->gpuva, vm_bo);
drm_gpuvm_bo_put(vm_bo);
+
+ xe_bo_vma_count_inc_locked(bo);
+ if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)
+ xe_bo_willneed_get_locked(bo);
} else /* userptr or null */ {
if (!is_null && !is_cpu_addr_mirror) {
struct xe_userptr_vma *uvma = to_userptr_vma(vma);
@@ -1208,7 +1231,10 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
xe_bo_assert_held(bo);
drm_gpuva_unlink(&vma->gpuva);
- xe_bo_recompute_purgeable_state(bo);
+
+ xe_bo_vma_count_dec_locked(bo);
+ if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)
+ xe_bo_willneed_put_locked(bo);
}
xe_vm_assert_held(vm);
@@ -3016,7 +3042,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
* @res_evict: Allow evicting resources during validation
* @validate: Perform BO validation
* @request_decompress: Request BO decompression
- * @check_purged: Reject operation if BO is purged
+ * @check_purged: Reject operation if BO is DONTNEED or PURGED
*/
struct xe_vma_lock_and_validate_flags {
u32 res_evict : 1;
@@ -3030,6 +3056,7 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma,
{
struct xe_bo *bo = xe_vma_bo(vma);
struct xe_vm *vm = xe_vma_vm(vma);
+ bool validate_bo = flags.validate;
int err = 0;
if (bo) {
@@ -3044,7 +3071,11 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma,
err = -EINVAL; /* BO already purged */
}
- if (!err && flags.validate)
+ /* Don't validate the BO for DONTNEED/PURGED remap remnants. */
+ if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED)
+ validate_bo = false;
+
+ if (!err && validate_bo)
err = xe_bo_validate(bo, vm,
xe_vm_allow_vm_eviction(vm) &&
flags.res_evict, exec);
@@ -3152,7 +3183,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
op->map.immediate,
.request_decompress =
op->map.request_decompress,
- .check_purged = true,
+ .check_purged = false,
});
break;
case DRM_GPUVA_OP_REMAP:
@@ -3174,7 +3205,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
.res_evict = res_evict,
.validate = true,
.request_decompress = false,
- .check_purged = true,
+ .check_purged = false,
});
if (!err && op->remap.next)
err = vma_lock_and_validate(exec, op->remap.next,
@@ -3182,7 +3213,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
.res_evict = res_evict,
.validate = true,
.request_decompress = false,
- .check_purged = true,
+ .check_purged = false,
});
break;
case DRM_GPUVA_OP_UNMAP:
@@ -3211,9 +3242,11 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
}
/*
- * Prefetch attempts to migrate BO's backing store without
- * repopulating it first. Purged BOs have no backing store
- * to migrate, so reject the operation.
+ * PREFETCH is the only op that still gates on BO purge state.
+ * MAP/REMAP handle this inside xe_vma_create() so partial
+ * unbind on a DONTNEED BO still works. PREFETCH skips
+ * xe_vma_create() and would migrate a BO with no backing
+ * store, so reject DONTNEED/PURGED here.
*/
err = vma_lock_and_validate(exec,
gpuva_to_vma(op->base.prefetch.va),
@@ -3658,6 +3691,8 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
+ XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE &&
+ is_cpu_addr_mirror) ||
XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) &&
(op == DRM_XE_VM_BIND_OP_MAP_USERPTR ||
is_cpu_addr_mirror) &&
@@ -4156,7 +4191,8 @@ int xe_vm_get_property_ioctl(struct drm_device *drm, void *data,
int ret = 0;
if (XE_IOCTL_DBG(xe, (args->reserved[0] || args->reserved[1] ||
- args->reserved[2])))
+ args->reserved[2] || args->extensions ||
+ args->pad)))
return -EINVAL;
vm = xe_vm_lookup(xef, args->vm_id);
diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c
index 66f00d3..c4fb290 100644
--- a/drivers/gpu/drm/xe/xe_vm_madvise.c
+++ b/drivers/gpu/drm/xe/xe_vm_madvise.c
@@ -186,147 +186,6 @@ static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm,
}
/**
- * xe_bo_is_dmabuf_shared() - Check if BO is shared via dma-buf
- * @bo: Buffer object
- *
- * Prevent marking imported or exported dma-bufs as purgeable.
- * For imported BOs, Xe doesn't own the backing store and cannot
- * safely reclaim pages (exporter or other devices may still be
- * using them). For exported BOs, external devices may have active
- * mappings we cannot track.
- *
- * Return: true if BO is imported or exported, false otherwise
- */
-static bool xe_bo_is_dmabuf_shared(struct xe_bo *bo)
-{
- struct drm_gem_object *obj = &bo->ttm.base;
-
- /* Imported: exporter owns backing store */
- if (drm_gem_is_imported(obj))
- return true;
-
- /* Exported: external devices may be accessing */
- if (obj->dma_buf)
- return true;
-
- return false;
-}
-
-/**
- * enum xe_bo_vmas_purge_state - VMA purgeable state aggregation
- *
- * Distinguishes whether a BO's VMAs are all DONTNEED, have at least
- * one WILLNEED, or have no VMAs at all.
- *
- * Enum values align with XE_MADV_PURGEABLE_* states for consistency.
- */
-enum xe_bo_vmas_purge_state {
- /** @XE_BO_VMAS_STATE_WILLNEED: At least one VMA is WILLNEED */
- XE_BO_VMAS_STATE_WILLNEED = 0,
- /** @XE_BO_VMAS_STATE_DONTNEED: All VMAs are DONTNEED */
- XE_BO_VMAS_STATE_DONTNEED = 1,
- /** @XE_BO_VMAS_STATE_NO_VMAS: BO has no VMAs */
- XE_BO_VMAS_STATE_NO_VMAS = 2,
-};
-
-/*
- * xe_bo_recompute_purgeable_state() casts between xe_bo_vmas_purge_state and
- * xe_madv_purgeable_state. Enforce that WILLNEED=0 and DONTNEED=1 match across
- * both enums so the single-line cast is always valid.
- */
-static_assert(XE_BO_VMAS_STATE_WILLNEED == (int)XE_MADV_PURGEABLE_WILLNEED,
- "VMA purge state WILLNEED must equal madv purgeable WILLNEED");
-static_assert(XE_BO_VMAS_STATE_DONTNEED == (int)XE_MADV_PURGEABLE_DONTNEED,
- "VMA purge state DONTNEED must equal madv purgeable DONTNEED");
-
-/**
- * xe_bo_all_vmas_dontneed() - Determine BO VMA purgeable state
- * @bo: Buffer object
- *
- * Check all VMAs across all VMs to determine aggregate purgeable state.
- * Shared BOs require unanimous DONTNEED state from all mappings.
- *
- * Caller must hold BO dma-resv lock.
- *
- * Return: XE_BO_VMAS_STATE_DONTNEED if all VMAs are DONTNEED,
- * XE_BO_VMAS_STATE_WILLNEED if at least one VMA is not DONTNEED,
- * XE_BO_VMAS_STATE_NO_VMAS if BO has no VMAs
- */
-static enum xe_bo_vmas_purge_state xe_bo_all_vmas_dontneed(struct xe_bo *bo)
-{
- struct drm_gpuvm_bo *vm_bo;
- struct drm_gpuva *gpuva;
- struct drm_gem_object *obj = &bo->ttm.base;
- bool has_vmas = false;
-
- xe_bo_assert_held(bo);
-
- /* Shared dma-bufs cannot be purgeable */
- if (xe_bo_is_dmabuf_shared(bo))
- return XE_BO_VMAS_STATE_WILLNEED;
-
- drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
- drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
- struct xe_vma *vma = gpuva_to_vma(gpuva);
-
- has_vmas = true;
-
- /* Any non-DONTNEED VMA prevents purging */
- if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_DONTNEED)
- return XE_BO_VMAS_STATE_WILLNEED;
- }
- }
-
- /*
- * No VMAs => preserve existing BO purgeable state.
- * Avoids incorrectly flipping DONTNEED -> WILLNEED when last VMA unmapped.
- */
- if (!has_vmas)
- return XE_BO_VMAS_STATE_NO_VMAS;
-
- return XE_BO_VMAS_STATE_DONTNEED;
-}
-
-/**
- * xe_bo_recompute_purgeable_state() - Recompute BO purgeable state from VMAs
- * @bo: Buffer object
- *
- * Walk all VMAs to determine if BO should be purgeable or not.
- * Shared BOs require unanimous DONTNEED state from all mappings.
- * If the BO has no VMAs the existing state is preserved.
- *
- * Locking: Caller must hold BO dma-resv lock. When iterating GPUVM lists,
- * VM lock must also be held (write) to prevent concurrent VMA modifications.
- * This is satisfied at both call sites:
- * - xe_vma_destroy(): holds vm->lock write
- * - madvise_purgeable(): holds vm->lock write (from madvise ioctl path)
- *
- * Return: nothing
- */
-void xe_bo_recompute_purgeable_state(struct xe_bo *bo)
-{
- enum xe_bo_vmas_purge_state vma_state;
-
- if (!bo)
- return;
-
- xe_bo_assert_held(bo);
-
- /*
- * Once purged, always purged. Cannot transition back to WILLNEED.
- * This matches i915 semantics where purged BOs are permanently invalid.
- */
- if (bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED)
- return;
-
- vma_state = xe_bo_all_vmas_dontneed(bo);
-
- if (vma_state != (enum xe_bo_vmas_purge_state)bo->madv_purgeable &&
- vma_state != XE_BO_VMAS_STATE_NO_VMAS)
- xe_bo_set_purgeable_state(bo, (enum xe_madv_purgeable_state)vma_state);
-}
-
-/**
* madvise_purgeable - Handle purgeable buffer object advice
* @xe: XE device
* @vm: VM
@@ -359,12 +218,6 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm,
/* BO must be locked before modifying madv state */
xe_bo_assert_held(bo);
- /* Skip shared dma-bufs - no PTEs to zap */
- if (xe_bo_is_dmabuf_shared(bo)) {
- vmas[i]->skip_invalidation = true;
- continue;
- }
-
/*
* Once purged, always purged. Cannot transition back to WILLNEED.
* This matches i915 semantics where purged BOs are permanently invalid.
@@ -377,13 +230,14 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm,
switch (op->purge_state_val.val) {
case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED:
- vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED;
vmas[i]->skip_invalidation = true;
-
- xe_bo_recompute_purgeable_state(bo);
+ /* Only act on a real DONTNEED -> WILLNEED transition. */
+ if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) {
+ vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED;
+ xe_bo_willneed_get_locked(bo);
+ }
break;
case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED:
- vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED;
/*
* Don't zap PTEs at DONTNEED time -- pages are still
* alive. The zap happens in xe_bo_move_notify() right
@@ -391,7 +245,11 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm,
*/
vmas[i]->skip_invalidation = true;
- xe_bo_recompute_purgeable_state(bo);
+ /* Only act on a real WILLNEED -> DONTNEED transition. */
+ if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) {
+ vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED;
+ xe_bo_willneed_put_locked(bo);
+ }
break;
default:
/* Should never hit - values validated in madvise_args_are_sane() */
@@ -621,6 +479,45 @@ static int xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details
return 0;
}
+static bool check_pat_args_are_sane(struct xe_device *xe,
+ struct xe_vmas_in_madvise_range *madvise_range,
+ u16 pat_index)
+{
+ u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
+ int i;
+
+ /*
+ * Using coh_none with CPU cached buffers is not allowed on iGPU.
+ * On iGPU the GPU shares the LLC with the CPU, so with coh_none
+ * the GPU bypasses CPU caches and reads directly from DRAM,
+ * potentially seeing stale sensitive data from previously freed
+ * pages. On dGPU this restriction does not apply, because the
+ * platform does not provide a non-coherent system memory access
+ * path that would violate the DMA coherency contract.
+ */
+ if (coh_mode != XE_COH_NONE || IS_DGFX(xe))
+ return true;
+
+ for (i = 0; i < madvise_range->num_vmas; i++) {
+ struct xe_vma *vma = madvise_range->vmas[i];
+ struct xe_bo *bo = xe_vma_bo(vma);
+
+ if (bo) {
+ /* BO with WB caching + COH_NONE is not allowed */
+ if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
+ return false;
+ /* Imported dma-buf without caching info, assume cached */
+ if (XE_IOCTL_DBG(xe, !bo->cpu_caching))
+ return false;
+ } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) ||
+ xe_vma_is_userptr(vma)))
+ /* System memory (userptr/SVM) is always CPU cached */
+ return false;
+ }
+
+ return true;
+}
+
static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas,
int num_vmas, u32 atomic_val)
{
@@ -750,6 +647,14 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
}
}
+ if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
+ if (!check_pat_args_are_sane(xe, &madvise_range,
+ args->pat_index.val)) {
+ err = -EINVAL;
+ goto free_vmas;
+ }
+ }
+
if (madvise_range.has_bo_vmas) {
if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) {
if (!check_bo_args_are_sane(vm, madvise_range.vmas,
diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h
index 39acd26..a3078f6 100644
--- a/drivers/gpu/drm/xe/xe_vm_madvise.h
+++ b/drivers/gpu/drm/xe/xe_vm_madvise.h
@@ -13,6 +13,4 @@ struct xe_bo;
int xe_vm_madvise_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
-void xe_bo_recompute_purgeable_state(struct xe_bo *bo);
-
#endif
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index 546296f..33df43d 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -651,7 +651,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
},
{ XE_RTP_NAME("18033852989"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
+ XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
},
{ XE_RTP_NAME("15016589081"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
@@ -743,14 +743,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
},
- { XE_RTP_NAME("14019988906"),
- XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
- },
- { XE_RTP_NAME("14019877138"),
- XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
- },
{ XE_RTP_NAME("14021490052"),
XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
XE_RTP_ACTIONS(SET(FF_MODE,
@@ -762,7 +754,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
},
{ XE_RTP_NAME("22021007897"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
+ XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
},
/* Xe3_LPG */
@@ -778,7 +770,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
},
{ XE_RTP_NAME("22021007897"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
- XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
+ XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
},
{ XE_RTP_NAME("14024681466"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c
index 50c7b45..d013065 100644
--- a/drivers/hid/bpf/hid_bpf_dispatch.c
+++ b/drivers/hid/bpf/hid_bpf_dispatch.c
@@ -24,7 +24,8 @@ EXPORT_SYMBOL(hid_ops);
u8 *
dispatch_hid_bpf_device_event(struct hid_device *hdev, enum hid_report_type type, u8 *data,
- u32 *size, int interrupt, u64 source, bool from_bpf)
+ size_t *buf_size, u32 *size, int interrupt, u64 source,
+ bool from_bpf)
{
struct hid_bpf_ctx_kern ctx_kern = {
.ctx = {
@@ -74,6 +75,7 @@ dispatch_hid_bpf_device_event(struct hid_device *hdev, enum hid_report_type type
*size = ret;
}
+ *buf_size = ctx_kern.ctx.allocated_size;
return ctx_kern.data;
}
EXPORT_SYMBOL_GPL(dispatch_hid_bpf_device_event);
@@ -505,7 +507,7 @@ __hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum hid_report_type type, u8 *b
if (ret)
return ret;
- return hid_ops->hid_input_report(ctx->hid, type, buf, size, 0, (u64)(long)ctx, true,
+ return hid_ops->hid_input_report(ctx->hid, type, buf, size, size, 0, (u64)(long)ctx, true,
lock_already_taken);
}
diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c
index 0fdc096..462010a 100644
--- a/drivers/hid/hid-appletb-kbd.c
+++ b/drivers/hid/hid-appletb-kbd.c
@@ -17,7 +17,7 @@
#include <linux/module.h>
#include <linux/string.h>
#include <linux/backlight.h>
-#include <linux/timer.h>
+#include <linux/workqueue.h>
#include <linux/input/sparse-keymap.h>
#include "hid-ids.h"
@@ -62,7 +62,8 @@ struct appletb_kbd {
struct input_handle kbd_handle;
struct input_handle tpd_handle;
struct backlight_device *backlight_dev;
- struct timer_list inactivity_timer;
+ struct delayed_work inactivity_work;
+ struct work_struct restore_brightness_work;
bool has_dimmed;
bool has_turned_off;
u8 saved_mode;
@@ -164,16 +165,18 @@ static int appletb_tb_key_to_slot(unsigned int code)
}
}
-static void appletb_inactivity_timer(struct timer_list *t)
+static void appletb_inactivity_work(struct work_struct *work)
{
- struct appletb_kbd *kbd = timer_container_of(kbd, t, inactivity_timer);
+ struct appletb_kbd *kbd = container_of(to_delayed_work(work),
+ struct appletb_kbd,
+ inactivity_work);
if (kbd->backlight_dev && appletb_tb_autodim) {
if (!kbd->has_dimmed) {
backlight_device_set_brightness(kbd->backlight_dev, 1);
kbd->has_dimmed = true;
- mod_timer(&kbd->inactivity_timer,
- jiffies + secs_to_jiffies(appletb_tb_idle_timeout));
+ mod_delayed_work(system_wq, &kbd->inactivity_work,
+ secs_to_jiffies(appletb_tb_idle_timeout));
} else if (!kbd->has_turned_off) {
backlight_device_set_brightness(kbd->backlight_dev, 0);
kbd->has_turned_off = true;
@@ -181,16 +184,25 @@ static void appletb_inactivity_timer(struct timer_list *t)
}
}
+static void appletb_restore_brightness_work(struct work_struct *work)
+{
+ struct appletb_kbd *kbd = container_of(work, struct appletb_kbd,
+ restore_brightness_work);
+
+ if (kbd->backlight_dev)
+ backlight_device_set_brightness(kbd->backlight_dev, 2);
+}
+
static void reset_inactivity_timer(struct appletb_kbd *kbd)
{
if (kbd->backlight_dev && appletb_tb_autodim) {
if (kbd->has_dimmed || kbd->has_turned_off) {
- backlight_device_set_brightness(kbd->backlight_dev, 2);
kbd->has_dimmed = false;
kbd->has_turned_off = false;
+ schedule_work(&kbd->restore_brightness_work);
}
- mod_timer(&kbd->inactivity_timer,
- jiffies + secs_to_jiffies(appletb_tb_dim_timeout));
+ mod_delayed_work(system_wq, &kbd->inactivity_work,
+ secs_to_jiffies(appletb_tb_dim_timeout));
}
}
@@ -408,9 +420,11 @@ static int appletb_kbd_probe(struct hid_device *hdev, const struct hid_device_id
dev_err_probe(dev, -ENODEV, "Failed to get backlight device\n");
} else {
backlight_device_set_brightness(kbd->backlight_dev, 2);
- timer_setup(&kbd->inactivity_timer, appletb_inactivity_timer, 0);
- mod_timer(&kbd->inactivity_timer,
- jiffies + secs_to_jiffies(appletb_tb_dim_timeout));
+ INIT_DELAYED_WORK(&kbd->inactivity_work, appletb_inactivity_work);
+ INIT_WORK(&kbd->restore_brightness_work,
+ appletb_restore_brightness_work);
+ mod_delayed_work(system_wq, &kbd->inactivity_work,
+ secs_to_jiffies(appletb_tb_dim_timeout));
}
kbd->inp_handler.event = appletb_kbd_inp_event;
@@ -440,13 +454,14 @@ static int appletb_kbd_probe(struct hid_device *hdev, const struct hid_device_id
unregister_handler:
input_unregister_handler(&kbd->inp_handler);
close_hw:
- if (kbd->backlight_dev) {
- put_device(&kbd->backlight_dev->dev);
- timer_delete_sync(&kbd->inactivity_timer);
- }
hid_hw_close(hdev);
stop_hw:
hid_hw_stop(hdev);
+ if (kbd->backlight_dev) {
+ cancel_delayed_work_sync(&kbd->inactivity_work);
+ cancel_work_sync(&kbd->restore_brightness_work);
+ put_device(&kbd->backlight_dev->dev);
+ }
return ret;
}
@@ -457,13 +472,14 @@ static void appletb_kbd_remove(struct hid_device *hdev)
appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_OFF);
input_unregister_handler(&kbd->inp_handler);
- if (kbd->backlight_dev) {
- put_device(&kbd->backlight_dev->dev);
- timer_delete_sync(&kbd->inactivity_timer);
- }
-
hid_hw_close(hdev);
hid_hw_stop(hdev);
+
+ if (kbd->backlight_dev) {
+ cancel_delayed_work_sync(&kbd->inactivity_work);
+ cancel_work_sync(&kbd->restore_brightness_work);
+ put_device(&kbd->backlight_dev->dev);
+ }
}
static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg)
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 61afec5..41a79e4 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -2033,24 +2033,32 @@ int __hid_request(struct hid_device *hid, struct hid_report *report,
}
EXPORT_SYMBOL_GPL(__hid_request);
-int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
- int interrupt)
+int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data,
+ size_t bufsize, u32 size, int interrupt)
{
struct hid_report_enum *report_enum = hid->report_enum + type;
struct hid_report *report;
struct hid_driver *hdrv;
int max_buffer_size = HID_MAX_BUFFER_SIZE;
u32 rsize, csize = size;
+ size_t bsize = bufsize;
u8 *cdata = data;
int ret = 0;
report = hid_get_report(report_enum, data);
if (!report)
- goto out;
+ return 0;
+
+ if (unlikely(bsize < csize)) {
+ hid_warn_ratelimited(hid, "Event data for report %d is incorrect (%d vs %zu)\n",
+ report->id, csize, bsize);
+ return -EINVAL;
+ }
if (report_enum->numbered) {
cdata++;
csize--;
+ bsize--;
}
rsize = hid_compute_report_size(report);
@@ -2063,11 +2071,16 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
else if (rsize > max_buffer_size)
rsize = max_buffer_size;
+ if (bsize < rsize) {
+ hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %zu)\n",
+ report->id, rsize, bsize);
+ return -EINVAL;
+ }
+
if (csize < rsize) {
- hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %d)\n",
- report->id, rsize, csize);
- ret = -EINVAL;
- goto out;
+ dbg_hid("report %d is too short, (%d < %d)\n", report->id,
+ csize, rsize);
+ memset(cdata + csize, 0, rsize - csize);
}
if ((hid->claimed & HID_CLAIMED_HIDDEV) && hid->hiddev_report_event)
@@ -2075,7 +2088,7 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
if (hid->claimed & HID_CLAIMED_HIDRAW) {
ret = hidraw_report_event(hid, data, size);
if (ret)
- goto out;
+ return ret;
}
if (hid->claimed != HID_CLAIMED_HIDRAW && report->maxfield) {
@@ -2087,15 +2100,15 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
if (hid->claimed & HID_CLAIMED_INPUT)
hidinput_report_event(hid, report);
-out:
+
return ret;
}
EXPORT_SYMBOL_GPL(hid_report_raw_event);
static int __hid_input_report(struct hid_device *hid, enum hid_report_type type,
- u8 *data, u32 size, int interrupt, u64 source, bool from_bpf,
- bool lock_already_taken)
+ u8 *data, size_t bufsize, u32 size, int interrupt, u64 source,
+ bool from_bpf, bool lock_already_taken)
{
struct hid_report_enum *report_enum;
struct hid_driver *hdrv;
@@ -2120,7 +2133,8 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type,
report_enum = hid->report_enum + type;
hdrv = hid->driver;
- data = dispatch_hid_bpf_device_event(hid, type, data, &size, interrupt, source, from_bpf);
+ data = dispatch_hid_bpf_device_event(hid, type, data, &bufsize, &size, interrupt,
+ source, from_bpf);
if (IS_ERR(data)) {
ret = PTR_ERR(data);
goto unlock;
@@ -2149,7 +2163,7 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type,
goto unlock;
}
- ret = hid_report_raw_event(hid, type, data, size, interrupt);
+ ret = hid_report_raw_event(hid, type, data, bufsize, size, interrupt);
unlock:
if (!lock_already_taken)
@@ -2167,16 +2181,41 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type,
* @interrupt: distinguish between interrupt and control transfers
*
* This is data entry for lower layers.
+ * Legacy, please use hid_safe_input_report() instead.
*/
int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
int interrupt)
{
- return __hid_input_report(hid, type, data, size, interrupt, 0,
+ return __hid_input_report(hid, type, data, size, size, interrupt, 0,
false, /* from_bpf */
false /* lock_already_taken */);
}
EXPORT_SYMBOL_GPL(hid_input_report);
+/**
+ * hid_safe_input_report - report data from lower layer (usb, bt...)
+ *
+ * @hid: hid device
+ * @type: HID report type (HID_*_REPORT)
+ * @data: report contents
+ * @bufsize: allocated size of the data buffer
+ * @size: useful size of data parameter
+ * @interrupt: distinguish between interrupt and control transfers
+ *
+ * This is data entry for lower layers.
+ * Please use this function instead of the non safe version because we provide
+ * here the size of the buffer, allowing hid-core to make smarter decisions
+ * regarding the incoming buffer.
+ */
+int hid_safe_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data,
+ size_t bufsize, u32 size, int interrupt)
+{
+ return __hid_input_report(hid, type, data, bufsize, size, interrupt, 0,
+ false, /* from_bpf */
+ false /* lock_already_taken */);
+}
+EXPORT_SYMBOL_GPL(hid_safe_input_report);
+
bool hid_match_one_id(const struct hid_device *hdev,
const struct hid_device_id *id)
{
diff --git a/drivers/hid/hid-elan.c b/drivers/hid/hid-elan.c
index 76d93fc..0190ad5 100644
--- a/drivers/hid/hid-elan.c
+++ b/drivers/hid/hid-elan.c
@@ -513,6 +513,7 @@ static const struct hid_device_id elan_devices[] = {
{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_HP_X2_10_COVER),
.driver_data = ELAN_HAS_LED },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_TOSHIBA_CLICK_L9W) },
+ { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_SB974D) },
{ }
};
MODULE_DEVICE_TABLE(hid, elan_devices);
diff --git a/drivers/hid/hid-ft260.c b/drivers/hid/hid-ft260.c
index 333341e..70e2eed 100644
--- a/drivers/hid/hid-ft260.c
+++ b/drivers/hid/hid-ft260.c
@@ -1068,10 +1068,22 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report,
struct ft260_device *dev = hid_get_drvdata(hdev);
struct ft260_i2c_input_report *xfer = (void *)data;
+ if (size < offsetof(struct ft260_i2c_input_report, data)) {
+ hid_err(hdev, "short report %d\n", size);
+ return -1;
+ }
+
if (xfer->report >= FT260_I2C_REPORT_MIN &&
xfer->report <= FT260_I2C_REPORT_MAX) {
- ft260_dbg("i2c resp: rep %#02x len %d\n", xfer->report,
- xfer->length);
+ ft260_dbg("i2c resp: rep %#02x len %d size %d\n",
+ xfer->report, xfer->length, size);
+
+ if (xfer->length > size -
+ offsetof(struct ft260_i2c_input_report, data)) {
+ hid_err(hdev, "report %#02x: length %d exceeds HID report size\n",
+ xfer->report, xfer->length);
+ return -1;
+ }
if ((dev->read_buf == NULL) ||
(xfer->length > dev->read_len - dev->read_idx)) {
diff --git a/drivers/hid/hid-gfrm.c b/drivers/hid/hid-gfrm.c
index 699186f..d2a56bf 100644
--- a/drivers/hid/hid-gfrm.c
+++ b/drivers/hid/hid-gfrm.c
@@ -66,7 +66,7 @@ static int gfrm_raw_event(struct hid_device *hdev, struct hid_report *report,
switch (data[1]) {
case GFRM100_SEARCH_KEY_DOWN:
ret = hid_report_raw_event(hdev, HID_INPUT_REPORT, search_key_dn,
- sizeof(search_key_dn), 1);
+ sizeof(search_key_dn), sizeof(search_key_dn), 1);
break;
case GFRM100_SEARCH_KEY_AUDIO_DATA:
@@ -74,7 +74,7 @@ static int gfrm_raw_event(struct hid_device *hdev, struct hid_report *report,
case GFRM100_SEARCH_KEY_UP:
ret = hid_report_raw_event(hdev, HID_INPUT_REPORT, search_key_up,
- sizeof(search_key_up), 1);
+ sizeof(search_key_up), sizeof(search_key_up), 1);
break;
default:
diff --git a/drivers/hid/hid-google-hammer.c b/drivers/hid/hid-google-hammer.c
index 1af477e..c99c3c0 100644
--- a/drivers/hid/hid-google-hammer.c
+++ b/drivers/hid/hid-google-hammer.c
@@ -496,7 +496,7 @@ static int hammer_probe(struct hid_device *hdev,
if (error)
return error;
- error = devm_add_action(&hdev->dev, hammer_stop, hdev);
+ error = devm_add_action_or_reset(&hdev->dev, hammer_stop, hdev);
if (error)
return error;
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 0cf6374..426ff78 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -277,6 +277,9 @@
#define USB_VENDOR_ID_BIGBEN 0x146b
#define USB_DEVICE_ID_BIGBEN_PS3OFMINIPAD 0x0902
+#define I2C_VENDOR_ID_BLTP 0x36b6
+#define I2C_PRODUCT_ID_BLTP7853 0xc001
+
#define USB_VENDOR_ID_BTC 0x046e
#define USB_DEVICE_ID_BTC_EMPREX_REMOTE 0x5578
#define USB_DEVICE_ID_BTC_EMPREX_REMOTE_2 0x5577
@@ -455,6 +458,7 @@
#define USB_DEVICE_ID_EDIFIER_QR30 0xa101 /* EDIFIER Hal0 2.0 SE */
#define USB_VENDOR_ID_ELAN 0x04f3
+#define USB_DEVICE_ID_SB974D 0x0400
#define USB_DEVICE_ID_TOSHIBA_CLICK_L9W 0x0401
#define USB_DEVICE_ID_HP_X2 0x074d
#define USB_DEVICE_ID_HP_X2_10_COVER 0x0755
@@ -1280,6 +1284,7 @@
#define USB_VENDOR_ID_SIGMA_MICRO 0x1c4f
#define USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD 0x0002
+#define USB_DEVICE_ID_SIGMA_MICRO_USB_MOUSE 0x0034
#define USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD2 0x0059
#define USB_VENDOR_ID_SIGMATEL 0x066F
diff --git a/drivers/hid/hid-lenovo-go-s.c b/drivers/hid/hid-lenovo-go-s.c
index 01c7bdd..a72f7f7 100644
--- a/drivers/hid/hid-lenovo-go-s.c
+++ b/drivers/hid/hid-lenovo-go-s.c
@@ -382,11 +382,9 @@ static int get_endpoint_address(struct hid_device *hdev)
struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
struct usb_host_endpoint *ep;
- if (intf) {
- ep = intf->cur_altsetting->endpoint;
- if (ep)
- return ep->desc.bEndpointAddress;
- }
+ ep = intf->cur_altsetting->endpoint;
+ if (ep)
+ return ep->desc.bEndpointAddress;
return -ENODEV;
}
@@ -1369,6 +1367,14 @@ static void cfg_setup(struct work_struct *work)
"Failed to retrieve IMU Manufacturer: %i\n", ret);
return;
}
+
+ ret = mcu_property_out(drvdata.hdev, GET_GAMEPAD_CFG, FEATURE_OS_MODE,
+ NULL, 0);
+ if (ret) {
+ dev_err(&drvdata.hdev->dev,
+ "Failed to retrieve OS Mode: %i\n", ret);
+ return;
+ }
}
static int hid_gos_cfg_probe(struct hid_device *hdev,
@@ -1427,11 +1433,35 @@ static void hid_gos_cfg_remove(struct hid_device *hdev)
hid_set_drvdata(hdev, NULL);
}
+static int hid_gos_cfg_reset_resume(struct hid_device *hdev)
+{
+ u8 os_mode = drvdata.os_mode;
+ int ret;
+
+ ret = mcu_property_out(drvdata.hdev, SET_GAMEPAD_CFG,
+ FEATURE_OS_MODE, &os_mode, 1);
+ if (ret < 0)
+ return ret;
+
+ ret = mcu_property_out(drvdata.hdev, GET_GAMEPAD_CFG,
+ FEATURE_OS_MODE, NULL, 0);
+ if (ret < 0)
+ return ret;
+
+ if (drvdata.os_mode != os_mode)
+ return -ENODEV;
+
+ return 0;
+}
+
static int hid_gos_probe(struct hid_device *hdev,
const struct hid_device_id *id)
{
int ret, ep;
+ if (!hid_is_usb(hdev))
+ return -EINVAL;
+
ret = hid_parse(hdev);
if (ret) {
hid_err(hdev, "Parse failed\n");
@@ -1481,6 +1511,20 @@ static void hid_gos_remove(struct hid_device *hdev)
}
}
+static int hid_gos_reset_resume(struct hid_device *hdev)
+{
+ int ep = get_endpoint_address(hdev);
+
+ switch (ep) {
+ case GO_S_CFG_INTF_IN:
+ return hid_gos_cfg_reset_resume(hdev);
+ default:
+ break;
+ }
+
+ return 0;
+}
+
static const struct hid_device_id hid_gos_devices[] = {
{ HID_USB_DEVICE(USB_VENDOR_ID_QHE,
USB_DEVICE_ID_LENOVO_LEGION_GO_S_XINPUT) },
@@ -1496,6 +1540,7 @@ static struct hid_driver hid_lenovo_go_s = {
.probe = hid_gos_probe,
.remove = hid_gos_remove,
.raw_event = hid_gos_raw_event,
+ .reset_resume = hid_gos_reset_resume,
};
module_hid_driver(hid_lenovo_go_s);
diff --git a/drivers/hid/hid-lenovo-go.c b/drivers/hid/hid-lenovo-go.c
index d4d26c7..e0c9d5e 100644
--- a/drivers/hid/hid-lenovo-go.c
+++ b/drivers/hid/hid-lenovo-go.c
@@ -641,9 +641,6 @@ static int get_endpoint_address(struct hid_device *hdev)
struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
struct usb_host_endpoint *ep;
- if (!intf)
- return -ENODEV;
-
ep = intf->cur_altsetting->endpoint;
if (!ep)
return -ENODEV;
@@ -2419,6 +2416,9 @@ static int hid_go_probe(struct hid_device *hdev, const struct hid_device_id *id)
{
int ret, ep;
+ if (!hid_is_usb(hdev))
+ return -EINVAL;
+
hdev->quirks |= HID_QUIRK_INPUT_PER_APP | HID_QUIRK_MULTI_INPUT;
ret = hid_parse(hdev);
diff --git a/drivers/hid/hid-lenovo.c b/drivers/hid/hid-lenovo.c
index a6b73e0..c11957a 100644
--- a/drivers/hid/hid-lenovo.c
+++ b/drivers/hid/hid-lenovo.c
@@ -30,6 +30,7 @@
#include <linux/hid.h>
#include <linux/input.h>
#include <linux/leds.h>
+#include <linux/unaligned.h>
#include <linux/workqueue.h>
#include "hid-ids.h"
@@ -793,8 +794,8 @@ static int lenovo_raw_event(struct hid_device *hdev,
*/
if (unlikely((hdev->product == USB_DEVICE_ID_LENOVO_X12_TAB
|| hdev->product == USB_DEVICE_ID_LENOVO_X12_TAB2)
- && size >= 3 && report->id == 0x03))
- return lenovo_raw_event_TP_X12_tab(hdev, le32_to_cpu(*(__le32 *)data));
+ && size >= 4 && report->id == 0x03))
+ return lenovo_raw_event_TP_X12_tab(hdev, get_unaligned_le32(data));
return 0;
}
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index b1330d2..ccbf288 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -3673,7 +3673,7 @@ static int hidpp10_consumer_keys_raw_event(struct hidpp_device *hidpp,
memcpy(&consumer_report[1], &data[3], 4);
/* We are called from atomic context */
hid_report_raw_event(hidpp->hid_dev, HID_INPUT_REPORT,
- consumer_report, 5, 1);
+ consumer_report, sizeof(consumer_report), 5, 1);
return 1;
}
@@ -4685,6 +4685,44 @@ static const struct hid_device_id hidpp_devices[] = {
HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb391) },
{ /* MX Master 4 mouse over Bluetooth */
HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb042) },
+ { /* Logitech Signature K650 over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb36f) },
+ { /* Logitech Signature K650 B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb370) },
+ { /* Logitech Pebble Keys 2 K380S over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb377) },
+ { /* Logitech Casa Pop-Up Desk over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb371) },
+ { /* Logitech Casa Pop-Up Desk B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb374) },
+ { /* Logitech Wave Keys over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb383) },
+ { /* Logitech Wave Keys B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb384) },
+ { /* Logitech Signature Slim K950 over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb386) },
+ { /* Logitech Signature Slim K950 B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb388) },
+ { /* Logitech MX Keys S over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb378) },
+ { /* Logitech MX Keys S B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb380) },
+ { /* Logitech Keys-To-Go 2 over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb38c) },
+ { /* Logitech Pop Icon Keys over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb38f) },
+ { /* Logitech MX Keys Mini over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb369) },
+ { /* Logitech MX Keys Mini B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb36e) },
+ { /* Logitech Signature Slim Solar+ K980 B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb394) },
+ { /* Logitech Bluetooth Keyboard K250/K251 over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb397) },
+ { /* Logitech Signature Comfort K880 over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb39c) },
+ { /* Logitech Signature Comfort K880 B2B over Bluetooth */
+ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb39d) },
{}
};
diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c
index e70bd3d..802a347 100644
--- a/drivers/hid/hid-magicmouse.c
+++ b/drivers/hid/hid-magicmouse.c
@@ -390,6 +390,10 @@ static int magicmouse_raw_event(struct hid_device *hdev,
struct input_dev *input = msc->input;
int x = 0, y = 0, ii, clicks = 0, npoints;
+ /* Protect against zero sized recursive calls from DOUBLE_REPORT_ID */
+ if (size < 1)
+ return 0;
+
switch (data[0]) {
case TRACKPAD_REPORT_ID:
case TRACKPAD2_BT_REPORT_ID:
@@ -490,6 +494,18 @@ static int magicmouse_raw_event(struct hid_device *hdev,
/* Sometimes the trackpad sends two touch reports in one
* packet.
*/
+
+ /* Ensure that we have at least 2 elements (report type and size) */
+ if (size < 2)
+ return 0;
+
+ if (size < data[1] + 2) {
+ hid_warn(hdev,
+ "received report length (%d) was smaller than specified (%d)",
+ size, data[1] + 2);
+ return 0;
+ }
+
magicmouse_raw_event(hdev, report, data + 2, data[1]);
magicmouse_raw_event(hdev, report, data + 2 + data[1],
size - 2 - data[1]);
diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c
index be80970a..e4ddd8e 100644
--- a/drivers/hid/hid-mcp2221.c
+++ b/drivers/hid/hid-mcp2221.c
@@ -128,6 +128,7 @@ struct mcp2221 {
u8 *rxbuf;
u8 txbuf[64];
int rxbuf_idx;
+ int rxbuf_size;
int status;
u8 cur_i2c_clk_div;
struct gpio_chip *gc;
@@ -330,12 +331,14 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp,
mcp->txbuf[3] = (u8)(msg->addr << 1);
total_len = msg->len;
mcp->rxbuf = msg->buf;
+ mcp->rxbuf_size = msg->len;
} else {
mcp->txbuf[1] = smbus_len;
mcp->txbuf[2] = 0;
mcp->txbuf[3] = (u8)(smbus_addr << 1);
total_len = smbus_len;
mcp->rxbuf = smbus_buf;
+ mcp->rxbuf_size = smbus_len;
}
ret = mcp_send_data_req_status(mcp, mcp->txbuf, 4);
@@ -919,6 +922,10 @@ static int mcp2221_raw_event(struct hid_device *hdev,
mcp->status = -EINVAL;
break;
}
+ if (mcp->rxbuf_idx + data[3] > mcp->rxbuf_size) {
+ mcp->status = -EINVAL;
+ break;
+ }
buf = mcp->rxbuf;
memcpy(&buf[mcp->rxbuf_idx], &data[4], data[3]);
mcp->rxbuf_idx = mcp->rxbuf_idx + data[3];
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index e82a3c4..eeab0b6 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -533,7 +533,7 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report)
}
ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf,
- size, 0);
+ size, size, 0);
if (ret)
dev_warn(&hdev->dev, "failed to report feature\n");
}
diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c
index c43caac..e485373 100644
--- a/drivers/hid/hid-playstation.c
+++ b/drivers/hid/hid-playstation.c
@@ -2384,7 +2384,8 @@ static int dualshock4_parse_report(struct ps_device *ps_dev, struct hid_report *
}
ds4_report = &usb->common;
- num_touch_reports = usb->num_touch_reports;
+ num_touch_reports = min_t(u8, usb->num_touch_reports,
+ ARRAY_SIZE(usb->touch_reports));
touch_reports = usb->touch_reports;
} else if (hdev->bus == BUS_BLUETOOTH && report->id == DS4_INPUT_REPORT_BT &&
size == DS4_INPUT_REPORT_BT_SIZE) {
@@ -2404,7 +2405,8 @@ static int dualshock4_parse_report(struct ps_device *ps_dev, struct hid_report *
}
ds4_report = &bt->common;
- num_touch_reports = bt->num_touch_reports;
+ num_touch_reports = min_t(u8, bt->num_touch_reports,
+ ARRAY_SIZE(bt->touch_reports));
touch_reports = bt->touch_reports;
} else if (hdev->bus == BUS_BLUETOOTH &&
report->id == DS4_INPUT_REPORT_BT_MINIMAL &&
diff --git a/drivers/hid/hid-primax.c b/drivers/hid/hid-primax.c
index e44d79d..8db0542 100644
--- a/drivers/hid/hid-primax.c
+++ b/drivers/hid/hid-primax.c
@@ -44,7 +44,7 @@ static int px_raw_event(struct hid_device *hid, struct hid_report *report,
data[0] |= (1 << (data[idx] - 0xE0));
data[idx] = 0;
}
- hid_report_raw_event(hid, HID_INPUT_REPORT, data, size, 0);
+ hid_report_raw_event(hid, HID_INPUT_REPORT, data, size, size, 0);
return 1;
default: /* unknown report */
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index 9e88c9d..57d8efd 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -187,6 +187,7 @@ static const struct hid_device_id hid_quirks[] = {
{ HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD), HID_QUIRK_NO_INIT_REPORTS },
{ HID_USB_DEVICE(USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB), HID_QUIRK_NOGET },
{ HID_USB_DEVICE(USB_VENDOR_ID_SIGMA_MICRO, USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD), HID_QUIRK_NO_INIT_REPORTS },
+ { HID_USB_DEVICE(USB_VENDOR_ID_SIGMA_MICRO, USB_DEVICE_ID_SIGMA_MICRO_USB_MOUSE), HID_QUIRK_ALWAYS_POLL },
{ HID_USB_DEVICE(USB_VENDOR_ID_SIGMATEL, USB_DEVICE_ID_SIGMATEL_STMP3780), HID_QUIRK_NOGET },
{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS1030_TOUCH), HID_QUIRK_NOGET },
{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS817_TOUCH), HID_QUIRK_NOGET },
@@ -235,7 +236,7 @@ static const struct hid_device_id hid_quirks[] = {
* used as a driver. See hid_scan_report().
*/
static const struct hid_device_id hid_have_special_driver[] = {
-#if IS_ENABLED(CONFIG_APPLEDISPLAY)
+#if IS_ENABLED(CONFIG_USB_APPLEDISPLAY)
{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9218) },
{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x9219) },
{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, 0x921c) },
diff --git a/drivers/hid/hid-sjoy.c b/drivers/hid/hid-sjoy.c
index bab93d7..963c451 100644
--- a/drivers/hid/hid-sjoy.c
+++ b/drivers/hid/hid-sjoy.c
@@ -91,17 +91,17 @@ static int sjoyff_init(struct hid_device *hid)
set_bit(FF_RUMBLE, dev->ffbit);
- error = input_ff_create_memless(dev, sjoyff, hid_sjoyff_play);
- if (error) {
- kfree(sjoyff);
- return error;
- }
-
sjoyff->report = report;
sjoyff->report->field[0]->value[0] = 0x01;
sjoyff->report->field[0]->value[1] = 0x00;
sjoyff->report->field[0]->value[2] = 0x00;
hid_hw_request(hid, sjoyff->report, HID_REQ_SET_REPORT);
+
+ error = input_ff_create_memless(dev, sjoyff, hid_sjoyff_play);
+ if (error) {
+ kfree(sjoyff);
+ return error;
+ }
}
hid_info(hid, "Force feedback for SmartJoy PLUS PS2/USB adapter\n");
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index b5e7246..3153434 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -1169,10 +1169,9 @@ static int sony_raw_event(struct hid_device *hdev, struct hid_report *report,
sixaxis_parse_report(sc, rd, size);
} else if ((sc->quirks & MOTION_CONTROLLER_BT) && rd[0] == 0x01 && size == 49) {
sixaxis_parse_report(sc, rd, size);
- } else if ((sc->quirks & NAVIGATION_CONTROLLER) && rd[0] == 0x01 &&
- size == 49) {
+ } else if ((sc->quirks & NAVIGATION_CONTROLLER) && rd[0] == 0x01 && size == 49) {
sixaxis_parse_report(sc, rd, size);
- } else if ((sc->quirks & NSG_MRXU_REMOTE) && rd[0] == 0x02) {
+ } else if ((sc->quirks & NSG_MRXU_REMOTE) && rd[0] == 0x02 && size >= 12) {
nsg_mrxu_parse_report(sc, rd, size);
return 1;
} else if ((sc->quirks & RB4_GUITAR_PS4_USB) && rd[0] == 0x01 && size == 64) {
@@ -1189,7 +1188,7 @@ static int sony_raw_event(struct hid_device *hdev, struct hid_report *report,
/* Rock Band 3 PS3 Pro instruments set rd[24] to 0xE0 when they're
* sending full reports, and 0x02 when only sending navigation.
*/
- if ((sc->quirks & RB3_PRO_INSTRUMENT) && rd[24] == 0x02) {
+ if ((sc->quirks & RB3_PRO_INSTRUMENT) && size >= 25 && rd[24] == 0x02) {
/* Only attempt to enable full report every 8 seconds */
if (time_after(jiffies, sc->rb3_pro_poke_jiffies)) {
sc->rb3_pro_poke_jiffies = jiffies + secs_to_jiffies(8);
@@ -1640,9 +1639,6 @@ static int sony_leds_init(struct sony_sc *sc)
u8 max_brightness[MAX_LEDS] = { [0 ... (MAX_LEDS - 1)] = 1 };
u8 use_hw_blink[MAX_LEDS] = { 0 };
- if (WARN_ON(!(sc->quirks & SONY_LED_SUPPORT)))
- return -EINVAL;
-
if (sc->quirks & BUZZ_CONTROLLER) {
sc->led_count = 4;
use_color_names = 0;
@@ -2456,11 +2452,10 @@ static void sony_remove(struct hid_device *hdev)
static int sony_suspend(struct hid_device *hdev, pm_message_t message)
{
#ifdef CONFIG_SONY_FF
+ struct sony_sc *sc = hid_get_drvdata(hdev);
/* On suspend stop any running force-feedback events */
- if (SONY_FF_SUPPORT) {
- struct sony_sc *sc = hid_get_drvdata(hdev);
-
+ if (sc->quirks & SONY_FF_SUPPORT) {
sc->left = sc->right = 0;
sony_send_output_report(sc);
}
diff --git a/drivers/hid/hid-u2fzero.c b/drivers/hid/hid-u2fzero.c
index 744a91e..82404b6 100644
--- a/drivers/hid/hid-u2fzero.c
+++ b/drivers/hid/hid-u2fzero.c
@@ -341,29 +341,33 @@ static int u2fzero_probe(struct hid_device *hdev,
if (ret)
return ret;
- u2fzero_fill_in_urb(dev);
+ ret = u2fzero_fill_in_urb(dev);
+ if (ret)
+ goto err_hid_hw_stop;
dev->present = true;
minor = ((struct hidraw *) hdev->hidraw)->minor;
ret = u2fzero_init_led(dev, minor);
- if (ret) {
- hid_hw_stop(hdev);
- return ret;
- }
+ if (ret)
+ goto err_free_urb;
hid_info(hdev, "%s LED initialised\n", hw_configs[dev->hw_revision].name);
ret = u2fzero_init_hwrng(dev, minor);
- if (ret) {
- hid_hw_stop(hdev);
- return ret;
- }
+ if (ret)
+ goto err_free_urb;
hid_info(hdev, "%s RNG initialised\n", hw_configs[dev->hw_revision].name);
return 0;
+
+err_free_urb:
+ usb_free_urb(dev->urb);
+err_hid_hw_stop:
+ hid_hw_stop(hdev);
+ return ret;
}
static void u2fzero_remove(struct hid_device *hdev)
diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c
index bd7f93e..b73f09d 100644
--- a/drivers/hid/hid-uclogic-core.c
+++ b/drivers/hid/hid-uclogic-core.c
@@ -184,7 +184,9 @@ static int uclogic_input_configured(struct hid_device *hdev,
suffix = "System Control";
break;
}
- } else {
+ }
+
+ if (suffix) {
hi->input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL,
"%s %s", hdev->name, suffix);
if (!hi->input->name)
diff --git a/drivers/hid/hid-vivaldi-common.c b/drivers/hid/hid-vivaldi-common.c
index bf73405..b12bb5c 100644
--- a/drivers/hid/hid-vivaldi-common.c
+++ b/drivers/hid/hid-vivaldi-common.c
@@ -85,7 +85,7 @@ void vivaldi_feature_mapping(struct hid_device *hdev,
}
ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, report_data,
- report_len, 0);
+ report_len, report_len, 0);
if (ret) {
dev_warn(&hdev->dev, "failed to report feature %d\n",
field->report->id);
diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c
index 5a183af..3adb163 100644
--- a/drivers/hid/i2c-hid/i2c-hid-core.c
+++ b/drivers/hid/i2c-hid/i2c-hid-core.c
@@ -149,6 +149,8 @@ static const struct i2c_hid_quirks {
I2C_HID_QUIRK_BOGUS_IRQ },
{ I2C_VENDOR_ID_GOODIX, I2C_DEVICE_ID_GOODIX_0D42,
I2C_HID_QUIRK_DELAY_WAKEUP_AFTER_RESUME },
+ { I2C_VENDOR_ID_BLTP, I2C_PRODUCT_ID_BLTP7853,
+ I2C_HID_QUIRK_NO_IRQ_AFTER_RESET },
{ 0, 0 }
};
@@ -574,9 +576,10 @@ static void i2c_hid_get_input(struct i2c_hid *ihid)
if (ihid->hid->group != HID_GROUP_RMI)
pm_wakeup_event(&ihid->client->dev, 0);
- hid_input_report(ihid->hid, HID_INPUT_REPORT,
- ihid->inbuf + sizeof(__le16),
- ret_size - sizeof(__le16), 1);
+ hid_safe_input_report(ihid->hid, HID_INPUT_REPORT,
+ ihid->inbuf + sizeof(__le16),
+ ihid->bufsize - sizeof(__le16),
+ ret_size - sizeof(__le16), 1);
}
return;
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
index 16f780b..cb19057 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
@@ -94,7 +94,7 @@ static int quickspi_get_device_descriptor(struct quickspi_device *qsdev)
dev_err_once(qsdev->dev, "Read DEVICE_DESCRIPTOR failed, ret = %d\n", ret);
dev_err_once(qsdev->dev, "DEVICE_DESCRIPTOR expected len = %u, actual read = %u\n",
input_len, read_len);
- return ret;
+ return ret ?: -EINVAL;
}
input_rep_type = ((struct input_report_body_header *)read_buf)->input_report_type;
@@ -318,7 +318,7 @@ int reset_tic(struct quickspi_device *qsdev)
dev_err_once(qsdev->dev, "Read RESET_RESPONSE body failed, ret = %d\n", ret);
dev_err_once(qsdev->dev, "RESET_RESPONSE body expected len = %u, actual = %u\n",
read_len, actual_read_len);
- return ret;
+ return ret ?: -EINVAL;
}
input_rep_type = FIELD_GET(HIDSPI_IN_REP_BDY_HDR_REP_TYPE, reset_response);
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index fbbfc0f..5af93b9 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -283,9 +283,9 @@ static void hid_irq_in(struct urb *urb)
break;
usbhid_mark_busy(usbhid);
if (!test_bit(HID_RESUME_RUNNING, &usbhid->iofl)) {
- hid_input_report(urb->context, HID_INPUT_REPORT,
- urb->transfer_buffer,
- urb->actual_length, 1);
+ hid_safe_input_report(urb->context, HID_INPUT_REPORT,
+ urb->transfer_buffer, urb->transfer_buffer_length,
+ urb->actual_length, 1);
/*
* autosuspend refused while keys are pressed
* because most keyboards don't wake up when
@@ -482,9 +482,10 @@ static void hid_ctrl(struct urb *urb)
switch (status) {
case 0: /* success */
if (usbhid->ctrl[usbhid->ctrltail].dir == USB_DIR_IN)
- hid_input_report(urb->context,
+ hid_safe_input_report(urb->context,
usbhid->ctrl[usbhid->ctrltail].report->type,
- urb->transfer_buffer, urb->actual_length, 0);
+ urb->transfer_buffer, urb->transfer_buffer_length,
+ urb->actual_length, 0);
break;
case -ESHUTDOWN: /* unplug */
unplug = 1;
diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c
index aee8a44..c45f182d 100644
--- a/drivers/hid/usbhid/hid-pidff.c
+++ b/drivers/hid/usbhid/hid-pidff.c
@@ -11,6 +11,7 @@
#include "hid-pidff.h"
#include <linux/hid.h>
#include <linux/input.h>
+#include <linux/math64.h>
#include <linux/minmax.h>
#include <linux/slab.h>
#include <linux/stringify.h>
@@ -326,8 +327,10 @@ static s32 pidff_clamp(s32 i, struct hid_field *field)
*/
static int pidff_rescale(int i, int max, struct hid_field *field)
{
- return i * (field->logical_maximum - field->logical_minimum) / max +
- field->logical_minimum;
+ /* 64 bits needed for big values during rescale */
+ s64 result = field->logical_maximum - field->logical_minimum;
+
+ return div_s64(result * i, max) + field->logical_minimum;
}
/*
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 0d1c6d9..2220168 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -90,7 +90,7 @@ static void wacom_wac_queue_flush(struct hid_device *hdev,
kfree(buf);
continue;
}
- err = hid_report_raw_event(hdev, HID_INPUT_REPORT, buf, size, false);
+ err = hid_report_raw_event(hdev, HID_INPUT_REPORT, buf, size, size, false);
if (err) {
hid_warn(hdev, "%s: unable to flush event due to error %d\n",
__func__, err);
@@ -334,7 +334,7 @@ static void wacom_feature_mapping(struct hid_device *hdev,
data, n, WAC_CMD_RETRIES);
if (ret == n && features->type == HID_GENERIC) {
ret = hid_report_raw_event(hdev,
- HID_FEATURE_REPORT, data, n, 0);
+ HID_FEATURE_REPORT, data, n, n, 0);
} else if (ret == 2 && features->type != HID_GENERIC) {
features->touch_max = data[1];
} else {
@@ -356,6 +356,7 @@ static void wacom_feature_mapping(struct hid_device *hdev,
hid_data->inputmode = field->report->id;
hid_data->inputmode_index = usage->usage_index;
+ hid_data->inputmode_field_index = field->index;
break;
case HID_UP_DIGITIZER:
@@ -395,7 +396,7 @@ static void wacom_feature_mapping(struct hid_device *hdev,
data, n, WAC_CMD_RETRIES);
if (ret == n) {
ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT,
- data, n, 0);
+ data, n, n, 0);
} else {
hid_warn(hdev, "%s: could not retrieve sensor offsets\n",
__func__);
@@ -571,9 +572,14 @@ static int wacom_hid_set_device_mode(struct hid_device *hdev)
re = &(hdev->report_enum[HID_FEATURE_REPORT]);
r = re->report_id_hash[hid_data->inputmode];
- if (r) {
- r->field[0]->value[hid_data->inputmode_index] = 2;
- hid_hw_request(hdev, r, HID_REQ_SET_REPORT);
+ if (r && hid_data->inputmode_field_index >= 0 &&
+ hid_data->inputmode_field_index < r->maxfield) {
+ struct hid_field *field = r->field[hid_data->inputmode_field_index];
+
+ if (field && hid_data->inputmode_index < field->report_count) {
+ field->value[hid_data->inputmode_index] = 2;
+ hid_hw_request(hdev, r, HID_REQ_SET_REPORT);
+ }
}
return 0;
}
@@ -2846,6 +2852,7 @@ static int wacom_probe(struct hid_device *hdev,
return -ENODEV;
wacom_wac->hid_data.inputmode = -1;
+ wacom_wac->hid_data.inputmode_field_index = -1;
wacom_wac->mode_report = -1;
if (hid_is_usb(hdev)) {
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index d4f7d8c..126bec6 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -295,6 +295,7 @@ struct wacom_shared {
struct hid_data {
__s16 inputmode; /* InputMode HID feature, -1 if non-existent */
__s16 inputmode_index; /* InputMode HID feature index in the report */
+ __s16 inputmode_field_index; /* InputMode HID feature field index in the report */
bool sense_state;
bool inrange_state;
bool eraser;
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 4788996..982ee2c6 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -201,7 +201,6 @@
obj-$(CONFIG_SENSORS_QNAP_MCU_HWMON) += qnap-mcu-hwmon.o
obj-$(CONFIG_SENSORS_RASPBERRYPI_HWMON) += raspberrypi-hwmon.o
obj-$(CONFIG_SENSORS_SBTSI) += sbtsi_temp.o
-obj-$(CONFIG_SENSORS_SBRMI) += sbrmi.o
obj-$(CONFIG_SENSORS_SCH56XX_COMMON)+= sch56xx-common.o
obj-$(CONFIG_SENSORS_SCH5627) += sch5627.o
obj-$(CONFIG_SENSORS_SCH5636) += sch5636.o
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index be7f702..0c9b9f4 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -884,10 +884,14 @@ static void acpi_power_meter_notify(acpi_handle handle, u32 event, void *data)
static int acpi_power_meter_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct acpi_power_meter_resource *resource;
+ struct acpi_device *device;
int res;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
resource = kzalloc_obj(*resource);
if (!resource)
return -ENOMEM;
diff --git a/drivers/hwmon/ads7871.c b/drivers/hwmon/ads7871.c
index 9bfdf9e..9ee3ce0 100644
--- a/drivers/hwmon/ads7871.c
+++ b/drivers/hwmon/ads7871.c
@@ -77,9 +77,13 @@ static int ads7871_read_reg8(struct spi_device *spi, int reg)
static int ads7871_read_reg16(struct spi_device *spi, int reg)
{
int ret;
+
reg = reg | INST_READ_BM | INST_16BIT_BM;
ret = spi_w8r16(spi, reg);
- return ret;
+ if (ret < 0)
+ return ret;
+
+ return le16_to_cpu((__force __le16)ret);
}
static int ads7871_write_reg8(struct spi_device *spi, int reg, u8 val)
diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c
index 5688ff5..109318b 100644
--- a/drivers/hwmon/asus_atk0110.c
+++ b/drivers/hwmon/asus_atk0110.c
@@ -1273,15 +1273,20 @@ static int atk_probe(struct platform_device *pdev)
struct acpi_buffer buf;
union acpi_object *obj;
struct atk_data *data;
+ acpi_handle handle;
dev_dbg(&pdev->dev, "adding...\n");
+ handle = ACPI_HANDLE(&pdev->dev);
+ if (!handle)
+ return -ENODEV;
+
data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
data->dev = &pdev->dev;
- data->atk_handle = ACPI_HANDLE(&pdev->dev);
+ data->atk_handle = handle;
INIT_LIST_HEAD(&data->sensor_list);
data->disable_ec = false;
diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
index dddbd24..76f3e1d 100644
--- a/drivers/hwmon/corsair-psu.c
+++ b/drivers/hwmon/corsair-psu.c
@@ -796,13 +796,13 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id
ret = corsairpsu_init(priv);
if (ret < 0) {
dev_err(&hdev->dev, "unable to initialize device (%d)\n", ret);
- goto fail_and_stop;
+ goto fail_and_close;
}
ret = corsairpsu_fwinfo(priv);
if (ret < 0) {
dev_err(&hdev->dev, "unable to query firmware (%d)\n", ret);
- goto fail_and_stop;
+ goto fail_and_close;
}
corsairpsu_get_criticals(priv);
diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
index efcbea2..ad4ed41 100644
--- a/drivers/hwmon/fam15h_power.c
+++ b/drivers/hwmon/fam15h_power.c
@@ -19,6 +19,7 @@
#include <linux/sched.h>
#include <linux/topology.h>
#include <asm/processor.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
MODULE_DESCRIPTION("AMD Family 15h CPU processor power monitor");
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index a5d8f45..de0760d 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -20,7 +20,9 @@
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
+
#include <asm/amd/node.h>
+#include <asm/cpuid/api.h>
#include <asm/processor.h>
MODULE_DESCRIPTION("AMD Family 10h+ CPU core temperature monitor");
diff --git a/drivers/hwmon/k8temp.c b/drivers/hwmon/k8temp.c
index 2b80ac4..5324116 100644
--- a/drivers/hwmon/k8temp.c
+++ b/drivers/hwmon/k8temp.c
@@ -15,6 +15,7 @@
#include <linux/err.h>
#include <linux/mutex.h>
#include <asm/processor.h>
+#include <asm/cpuid/api.h>
#define TEMP_FROM_REG(val) (((((val) >> 16) & 0xff) - 49) * 1000)
#define REG_TEMP 0xe4
diff --git a/drivers/hwmon/lenovo-ec-sensors.c b/drivers/hwmon/lenovo-ec-sensors.c
index 8681bbf..24a182a 100644
--- a/drivers/hwmon/lenovo-ec-sensors.c
+++ b/drivers/hwmon/lenovo-ec-sensors.c
@@ -519,8 +519,8 @@ static int lenovo_ec_probe(struct platform_device *pdev)
if (!ec_data)
return -ENOMEM;
- if (!request_region(IO_REGION_START, IO_REGION_LENGTH, "LNV-WKS")) {
- pr_err(":request fail\n");
+ if (!devm_request_region(dev, IO_REGION_START, IO_REGION_LENGTH, "LNV-WKS")) {
+ dev_err(dev, "Failed to request I/O region\n");
return -EIO;
}
@@ -537,13 +537,11 @@ static int lenovo_ec_probe(struct platform_device *pdev)
outw_p(MCHP_SING_IDX, MCHP_EMI0_EC_ADDRESS);
mutex_unlock(&ec_data->mec_mutex);
- if ((inb_p(MCHP_EMI0_EC_DATA_BYTE0) != 'M') &&
- (inb_p(MCHP_EMI0_EC_DATA_BYTE1) != 'C') &&
- (inb_p(MCHP_EMI0_EC_DATA_BYTE2) != 'H') &&
- (inb_p(MCHP_EMI0_EC_DATA_BYTE3) != 'P')) {
- release_region(IO_REGION_START, IO_REGION_LENGTH);
+ if ((inb_p(MCHP_EMI0_EC_DATA_BYTE0) != 'M') ||
+ (inb_p(MCHP_EMI0_EC_DATA_BYTE1) != 'C') ||
+ (inb_p(MCHP_EMI0_EC_DATA_BYTE2) != 'H') ||
+ (inb_p(MCHP_EMI0_EC_DATA_BYTE3) != 'P'))
return -ENODEV;
- }
dmi_id = dmi_first_match(thinkstation_dmi_table);
@@ -577,7 +575,6 @@ static int lenovo_ec_probe(struct platform_device *pdev)
lenovo_ec_chip_info.info = lenovo_ec_hwmon_info_p8;
break;
default:
- release_region(IO_REGION_START, IO_REGION_LENGTH);
return -ENODEV;
}
@@ -606,10 +603,8 @@ static int __init lenovo_ec_init(void)
platform_create_bundle(&lenovo_ec_sensors_platform_driver,
lenovo_ec_probe, NULL, 0, NULL, 0);
- if (IS_ERR(lenovo_ec_sensors_platform_device)) {
- release_region(IO_REGION_START, IO_REGION_LENGTH);
+ if (IS_ERR(lenovo_ec_sensors_platform_device))
return PTR_ERR(lenovo_ec_sensors_platform_device);
- }
return 0;
}
@@ -617,7 +612,6 @@ module_init(lenovo_ec_init);
static void __exit lenovo_ec_exit(void)
{
- release_region(IO_REGION_START, IO_REGION_LENGTH);
platform_device_unregister(lenovo_ec_sensors_platform_device);
platform_driver_unregister(&lenovo_ec_sensors_platform_driver);
}
diff --git a/drivers/hwmon/lm63.c b/drivers/hwmon/lm63.c
index 035176a..30500b4 100644
--- a/drivers/hwmon/lm63.c
+++ b/drivers/hwmon/lm63.c
@@ -333,7 +333,13 @@ static ssize_t show_fan(struct device *dev, struct device_attribute *devattr,
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct lm63_data *data = lm63_update_device(dev);
- return sprintf(buf, "%d\n", FAN_FROM_REG(data->fan[attr->index]));
+ int fan;
+
+ mutex_lock(&data->update_lock);
+ fan = FAN_FROM_REG(data->fan[attr->index]);
+ mutex_unlock(&data->update_lock);
+
+ return sprintf(buf, "%d\n", fan);
}
static ssize_t set_fan(struct device *dev, struct device_attribute *dummy,
@@ -366,12 +372,14 @@ static ssize_t show_pwm1(struct device *dev, struct device_attribute *devattr,
int nr = attr->index;
int pwm;
+ mutex_lock(&data->update_lock);
if (data->pwm_highres)
pwm = data->pwm1[nr];
else
pwm = data->pwm1[nr] >= 2 * data->pwm1_freq ?
255 : (data->pwm1[nr] * 255 + data->pwm1_freq) /
(2 * data->pwm1_freq);
+ mutex_unlock(&data->update_lock);
return sprintf(buf, "%d\n", pwm);
}
@@ -529,6 +537,7 @@ static ssize_t show_temp11(struct device *dev, struct device_attribute *devattr,
int nr = attr->index;
int temp;
+ mutex_lock(&data->update_lock);
if (!nr) {
/*
* Use unsigned temperature unless its value is zero.
@@ -544,7 +553,10 @@ static ssize_t show_temp11(struct device *dev, struct device_attribute *devattr,
else
temp = TEMP11_FROM_REG(data->temp11[nr]);
}
- return sprintf(buf, "%d\n", temp + data->temp2_offset);
+ temp += data->temp2_offset;
+ mutex_unlock(&data->update_lock);
+
+ return sprintf(buf, "%d\n", temp);
}
static ssize_t set_temp11(struct device *dev, struct device_attribute *devattr,
@@ -592,9 +604,14 @@ static ssize_t temp2_crit_hyst_show(struct device *dev,
struct device_attribute *dummy, char *buf)
{
struct lm63_data *data = lm63_update_device(dev);
- return sprintf(buf, "%d\n", temp8_from_reg(data, 2)
- + data->temp2_offset
- - TEMP8_FROM_REG(data->temp2_crit_hyst));
+ int temp;
+
+ mutex_lock(&data->update_lock);
+ temp = temp8_from_reg(data, 2) + data->temp2_offset
+ - TEMP8_FROM_REG(data->temp2_crit_hyst);
+ mutex_unlock(&data->update_lock);
+
+ return sprintf(buf, "%d\n", temp);
}
static ssize_t show_lut_temp_hyst(struct device *dev,
@@ -602,10 +619,14 @@ static ssize_t show_lut_temp_hyst(struct device *dev,
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct lm63_data *data = lm63_update_device(dev);
+ int temp;
- return sprintf(buf, "%d\n", lut_temp_from_reg(data, attr->index)
- + data->temp2_offset
- - TEMP8_FROM_REG(data->lut_temp_hyst));
+ mutex_lock(&data->update_lock);
+ temp = lut_temp_from_reg(data, attr->index) + data->temp2_offset
+ - TEMP8_FROM_REG(data->lut_temp_hyst);
+ mutex_unlock(&data->update_lock);
+
+ return sprintf(buf, "%d\n", temp);
}
/*
@@ -616,7 +637,7 @@ static ssize_t temp2_crit_hyst_store(struct device *dev,
struct device_attribute *dummy,
const char *buf, size_t count)
{
- struct lm63_data *data = dev_get_drvdata(dev);
+ struct lm63_data *data = lm63_update_device(dev);
struct i2c_client *client = data->client;
long val;
int err;
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index f1a1e5b..c283443 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -137,7 +137,7 @@ static const struct lm75_params device_params[] = {
},
[as6200] = {
.config_reg_16bits = true,
- .set_mask = 0x94C0, /* 8 sample/s, 4 CF, positive polarity */
+ .set_mask = 0xC010, /* 8 sample/s, 4 CF */
.default_resolution = 12,
.default_sample_time = 125,
.num_sample_times = 4,
@@ -286,8 +286,8 @@ static const struct lm75_params device_params[] = {
},
[tmp112] = {
.config_reg_16bits = true,
- .set_mask = 0x60C0, /* 12-bit mode, 8 samples / second */
- .clr_mask = 1 << 15, /* no one-shot mode*/
+ .set_mask = 0xC060, /* 12-bit mode, 8 samples / second */
+ .clr_mask = 1 << 7, /* no one-shot mode*/
.default_resolution = 12,
.default_sample_time = 125,
.num_sample_times = 4,
@@ -353,7 +353,7 @@ static inline int lm75_write_config(struct lm75_data *data, u16 set_mask,
u16 clr_mask)
{
return regmap_update_bits(data->regmap, LM75_REG_CONF,
- clr_mask | LM75_SHUTDOWN, set_mask);
+ clr_mask | set_mask | LM75_SHUTDOWN, set_mask);
}
static irqreturn_t lm75_alarm_handler(int irq, void *private)
@@ -416,7 +416,7 @@ static int lm75_read(struct device *dev, enum hwmon_sensor_types type,
switch (data->kind) {
case as6200:
case tmp112:
- *val = (regval >> 13) & 0x1;
+ *val = !!(regval & BIT(13)) == !!(regval & BIT(2));
break;
default:
return -EINVAL;
diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
index 3c10a50..1eeb608 100644
--- a/drivers/hwmon/lm90.c
+++ b/drivers/hwmon/lm90.c
@@ -736,6 +736,7 @@ struct lm90_data {
struct hwmon_chip_info chip;
struct delayed_work alert_work;
struct work_struct report_work;
+ bool shutdown; /* true if shutting down */
bool valid; /* true if register values are valid */
bool alarms_valid; /* true if status register values are valid */
unsigned long last_updated; /* in jiffies */
@@ -1154,6 +1155,9 @@ static void lm90_report_alarms(struct work_struct *work)
static int lm90_update_alarms_locked(struct lm90_data *data, bool force)
{
+ if (data->shutdown)
+ return 0;
+
if (force || !data->alarms_valid ||
time_after(jiffies, data->alarms_updated + msecs_to_jiffies(data->update_interval))) {
struct i2c_client *client = data->client;
@@ -2584,15 +2588,23 @@ static void lm90_restore_conf(void *_data)
struct lm90_data *data = _data;
struct i2c_client *client = data->client;
- cancel_delayed_work_sync(&data->alert_work);
- cancel_work_sync(&data->report_work);
-
/* Restore initial configuration */
if (data->flags & LM90_HAVE_CONVRATE)
lm90_write_convrate(data, data->convrate_orig);
lm90_write_reg(client, LM90_REG_CONFIG1, data->config_orig);
}
+static void lm90_stop_work(void *_data)
+{
+ struct lm90_data *data = _data;
+
+ hwmon_lock(data->hwmon_dev);
+ data->shutdown = true;
+ hwmon_unlock(data->hwmon_dev);
+ cancel_delayed_work_sync(&data->alert_work);
+ cancel_work_sync(&data->report_work);
+}
+
static int lm90_init_client(struct i2c_client *client, struct lm90_data *data)
{
struct device_node *np = client->dev.of_node;
@@ -2902,6 +2914,10 @@ static int lm90_probe(struct i2c_client *client)
data->hwmon_dev = hwmon_dev;
+ err = devm_add_action_or_reset(&client->dev, lm90_stop_work, data);
+ if (err)
+ return err;
+
if (client->irq) {
dev_dbg(dev, "IRQ: %d\n", client->irq);
err = devm_request_threaded_irq(dev, client->irq,
@@ -2930,7 +2946,8 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type,
*/
struct lm90_data *data = i2c_get_clientdata(client);
- if ((data->flags & LM90_HAVE_BROKEN_ALERT) &&
+ hwmon_lock(data->hwmon_dev);
+ if (!data->shutdown && (data->flags & LM90_HAVE_BROKEN_ALERT) &&
(data->current_alarms & data->alert_alarms)) {
if (!(data->config & 0x80)) {
dev_dbg(&client->dev, "Disabling ALERT#\n");
@@ -2939,6 +2956,7 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type,
schedule_delayed_work(&data->alert_work,
max_t(int, HZ, msecs_to_jiffies(data->update_interval)));
}
+ hwmon_unlock(data->hwmon_dev);
} else {
dev_dbg(&client->dev, "Everything OK\n");
}
diff --git a/drivers/hwmon/ltc2992.c b/drivers/hwmon/ltc2992.c
index 1fcd320..2617c45 100644
--- a/drivers/hwmon/ltc2992.c
+++ b/drivers/hwmon/ltc2992.c
@@ -431,10 +431,16 @@ static int ltc2992_get_voltage(struct ltc2992_state *st, u32 reg, u32 scale, lon
static int ltc2992_set_voltage(struct ltc2992_state *st, u32 reg, u32 scale, long val)
{
- val = DIV_ROUND_CLOSEST(val * 1000, scale);
- val = val << 4;
+ u32 reg_val;
+ long vmax;
- return ltc2992_write_reg(st, reg, 2, val);
+ vmax = DIV_ROUND_CLOSEST_ULL(0xFFFULL * scale, 1000);
+ val = max(val, 0L);
+ val = min(val, vmax);
+ reg_val = min(DIV_ROUND_CLOSEST_ULL((u64)val * 1000, scale),
+ 0xFFFULL) << 4;
+
+ return ltc2992_write_reg(st, reg, 2, reg_val);
}
static int ltc2992_read_gpio_alarm(struct ltc2992_state *st, int nr_gpio, u32 attr, long *val)
@@ -559,9 +565,15 @@ static int ltc2992_get_current(struct ltc2992_state *st, u32 reg, u32 channel, l
static int ltc2992_set_current(struct ltc2992_state *st, u32 reg, u32 channel, long val)
{
u32 reg_val;
+ long cmax;
- reg_val = DIV_ROUND_CLOSEST(val * st->r_sense_uohm[channel], LTC2992_IADC_NANOV_LSB);
- reg_val = reg_val << 4;
+ cmax = DIV_ROUND_CLOSEST_ULL(0xFFFULL * LTC2992_IADC_NANOV_LSB,
+ st->r_sense_uohm[channel]);
+ val = max(val, 0L);
+ val = min(val, cmax);
+ reg_val = min(DIV_ROUND_CLOSEST_ULL((u64)val * st->r_sense_uohm[channel],
+ LTC2992_IADC_NANOV_LSB),
+ 0xFFFULL) << 4;
return ltc2992_write_reg(st, reg, 2, reg_val);
}
@@ -625,8 +637,10 @@ static int ltc2992_get_power(struct ltc2992_state *st, u32 reg, u32 channel, lon
if (reg_val < 0)
return reg_val;
- *val = mul_u64_u32_div(reg_val, LTC2992_VADC_UV_LSB * LTC2992_IADC_NANOV_LSB,
- st->r_sense_uohm[channel] * 1000);
+ *val = mul_u64_u32_div(reg_val,
+ LTC2992_VADC_UV_LSB / 1000 *
+ LTC2992_IADC_NANOV_LSB,
+ st->r_sense_uohm[channel]);
return 0;
}
@@ -634,9 +648,18 @@ static int ltc2992_get_power(struct ltc2992_state *st, u32 reg, u32 channel, lon
static int ltc2992_set_power(struct ltc2992_state *st, u32 reg, u32 channel, long val)
{
u32 reg_val;
+ u64 pmax, uval;
- reg_val = mul_u64_u32_div(val, st->r_sense_uohm[channel] * 1000,
- LTC2992_VADC_UV_LSB * LTC2992_IADC_NANOV_LSB);
+ uval = max(val, 0L);
+ pmax = mul_u64_u32_div(0xFFFFFFULL,
+ LTC2992_VADC_UV_LSB / 1000 *
+ LTC2992_IADC_NANOV_LSB,
+ st->r_sense_uohm[channel]);
+ uval = min(uval, pmax);
+ reg_val = min(mul_u64_u32_div(uval, st->r_sense_uohm[channel],
+ LTC2992_VADC_UV_LSB / 1000 *
+ LTC2992_IADC_NANOV_LSB),
+ 0xFFFFFFULL);
return ltc2992_write_reg(st, reg, 3, reg_val);
}
diff --git a/drivers/hwmon/pmbus/adm1266.c b/drivers/hwmon/pmbus/adm1266.c
index d90f8f80..6f6ad7b 100644
--- a/drivers/hwmon/pmbus/adm1266.c
+++ b/drivers/hwmon/pmbus/adm1266.c
@@ -46,6 +46,7 @@
#define ADM1266_BLACKBOX_OFFSET 0
#define ADM1266_BLACKBOX_SIZE 64
+#define ADM1266_BLACKBOX_MAX_RECORDS 32
#define ADM1266_PMBUS_BLOCK_MAX 255
@@ -60,7 +61,7 @@ struct adm1266_data {
u8 *dev_mem;
struct mutex buf_mutex;
u8 write_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
- u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
+ u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 2] ____cacheline_aligned;
};
static const struct nvmem_cell_info adm1266_nvmem_cells[] = {
@@ -172,9 +173,13 @@ static int adm1266_gpio_get(struct gpio_chip *chip, unsigned int offset)
else
pmbus_cmd = ADM1266_PDIO_STATUS;
+ guard(pmbus_lock)(data->client);
+
ret = i2c_smbus_read_block_data(data->client, pmbus_cmd, read_buf);
if (ret < 0)
return ret;
+ if (ret < 2)
+ return -EIO;
pins_status = read_buf[0] + (read_buf[1] << 8);
if (offset < ADM1266_GPIO_NR)
@@ -192,9 +197,13 @@ static int adm1266_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask
unsigned int gpio_nr;
int ret;
+ guard(pmbus_lock)(data->client);
+
ret = i2c_smbus_read_block_data(data->client, ADM1266_GPIO_STATUS, read_buf);
if (ret < 0)
return ret;
+ if (ret < 2)
+ return -EIO;
status = read_buf[0] + (read_buf[1] << 8);
@@ -207,11 +216,12 @@ static int adm1266_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask
ret = i2c_smbus_read_block_data(data->client, ADM1266_PDIO_STATUS, read_buf);
if (ret < 0)
return ret;
+ if (ret < 2)
+ return -EIO;
status = read_buf[0] + (read_buf[1] << 8);
- *bits = 0;
- for_each_set_bit_from(gpio_nr, mask, ADM1266_GPIO_NR + ADM1266_PDIO_STATUS) {
+ for_each_set_bit_from(gpio_nr, mask, ADM1266_GPIO_NR + ADM1266_PDIO_NR) {
if (test_bit(gpio_nr - ADM1266_GPIO_NR, &status))
set_bit(gpio_nr, bits);
}
@@ -230,6 +240,8 @@ static void adm1266_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
int ret;
int i;
+ guard(pmbus_lock)(data->client);
+
for (i = 0; i < ADM1266_GPIO_NR; i++) {
write_cmd = adm1266_gpio_mapping[i][1];
ret = adm1266_pmbus_block_xfer(data, ADM1266_GPIO_CONFIG, 1, &write_cmd, read_buf);
@@ -290,8 +302,9 @@ static int adm1266_config_gpio(struct adm1266_data *data)
int i;
for (i = 0; i < ARRAY_SIZE(data->gpio_names); i++) {
- gpio_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "adm1266-%x-%s",
- data->client->addr, adm1266_names[i]);
+ gpio_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "adm1266-%d-%x-%s",
+ data->client->adapter->nr, data->client->addr,
+ adm1266_names[i]);
if (!gpio_name)
return -ENOMEM;
@@ -322,6 +335,7 @@ static int adm1266_state_read(struct seq_file *s, void *pdata)
struct i2c_client *client = to_i2c_client(dev);
int ret;
+ guard(pmbus_lock)(client);
ret = i2c_smbus_read_word_data(client, ADM1266_READ_STATE);
if (ret < 0)
return ret;
@@ -347,9 +361,10 @@ static void adm1266_init_debugfs(struct adm1266_data *data)
static int adm1266_nvmem_read_blackbox(struct adm1266_data *data, u8 *read_buff)
{
+ u8 record[ADM1266_PMBUS_BLOCK_MAX];
int record_count;
char index;
- u8 buf[5];
+ u8 buf[I2C_SMBUS_BLOCK_MAX];
int ret;
ret = i2c_smbus_read_block_data(data->client, ADM1266_BLACKBOX_INFO, buf);
@@ -360,15 +375,18 @@ static int adm1266_nvmem_read_blackbox(struct adm1266_data *data, u8 *read_buff)
return -EIO;
record_count = buf[3];
+ if (record_count > ADM1266_BLACKBOX_MAX_RECORDS)
+ return -EIO;
for (index = 0; index < record_count; index++) {
- ret = adm1266_pmbus_block_xfer(data, ADM1266_READ_BLACKBOX, 1, &index, read_buff);
+ ret = adm1266_pmbus_block_xfer(data, ADM1266_READ_BLACKBOX, 1, &index, record);
if (ret < 0)
return ret;
if (ret != ADM1266_BLACKBOX_SIZE)
return -EIO;
+ memcpy(read_buff, record, ADM1266_BLACKBOX_SIZE);
read_buff += ADM1266_BLACKBOX_SIZE;
}
@@ -383,6 +401,8 @@ static int adm1266_nvmem_read(void *priv, unsigned int offset, void *val, size_t
if (offset + bytes > data->nvmem_config.size)
return -EINVAL;
+ guard(pmbus_lock)(data->client);
+
if (offset == 0) {
memset(data->dev_mem, 0, data->nvmem_config.size);
@@ -432,7 +452,7 @@ static int adm1266_set_rtc(struct adm1266_data *data)
char write_buf[6];
int i;
- kt = ktime_get_seconds();
+ kt = ktime_get_real_seconds();
memset(write_buf, 0, sizeof(write_buf));
@@ -462,20 +482,20 @@ static int adm1266_probe(struct i2c_client *client)
crc8_populate_msb(pmbus_crc_table, 0x7);
mutex_init(&data->buf_mutex);
- ret = adm1266_config_gpio(data);
+ ret = adm1266_set_rtc(data);
if (ret < 0)
return ret;
- ret = adm1266_set_rtc(data);
- if (ret < 0)
+ ret = pmbus_do_probe(client, &data->info);
+ if (ret)
return ret;
ret = adm1266_config_nvmem(data);
if (ret < 0)
return ret;
- ret = pmbus_do_probe(client, &data->info);
- if (ret)
+ ret = adm1266_config_gpio(data);
+ if (ret < 0)
return ret;
adm1266_init_debugfs(data);
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index a773ba0..66c2353 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -117,7 +117,7 @@
/* timeout for pm runtime autosuspend */
#define DAVINCI_I2C_PM_TIMEOUT 1000 /* ms */
-#define DAVINCI_I2C_DEFAULT_BUS_FREQ 100
+#define DAVINCI_I2C_DEFAULT_BUS_FREQ 100000
struct davinci_i2c_dev {
struct device *dev;
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
index 70cb582..53d9df7 100644
--- a/drivers/i2c/busses/i2c-stm32f7.c
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -895,8 +895,6 @@ static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
f7_msg->result = 0;
f7_msg->stop = (i2c_dev->msg_id >= i2c_dev->msg_num - 1);
- reinit_completion(&i2c_dev->complete);
-
cr1 = readl_relaxed(base + STM32F7_I2C_CR1);
cr2 = readl_relaxed(base + STM32F7_I2C_CR2);
@@ -1728,6 +1726,8 @@ static int stm32f7_i2c_xfer_core(struct i2c_adapter *i2c_adap,
if (ret)
goto pm_free;
+ reinit_completion(&i2c_dev->complete);
+
stm32f7_i2c_xfer_msg(i2c_dev, msgs);
if (!i2c_dev->atomic)
@@ -2253,7 +2253,7 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
snprintf(adap->name, sizeof(adap->name), "STM32F7 I2C(%pa)",
&res->start);
adap->owner = THIS_MODULE;
- adap->timeout = 2 * HZ;
+ adap->timeout = 8 * HZ;
adap->retries = 3;
adap->algo = &stm32f7_i2c_algo;
adap->dev.parent = &pdev->dev;
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 9fd5ade..479a166 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -589,25 +589,22 @@ static int tegra_i2c_mutex_lock(struct tegra_i2c_dev *i2c_dev)
return ret;
}
-static int tegra_i2c_mutex_unlock(struct tegra_i2c_dev *i2c_dev)
+static void tegra_i2c_mutex_unlock(struct tegra_i2c_dev *i2c_dev)
{
unsigned int reg = i2c_dev->hw->regs->sw_mutex;
u32 val, id;
if (!i2c_dev->hw->has_mutex)
- return 0;
+ return;
val = readl(i2c_dev->base + reg);
id = FIELD_GET(I2C_SW_MUTEX_GRANT, val);
- if (id && id != I2C_SW_MUTEX_ID_CCPLEX) {
- dev_warn(i2c_dev->dev, "unable to unlock mutex, mutex is owned by: %u\n", id);
- return -EPERM;
- }
+ if (WARN(id && id != I2C_SW_MUTEX_ID_CCPLEX,
+ "unable to unlock mutex, mutex is owned by: %u\n", id))
+ return;
writel(0, i2c_dev->base + reg);
-
- return 0;
}
static void tegra_i2c_mask_irq(struct tegra_i2c_dev *i2c_dev, u32 mask)
@@ -1666,8 +1663,10 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
}
ret = tegra_i2c_mutex_lock(i2c_dev);
- if (ret)
+ if (ret) {
+ pm_runtime_put(i2c_dev->dev);
return ret;
+ }
for (i = 0; i < num; i++) {
enum msg_end_type end_type = MSG_END_STOP;
@@ -1698,7 +1697,7 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
break;
}
- ret = tegra_i2c_mutex_unlock(i2c_dev);
+ tegra_i2c_mutex_unlock(i2c_dev);
pm_runtime_put(i2c_dev->dev);
return ret ?: i;
diff --git a/drivers/i2c/busses/i2c-virtio.c b/drivers/i2c/busses/i2c-virtio.c
index 7b0b0bf..5da6fef 100644
--- a/drivers/i2c/busses/i2c-virtio.c
+++ b/drivers/i2c/busses/i2c-virtio.c
@@ -222,6 +222,8 @@ static int virtio_i2c_probe(struct virtio_device *vdev)
*/
ACPI_COMPANION_SET(&vi->adap.dev, ACPI_COMPANION(vdev->dev.parent));
+ virtio_device_ready(vdev);
+
ret = i2c_add_adapter(&vi->adap);
if (ret)
virtio_i2c_del_vqs(vdev);
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index 2cbd31f..28c0e48 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -371,6 +371,7 @@ static const struct acpi_device_id i2c_acpi_force_100khz_device_ids[] = {
* a 400KHz frequency. The root cause of the issue is not known.
*/
{ "DLL0945", 0 },
+ { "ELAN0678", 0 },
{ "ELAN06FA", 0 },
{}
};
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 9c46147..a2132d7 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -445,8 +445,7 @@ static int i2c_init_recovery(struct i2c_adapter *adap)
bri->set_scl = set_scl_gpio_value;
if (bri->sda_gpiod) {
bri->get_sda = get_sda_gpio_value;
- /* FIXME: add proper flag instead of '0' once available */
- if (gpiod_get_direction(bri->sda_gpiod) == 0)
+ if (gpiod_get_direction(bri->sda_gpiod) == GPIO_LINE_DIRECTION_OUT)
bri->set_sda = set_sda_gpio_value;
}
} else if (bri->recover_bus == i2c_generic_scl_recovery) {
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index 71eb1ef..fa63bee 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -353,6 +353,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
&& size != I2C_SMBUS_I2C_BLOCK_DATA);
msgbuf0[0] = command;
+ msgbuf1[0] = 0;
switch (size) {
case I2C_SMBUS_QUICK:
msg[0].len = 0;
@@ -566,6 +567,18 @@ s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
if (res)
return res;
+ /* Reject invalid caller-supplied block lengths before any
+ * tracepoint or native smbus_xfer callback runs.
+ */
+ if (data &&
+ (protocol == I2C_SMBUS_I2C_BLOCK_DATA ||
+ protocol == I2C_SMBUS_BLOCK_PROC_CALL ||
+ (protocol == I2C_SMBUS_BLOCK_DATA &&
+ read_write == I2C_SMBUS_WRITE)) &&
+ (data->block[0] == 0 ||
+ data->block[0] > I2C_SMBUS_BLOCK_MAX))
+ return -EINVAL;
+
/* If enabled, the following two tracepoints are conditional on
* read_write and protocol.
*/
diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c
index 7bbe026..ccaac5e 100644
--- a/drivers/i2c/i2c-dev.c
+++ b/drivers/i2c/i2c-dev.c
@@ -487,12 +487,13 @@ static long i2cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
client->adapter->retries = arg;
break;
case I2C_TIMEOUT:
- if (arg > INT_MAX)
+ /*
+ * For historical reasons, user-space sets the timeout value in
+ * units of 10 ms.
+ */
+ if (arg > INT_MAX / 10)
return -EINVAL;
- /* For historical reasons, user-space sets the timeout
- * value in units of 10 ms.
- */
client->adapter->timeout = msecs_to_jiffies(arg * 10);
break;
default:
diff --git a/drivers/i2c/i2c-slave-testunit.c b/drivers/i2c/i2c-slave-testunit.c
index 6de4307..871c584 100644
--- a/drivers/i2c/i2c-slave-testunit.c
+++ b/drivers/i2c/i2c-slave-testunit.c
@@ -15,7 +15,7 @@
#include <linux/module.h>
#include <linux/of.h>
#include <linux/slab.h>
-#include <linux/workqueue.h> /* FIXME: is system_long_wq the best choice? */
+#include <linux/workqueue.h>
#define TU_VERSION_MAX_LENGTH 128
@@ -124,7 +124,7 @@ static int i2c_slave_testunit_slave_cb(struct i2c_client *client,
case I2C_SLAVE_STOP:
if (tu->reg_idx == TU_NUM_REGS) {
set_bit(TU_FLAG_IN_PROCESS, &tu->flags);
- queue_delayed_work(system_long_wq, &tu->worker,
+ queue_delayed_work(system_dfl_long_wq, &tu->worker,
msecs_to_jiffies(10 * tu->regs[TU_REG_DELAY]));
}
diff --git a/drivers/i2c/i2c-stub.c b/drivers/i2c/i2c-stub.c
index fbb0db4..04314e3 100644
--- a/drivers/i2c/i2c-stub.c
+++ b/drivers/i2c/i2c-stub.c
@@ -214,6 +214,11 @@ static s32 stub_xfer(struct i2c_adapter *adap, u16 addr, unsigned short flags,
* We ignore banks here, because banked chips don't use I2C
* block transfers
*/
+ if (data->block[0] == 0 ||
+ data->block[0] > I2C_SMBUS_BLOCK_MAX) {
+ ret = -EINVAL;
+ break;
+ }
if (data->block[0] > 256 - command) /* Avoid overrun */
data->block[0] = 256 - command;
len = data->block[0];
diff --git a/drivers/iio/adc/ad4695.c b/drivers/iio/adc/ad4695.c
index cda4196..53642de 100644
--- a/drivers/iio/adc/ad4695.c
+++ b/drivers/iio/adc/ad4695.c
@@ -876,14 +876,14 @@ static int ad4695_offload_buffer_postenable(struct iio_dev *indio_dev)
if (ret)
goto err_unoptimize_message;
- ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
- &config);
+ ret = ad4695_enter_advanced_sequencer_mode(st, num_slots);
if (ret)
goto err_disable_busy_output;
- ret = ad4695_enter_advanced_sequencer_mode(st, num_slots);
+ ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
+ &config);
if (ret)
- goto err_offload_trigger_disable;
+ goto err_exit_conversion_mode;
mutex_lock(&st->cnv_pwm_lock);
pwm_get_state(st->cnv_pwm, &state);
@@ -895,23 +895,16 @@ static int ad4695_offload_buffer_postenable(struct iio_dev *indio_dev)
ret = pwm_apply_might_sleep(st->cnv_pwm, &state);
mutex_unlock(&st->cnv_pwm_lock);
if (ret)
- goto err_offload_exit_conversion_mode;
+ goto err_offload_trigger_disable;
return 0;
-err_offload_exit_conversion_mode:
- /*
- * We have to unwind in a different order to avoid triggering offload.
- * ad4695_exit_conversion_mode() triggers a conversion, so it has to be
- * done after spi_offload_trigger_disable().
- */
- spi_offload_trigger_disable(st->offload, st->offload_trigger);
- ad4695_exit_conversion_mode(st);
- goto err_disable_busy_output;
-
err_offload_trigger_disable:
spi_offload_trigger_disable(st->offload, st->offload_trigger);
+err_exit_conversion_mode:
+ ad4695_exit_conversion_mode(st);
+
err_disable_busy_output:
regmap_clear_bits(st->regmap, AD4695_REG_GP_MODE,
AD4695_REG_GP_MODE_BUSY_GP_EN);
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 23991a36..000e39c 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -817,9 +817,11 @@ static int meson_sar_adc_temp_sensor_init(struct iio_dev *indio_dev)
}
priv->tsc_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "amlogic,hhi-sysctrl");
- if (IS_ERR(priv->tsc_regmap))
+ if (IS_ERR(priv->tsc_regmap)) {
+ kfree(buf);
return dev_err_probe(dev, PTR_ERR(priv->tsc_regmap),
"failed to get amlogic,hhi-sysctrl regmap\n");
+ }
trimming_bits = priv->param->temperature_trimming_bits;
trimming_mask = BIT(trimming_bits) - 1;
diff --git a/drivers/iio/adc/mt6359-auxadc.c b/drivers/iio/adc/mt6359-auxadc.c
index 6b9ed9b..1d9724e 100644
--- a/drivers/iio/adc/mt6359-auxadc.c
+++ b/drivers/iio/adc/mt6359-auxadc.c
@@ -497,6 +497,7 @@ static int mt6358_read_imp(struct mt6359_auxadc *adc_dev,
return ret;
/* Read the params before stopping */
+ val_v = 0;
regmap_read(regmap, reg_adc0 + (cinfo->imp_adc_num << 1), &val_v);
mt6358_stop_imp_conv(adc_dev);
diff --git a/drivers/iio/adc/npcm_adc.c b/drivers/iio/adc/npcm_adc.c
index ddabb96..61c8b82 100644
--- a/drivers/iio/adc/npcm_adc.c
+++ b/drivers/iio/adc/npcm_adc.c
@@ -231,7 +231,7 @@ static int npcm_adc_probe(struct platform_device *pdev)
if (IS_ERR(info->reset))
return PTR_ERR(info->reset);
- info->adc_clk = devm_clk_get(&pdev->dev, NULL);
+ info->adc_clk = devm_clk_get_enabled(&pdev->dev, NULL);
if (IS_ERR(info->adc_clk)) {
dev_warn(&pdev->dev, "ADC clock failed: can't read clk\n");
return PTR_ERR(info->adc_clk);
@@ -244,17 +244,13 @@ static int npcm_adc_probe(struct platform_device *pdev)
info->adc_sample_hz = clk_get_rate(info->adc_clk) / ((div + 1) * 2);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- ret = irq;
- goto err_disable_clk;
- }
+ if (irq < 0)
+ return irq;
ret = devm_request_irq(&pdev->dev, irq, npcm_adc_isr, 0,
"NPCM_ADC", indio_dev);
- if (ret < 0) {
- dev_err(dev, "failed requesting interrupt\n");
- goto err_disable_clk;
- }
+ if (ret < 0)
+ return ret;
reg_con = ioread32(info->regs + NPCM_ADCCON);
info->vref = devm_regulator_get_optional(&pdev->dev, "vref");
@@ -262,7 +258,7 @@ static int npcm_adc_probe(struct platform_device *pdev)
ret = regulator_enable(info->vref);
if (ret) {
dev_err(&pdev->dev, "Can't enable ADC reference voltage\n");
- goto err_disable_clk;
+ return ret;
}
iowrite32(reg_con & ~NPCM_ADCCON_REFSEL,
@@ -272,10 +268,8 @@ static int npcm_adc_probe(struct platform_device *pdev)
* Any error which is not ENODEV indicates the regulator
* has been specified and so is a failure case.
*/
- if (PTR_ERR(info->vref) != -ENODEV) {
- ret = PTR_ERR(info->vref);
- goto err_disable_clk;
- }
+ if (PTR_ERR(info->vref) != -ENODEV)
+ return PTR_ERR(info->vref);
/* Use internal reference */
iowrite32(reg_con | NPCM_ADCCON_REFSEL,
@@ -314,8 +308,6 @@ static int npcm_adc_probe(struct platform_device *pdev)
iowrite32(reg_con & ~NPCM_ADCCON_ADC_EN, info->regs + NPCM_ADCCON);
if (!IS_ERR(info->vref))
regulator_disable(info->vref);
-err_disable_clk:
- clk_disable_unprepare(info->adc_clk);
return ret;
}
@@ -332,7 +324,6 @@ static void npcm_adc_remove(struct platform_device *pdev)
iowrite32(regtemp & ~NPCM_ADCCON_ADC_EN, info->regs + NPCM_ADCCON);
if (!IS_ERR(info->vref))
regulator_disable(info->vref);
- clk_disable_unprepare(info->adc_clk);
}
static struct platform_driver npcm_adc_driver = {
diff --git a/drivers/iio/adc/nxp-sar-adc.c b/drivers/iio/adc/nxp-sar-adc.c
index 9d9f2c7..8f4ed3d 100644
--- a/drivers/iio/adc/nxp-sar-adc.c
+++ b/drivers/iio/adc/nxp-sar-adc.c
@@ -198,6 +198,15 @@ static void nxp_sar_adc_irq_cfg(struct nxp_sar_adc *info, bool enable)
writel(0, NXP_SAR_ADC_IMR(info->regs));
}
+static void nxp_sar_adc_wait_for(struct nxp_sar_adc *info, unsigned int cycles)
+{
+ u64 rate;
+
+ rate = clk_get_rate(info->clk);
+ if (rate)
+ ndelay(div64_u64(NSEC_PER_SEC, rate * cycles));
+}
+
static bool nxp_sar_adc_set_enabled(struct nxp_sar_adc *info, bool enable)
{
u32 mcr;
@@ -221,7 +230,7 @@ static bool nxp_sar_adc_set_enabled(struct nxp_sar_adc *info, bool enable)
* configuration of NCMR and the setting of NSTART.
*/
if (enable)
- ndelay(div64_u64(NSEC_PER_SEC, clk_get_rate(info->clk) * 3));
+ nxp_sar_adc_wait_for(info, 3);
return pwdn;
}
@@ -469,7 +478,7 @@ static void nxp_sar_adc_stop_conversion(struct nxp_sar_adc *info)
* only when the capture finishes. The delay will be very
* short, usec-ish, which is acceptable in the atomic context.
*/
- ndelay(div64_u64(NSEC_PER_SEC, clk_get_rate(info->clk)) * 80);
+ nxp_sar_adc_wait_for(info, 80);
}
static int nxp_sar_adc_start_conversion(struct nxp_sar_adc *info, bool raw)
@@ -560,6 +569,9 @@ static int nxp_sar_adc_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
switch (mask) {
case IIO_CHAN_INFO_SAMP_FREQ:
+ if (val <= 0)
+ return -EINVAL;
+
/*
* Configures the sample period duration in terms of the SAR
* controller clock. The minimum acceptable value is 8.
@@ -568,7 +580,11 @@ static int nxp_sar_adc_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
* sampling timing which gives us the number of cycles expected.
* The value is 8-bit wide, consequently the max value is 0xFF.
*/
- inpsamp = clk_get_rate(info->clk) / val - NXP_SAR_ADC_CONV_TIME;
+ inpsamp = clk_get_rate(info->clk) / val;
+ if (inpsamp < NXP_SAR_ADC_CONV_TIME)
+ return -EINVAL;
+
+ inpsamp -= NXP_SAR_ADC_CONV_TIME;
nxp_sar_adc_conversion_timing_set(info, inpsamp);
return 0;
@@ -660,7 +676,7 @@ static void nxp_sar_adc_dma_cb(void *data)
static int nxp_sar_adc_start_cyclic_dma(struct iio_dev *indio_dev)
{
struct nxp_sar_adc *info = iio_priv(indio_dev);
- struct dma_slave_config config;
+ struct dma_slave_config config = { };
struct dma_async_tx_descriptor *desc;
int ret;
diff --git a/drivers/iio/adc/qcom-spmi-adc5-gen3.c b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
index f8168a1..48c793b 100644
--- a/drivers/iio/adc/qcom-spmi-adc5-gen3.c
+++ b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
@@ -482,7 +482,7 @@ static int adc5_gen3_get_fw_channel_data(struct adc5_chip *adc,
sid = FIELD_GET(ADC5_GEN3_VIRTUAL_SID_MASK, chan);
chan = FIELD_GET(ADC5_GEN3_CHANNEL_MASK, chan);
- if (chan > ADC5_MAX_CHANNEL)
+ if (chan >= ADC5_MAX_CHANNEL)
return dev_err_probe(dev, -EINVAL,
"%s invalid channel number %d\n",
name, chan);
diff --git a/drivers/iio/adc/viperboard_adc.c b/drivers/iio/adc/viperboard_adc.c
index 9bb0b83..6efe1c61 100644
--- a/drivers/iio/adc/viperboard_adc.c
+++ b/drivers/iio/adc/viperboard_adc.c
@@ -70,8 +70,10 @@ static int vprbrd_iio_read_raw(struct iio_dev *iio_dev,
VPRBRD_USB_TYPE_OUT, 0x0000, 0x0000, admsg,
sizeof(struct vprbrd_adc_msg), VPRBRD_USB_TIMEOUT_MS);
if (ret != sizeof(struct vprbrd_adc_msg)) {
- dev_err(&iio_dev->dev, "usb send error on adc read\n");
+ mutex_unlock(&vb->lock);
error = -EREMOTEIO;
+ dev_err(&iio_dev->dev, "usb send error on adc read\n");
+ goto error;
}
ret = usb_control_msg(vb->usb_dev,
diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c
index e257c1b..3980dfa 100644
--- a/drivers/iio/adc/xilinx-xadc-core.c
+++ b/drivers/iio/adc/xilinx-xadc-core.c
@@ -817,6 +817,7 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
{
struct xadc *xadc = iio_priv(indio_dev);
unsigned long scan_mask;
+ int seq_mode;
int ret;
int i;
@@ -824,6 +825,12 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
for (i = 0; i < indio_dev->num_channels; i++)
scan_mask |= BIT(indio_dev->channels[i].scan_index);
+ /*
+ * Use the correct sequencer mode for the idle state: simultaneous
+ * mode for dual external mux configurations, continuous otherwise.
+ */
+ seq_mode = xadc_get_seq_mode(xadc, scan_mask);
+
/* Enable all channels and calibration */
ret = xadc_write_adc_reg(xadc, XADC_REG_SEQ(0), scan_mask & 0xffff);
if (ret)
@@ -834,11 +841,11 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
return ret;
ret = xadc_update_adc_reg(xadc, XADC_REG_CONF1, XADC_CONF1_SEQ_MASK,
- XADC_CONF1_SEQ_CONTINUOUS);
+ seq_mode);
if (ret)
return ret;
- return xadc_power_adc_b(xadc, XADC_CONF1_SEQ_CONTINUOUS);
+ return xadc_power_adc_b(xadc, seq_mode);
}
static int xadc_preenable(struct iio_dev *indio_dev)
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
index 24d7df6..700528c 100644
--- a/drivers/iio/buffer/industrialio-hw-consumer.c
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -85,7 +85,7 @@ static struct hw_consumer_buffer *iio_hw_consumer_get_buffer(
*/
struct iio_hw_consumer *iio_hw_consumer_alloc(struct device *dev)
{
- struct hw_consumer_buffer *buf;
+ struct hw_consumer_buffer *buf, *tmp;
struct iio_hw_consumer *hwc;
struct iio_channel *chan;
int ret;
@@ -116,7 +116,7 @@ struct iio_hw_consumer *iio_hw_consumer_alloc(struct device *dev)
return hwc;
err_put_buffers:
- list_for_each_entry(buf, &hwc->buffers, head)
+ list_for_each_entry_safe(buf, tmp, &hwc->buffers, head)
iio_buffer_put(&buf->buffer);
iio_channel_release_all(hwc->channels);
err_free_hwc:
diff --git a/drivers/iio/chemical/mhz19b.c b/drivers/iio/chemical/mhz19b.c
index 3c64154..9d4cf43 100644
--- a/drivers/iio/chemical/mhz19b.c
+++ b/drivers/iio/chemical/mhz19b.c
@@ -52,6 +52,8 @@ struct mhz19b_state {
struct completion buf_ready;
u8 buf_idx;
+ bool buf_overflow;
+
/*
* Serdev receive buffer.
* When data is received from the MH-Z19B,
@@ -106,6 +108,10 @@ static int mhz19b_serdev_cmd(struct iio_dev *indio_dev, int cmd, u16 arg)
cmd_buf[8] = mhz19b_get_checksum(cmd_buf);
/* Write buf to uart ctrl synchronously */
+ st->buf_idx = 0;
+ st->buf_overflow = false;
+ reinit_completion(&st->buf_ready);
+
ret = serdev_device_write(serdev, cmd_buf, MHZ19B_CMD_SIZE, 0);
if (ret < 0)
return ret;
@@ -121,6 +127,9 @@ static int mhz19b_serdev_cmd(struct iio_dev *indio_dev, int cmd, u16 arg)
if (!ret)
return -ETIMEDOUT;
+ if (st->buf_overflow)
+ return -EMSGSIZE;
+
if (st->buf[8] != mhz19b_get_checksum(st->buf)) {
dev_err(dev, "checksum err");
return -EINVAL;
@@ -240,6 +249,14 @@ static size_t mhz19b_receive_buf(struct serdev_device *serdev,
{
struct iio_dev *indio_dev = dev_get_drvdata(&serdev->dev);
struct mhz19b_state *st = iio_priv(indio_dev);
+ size_t remaining = MHZ19B_CMD_SIZE - st->buf_idx;
+
+ if (len > remaining) {
+ st->buf_idx = 0;
+ st->buf_overflow = true;
+ complete(&st->buf_ready);
+ return len;
+ }
memcpy(st->buf + st->buf_idx, data, len);
st->buf_idx += len;
diff --git a/drivers/iio/chemical/scd30_core.c b/drivers/iio/chemical/scd30_core.c
index a665fcb..11d6bc1 100644
--- a/drivers/iio/chemical/scd30_core.c
+++ b/drivers/iio/chemical/scd30_core.c
@@ -256,7 +256,7 @@ static int scd30_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const
guard(mutex)(&state->lock);
switch (mask) {
case IIO_CHAN_INFO_SAMP_FREQ:
- if (val)
+ if (val || !val2)
return -EINVAL;
val = 1000000000 / val2;
diff --git a/drivers/iio/common/ssp_sensors/ssp_dev.c b/drivers/iio/common/ssp_sensors/ssp_dev.c
index da09c9f..e2538a8 100644
--- a/drivers/iio/common/ssp_sensors/ssp_dev.c
+++ b/drivers/iio/common/ssp_sensors/ssp_dev.c
@@ -590,6 +590,7 @@ static void ssp_remove(struct spi_device *spi)
ssp_clean_pending_list(data);
free_irq(data->spi->irq, data);
+ cancel_delayed_work_sync(&data->work_refresh);
timer_delete_sync(&data->wdt_timer);
cancel_work_sync(&data->work_wdt);
diff --git a/drivers/iio/dac/ad3530r.c b/drivers/iio/dac/ad3530r.c
index b97b460..d9db322 100644
--- a/drivers/iio/dac/ad3530r.c
+++ b/drivers/iio/dac/ad3530r.c
@@ -105,6 +105,12 @@ static const char * const ad3530r_powerdown_modes[] = {
"32kohm_to_gnd",
};
+static const char * const ad3531r_powerdown_modes[] = {
+ "500ohm_to_gnd",
+ "3.85kohm_to_gnd",
+ "16kohm_to_gnd",
+};
+
static int ad3530r_get_powerdown_mode(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan)
{
@@ -133,6 +139,13 @@ static const struct iio_enum ad3530r_powerdown_mode_enum = {
.set = ad3530r_set_powerdown_mode,
};
+static const struct iio_enum ad3531r_powerdown_mode_enum = {
+ .items = ad3531r_powerdown_modes,
+ .num_items = ARRAY_SIZE(ad3531r_powerdown_modes),
+ .get = ad3530r_get_powerdown_mode,
+ .set = ad3530r_set_powerdown_mode,
+};
+
static ssize_t ad3530r_get_dac_powerdown(struct iio_dev *indio_dev,
uintptr_t private,
const struct iio_chan_spec *chan,
@@ -276,7 +289,20 @@ static const struct iio_chan_spec_ext_info ad3530r_ext_info[] = {
{ }
};
-#define AD3530R_CHAN(_chan) \
+static const struct iio_chan_spec_ext_info ad3531r_ext_info[] = {
+ {
+ .name = "powerdown",
+ .shared = IIO_SEPARATE,
+ .read = ad3530r_get_dac_powerdown,
+ .write = ad3530r_set_dac_powerdown,
+ },
+ IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad3531r_powerdown_mode_enum),
+ IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE,
+ &ad3531r_powerdown_mode_enum),
+ { }
+};
+
+#define AD3530R_CHAN(_chan, _ext_info) \
{ \
.type = IIO_VOLTAGE, \
.indexed = 1, \
@@ -284,25 +310,25 @@ static const struct iio_chan_spec_ext_info ad3530r_ext_info[] = {
.output = 1, \
.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
BIT(IIO_CHAN_INFO_SCALE), \
- .ext_info = ad3530r_ext_info, \
+ .ext_info = _ext_info, \
}
static const struct iio_chan_spec ad3530r_channels[] = {
- AD3530R_CHAN(0),
- AD3530R_CHAN(1),
- AD3530R_CHAN(2),
- AD3530R_CHAN(3),
- AD3530R_CHAN(4),
- AD3530R_CHAN(5),
- AD3530R_CHAN(6),
- AD3530R_CHAN(7),
+ AD3530R_CHAN(0, ad3530r_ext_info),
+ AD3530R_CHAN(1, ad3530r_ext_info),
+ AD3530R_CHAN(2, ad3530r_ext_info),
+ AD3530R_CHAN(3, ad3530r_ext_info),
+ AD3530R_CHAN(4, ad3530r_ext_info),
+ AD3530R_CHAN(5, ad3530r_ext_info),
+ AD3530R_CHAN(6, ad3530r_ext_info),
+ AD3530R_CHAN(7, ad3530r_ext_info),
};
static const struct iio_chan_spec ad3531r_channels[] = {
- AD3530R_CHAN(0),
- AD3530R_CHAN(1),
- AD3530R_CHAN(2),
- AD3530R_CHAN(3),
+ AD3530R_CHAN(0, ad3531r_ext_info),
+ AD3530R_CHAN(1, ad3531r_ext_info),
+ AD3530R_CHAN(2, ad3531r_ext_info),
+ AD3530R_CHAN(3, ad3531r_ext_info),
};
static const struct ad3530r_chip_info ad3530_chip = {
diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c
index 4b18498..a7213bc 100644
--- a/drivers/iio/dac/ad5686.c
+++ b/drivers/iio/dac/ad5686.c
@@ -25,22 +25,37 @@ static const char * const ad5686_powerdown_modes[] = {
"three_state"
};
+static inline unsigned int ad5686_pd_mask_shift(const struct iio_chan_spec *chan)
+{
+ if (chan->channel == chan->address)
+ return chan->channel * 2;
+
+ /* one-hot encoding is used in dual/quad channel devices */
+ return __ffs(chan->address) * 2;
+}
+
static int ad5686_get_powerdown_mode(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan)
{
+ unsigned int shift = ad5686_pd_mask_shift(chan);
struct ad5686_state *st = iio_priv(indio_dev);
- return ((st->pwr_down_mode >> (chan->channel * 2)) & 0x3) - 1;
+ guard(mutex)(&st->lock);
+
+ return ((st->pwr_down_mode >> shift) & 0x3U) - 1;
}
static int ad5686_set_powerdown_mode(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan,
unsigned int mode)
{
+ unsigned int shift = ad5686_pd_mask_shift(chan);
struct ad5686_state *st = iio_priv(indio_dev);
- st->pwr_down_mode &= ~(0x3 << (chan->channel * 2));
- st->pwr_down_mode |= ((mode + 1) << (chan->channel * 2));
+ guard(mutex)(&st->lock);
+
+ st->pwr_down_mode &= ~(0x3U << shift);
+ st->pwr_down_mode |= (mode + 1) << shift;
return 0;
}
@@ -55,10 +70,12 @@ static const struct iio_enum ad5686_powerdown_mode_enum = {
static ssize_t ad5686_read_dac_powerdown(struct iio_dev *indio_dev,
uintptr_t private, const struct iio_chan_spec *chan, char *buf)
{
+ unsigned int shift = ad5686_pd_mask_shift(chan);
struct ad5686_state *st = iio_priv(indio_dev);
- return sysfs_emit(buf, "%d\n", !!(st->pwr_down_mask &
- (0x3 << (chan->channel * 2))));
+ guard(mutex)(&st->lock);
+
+ return sysfs_emit(buf, "%d\n", !!(st->pwr_down_mask & (0x3U << shift)));
}
static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
@@ -77,10 +94,12 @@ static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
if (ret)
return ret;
+ guard(mutex)(&st->lock);
+
if (readin)
- st->pwr_down_mask |= (0x3 << (chan->channel * 2));
+ st->pwr_down_mask |= 0x3U << ad5686_pd_mask_shift(chan);
else
- st->pwr_down_mask &= ~(0x3 << (chan->channel * 2));
+ st->pwr_down_mask &= ~(0x3U << ad5686_pd_mask_shift(chan));
switch (st->chip_info->regmap_type) {
case AD5310_REGMAP:
@@ -154,7 +173,7 @@ static int ad5686_write_raw(struct iio_dev *indio_dev,
switch (mask) {
case IIO_CHAN_INFO_RAW:
- if (val > (1 << chan->scan_type.realbits) || val < 0)
+ if (val >= (1 << chan->scan_type.realbits) || val < 0)
return -EINVAL;
mutex_lock(&st->lock);
@@ -460,7 +479,7 @@ int ad5686_probe(struct device *dev,
{
struct ad5686_state *st;
struct iio_dev *indio_dev;
- unsigned int val, ref_bit_msk;
+ unsigned int val, ref_bit_msk, shift;
bool has_external_vref;
u8 cmd;
int ret, i;
@@ -484,9 +503,18 @@ int ad5686_probe(struct device *dev,
has_external_vref = ret != -ENODEV;
st->vref_mv = has_external_vref ? ret / 1000 : st->chip_info->int_vref_mv;
+ /* Initialize masks to all ones provided the max shift (last channel) */
+ shift = ad5686_pd_mask_shift(&st->chip_info->channels[st->chip_info->num_channels - 1]);
+ st->pwr_down_mask = GENMASK(shift + 1, 0);
+ st->pwr_down_mode = GENMASK(shift + 1, 0);
+
/* Set all the power down mode for all channels to 1K pulldown */
- for (i = 0; i < st->chip_info->num_channels; i++)
- st->pwr_down_mode |= (0x01 << (i * 2));
+ for (i = 0; i < st->chip_info->num_channels; i++) {
+ shift = ad5686_pd_mask_shift(&st->chip_info->channels[i]);
+ st->pwr_down_mask &= ~(0x3U << shift); /* powered up state */
+ st->pwr_down_mode &= ~(0x3U << shift);
+ st->pwr_down_mode |= 0x01U << shift;
+ }
indio_dev->name = name;
indio_dev->info = &ad5686_info;
@@ -509,7 +537,7 @@ int ad5686_probe(struct device *dev,
break;
case AD5686_REGMAP:
cmd = AD5686_CMD_INTERNAL_REFER_SETUP;
- ref_bit_msk = 0;
+ ref_bit_msk = AD5686_REF_BIT_MSK;
break;
case AD5693_REGMAP:
cmd = AD5686_CMD_CONTROL_REG;
@@ -520,9 +548,9 @@ int ad5686_probe(struct device *dev,
return -EINVAL;
}
- val = (has_external_vref | ref_bit_msk);
+ val = has_external_vref ? ref_bit_msk : 0;
- ret = st->write(st, cmd, 0, !!val);
+ ret = st->write(st, cmd, 0, val);
if (ret)
return ret;
diff --git a/drivers/iio/dac/ad5686.h b/drivers/iio/dac/ad5686.h
index e7d36ba..36e16c5 100644
--- a/drivers/iio/dac/ad5686.h
+++ b/drivers/iio/dac/ad5686.h
@@ -46,6 +46,7 @@
#define AD5310_REF_BIT_MSK BIT(8)
#define AD5683_REF_BIT_MSK BIT(12)
+#define AD5686_REF_BIT_MSK BIT(0)
#define AD5693_REF_BIT_MSK BIT(12)
/**
diff --git a/drivers/iio/dac/max5821.c b/drivers/iio/dac/max5821.c
index e7e2935..dd4e354 100644
--- a/drivers/iio/dac/max5821.c
+++ b/drivers/iio/dac/max5821.c
@@ -90,6 +90,7 @@ static int max5821_sync_powerdown_mode(struct max5821_data *data,
const struct iio_chan_spec *chan)
{
u8 outbuf[2];
+ int ret;
outbuf[0] = MAX5821_EXTENDED_COMMAND_MODE;
@@ -103,7 +104,13 @@ static int max5821_sync_powerdown_mode(struct max5821_data *data,
else
outbuf[1] |= MAX5821_EXTENDED_POWER_UP;
- return i2c_master_send(data->client, outbuf, 2);
+ ret = i2c_master_send(data->client, outbuf, sizeof(outbuf));
+ if (ret < 0)
+ return ret;
+ if (ret != sizeof(outbuf))
+ return -EIO;
+
+ return 0;
}
static ssize_t max5821_write_dac_powerdown(struct iio_dev *indio_dev,
diff --git a/drivers/iio/gyro/adis16260.c b/drivers/iio/gyro/adis16260.c
index 586e6cf..91b9c5f 100644
--- a/drivers/iio/gyro/adis16260.c
+++ b/drivers/iio/gyro/adis16260.c
@@ -287,6 +287,9 @@ static int adis16260_write_raw(struct iio_dev *indio_dev,
addr = adis16260_addresses[chan->scan_index][1];
return adis_write_reg_16(adis, addr, val);
case IIO_CHAN_INFO_SAMP_FREQ:
+ if (val <= 0)
+ return -EINVAL;
+
if (spi_get_device_id(adis->spi)->driver_data)
t = 256 / val;
else
diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c
index cf97adf..87efa2c 100644
--- a/drivers/iio/gyro/itg3200_buffer.c
+++ b/drivers/iio/gyro/itg3200_buffer.c
@@ -34,7 +34,7 @@ static int itg3200_read_all_channels(struct i2c_client *i2c, __be16 *buf)
.addr = i2c->addr,
.flags = i2c->flags | I2C_M_RD,
.len = ITG3200_SCAN_ELEMENTS * sizeof(s16),
- .buf = (char *)&buf,
+ .buf = (char *)buf,
},
};
diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c
index 1f2af50..7567961 100644
--- a/drivers/iio/imu/adis16550.c
+++ b/drivers/iio/imu/adis16550.c
@@ -836,7 +836,7 @@ static irqreturn_t adis16550_trigger_handler(int irq, void *p)
u16 dummy;
bool valid;
struct iio_poll_func *pf = p;
- __be32 data[ADIS16550_MAX_SCAN_DATA] __aligned(8);
+ __be32 data[ADIS16550_MAX_SCAN_DATA] __aligned(8) = { };
struct iio_dev *indio_dev = pf->indio_dev;
struct adis16550 *st = iio_priv(indio_dev);
struct adis *adis = iio_device_get_drvdata(indio_dev);
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 5b28a3f..4829120 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -609,7 +609,7 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw)
* must be passed a buffer that is aligned to 8 bytes so
* as to allow insertion of a naturally aligned timestamp.
*/
- u8 iio_buff[ST_LSM6DSX_IIO_BUFF_SIZE] __aligned(8);
+ u8 iio_buff[ST_LSM6DSX_IIO_BUFF_SIZE] __aligned(8) = { };
u8 tag;
bool reset_ts = false;
int i, err, read_len;
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 46f36a6..5c3df99 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -1909,6 +1909,7 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib,
dma_resv_add_fence(dmabuf->resv, &fence->base,
dma_to_ram ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ);
+ dma_fence_put(&fence->base);
dma_resv_unlock(dmabuf->resv);
cookie = dma_fence_begin_signalling();
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index 0df0ab3..9ce20cb 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -738,7 +738,11 @@ int iio_read_channel_processed_scale(struct iio_channel *chan, int *val,
if (ret < 0)
return ret;
- return iio_multiply_value(val, scale, ret, pval, pval2);
+ ret = iio_multiply_value(val, scale, ret, pval, pval2);
+ if (ret < 0)
+ return ret;
+
+ return 0;
} else {
ret = iio_channel_read(chan, val, NULL, IIO_CHAN_INFO_RAW);
if (ret < 0)
diff --git a/drivers/iio/light/cm3323.c b/drivers/iio/light/cm3323.c
index 79ad6e2..0fe61b8 100644
--- a/drivers/iio/light/cm3323.c
+++ b/drivers/iio/light/cm3323.c
@@ -89,15 +89,14 @@ static int cm3323_init(struct iio_dev *indio_dev)
/* enable sensor and set auto force mode */
ret &= ~(CM3323_CONF_SD_BIT | CM3323_CONF_AF_BIT);
+ data->reg_conf = ret;
- ret = i2c_smbus_write_word_data(data->client, CM3323_CMD_CONF, ret);
+ ret = i2c_smbus_write_word_data(data->client, CM3323_CMD_CONF, data->reg_conf);
if (ret < 0) {
dev_err(&data->client->dev, "Error writing reg_conf\n");
return ret;
}
- data->reg_conf = ret;
-
return 0;
}
diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c
index 74d7246..4bbd86d 100644
--- a/drivers/iio/light/veml6070.c
+++ b/drivers/iio/light/veml6070.c
@@ -245,13 +245,6 @@ static const struct iio_info veml6070_info = {
.write_raw = veml6070_write_raw,
};
-static void veml6070_i2c_unreg(void *p)
-{
- struct veml6070_data *data = p;
-
- i2c_unregister_device(data->client2);
-}
-
static int veml6070_probe(struct i2c_client *client)
{
struct veml6070_data *data;
@@ -281,7 +274,8 @@ static int veml6070_probe(struct i2c_client *client)
if (ret < 0)
return ret;
- data->client2 = i2c_new_dummy_device(client->adapter, VEML6070_ADDR_DATA_LSB);
+ data->client2 = devm_i2c_new_dummy_device(&client->dev, client->adapter,
+ VEML6070_ADDR_DATA_LSB);
if (IS_ERR(data->client2))
return dev_err_probe(&client->dev, PTR_ERR(data->client2),
"i2c device for second chip address failed\n");
@@ -292,10 +286,6 @@ static int veml6070_probe(struct i2c_client *client)
if (ret < 0)
return ret;
- ret = devm_add_action_or_reset(&client->dev, veml6070_i2c_unreg, data);
- if (ret < 0)
- return ret;
-
return devm_iio_device_register(&client->dev, indio_dev);
}
diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c
index ef348d3..7644bd04 100644
--- a/drivers/iio/magnetometer/st_magn_core.c
+++ b/drivers/iio/magnetometer/st_magn_core.c
@@ -506,6 +506,11 @@ static const struct st_sensors_platform_data default_magn_pdata = {
.drdy_int_pin = 2,
};
+/* LIS2MDL only supports DRDY on INT1 */
+static const struct st_sensors_platform_data alt_magn_pdata = {
+ .drdy_int_pin = 1,
+};
+
static int st_magn_read_raw(struct iio_dev *indio_dev,
struct iio_chan_spec const *ch, int *val,
int *val2, long mask)
@@ -628,8 +633,12 @@ int st_magn_common_probe(struct iio_dev *indio_dev)
mdata->current_fullscale = &mdata->sensor_settings->fs.fs_avl[0];
mdata->odr = mdata->sensor_settings->odr.odr_avl[0].hz;
- if (!pdata)
- pdata = (struct st_sensors_platform_data *)&default_magn_pdata;
+ if (!pdata) {
+ if (mdata->sensor_settings->drdy_irq.int2.mask)
+ pdata = (struct st_sensors_platform_data *)&default_magn_pdata;
+ else
+ pdata = (struct st_sensors_platform_data *)&alt_magn_pdata;
+ }
err = st_sensors_init_sensor(indio_dev, pdata);
if (err < 0)
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index d983ce9..9b48976 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -2616,7 +2616,7 @@ static irqreturn_t bmp580_trigger_handler(int irq, void *p)
__le32 comp_temp;
__le32 comp_press;
aligned_s64 timestamp;
- } buffer;
+ } buffer = { };
int ret;
guard(mutex)(&data->lock);
diff --git a/drivers/iio/temperature/tsys01.c b/drivers/iio/temperature/tsys01.c
index 334bba6..104dd45 100644
--- a/drivers/iio/temperature/tsys01.c
+++ b/drivers/iio/temperature/tsys01.c
@@ -119,7 +119,7 @@ static bool tsys01_crc_valid(u16 *n_prom)
u8 sum = 0;
for (cnt = 0; cnt < TSYS01_PROM_WORDS_NB; cnt++)
- sum += ((n_prom[0] >> 8) + (n_prom[0] & 0xFF));
+ sum += ((n_prom[cnt] >> 8) + (n_prom[cnt] & 0xFF));
return (sum == 0);
}
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index a40a765..27992c3 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -149,7 +149,7 @@ static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,
attrtype = RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_IPV6;
}
- len = nla_total_size(sizeof(size));
+ len = nla_total_size(size);
len += NLMSG_ALIGN(sizeof(*header));
skb = nlmsg_new(len, GFP_KERNEL);
diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c
index 1f7a5c1..8a0e6fa 100644
--- a/drivers/infiniband/core/ib_core_uverbs.c
+++ b/drivers/infiniband/core/ib_core_uverbs.c
@@ -9,6 +9,7 @@
#include <linux/dma-resv.h>
#include "uverbs.h"
#include "core_priv.h"
+#include "rdma_core.h"
MODULE_IMPORT_NS("DMA_BUF");
@@ -416,3 +417,89 @@ struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
}
EXPORT_SYMBOL(rdma_udata_to_dev);
+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
+uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
+{
+ struct uverbs_attr_bundle *bundle =
+ rdma_udata_to_uverbs_attr_bundle(udata);
+
+ lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
+
+ return srcu_dereference(bundle->method_elm->handler,
+ &bundle->ufile->device->disassociate_srcu);
+}
+
+int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
+ size_t kernel_size, size_t minimum_size)
+{
+ int err;
+
+ if (udata->inlen < minimum_size) {
+ ibdev_dbg(
+ rdma_udata_to_dev(udata),
+ "System call driver input udata too small (%zu < %zu) for ioctl %ps called by %pSR\n",
+ udata->inlen, minimum_size,
+ uverbs_get_handler_fn(udata),
+ __builtin_return_address(0));
+ return -EINVAL;
+ }
+
+ err = copy_struct_from_user(req, kernel_size, udata->inbuf,
+ udata->inlen);
+ if (err) {
+ if (err == -E2BIG) {
+ ibdev_dbg(
+ rdma_udata_to_dev(udata),
+ "System call driver input udata not zero from %zu -> %zu for ioctl %ps called by %pSR\n",
+ minimum_size, udata->inlen,
+ uverbs_get_handler_fn(udata),
+ __builtin_return_address(0));
+ return -EOPNOTSUPP;
+ }
+ ibdev_dbg(
+ rdma_udata_to_dev(udata),
+ "System call driver input udata EFAULT for ioctl %ps called by %pSR\n",
+ uverbs_get_handler_fn(udata),
+ __builtin_return_address(0));
+ return err;
+ }
+ return 0;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_in);
+
+int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
+ u64 valid_cm)
+{
+ ibdev_dbg(
+ rdma_udata_to_dev(udata),
+ "System call driver input udata has unsupported comp_mask %llx & ~%llx = %llx for ioctl %ps called by %pSR\n",
+ req_cm, valid_cm, req_cm & ~valid_cm,
+ uverbs_get_handler_fn(udata), __builtin_return_address(0));
+ return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_cm_fail);
+
+int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
+{
+ size_t copy_len;
+
+ /* 0 length copy_len is a NOP for copy_to_user() and doesn't fail. */
+ copy_len = min(len, udata->outlen);
+ if (copy_to_user(udata->outbuf, src, copy_len))
+ goto err_fault;
+ if (copy_len < udata->outlen) {
+ if (clear_user(udata->outbuf + copy_len,
+ udata->outlen - copy_len))
+ goto err_fault;
+ }
+ return 0;
+err_fault:
+ ibdev_dbg(
+ rdma_udata_to_dev(udata),
+ "System call driver out udata has EFAULT (%zu into %zu) for ioctl %ps called by %pSR\n",
+ len, udata->outlen, uverbs_get_handler_fn(udata),
+ __builtin_return_address(0));
+ return -EFAULT;
+}
+EXPORT_SYMBOL(_ib_respond_udata);
+#endif
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 96c745d..5aaba2b 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -51,6 +51,7 @@
* a controlled QKEY.
*/
static bool privileged_qkey;
+static DEFINE_MUTEX(nldev_dellink_mutex);
typedef int (*res_fill_func_t)(struct sk_buff*, bool,
struct rdma_restrack_entry*, uint32_t);
@@ -1846,7 +1847,9 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
* implicitly scoped to the driver supporting dynamic link deletion like RXE.
*/
if (device->link_ops && device->link_ops->dellink) {
+ mutex_lock(&nldev_dellink_mutex);
err = device->link_ops->dellink(device);
+ mutex_unlock(&nldev_dellink_mutex);
if (err)
return err;
}
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 6d42952..f2e192b 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -229,6 +229,40 @@ int uverbs_dealloc_mw(struct ib_mw *mw);
void ib_uverbs_detach_umcast(struct ib_qp *qp,
struct ib_uqp_object *uobj);
+struct bundle_alloc_head {
+ struct_group_tagged(bundle_alloc_head_hdr, hdr,
+ struct bundle_alloc_head *next;
+ );
+ u8 data[];
+};
+
+struct bundle_priv {
+ /* Must be first */
+ struct bundle_alloc_head_hdr alloc_head;
+ struct bundle_alloc_head *allocated_mem;
+ size_t internal_avail;
+ size_t internal_used;
+
+ struct radix_tree_root *radix;
+ void __rcu **radix_slots;
+ unsigned long radix_slots_len;
+ u32 method_key;
+
+ struct ib_uverbs_attr __user *user_attrs;
+ struct ib_uverbs_attr *uattrs;
+
+ DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
+ DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
+ DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
+
+ /*
+ * Must be last. bundle ends in a flex array which overlaps
+ * internal_buffer.
+ */
+ struct uverbs_attr_bundle_hdr bundle;
+ u64 internal_buffer[32];
+};
+
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
struct ib_uverbs_flow_spec {
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index a768436..91a62d2 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -778,6 +778,7 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
struct ib_pd *orig_pd;
struct ib_pd *new_pd;
struct ib_mr *new_mr;
+ u32 lkey, rkey;
ret = uverbs_request(attrs, &cmd, sizeof(cmd));
if (ret)
@@ -846,6 +847,8 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
new_mr->uobject = uobj;
atomic_inc(&new_pd->usecnt);
new_uobj->object = new_mr;
+ lkey = new_mr->lkey;
+ rkey = new_mr->rkey;
rdma_restrack_new(&new_mr->res, RDMA_RESTRACK_MR);
rdma_restrack_set_name(&new_mr->res, NULL);
@@ -871,11 +874,13 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
mr->iova = cmd.hca_va;
mr->length = cmd.length;
}
+ lkey = mr->lkey;
+ rkey = mr->rkey;
}
memset(&resp, 0, sizeof(resp));
- resp.lkey = mr->lkey;
- resp.rkey = mr->rkey;
+ resp.lkey = lkey;
+ resp.rkey = rkey;
ret = uverbs_response(attrs, &resp, sizeof(resp));
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index b61af62..2552a7e 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -35,54 +35,6 @@
#include "rdma_core.h"
#include "uverbs.h"
-struct bundle_alloc_head {
- struct_group_tagged(bundle_alloc_head_hdr, hdr,
- struct bundle_alloc_head *next;
- );
- u8 data[];
-};
-
-struct bundle_priv {
- /* Must be first */
- struct bundle_alloc_head_hdr alloc_head;
- struct bundle_alloc_head *allocated_mem;
- size_t internal_avail;
- size_t internal_used;
-
- struct radix_tree_root *radix;
- const struct uverbs_api_ioctl_method *method_elm;
- void __rcu **radix_slots;
- unsigned long radix_slots_len;
- u32 method_key;
-
- struct ib_uverbs_attr __user *user_attrs;
- struct ib_uverbs_attr *uattrs;
-
- DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
- DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
- DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
-
- /*
- * Must be last. bundle ends in a flex array which overlaps
- * internal_buffer.
- */
- struct uverbs_attr_bundle_hdr bundle;
- u64 internal_buffer[32];
-};
-
-uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
-{
- struct uverbs_attr_bundle *bundle =
- rdma_udata_to_uverbs_attr_bundle(udata);
- struct bundle_priv *pbundle =
- container_of(&bundle->hdr, struct bundle_priv, bundle);
-
- lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
-
- return srcu_dereference(pbundle->method_elm->handler,
- &bundle->ufile->device->disassociate_srcu);
-}
-
/*
* Each method has an absolute minimum amount of memory it needs to allocate,
* precompute that amount and determine if the onstack memory can be used or
@@ -445,13 +397,13 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
struct uverbs_attr_bundle *bundle =
container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
- unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
+ unsigned int destroy_bkey = bundle->method_elm->destroy_bkey;
unsigned int i;
int ret;
/* See uverbs_disassociate_api() */
handler = srcu_dereference(
- pbundle->method_elm->handler,
+ bundle->method_elm->handler,
&pbundle->bundle.ufile->device->disassociate_srcu);
if (!handler)
return -EIO;
@@ -469,12 +421,12 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
}
/* User space did not provide all the mandatory attributes */
- if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
+ if (unlikely(!bitmap_subset(bundle->method_elm->attr_mandatory,
pbundle->bundle.attr_present,
- pbundle->method_elm->key_bitmap_len)))
+ bundle->method_elm->key_bitmap_len)))
return -EINVAL;
- if (pbundle->method_elm->has_udata)
+ if (bundle->method_elm->has_udata)
uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,
UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
else
@@ -499,7 +451,7 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
* assume that the driver wrote to its UHW_OUT and flag userspace
* appropriately.
*/
- if (!ret && pbundle->method_elm->has_udata) {
+ if (!ret && bundle->method_elm->has_udata) {
const struct uverbs_attr *attr =
uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);
@@ -520,7 +472,7 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
{
- unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
+ unsigned int key_bitmap_len = pbundle->bundle.method_elm->key_bitmap_len;
struct uverbs_attr_bundle *bundle =
container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
struct bundle_alloc_head *memblock;
@@ -608,7 +560,7 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
}
/* Space for the pbundle->bundle.attrs flex array */
- pbundle->method_elm = method_elm;
+ pbundle->bundle.method_elm = method_elm;
pbundle->method_key = attrs_iter.index;
pbundle->bundle.ufile = ufile;
pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
@@ -617,10 +569,12 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
pbundle->user_attrs = user_attrs;
- pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
- sizeof(*container_of(&pbundle->bundle,
- struct uverbs_attr_bundle, hdr)->attrs),
- sizeof(*pbundle->internal_buffer));
+ pbundle->internal_used = ALIGN(
+ pbundle->bundle.method_elm->key_bitmap_len *
+ sizeof(*container_of(&pbundle->bundle,
+ struct uverbs_attr_bundle, hdr)
+ ->attrs),
+ sizeof(*pbundle->internal_buffer));
memset(pbundle->bundle.attr_present, 0,
sizeof(pbundle->bundle.attr_present));
memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
@@ -860,77 +814,3 @@ void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
pbundle->uobj_hw_obj_valid);
}
EXPORT_SYMBOL(uverbs_finalize_uobj_create);
-
-int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
- size_t kernel_size, size_t minimum_size)
-{
- int err;
-
- if (udata->inlen < minimum_size) {
- ibdev_dbg(
- rdma_udata_to_dev(udata),
- "System call driver input udata too small (%zu < %zu) for ioctl %ps called by %pSR\n",
- udata->inlen, minimum_size,
- uverbs_get_handler_fn(udata),
- __builtin_return_address(0));
- return -EINVAL;
- }
-
- err = copy_struct_from_user(req, kernel_size, udata->inbuf,
- udata->inlen);
- if (err) {
- if (err == -E2BIG) {
- ibdev_dbg(
- rdma_udata_to_dev(udata),
- "System call driver input udata not zero from %zu -> %zu for ioctl %ps called by %pSR\n",
- minimum_size, udata->inlen,
- uverbs_get_handler_fn(udata),
- __builtin_return_address(0));
- return -EOPNOTSUPP;
- }
- ibdev_dbg(
- rdma_udata_to_dev(udata),
- "System call driver input udata EFAULT for ioctl %ps called by %pSR\n",
- uverbs_get_handler_fn(udata),
- __builtin_return_address(0));
- return err;
- }
- return 0;
-}
-EXPORT_SYMBOL(_ib_copy_validate_udata_in);
-
-int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
- u64 valid_cm)
-{
- ibdev_dbg(
- rdma_udata_to_dev(udata),
- "System call driver input udata has unsupported comp_mask %llx & ~%llx = %llx for ioctl %ps called by %pSR\n",
- req_cm, valid_cm, req_cm & ~valid_cm,
- uverbs_get_handler_fn(udata), __builtin_return_address(0));
- return -EOPNOTSUPP;
-}
-EXPORT_SYMBOL(_ib_copy_validate_udata_cm_fail);
-
-int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
-{
- size_t copy_len;
-
- /* 0 length copy_len is a NOP for copy_to_user() and doesn't fail. */
- copy_len = min(len, udata->outlen);
- if (copy_to_user(udata->outbuf, src, copy_len))
- goto err_fault;
- if (copy_len < udata->outlen) {
- if (clear_user(udata->outbuf + copy_len,
- udata->outlen - copy_len))
- goto err_fault;
- }
- return 0;
-err_fault:
- ibdev_dbg(
- rdma_udata_to_dev(udata),
- "System call driver out udata has EFAULT (%zu into %zu) for ioctl %ps called by %pSR\n",
- len, udata->outlen, uverbs_get_handler_fn(udata),
- __builtin_return_address(0));
- return -EFAULT;
-}
-EXPORT_SYMBOL(_ib_respond_udata);
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 7ed2945..365ec27 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -4638,7 +4638,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
uctx->rdev = rdev;
- uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
+ uctx->shpg = (void *)get_zeroed_page(GFP_KERNEL);
if (!uctx->shpg) {
rc = -ENOMEM;
goto fail;
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 51afaac..9121d83 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -1942,13 +1942,16 @@ int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts)
void free_pio_map(struct hfi1_devdata *dd)
{
+ struct pio_vl_map *map;
+
/* Free PIO map if allocated */
if (rcu_access_pointer(dd->pio_map)) {
spin_lock_irq(&dd->pio_map_lock);
- pio_map_free(rcu_access_pointer(dd->pio_map));
+ map = rcu_access_pointer(dd->pio_map);
RCU_INIT_POINTER(dd->pio_map, NULL);
spin_unlock_irq(&dd->pio_map_lock);
synchronize_rcu();
+ pio_map_free(map);
}
kfree(dd->kernel_send_context);
dd->kernel_send_context = NULL;
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index e5f4429..cfd9dd0 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -1255,6 +1255,7 @@ void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
{
size_t i;
struct sdma_engine *sde;
+ struct sdma_vl_map *map;
if (dd->sdma_pad_dma) {
dma_free_coherent(&dd->pcidev->dev, SDMA_PAD,
@@ -1291,10 +1292,11 @@ void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
}
if (rcu_access_pointer(dd->sdma_map)) {
spin_lock_irq(&dd->sde_map_lock);
- sdma_map_free(rcu_access_pointer(dd->sdma_map));
+ map = rcu_access_pointer(dd->sdma_map);
RCU_INIT_POINTER(dd->sdma_map, NULL);
spin_unlock_irq(&dd->sde_map_lock);
synchronize_rcu();
+ sdma_map_free(map);
}
kfree(dd->per_sdma);
dd->per_sdma = NULL;
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index a27ea85..bf04ee8 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -47,8 +47,8 @@ static struct hns_roce_qp *hns_roce_qp_lookup(struct hns_roce_dev *hr_dev,
xa_lock_irqsave(&hr_dev->qp_table_xa, flags);
qp = __hns_roce_qp_lookup(hr_dev, qpn);
- if (qp)
- refcount_inc(&qp->refcount);
+ if (qp && !refcount_inc_not_zero(&qp->refcount))
+ qp = NULL;
xa_unlock_irqrestore(&hr_dev->qp_table_xa, flags);
if (!qp)
@@ -1171,6 +1171,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
struct hns_roce_ib_create_qp_resp resp = {};
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_ib_create_qp ucmd = {};
+ unsigned long flags;
int ret;
mutex_init(&hr_qp->mutex);
@@ -1251,13 +1252,19 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
hr_qp->ibqp.qp_num = hr_qp->qpn;
hr_qp->event = hns_roce_ib_qp_event;
- refcount_set(&hr_qp->refcount, 1);
init_completion(&hr_qp->free);
+ refcount_set_release(&hr_qp->refcount, 1);
return 0;
err_flow_ctrl:
+ spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
+ hns_roce_lock_cqs(init_attr->send_cq ? to_hr_cq(init_attr->send_cq) : NULL,
+ init_attr->recv_cq ? to_hr_cq(init_attr->recv_cq) : NULL);
hns_roce_qp_remove(hr_dev, hr_qp);
+ hns_roce_unlock_cqs(init_attr->send_cq ? to_hr_cq(init_attr->send_cq) : NULL,
+ init_attr->recv_cq ? to_hr_cq(init_attr->recv_cq) : NULL);
+ spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
err_store:
free_qpc(hr_dev, hr_qp);
err_qpc:
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index cb848e8..8b94cbd 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -16,8 +16,8 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)
xa_lock(&srq_table->xa);
srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1));
- if (srq)
- refcount_inc(&srq->refcount);
+ if (srq && !refcount_inc_not_zero(&srq->refcount))
+ srq = NULL;
xa_unlock(&srq_table->xa);
if (!srq) {
@@ -470,6 +470,10 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
if (ret)
goto err_srqn;
+ srq->event = hns_roce_ib_srq_event;
+ init_completion(&srq->free);
+ refcount_set_release(&srq->refcount, 1);
+
if (udata) {
resp.cap_flags = srq->cap_flags;
resp.srqn = srq->srqn;
@@ -480,10 +484,6 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
}
}
- srq->event = hns_roce_ib_srq_event;
- refcount_set(&srq->refcount, 1);
- init_completion(&srq->free);
-
return 0;
err_srqc:
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c
index 0382a64..73a616a 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.c
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c
@@ -185,7 +185,7 @@ static ssize_t hca_type_show(struct device *device,
struct ionic_ibdev *dev =
rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev);
- return sysfs_emit(buf, "%s.64\n", dev->ibdev.node_desc);
+ return sysfs_emit(buf, "%.64s\n", dev->ibdev.node_desc);
}
static DEVICE_ATTR_RO(hca_type);
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index f4cbe21..2d68242 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -137,8 +137,9 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
if (cq->queue.id >= gc->max_num_cqs)
return -EINVAL;
- /* Create CQ table entry */
- WARN_ON(gc->cq_table[cq->queue.id]);
+ /* Create CQ table entry, sharing a CQ between WQs is not supported */
+ if (gc->cq_table[cq->queue.id])
+ return -EINVAL;
if (cq->queue.kmem)
gdma_cq = cq->queue.kmem;
else
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index ac5e75d..afc2fc1 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -606,6 +606,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
if (mana_ib_is_rnic(dev)) {
props->gid_tbl_len = 16;
props->ip_gids = true;
+ props->max_msg_sz = SZ_16M;
if (port == 1)
props->port_cap_flags = IB_PORT_CM_SUP;
}
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 6455813..0fbcf44 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -21,6 +21,9 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
gc = mdev_to_gc(dev);
+ if (rx_hash_key_len > sizeof(req->hashkey))
+ return -EINVAL;
+
req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE);
req = kzalloc(req_buf_size, GFP_KERNEL);
if (!req)
@@ -173,11 +176,8 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
&wq_spec, &cq_spec, &wq->rx_object);
- if (ret) {
- /* Do cleanup starting with index i-1 */
- i--;
+ if (ret)
goto fail;
- }
/* The GDMA regions are now owned by the WQ object */
wq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
@@ -197,8 +197,10 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
/* Create CQ table entry */
ret = mana_ib_install_cq_cb(mdev, cq);
- if (ret)
+ if (ret) {
+ mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
goto fail;
+ }
}
resp.num_entries = i;
@@ -215,13 +217,15 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
ibdev_dbg(&mdev->ib_dev,
"Failed to copy to udata create rss-qp, %d\n",
ret);
- goto fail;
+ goto err_disable_vport_rx;
}
kfree(mana_ind_table);
return 0;
+err_disable_vport_rx:
+ mana_disable_vport_rx(mpc);
fail:
while (i-- > 0) {
ibwq = ind_tbl->ind_tbl[i];
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 5b23e5f..7678407 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -194,13 +194,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,
if (udata)
if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
err = -EFAULT;
- goto err_wrid;
+ goto err_srq;
}
init_attr->attr.max_wr = srq->msrq.max - 1;
return 0;
+err_srq:
+ mlx4_srq_free(dev->dev, &srq->msrq);
err_wrid:
if (udata)
mlx4_ib_db_unmap_user(ucontext, &srq->db);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 109661c..6107828 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3310,7 +3310,7 @@ int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
* devr->c0 is set once, never changed until device unload.
* Avoid taking the mutex if initialization is already done.
*/
- if (devr->c0)
+ if (smp_load_acquire(&devr->c0))
return 0;
mutex_lock(&devr->cq_lock);
@@ -3336,7 +3336,7 @@ int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
}
devr->p0 = pd;
- devr->c0 = cq;
+ smp_store_release(&devr->c0, cq);
unlock:
mutex_unlock(&devr->cq_lock);
@@ -3354,7 +3354,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
* devr->s1 is set once, never changed until device unload.
* Avoid taking the mutex if initialization is already done.
*/
- if (devr->s1)
+ if (smp_load_acquire(&devr->s1))
return 0;
mutex_lock(&devr->srq_lock);
@@ -3392,10 +3392,11 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
"Couldn't create SRQ 1 for res init, err=%pe\n",
s1);
ib_destroy_srq(s0);
+ goto unlock;
}
devr->s0 = s0;
- devr->s1 = s1;
+ smp_store_release(&devr->s1, s1);
unlock:
mutex_unlock(&devr->srq_lock);
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 8f50e73..8fd0553 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1603,6 +1603,11 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
}
if (qp->rq.wqe_cnt) {
+ if (!rq->base.ubuffer.umem) {
+ err = -EINVAL;
+ goto err_destroy_sq;
+ }
+
rq->base.container_mibqp = qp;
if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
@@ -4692,7 +4697,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
struct mlx5_ib_modify_qp_resp resp = {};
struct mlx5_ib_qp *qp = to_mqp(ibqp);
- struct mlx5_ib_modify_qp ucmd;
+ struct mlx5_ib_modify_qp ucmd = {};
enum ib_qp_type qp_type;
enum ib_qp_state cur_state, new_state;
int err = -EINVAL;
diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c
index 146d03a..a7a4f94 100644
--- a/drivers/infiniband/hw/mlx5/qpc.c
+++ b/drivers/infiniband/hw/mlx5/qpc.c
@@ -314,7 +314,14 @@ int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,
xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, XA_ZERO_ENTRY, dct, 0);
return err;
}
- xa_erase_irq(&table->dct_xa, dct->mqp.qpn);
+
+ /*
+ * A race can occur where a concurrent create gets the same dctn
+ * (after hardware released it) and overwrites XA_ZERO_ENTRY with
+ * its new DCT before we reach here. In that case, we must not erase
+ * the entry as it now belongs to the new DCT.
+ */
+ xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, XA_ZERO_ENTRY, NULL, 0);
return 0;
}
diff --git a/drivers/infiniband/hw/mlx5/srq_cmd.c b/drivers/infiniband/hw/mlx5/srq_cmd.c
index 8b33853..c1a0881 100644
--- a/drivers/infiniband/hw/mlx5/srq_cmd.c
+++ b/drivers/infiniband/hw/mlx5/srq_cmd.c
@@ -683,7 +683,14 @@ int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0);
return err;
}
- xa_erase_irq(&table->array, srq->srqn);
+
+ /*
+ * A race can occur where a concurrent create gets the same srqn
+ * (after hardware released it) and overwrites XA_ZERO_ENTRY with
+ * its new SRQ before we reach here. In that case, we must not erase
+ * the entry as it now belongs to the new SRQ.
+ */
+ xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, NULL, 0);
mlx5_core_res_put(&srq->common);
wait_for_completion(&srq->common.free);
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 29488fb..f213947 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -147,7 +147,7 @@ int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev)
* UMR qp is set once, never changed until device unload.
* Avoid taking the mutex if initialization is already done.
*/
- if (dev->umrc.qp)
+ if (smp_load_acquire(&dev->umrc.qp))
return 0;
mutex_lock(&dev->umrc.init_lock);
@@ -185,7 +185,7 @@ int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev)
sema_init(&dev->umrc.sem, MAX_UMR_WR);
mutex_init(&dev->umrc.lock);
dev->umrc.state = MLX5_UMR_STATE_ACTIVE;
- dev->umrc.qp = qp;
+ smp_store_release(&dev->umrc.qp, qp);
mutex_unlock(&dev->umrc.init_lock);
return 0;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index c17e2a5..a88cc5d 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -215,7 +215,7 @@ static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
mutex_lock(&uctx->mm_list_lock);
list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
- if (len != mm->key.len && phy_addr != mm->key.phy_addr)
+ if (len != mm->key.len || phy_addr != mm->key.phy_addr)
continue;
list_del(&mm->entry);
@@ -233,7 +233,7 @@ static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
mutex_lock(&uctx->mm_list_lock);
list_for_each_entry(mm, &uctx->mm_head, entry) {
- if (len != mm->key.len && phy_addr != mm->key.phy_addr)
+ if (len != mm->key.len || phy_addr != mm->key.phy_addr)
continue;
found = true;
@@ -620,9 +620,9 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
ucopy_err:
if (pd->dpp_enabled)
- ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
+ ocrdma_del_mmap(uctx, dpp_page_addr, PAGE_SIZE);
dpp_map_err:
- ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
+ ocrdma_del_mmap(uctx, db_page_addr, db_page_size);
return status;
}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index bcd43dc..c7c2b41 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -322,7 +322,7 @@ int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
uresp.qp_tab_size = vdev->dsr->caps.max_qp;
ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
if (ret) {
- pvrdma_uar_free(vdev, &context->uar);
+ /* pvrdma_dealloc_ucontext() also frees the UAR */
pvrdma_dealloc_ucontext(&context->ibucontext);
return -EFAULT;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index f792147..2d5e701 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -330,6 +330,17 @@ void rxe_rcv(struct sk_buff *skb)
pkt->qp = NULL;
pkt->mask |= rxe_opcode[pkt->opcode].mask;
+ /*
+ * Unknown opcodes have a zero-initialized rxe_opcode[] entry, so
+ * both mask and length are 0. Reject them before any length math:
+ * rxe_icrc_hdr() would otherwise compute length - RXE_BTH_BYTES
+ * and pass the underflowed value to rxe_crc32(), producing an
+ * out-of-bounds read.
+ */
+ if (unlikely(!rxe_opcode[pkt->opcode].mask ||
+ !rxe_opcode[pkt->opcode].length))
+ goto drop;
+
if (unlikely(pkt->paylen < header_size(pkt) + bth_pad(pkt) +
RXE_ICRC_SIZE))
goto drop;
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 9faf8c0..9cb2f6f 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -540,7 +540,19 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
}
skip_check_range:
- if (pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) {
+ if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
+ /* IBA oA19-28: ATOMIC_WRITE payload is exactly 8 bytes.
+ * Reject any other length before the responder reads
+ * sizeof(u64) bytes from payload_addr(pkt); a shorter
+ * payload would read past the logical end of the packet
+ * into skb->head tailroom.
+ */
+ if (resid != sizeof(u64) || pktlen != sizeof(u64) ||
+ bth_pad(pkt)) {
+ state = RESPST_ERR_LENGTH;
+ goto err;
+ }
+ } else if (pkt->mask & RXE_WRITE_MASK) {
if (resid > mtu) {
if (pktlen != mtu || bth_pad(pkt)) {
state = RESPST_ERR_LENGTH;
diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c
index e8a88b3..34d03584 100644
--- a/drivers/infiniband/sw/siw/siw_qp_rx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_rx.c
@@ -1082,6 +1082,21 @@ static int siw_get_hdr(struct siw_rx_stream *srx)
}
/*
+ * Peer-controlled mpa_len must not underflow srx->fpdu_part_rem
+ * in siw_tcp_rx_data(); a negative value flows as a signed copy
+ * length into siw_check_mem() and skb_copy_bits().
+ */
+ if (unlikely(be16_to_cpu(c_hdr->mpa_len) + MPA_HDR_SIZE <
+ iwarp_pktinfo[opcode].hdr_len)) {
+ pr_warn_ratelimited("siw: short mpa_len %u for opcode %u (hdr_len %u)\n",
+ be16_to_cpu(c_hdr->mpa_len), opcode,
+ iwarp_pktinfo[opcode].hdr_len);
+ siw_init_terminate(rx_qp(srx), TERM_ERROR_LAYER_LLP,
+ LLP_ETYPE_MPA, LLP_ECODE_FPDU_START, 0);
+ return -EINVAL;
+ }
+
+ /*
* DDP/RDMAP header receive completed. Check if the current
* DDP segment starts a new RDMAP message or continues a previously
* started RDMAP message.
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 4026715..3e1e1e8 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1297,7 +1297,9 @@ static int ipoib_hard_header(struct sk_buff *skb,
return IPOIB_HARD_LEN;
}
-static void ipoib_set_mcast_list(struct net_device *dev)
+static void ipoib_set_rx_mode_async(struct net_device *dev,
+ struct netdev_hw_addr_list *uc,
+ struct netdev_hw_addr_list *mc)
{
struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -2160,7 +2162,7 @@ static const struct net_device_ops ipoib_netdev_ops_pf = {
.ndo_fix_features = ipoib_fix_features,
.ndo_start_xmit = ipoib_start_xmit,
.ndo_tx_timeout = ipoib_timeout,
- .ndo_set_rx_mode = ipoib_set_mcast_list,
+ .ndo_set_rx_mode_async = ipoib_set_rx_mode_async,
.ndo_get_iflink = ipoib_get_iflink,
.ndo_set_vf_link_state = ipoib_set_vf_link_state,
.ndo_get_vf_config = ipoib_get_vf_config,
@@ -2183,7 +2185,7 @@ static const struct net_device_ops ipoib_netdev_ops_vf = {
.ndo_fix_features = ipoib_fix_features,
.ndo_start_xmit = ipoib_start_xmit,
.ndo_tx_timeout = ipoib_timeout,
- .ndo_set_rx_mode = ipoib_set_mcast_list,
+ .ndo_set_rx_mode_async = ipoib_set_rx_mode_async,
.ndo_get_iflink = ipoib_get_iflink,
.ndo_get_stats64 = ipoib_get_stats,
.ndo_eth_ioctl = ipoib_ioctl,
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
index 51727c7..9dd9141 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
@@ -295,8 +295,8 @@ int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path)
put_kobj:
kobject_del(&srv_path->kobj);
destroy_root:
- kobject_put(&srv_path->kobj);
rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
+ kobject_put(&srv_path->kobj);
return err;
}
diff --git a/drivers/input/gameport/fm801-gp.c b/drivers/input/gameport/fm801-gp.c
index 423cccd..1e8c6c0 100644
--- a/drivers/input/gameport/fm801-gp.c
+++ b/drivers/input/gameport/fm801-gp.c
@@ -125,8 +125,8 @@ static void fm801_gp_remove(struct pci_dev *pci)
}
static const struct pci_device_id fm801_gp_id_table[] = {
- { PCI_VENDOR_ID_FORTEMEDIA, PCI_DEVICE_ID_FM801_GP, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
- { 0 }
+ { PCI_VDEVICE(FORTEMEDIA, PCI_DEVICE_ID_FM801_GP) },
+ { }
};
MODULE_DEVICE_TABLE(pci, fm801_gp_id_table);
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index 0549fdc..feb8f36 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -186,6 +186,10 @@ static const struct xpad_device {
{ 0x07ff, 0xffff, "Mad Catz GamePad", 0, XTYPE_XBOX360 },
{ 0x0b05, 0x1a38, "ASUS ROG RAIKIRI", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
{ 0x0b05, 0x1abb, "ASUS ROG RAIKIRI PRO", 0, XTYPE_XBOXONE },
+ { 0x0b05, 0x1c91, "ASUS ROG RAIKIRI II", 0, XTYPE_XBOX360 },
+ { 0x0b05, 0x1c92, "ASUS ROG RAIKIRI II WIRELESS", 0, XTYPE_XBOX360 },
+ { 0x0b05, 0x1c96, "ASUS ROG RAIKIRI II XBOX", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
+ { 0x0b05, 0x1d04, "ASUS ROG RAIKIRI II XBOX WIRELESS", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
{ 0x0c12, 0x0005, "Intec wireless", 0, XTYPE_XBOX },
{ 0x0c12, 0x8801, "Nyko Xbox Controller", 0, XTYPE_XBOX },
{ 0x0c12, 0x8802, "Zeroplus Xbox Controller", 0, XTYPE_XBOX },
@@ -391,6 +395,7 @@ static const struct xpad_device {
{ 0x3285, 0x0662, "Nacon Revolution5 Pro", 0, XTYPE_XBOX360 },
{ 0x3285, 0x0663, "Nacon Evol-X", 0, XTYPE_XBOXONE },
{ 0x3537, 0x1004, "GameSir T4 Kaleid", 0, XTYPE_XBOX360 },
+ { 0x3537, 0x100f, "GameSir Nova 2 Lite", 0, XTYPE_XBOX360 },
{ 0x3537, 0x1010, "GameSir G7 SE", 0, XTYPE_XBOXONE },
{ 0x3651, 0x1000, "CRKD SG", 0, XTYPE_XBOX360 },
{ 0x366c, 0x0005, "ByoWave Proteus Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE, FLAG_DELAY_INIT },
@@ -507,6 +512,7 @@ static const struct usb_device_id xpad_table[] = {
{ USB_DEVICE(0x0738, 0x4540) }, /* Mad Catz Beat Pad */
XPAD_XBOXONE_VENDOR(0x0738), /* Mad Catz FightStick TE 2 */
XPAD_XBOX360_VENDOR(0x07ff), /* Mad Catz Gamepad */
+ XPAD_XBOX360_VENDOR(0x0b05), /* ASUS controllers */
XPAD_XBOXONE_VENDOR(0x0b05), /* ASUS controllers */
XPAD_XBOX360_VENDOR(0x0c12), /* Zeroplus X-Box 360 controllers */
XPAD_XBOX360_VENDOR(0x0db0), /* Micro Star International X-Box 360 controllers */
@@ -1077,10 +1083,10 @@ static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char
input_report_key(dev, BTN_START, data[4] & BIT(2));
input_report_key(dev, BTN_SELECT, data[4] & BIT(3));
if (xpad->mapping & MAP_SHARE_BUTTON) {
- if (xpad->mapping & MAP_SHARE_OFFSET)
- input_report_key(dev, KEY_RECORD, data[len - 26] & BIT(0));
- else
- input_report_key(dev, KEY_RECORD, data[len - 18] & BIT(0));
+ u32 offset = (xpad->mapping & MAP_SHARE_OFFSET) ? 26 : 18;
+
+ if (len >= offset)
+ input_report_key(dev, KEY_RECORD, data[len - offset] & BIT(0));
}
/* buttons A,B,X,Y */
diff --git a/drivers/input/misc/atlas_btns.c b/drivers/input/misc/atlas_btns.c
index 47b3172..835ad45 100644
--- a/drivers/input/misc/atlas_btns.c
+++ b/drivers/input/misc/atlas_btns.c
@@ -60,11 +60,15 @@ static acpi_status acpi_atlas_button_handler(u32 function,
static int atlas_acpi_button_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
acpi_status status;
int i;
int err;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
input_dev = input_allocate_device();
if (!input_dev) {
pr_err("unable to allocate input device\n");
diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c
index 4c022a3..7a1cb93 100644
--- a/drivers/input/misc/ims-pcu.c
+++ b/drivers/input/misc/ims-pcu.c
@@ -1624,7 +1624,7 @@ static void ims_pcu_buffers_free(struct ims_pcu *pcu)
usb_kill_urb(pcu->urb_in);
usb_free_urb(pcu->urb_in);
- usb_free_coherent(pcu->udev, pcu->max_out_size,
+ usb_free_coherent(pcu->udev, pcu->max_in_size,
pcu->urb_in_buf, pcu->read_dma);
kfree(pcu->urb_out_buf);
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index fee1796..5cba02a 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -162,6 +162,9 @@ static int elan_get_fwinfo(u16 ic_type, u8 iap_version, u16 *validpage_count,
case 0x15:
*validpage_count = 1024;
break;
+ case 0x19:
+ *validpage_count = 2032;
+ break;
default:
/* unknown ic type clear value */
*validpage_count = 0;
@@ -645,6 +648,11 @@ static ssize_t elan_sysfs_update_fw(struct device *dev,
return error;
}
+ if (fw->size < data->fw_signature_address + sizeof(signature)) {
+ dev_err(dev, "firmware file too small\n");
+ return -EBADF;
+ }
+
/* Firmware file must match signature data */
fw_signature = &fw->data[data->fw_signature_address];
if (memcmp(fw_signature, signature, sizeof(signature)) != 0) {
diff --git a/drivers/input/mouse/elan_i2c_i2c.c b/drivers/input/mouse/elan_i2c_i2c.c
index a9057d1..88d4070 100644
--- a/drivers/input/mouse/elan_i2c_i2c.c
+++ b/drivers/input/mouse/elan_i2c_i2c.c
@@ -690,7 +690,7 @@ static int elan_i2c_finish_fw_update(struct i2c_client *client,
if (error) {
dev_err(dev, "device reset failed: %d\n", error);
} else if (!wait_for_completion_timeout(completion,
- msecs_to_jiffies(300))) {
+ msecs_to_jiffies(700))) {
dev_err(dev, "timeout waiting for device reset\n");
error = -ETIMEDOUT;
}
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 2607112..c70502e2 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -190,6 +190,7 @@ static const char * const smbus_pnp_ids[] = {
"LEN2044", /* L470 */
"LEN2054", /* E480 */
"LEN2055", /* E580 */
+ "LEN2058", /* E490 */
"LEN2068", /* T14 Gen 1 */
"SYN1221", /* TUXEDO InfinityBook Pro 14 v5 */
"SYN3003", /* HP EliteBook 850 G1 */
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index 87c6a10..f21bf28 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -275,8 +275,8 @@ struct mxt_cfg {
off_t raw_pos;
u8 *mem;
- size_t mem_size;
- int start_ofs;
+ u16 mem_size;
+ u16 start_ofs;
struct mxt_info info;
};
@@ -1473,7 +1473,7 @@ static int mxt_prepare_cfg_mem(struct mxt_data *data, struct mxt_cfg *cfg)
}
cfg->raw_pos += offset;
- if (i > mxt_obj_size(object))
+ if (i >= mxt_obj_size(object))
continue;
byte_offset = reg + i - cfg->start_ofs;
@@ -1627,6 +1627,13 @@ static int mxt_update_cfg(struct mxt_data *data, const struct firmware *fw)
cfg.start_ofs = MXT_OBJECT_START +
data->info->object_num * sizeof(struct mxt_object) +
MXT_INFO_CHECKSUM_SIZE;
+
+ if (data->mem_size <= cfg.start_ofs) {
+ dev_err(dev, "Memory size too small: %u < %u\n",
+ data->mem_size, cfg.start_ofs);
+ return -EINVAL;
+ }
+
cfg.mem_size = data->mem_size - cfg.start_ofs;
u8 *mem_buf __free(kfree) = cfg.mem = kzalloc(cfg.mem_size, GFP_KERNEL);
diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c
index daa2813..0bbacb5 100644
--- a/drivers/input/touchscreen/usbtouchscreen.c
+++ b/drivers/input/touchscreen/usbtouchscreen.c
@@ -1067,6 +1067,11 @@ static int nexio_read_data(struct usbtouch_usb *usbtouch, unsigned char *pkt)
if (x_len > 0xff)
x_len -= 0x80;
+ if (data_len > usbtouch->data_size - sizeof(*packet))
+ data_len = usbtouch->data_size - sizeof(*packet);
+ if (x_len > data_len)
+ x_len = data_len;
+
/* send ACK */
ret = usb_submit_urb(priv->ack, GFP_ATOMIC);
if (ret)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 1342e76..834d8fa 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -11,6 +11,9 @@
#include "amd_iommu_types.h"
+extern int amd_iommu_evtlog_size;
+extern int amd_iommu_pprlog_size;
+
irqreturn_t amd_iommu_int_thread(int irq, void *data);
irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data);
irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data);
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index c685d37..f9f7180 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -15,6 +15,7 @@
#include <linux/mutex.h>
#include <linux/msi.h>
#include <linux/list.h>
+#include <linux/sizes.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/iommufd.h>
@@ -141,7 +142,6 @@
#define MMIO_STATUS_GALOG_INT_MASK BIT(10)
/* event logging constants */
-#define EVENT_ENTRY_SIZE 0x10
#define EVENT_TYPE_SHIFT 28
#define EVENT_TYPE_MASK 0xf
#define EVENT_TYPE_ILL_DEV 0x1
@@ -259,15 +259,20 @@
#define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x))
/* constants for event buffer handling */
-#define EVT_BUFFER_SIZE 8192 /* 512 entries */
-#define EVT_LEN_MASK (0x9ULL << 56)
+#define EVTLOG_ENTRY_SIZE 0x10
+#define EVTLOG_SIZE_SHIFT 56
+#define EVTLOG_SIZE_DEF SZ_8K /* 512 entries */
+#define EVTLOG_LEN_MASK_DEF (0x9ULL << EVTLOG_SIZE_SHIFT)
+#define EVTLOG_SIZE_MAX SZ_512K /* 32K entries */
+#define EVTLOG_LEN_MASK_MAX (0xFULL << EVTLOG_SIZE_SHIFT)
/* Constants for PPR Log handling */
-#define PPR_LOG_ENTRIES 512
-#define PPR_LOG_SIZE_SHIFT 56
-#define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
-#define PPR_ENTRY_SIZE 16
-#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
+#define PPRLOG_ENTRY_SIZE 0x10
+#define PPRLOG_SIZE_SHIFT 56
+#define PPRLOG_SIZE_DEF SZ_8K /* 512 entries */
+#define PPRLOG_LEN_MASK_DEF (0x9ULL << PPRLOG_SIZE_SHIFT)
+#define PPRLOG_SIZE_MAX SZ_512K /* 32K entries */
+#define PPRLOG_LEN_MASK_MAX (0xFULL << PPRLOG_SIZE_SHIFT)
/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
#define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c
index 4e66473..4c53b63 100644
--- a/drivers/iommu/amd/debugfs.c
+++ b/drivers/iommu/amd/debugfs.c
@@ -31,11 +31,12 @@ static ssize_t iommu_mmio_write(struct file *filp, const char __user *ubuf,
if (cnt > OFS_IN_SZ)
return -EINVAL;
- ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_mmio_offset);
+ ret = kstrtos32_from_user(ubuf, cnt, 0, &dbg_mmio_offset);
if (ret)
return ret;
- if (dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64))
+ if (dbg_mmio_offset < 0 || dbg_mmio_offset >
+ iommu->mmio_phys_end - sizeof(u64))
return -EINVAL;
iommu->dbg_mmio_offset = dbg_mmio_offset;
@@ -71,12 +72,12 @@ static ssize_t iommu_capability_write(struct file *filp, const char __user *ubuf
if (cnt > OFS_IN_SZ)
return -EINVAL;
- ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_cap_offset);
+ ret = kstrtos32_from_user(ubuf, cnt, 0, &dbg_cap_offset);
if (ret)
return ret;
/* Capability register at offset 0x14 is the last IOMMU capability register. */
- if (dbg_cap_offset > 0x14)
+ if (dbg_cap_offset < 0 || dbg_cap_offset > 0x14)
return -EINVAL;
iommu->dbg_cap_offset = dbg_cap_offset;
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 56ad020..3bdb380 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -132,6 +132,9 @@ struct ivhd_entry {
u8 uid;
} __attribute__((packed));
+int amd_iommu_evtlog_size = EVTLOG_SIZE_DEF;
+int amd_iommu_pprlog_size = PPRLOG_SIZE_DEF;
+
/*
* An AMD IOMMU memory definition structure. It defines things like exclusion
* ranges for devices and regions that should be unity mapped.
@@ -865,35 +868,47 @@ void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
}
/* allocates the memory where the IOMMU will log its events to */
-static int __init alloc_event_buffer(struct amd_iommu *iommu)
+static int __init alloc_event_buffer(void)
{
- iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
- EVT_BUFFER_SIZE);
+ struct amd_iommu *iommu;
- return iommu->evt_buf ? 0 : -ENOMEM;
-}
-
-static void iommu_enable_event_buffer(struct amd_iommu *iommu)
-{
- u64 entry;
-
- BUG_ON(iommu->evt_buf == NULL);
-
- if (!is_kdump_kernel()) {
- /*
- * Event buffer is re-used for kdump kernel and setting
- * of MMIO register is not required.
- */
- entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
- memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
- &entry, sizeof(entry));
+ for_each_iommu(iommu) {
+ iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
+ amd_iommu_evtlog_size);
+ if (!iommu->evt_buf)
+ return -ENOMEM;
}
- /* set head and tail to zero manually */
- writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
- writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
+ return 0;
+}
- iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
+static void iommu_enable_event_buffer(void)
+{
+ struct amd_iommu *iommu;
+ u64 entry;
+
+ for_each_iommu(iommu) {
+ BUG_ON(iommu->evt_buf == NULL);
+
+ if (!is_kdump_kernel()) {
+ /*
+ * Event buffer is re-used for kdump kernel and setting
+ * of MMIO register is not required.
+ */
+ entry = iommu_virt_to_phys(iommu->evt_buf);
+ entry |= (amd_iommu_evtlog_size == EVTLOG_SIZE_DEF) ?
+ EVTLOG_LEN_MASK_DEF : EVTLOG_LEN_MASK_MAX;
+
+ memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
+ &entry, sizeof(entry));
+ }
+
+ /* set head and tail to zero manually */
+ writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
+ writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
+
+ iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
+ }
}
/*
@@ -984,15 +999,20 @@ static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
return 0;
}
-static int __init remap_event_buffer(struct amd_iommu *iommu)
+static int __init remap_event_buffer(void)
{
+ struct amd_iommu *iommu;
u64 paddr;
pr_info_once("Re-using event buffer from the previous kernel\n");
- paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK;
- iommu->evt_buf = iommu_memremap(paddr, EVT_BUFFER_SIZE);
+ for_each_iommu(iommu) {
+ paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK;
+ iommu->evt_buf = iommu_memremap(paddr, amd_iommu_evtlog_size);
+ if (!iommu->evt_buf)
+ return -ENOMEM;
+ }
- return iommu->evt_buf ? 0 : -ENOMEM;
+ return 0;
}
static int __init remap_command_buffer(struct amd_iommu *iommu)
@@ -1044,10 +1064,6 @@ static int __init alloc_iommu_buffers(struct amd_iommu *iommu)
ret = remap_command_buffer(iommu);
if (ret)
return ret;
-
- ret = remap_event_buffer(iommu);
- if (ret)
- return ret;
} else {
ret = alloc_cwwb_sem(iommu);
if (ret)
@@ -1056,10 +1072,6 @@ static int __init alloc_iommu_buffers(struct amd_iommu *iommu)
ret = alloc_command_buffer(iommu);
if (ret)
return ret;
-
- ret = alloc_event_buffer(iommu);
- if (ret)
- return ret;
}
return 0;
@@ -2893,7 +2905,6 @@ static void early_enable_iommu(struct amd_iommu *iommu)
iommu_init_flags(iommu);
iommu_set_device_table(iommu);
iommu_enable_command_buffer(iommu);
- iommu_enable_event_buffer(iommu);
iommu_set_exclusion_range(iommu);
iommu_enable_gt(iommu);
iommu_enable_ga(iommu);
@@ -2957,7 +2968,6 @@ static void early_enable_iommus(void)
iommu_disable_event_buffer(iommu);
iommu_disable_irtcachedis(iommu);
iommu_enable_command_buffer(iommu);
- iommu_enable_event_buffer(iommu);
iommu_enable_ga(iommu);
iommu_enable_xt(iommu);
iommu_enable_irtcachedis(iommu);
@@ -3070,6 +3080,7 @@ static void amd_iommu_resume(void *data)
for_each_iommu(iommu)
early_enable_iommu(iommu);
+ iommu_enable_event_buffer();
amd_iommu_enable_interrupts();
}
@@ -3399,6 +3410,33 @@ static __init void iommu_snp_enable(void)
#endif
}
+static void amd_iommu_apply_erratum_snp(void)
+{
+#ifdef CONFIG_KVM_AMD_SEV
+ if (!amd_iommu_snp_en)
+ return;
+
+ /* Errata fix for Family 0x19 */
+ if (boot_cpu_data.x86 != 0x19)
+ return;
+
+ /* Set event log buffer size to max */
+ amd_iommu_evtlog_size = EVTLOG_SIZE_MAX;
+ pr_info("Applying erratum: Increase Event log size to 0x%x\n",
+ amd_iommu_evtlog_size);
+
+ /*
+ * Set PPR log buffer size to max.
+ * (Family 0x19, model < 0x10 doesn't support PPR when SNP is enabled).
+ */
+ if (boot_cpu_data.x86_model >= 0x10) {
+ amd_iommu_pprlog_size = PPRLOG_SIZE_MAX;
+ pr_info("Applying erratum: Increase PPR log size to 0x%x\n",
+ amd_iommu_pprlog_size);
+ }
+#endif
+}
+
/****************************************************************************
*
* AMD IOMMU Initialization State Machine
@@ -3435,6 +3473,21 @@ static int __init state_next(void)
case IOMMU_ENABLED:
register_syscore(&amd_iommu_syscore);
iommu_snp_enable();
+
+ amd_iommu_apply_erratum_snp();
+
+ /* Allocate/enable event log buffer */
+ if (is_kdump_kernel())
+ ret = remap_event_buffer();
+ else
+ ret = alloc_event_buffer();
+
+ if (ret) {
+ init_state = IOMMU_INIT_ERROR;
+ break;
+ }
+ iommu_enable_event_buffer();
+
ret = amd_iommu_init_pci();
init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
break;
@@ -4037,11 +4090,11 @@ int amd_iommu_snp_disable(void)
return 0;
for_each_iommu(iommu) {
- ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE);
+ ret = iommu_make_shared(iommu->evt_buf, amd_iommu_evtlog_size);
if (ret)
return ret;
- ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE);
+ ret = iommu_make_shared(iommu->ppr_log, amd_iommu_pprlog_size);
if (ret)
return ret;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 0117136..57dc8fa 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -351,8 +351,12 @@ static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)
struct amd_iommu_pci_seg *pci_seg;
for_each_pci_segment(pci_seg) {
- if (pci_seg->id == seg)
- return pci_seg->rlookup_table[devid];
+ if (pci_seg->id != seg)
+ continue;
+ /* IVRS may not describe every device on the bus */
+ if (devid > pci_seg->last_bdf)
+ return NULL;
+ return pci_seg->rlookup_table[devid];
}
return NULL;
}
@@ -1010,7 +1014,7 @@ static void iommu_poll_events(struct amd_iommu *iommu)
iommu_print_event(iommu, iommu->evt_buf + head);
/* Update head pointer of hardware ring-buffer */
- head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
+ head = (head + EVTLOG_ENTRY_SIZE) % amd_iommu_evtlog_size;
writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
}
@@ -2149,7 +2153,8 @@ static void set_dte_passthrough(struct iommu_dev_data *dev_data,
new->data[0] |= DTE_FLAG_TV | DTE_FLAG_IR | DTE_FLAG_IW;
new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domain->id) |
- (dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0;
+ (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0);
+
}
static void set_dte_entry(struct amd_iommu *iommu,
diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c
index e6767c0..1f8d282 100644
--- a/drivers/iommu/amd/ppr.c
+++ b/drivers/iommu/amd/ppr.c
@@ -20,7 +20,7 @@
int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)
{
iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
- PPR_LOG_SIZE);
+ amd_iommu_pprlog_size);
return iommu->ppr_log ? 0 : -ENOMEM;
}
@@ -33,7 +33,9 @@ void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)
iommu_feature_enable(iommu, CONTROL_PPR_EN);
- entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
+ entry = iommu_virt_to_phys(iommu->ppr_log);
+ entry |= (amd_iommu_pprlog_size == PPRLOG_SIZE_DEF) ?
+ PPRLOG_LEN_MASK_DEF : PPRLOG_LEN_MASK_MAX;
memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
&entry, sizeof(entry));
@@ -201,7 +203,7 @@ void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
raw[0] = raw[1] = 0UL;
/* Update head pointer of hardware ring-buffer */
- head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
+ head = (head + PPRLOG_ENTRY_SIZE) % amd_iommu_pprlog_size;
writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
/* Handle PPR entry */
diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 19b6daf..dc91fb4 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -534,10 +534,12 @@ static int __map_range_leaf(struct pt_range *range, void *arg,
struct pt_state pts = pt_init(range, level, table);
struct pt_iommu_map_args *map = arg;
unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2;
+ unsigned int leaves_avail;
unsigned int start_index;
pt_oaddr_t oa = map->oa;
- unsigned int num_leaves;
+ pt_vaddr_t num_leaves;
unsigned int orig_end;
+ unsigned int step_lg2;
pt_vaddr_t last_va;
unsigned int step;
bool need_contig;
@@ -546,21 +548,25 @@ static int __map_range_leaf(struct pt_range *range, void *arg,
PT_WARN_ON(map->leaf_level != level);
PT_WARN_ON(!pt_can_have_leaf(&pts));
- step = log2_to_int_t(unsigned int,
- leaf_pgsize_lg2 - pt_table_item_lg2sz(&pts));
- need_contig = leaf_pgsize_lg2 != pt_table_item_lg2sz(&pts);
+ step_lg2 = leaf_pgsize_lg2 - pt_table_item_lg2sz(&pts);
+ step = log2_to_int_t(unsigned int, step_lg2);
+ need_contig = step_lg2 != 0;
_pt_iter_first(&pts);
start_index = pts.index;
orig_end = pts.end_index;
- if (pts.index + map->num_leaves < pts.end_index) {
+ leaves_avail =
+ log2_div_t(unsigned int, pts.end_index - pts.index, step_lg2);
+ if (map->num_leaves <= leaves_avail) {
/* Need to stop in the middle of the table to change sizes */
- pts.end_index = pts.index + map->num_leaves;
+ pts.end_index = pts.index + log2_mul(map->num_leaves, step_lg2);
num_leaves = 0;
} else {
- num_leaves = map->num_leaves - (pts.end_index - pts.index);
+ num_leaves = map->num_leaves - leaves_avail;
}
+ PT_WARN_ON(
+ log2_mod_t(unsigned int, pts.end_index - pts.index, step_lg2));
do {
pts.type = pt_load_entry_raw(&pts);
if (pts.type != PT_ENTRY_EMPTY || need_contig) {
@@ -920,8 +926,8 @@ static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
return ret;
/* Calculate target page size and level for the leaves */
- if (pt_has_system_page_size(common) && len == PAGE_SIZE) {
- PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE));
+ if (pt_has_system_page_size(common) && len == PAGE_SIZE &&
+ likely(pgsize_bitmap & PAGE_SIZE)) {
if (log2_mod(iova | paddr, PAGE_SHIFT))
return -ENXIO;
map.leaf_pgsize_lg2 = PAGE_SHIFT;
diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index be8410f..fdc8881 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -254,37 +254,29 @@ void cache_tag_unassign_domain(struct dmar_domain *domain,
}
static unsigned long calculate_psi_aligned_address(unsigned long start,
- unsigned long end,
- unsigned long *_mask)
+ unsigned long last,
+ unsigned long *size_order)
{
- unsigned long pages = aligned_nrpages(start, end - start + 1);
- unsigned long aligned_pages = __roundup_pow_of_two(pages);
- unsigned long bitmask = aligned_pages - 1;
- unsigned long mask = ilog2(aligned_pages);
- unsigned long pfn = IOVA_PFN(start);
+ unsigned int sz_lg2;
- /*
- * PSI masks the low order bits of the base address. If the
- * address isn't aligned to the mask, then compute a mask value
- * needed to ensure the target range is flushed.
- */
- if (unlikely(bitmask & pfn)) {
- unsigned long end_pfn = pfn + pages - 1, shared_bits;
-
+ /* Compute a sz_lg2 that spans start and last */
+ start &= GENMASK(BITS_PER_LONG - 1, VTD_PAGE_SHIFT);
+ sz_lg2 = fls_long(start ^ last);
+ if (sz_lg2 <= 12) {
+ *size_order = 0;
+ return start;
+ }
+ if (unlikely(sz_lg2 >= BITS_PER_LONG)) {
/*
- * Since end_pfn <= pfn + bitmask, the only way bits
- * higher than bitmask can differ in pfn and end_pfn is
- * by carrying. This means after masking out bitmask,
- * high bits starting with the first set bit in
- * shared_bits are all equal in both pfn and end_pfn.
+ * MAX_AGAW_PFN_WIDTH triggers full invalidation in all
+ * downstream users.
*/
- shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
- mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH;
+ *size_order = MAX_AGAW_PFN_WIDTH;
+ return 0;
}
- *_mask = mask;
-
- return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask);
+ *size_order = sz_lg2 - VTD_PAGE_SHIFT;
+ return start & GENMASK(BITS_PER_LONG - 1, sz_lg2);
}
static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)
@@ -441,12 +433,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
struct cache_tag *tag;
unsigned long flags;
- if (start == 0 && end == ULONG_MAX) {
- addr = 0;
- mask = MAX_AGAW_PFN_WIDTH;
- } else {
- addr = calculate_psi_aligned_address(start, end, &mask);
- }
+ addr = calculate_psi_aligned_address(start, end, &mask);
spin_lock_irqsave(&domain->cache_lock, flags);
list_for_each_entry(tag, &domain->cache_tags, node) {
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index c3d18cd..4d0e65b 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -3530,8 +3530,8 @@ void domain_remove_dev_pasid(struct iommu_domain *domain,
if (!domain)
return;
- /* Identity domain has no meta data for pasid. */
- if (domain->type == IOMMU_DOMAIN_IDENTITY)
+ /* Identity domain and blocked domain have no meta data for pasid. */
+ if (domain->type == IOMMU_DOMAIN_IDENTITY || domain->type == IOMMU_DOMAIN_BLOCKED)
return;
dmar_domain = to_dmar_domain(domain);
@@ -3545,12 +3545,13 @@ void domain_remove_dev_pasid(struct iommu_domain *domain,
}
spin_unlock_irqrestore(&dmar_domain->lock, flags);
+ if (WARN_ON_ONCE(!dev_pasid))
+ return;
+
cache_tag_unassign_domain(dmar_domain, dev, pasid);
domain_detach_iommu(dmar_domain, iommu);
- if (!WARN_ON_ONCE(!dev_pasid)) {
- intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
- kfree(dev_pasid);
- }
+ intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
+ kfree(dev_pasid);
}
static int blocking_domain_set_dev_pasid(struct iommu_domain *domain,
@@ -3937,6 +3938,9 @@ static void quirk_iommu_igfx(struct pci_dev *dev)
disable_igfx_iommu = 1;
}
+/* Q35 integrated gfx dmar support is totally busted. */
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x29b2, quirk_iommu_igfx);
+
/* G4x/GM45 integrated gfx dmar support is totally busted. */
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index 40e3325..1dbef8c 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -777,21 +777,27 @@ struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns = {
static struct io_pgtable_cfg *cfg_cookie __initdata;
-static void __init dummy_tlb_flush_all(void *cookie)
+/*
+ * __noipa prevents gcc from turning indirect iommu_flush_ops calls
+ * into direct calls from a specialized __arm_v7s_unmap() that triggers
+ * a build time section mismatch assertion.
+ */
+static __noipa void __init dummy_tlb_flush_all(void *cookie)
{
WARN_ON(cookie != cfg_cookie);
}
-static void __init dummy_tlb_flush(unsigned long iova, size_t size,
- size_t granule, void *cookie)
+static __noipa void __init dummy_tlb_flush(unsigned long iova, size_t size,
+ size_t granule, void *cookie)
{
WARN_ON(cookie != cfg_cookie);
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
}
-static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
- unsigned long iova, size_t granule,
- void *cookie)
+static __noipa void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
+ unsigned long iova,
+ size_t granule,
+ void *cookie)
{
dummy_tlb_flush(iova, granule, granule, cookie);
}
diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h
index ae9da4f..e9e605b 100644
--- a/drivers/iommu/iommu-pages.h
+++ b/drivers/iommu/iommu-pages.h
@@ -137,7 +137,7 @@ static inline void iommu_pages_flush_incoherent(struct device *dma_dev,
void *virt, size_t offset,
size_t len)
{
- dma_sync_single_for_device(dma_dev, (uintptr_t)virt + offset, len,
+ dma_sync_single_for_device(dma_dev, virt_to_phys(virt) + offset, len,
DMA_TO_DEVICE);
}
void iommu_pages_stop_incoherent_list(struct iommu_pages_list *list,
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 61c12ba..d1a9e71 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -62,14 +62,14 @@ struct iommu_group {
int id;
struct iommu_domain *default_domain;
struct iommu_domain *blocking_domain;
- /*
- * During a group device reset, @resetting_domain points to the physical
- * domain, while @domain points to the attached domain before the reset.
- */
- struct iommu_domain *resetting_domain;
struct iommu_domain *domain;
struct list_head entry;
unsigned int owner_cnt;
+ /*
+ * Number of devices in the group undergoing or awaiting recovery.
+ * If non-zero, concurrent domain attachments are rejected.
+ */
+ unsigned int recovery_cnt;
void *owner;
};
@@ -77,12 +77,33 @@ struct group_device {
struct list_head list;
struct device *dev;
char *name;
+ /*
+ * Device is blocked for a pending recovery while its group->domain is
+ * retained. This can happen when:
+ * - Device is undergoing a reset
+ */
+ bool blocked;
+ unsigned int reset_depth;
};
/* Iterate over each struct group_device in a struct iommu_group */
#define for_each_group_device(group, pos) \
list_for_each_entry(pos, &(group)->devices, list)
+static struct group_device *__dev_to_gdev(struct device *dev)
+{
+ struct iommu_group *group = dev->iommu_group;
+ struct group_device *gdev;
+
+ lockdep_assert_held(&group->mutex);
+
+ for_each_group_device(group, gdev) {
+ if (gdev->dev == dev)
+ return gdev;
+ }
+ return NULL;
+}
+
struct iommu_group_attribute {
struct attribute attr;
ssize_t (*show)(struct iommu_group *group, char *buf);
@@ -2196,6 +2217,8 @@ EXPORT_SYMBOL_GPL(iommu_attach_device);
int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
{
+ struct group_device *gdev;
+
/*
* This is called on the dma mapping fast path so avoid locking. This is
* racy, but we have an expectation that the driver will setup its DMAs
@@ -2206,14 +2229,18 @@ int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
guard(mutex)(&dev->iommu_group->mutex);
+ gdev = __dev_to_gdev(dev);
+ if (WARN_ON(!gdev))
+ return -ENODEV;
+
/*
- * This is a concurrent attach during a device reset. Reject it until
+ * This is a concurrent attach during device recovery. Reject it until
* pci_dev_reset_iommu_done() attaches the device to group->domain.
*
* Note that this might fail the iommu_dma_map(). But there's nothing
* more we can do here.
*/
- if (dev->iommu_group->resetting_domain)
+ if (gdev->blocked)
return -EBUSY;
return __iommu_attach_device(domain, dev, NULL);
}
@@ -2270,19 +2297,24 @@ EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
+ struct group_device *gdev;
lockdep_assert_held(&group->mutex);
+ gdev = __dev_to_gdev(dev);
+ if (WARN_ON(!gdev))
+ return NULL;
+
/*
* Driver handles the low-level __iommu_attach_device(), including the
* one invoked by pci_dev_reset_iommu_done() re-attaching the device to
* the cached group->domain. In this case, the driver must get the old
- * domain from group->resetting_domain rather than group->domain. This
+ * domain from group->blocking_domain rather than group->domain. This
* prevents it from re-attaching the device from group->domain (old) to
* group->domain (new).
*/
- if (group->resetting_domain)
- return group->resetting_domain;
+ if (gdev->blocked)
+ return group->blocking_domain;
return group->domain;
}
@@ -2441,10 +2473,11 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group,
return -EINVAL;
/*
- * This is a concurrent attach during a device reset. Reject it until
- * pci_dev_reset_iommu_done() attaches the device to group->domain.
+ * This is a concurrent attach during device recovery. Reject it until
+ * pci_dev_reset_iommu_done() attaches the device to group->domain, if
+ * IOMMU_SET_DOMAIN_MUST_SUCCEED is not set.
*/
- if (group->resetting_domain)
+ if (group->recovery_cnt && !(flags & IOMMU_SET_DOMAIN_MUST_SUCCEED))
return -EBUSY;
/*
@@ -2455,6 +2488,13 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group,
*/
result = 0;
for_each_group_device(group, gdev) {
+ /*
+ * Device under recovery is attached to group->blocking_domain.
+ * Don't change that. pci_dev_reset_iommu_done() will re-attach
+ * its domain to the updated group->domain, after the recovery.
+ */
+ if (gdev->blocked)
+ continue;
ret = __iommu_device_set_domain(group, gdev->dev, new_domain,
group->domain, flags);
if (ret) {
@@ -2575,27 +2615,16 @@ static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
static int __iommu_map_domain_pgtbl(struct iommu_domain *domain,
unsigned long iova, phys_addr_t paddr,
- size_t size, int prot, gfp_t gfp)
+ size_t size, int prot, gfp_t gfp,
+ size_t *mapped)
{
const struct iommu_domain_ops *ops = domain->ops;
- unsigned long orig_iova = iova;
unsigned int min_pagesz;
- size_t orig_size = size;
int ret = 0;
- might_sleep_if(gfpflags_allow_blocking(gfp));
-
- if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
- return -EINVAL;
-
- if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
+ if (WARN_ON(!ops->map_pages))
return -ENODEV;
- /* Discourage passing strange GFP flags */
- if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
- __GFP_HIGHMEM)))
- return -EINVAL;
-
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
@@ -2613,31 +2642,25 @@ static int __iommu_map_domain_pgtbl(struct iommu_domain *domain,
pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
while (size) {
- size_t pgsize, count, mapped = 0;
+ size_t pgsize, count, op_mapped = 0;
pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
iova, &paddr, pgsize, count);
ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
- gfp, &mapped);
+ gfp, &op_mapped);
/*
* Some pages may have been mapped, even if an error occurred,
* so we should account for those so they can be unmapped.
*/
- size -= mapped;
-
+ *mapped += op_mapped;
if (ret)
- break;
+ return ret;
- iova += mapped;
- paddr += mapped;
- }
-
- /* unroll mapping in case something went wrong */
- if (ret) {
- iommu_unmap(domain, orig_iova, orig_size - size);
- return ret;
+ size -= op_mapped;
+ iova += op_mapped;
+ paddr += op_mapped;
}
return 0;
}
@@ -2655,25 +2678,31 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
struct pt_iommu *pt = iommupt_from_domain(domain);
+ size_t mapped = 0;
int ret;
- if (pt) {
- size_t mapped = 0;
+ might_sleep_if(gfpflags_allow_blocking(gfp));
+ /* Discourage passing strange GFP flags or illegal domains */
+ if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING) ||
+ !domain->pgsize_bitmap ||
+ (gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
+ __GFP_HIGHMEM))))
+ return -EINVAL;
+
+ if (pt)
ret = pt->ops->map_range(pt, iova, paddr, size, prot, gfp,
&mapped);
- if (ret) {
- iommu_unmap(domain, iova, mapped);
- return ret;
- }
- return 0;
- }
- ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp);
- if (!ret)
- return ret;
+ else
+ ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot,
+ gfp, &mapped);
- trace_map(iova, paddr, size);
- iommu_debug_map(domain, paddr, size);
+ trace_map(iova, paddr, mapped);
+ iommu_debug_map(domain, paddr, mapped);
+ if (ret) {
+ iommu_unmap(domain, iova, mapped);
+ return ret;
+ }
return 0;
}
@@ -2702,10 +2731,7 @@ __iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova,
size_t unmapped_page, unmapped = 0;
unsigned int min_pagesz;
- if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
- return 0;
-
- if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
+ if (WARN_ON(!ops->unmap_pages))
return 0;
/* find out the minimum page size supported */
@@ -2724,8 +2750,6 @@ __iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova,
pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
- iommu_debug_unmap_begin(domain, iova, size);
-
/*
* Keep iterating until we either unmap 'size' bytes (or more)
* or we hit an area that isn't mapped.
@@ -2761,6 +2785,12 @@ static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova,
struct pt_iommu *pt = iommupt_from_domain(domain);
size_t unmapped;
+ if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING) ||
+ !domain->pgsize_bitmap))
+ return 0;
+
+ iommu_debug_unmap_begin(domain, iova, size);
+
if (pt)
unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather);
else
@@ -3570,7 +3600,12 @@ static void __iommu_remove_group_pasid(struct iommu_group *group,
struct group_device *device;
for_each_group_device(group, device) {
- if (device->dev->iommu->max_pasids > 0)
+ /*
+ * A group-level detach cannot fail, even if there is a blocked
+ * device. In fact, blocked devices must be already detached for
+ * a pending device recovery.
+ */
+ if (!device->blocked && device->dev->iommu->max_pasids > 0)
iommu_remove_dev_pasid(device->dev, pasid, domain);
}
}
@@ -3615,10 +3650,10 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
mutex_lock(&group->mutex);
/*
- * This is a concurrent attach during a device reset. Reject it until
+ * This is a concurrent attach during device recovery. Reject it until
* pci_dev_reset_iommu_done() attaches the device to group->domain.
*/
- if (group->resetting_domain) {
+ if (group->recovery_cnt) {
ret = -EBUSY;
goto out_unlock;
}
@@ -3708,10 +3743,10 @@ int iommu_replace_device_pasid(struct iommu_domain *domain,
mutex_lock(&group->mutex);
/*
- * This is a concurrent attach during a device reset. Reject it until
+ * This is a concurrent attach during device recovery. Reject it until
* pci_dev_reset_iommu_done() attaches the device to group->domain.
*/
- if (group->resetting_domain) {
+ if (group->recovery_cnt) {
ret = -EBUSY;
goto out_unlock;
}
@@ -3982,12 +4017,12 @@ EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
* routine wants to block any IOMMU activity: translation and ATS invalidation.
*
* This function attaches the device's RID/PASID(s) the group->blocking_domain,
- * setting the group->resetting_domain. This allows the IOMMU driver pausing any
+ * incrementing the group->recovery_cnt, to allow the IOMMU driver pausing any
* IOMMU activity while leaving the group->domain pointer intact. Later when the
* reset is finished, pci_dev_reset_iommu_done() can restore everything.
*
* Caller must use pci_dev_reset_iommu_prepare() with pci_dev_reset_iommu_done()
- * before/after the core-level reset routine, to unset the resetting_domain.
+ * before/after the core-level reset routine, to decrement the recovery_cnt.
*
* Return: 0 on success or negative error code if the preparation failed.
*
@@ -4000,6 +4035,7 @@ EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
{
struct iommu_group *group = pdev->dev.iommu_group;
+ struct group_device *gdev;
unsigned long pasid;
void *entry;
int ret;
@@ -4009,45 +4045,99 @@ int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
guard(mutex)(&group->mutex);
- /* Re-entry is not allowed */
- if (WARN_ON(group->resetting_domain))
- return -EBUSY;
+ gdev = __dev_to_gdev(&pdev->dev);
+ if (WARN_ON(!gdev))
+ return -ENODEV;
+
+ if (gdev->reset_depth++)
+ return 0;
ret = __iommu_group_alloc_blocking_domain(group);
- if (ret)
+ if (ret) {
+ gdev->reset_depth--;
return ret;
+ }
/* Stage RID domain at blocking_domain while retaining group->domain */
if (group->domain != group->blocking_domain) {
ret = __iommu_attach_device(group->blocking_domain, &pdev->dev,
group->domain);
- if (ret)
+ if (ret) {
+ gdev->reset_depth--;
return ret;
+ }
}
/*
+ * Update gdev->blocked upon the domain change, as it is used to return
+ * the correct domain in iommu_driver_get_domain_for_dev() that might be
+ * called in a set_dev_pasid callback function.
+ */
+ gdev->blocked = true;
+
+ /*
* Stage PASID domains at blocking_domain while retaining pasid_array.
*
* The pasid_array is mostly fenced by group->mutex, except one reader
* in iommu_attach_handle_get(), so it's safe to read without xa_lock.
*/
- xa_for_each_start(&group->pasid_array, pasid, entry, 1)
- iommu_remove_dev_pasid(&pdev->dev, pasid,
- pasid_array_entry_to_domain(entry));
+ if (pdev->dev.iommu->max_pasids > 0) {
+ xa_for_each_start(&group->pasid_array, pasid, entry, 1) {
+ struct iommu_domain *pasid_dom =
+ pasid_array_entry_to_domain(entry);
- group->resetting_domain = group->blocking_domain;
+ iommu_remove_dev_pasid(&pdev->dev, pasid, pasid_dom);
+ }
+ }
+
+ group->recovery_cnt++;
return ret;
}
EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare);
+static int __group_device_cmp_dma_alias(struct pci_dev *dev, u16 alias,
+ void *data)
+{
+ return alias == *(u16 *)data;
+}
+
+static int group_device_cmp_dma_alias(struct pci_dev *dev, u16 alias,
+ void *data)
+{
+ return pci_for_each_dma_alias(data, __group_device_cmp_dma_alias,
+ &alias);
+}
+
+static bool group_device_dma_alias_is_blocked(struct iommu_group *group,
+ struct group_device *gdev)
+{
+ struct group_device *sibling;
+
+ lockdep_assert_held(&group->mutex);
+
+ if (!dev_is_pci(gdev->dev))
+ return false;
+
+ for_each_group_device(group, sibling) {
+ if (sibling == gdev || !sibling->blocked ||
+ !dev_is_pci(sibling->dev))
+ continue;
+ if (pci_for_each_dma_alias(to_pci_dev(gdev->dev),
+ group_device_cmp_dma_alias,
+ to_pci_dev(sibling->dev)))
+ return true;
+ }
+ return false;
+}
+
/**
* pci_dev_reset_iommu_done() - Restore IOMMU after a PCI device reset is done
* @pdev: PCI device that has finished a reset routine
*
* After a PCIe device finishes a reset routine, it wants to restore its IOMMU
- * IOMMU activity, including new translation as well as cache invalidation, by
- * re-attaching all RID/PASID of the device's back to the domains retained in
- * the core-level structure.
+ * activity, including new translation and cache invalidation, by re-attaching
+ * all RID/PASID of the device back to the domains retained in the core-level
+ * structure.
*
* Caller must pair it with a successful pci_dev_reset_iommu_prepare().
*
@@ -4057,6 +4147,7 @@ EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare);
void pci_dev_reset_iommu_done(struct pci_dev *pdev)
{
struct iommu_group *group = pdev->dev.iommu_group;
+ struct group_device *gdev;
unsigned long pasid;
void *entry;
@@ -4065,32 +4156,70 @@ void pci_dev_reset_iommu_done(struct pci_dev *pdev)
guard(mutex)(&group->mutex);
- /* pci_dev_reset_iommu_prepare() was bypassed for the device */
- if (!group->resetting_domain)
+ gdev = __dev_to_gdev(&pdev->dev);
+ if (WARN_ON(!gdev))
return;
- /* pci_dev_reset_iommu_prepare() was not successfully called */
+ /* Unbalanced done() calls would underflow the counter */
+ if (WARN_ON(gdev->reset_depth == 0))
+ return;
+ if (--gdev->reset_depth)
+ return;
+
if (WARN_ON(!group->blocking_domain))
return;
- /* Re-attach RID domain back to group->domain */
- if (group->domain != group->blocking_domain) {
+ if (group_device_dma_alias_is_blocked(group, gdev)) {
+ /*
+ * FIXME: DMA aliased devices share the same RID, which would be
+ * convoluted to handle, as "gdev->blocked" is not sufficient:
+ * - "blocked" state is effectively shared across these devices
+ * - if the core skipped the blocking on the second device, the
+ * IOMMU driver's attachment state would diverge from the HW
+ * state
+ * For now, just warn and see whether real ATS use cases hit it.
+ */
+ pci_warn(pdev,
+ "DMA-aliased sibling may be prematurely unblocked\n");
+ }
+
+ /*
+ * Re-attach RID domain back to group->domain
+ *
+ * Leave the device parked in the blocking_domain if group->domain isn't
+ * initialized yet
+ */
+ if (group->domain && group->domain != group->blocking_domain) {
WARN_ON(__iommu_attach_device(group->domain, &pdev->dev,
group->blocking_domain));
}
/*
+ * Update gdev->blocked upon the domain change, as it is used to return
+ * the correct domain in iommu_driver_get_domain_for_dev() that might be
+ * called in a set_dev_pasid callback function.
+ */
+ gdev->blocked = false;
+
+ /*
* Re-attach PASID domains back to the domains retained in pasid_array.
*
* The pasid_array is mostly fenced by group->mutex, except one reader
* in iommu_attach_handle_get(), so it's safe to read without xa_lock.
*/
- xa_for_each_start(&group->pasid_array, pasid, entry, 1)
- WARN_ON(__iommu_set_group_pasid(
- pasid_array_entry_to_domain(entry), group, pasid,
- group->blocking_domain));
+ if (pdev->dev.iommu->max_pasids > 0) {
+ xa_for_each_start(&group->pasid_array, pasid, entry, 1) {
+ struct iommu_domain *pasid_dom =
+ pasid_array_entry_to_domain(entry);
- group->resetting_domain = NULL;
+ WARN_ON(pasid_dom->ops->set_dev_pasid(
+ pasid_dom, &pdev->dev, pasid,
+ group->blocking_domain));
+ }
+ }
+
+ if (!WARN_ON(group->recovery_cnt == 0))
+ group->recovery_cnt--;
}
EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_done);
diff --git a/drivers/irqchip/.kunitconfig b/drivers/irqchip/.kunitconfig
new file mode 100644
index 0000000..00a1270
--- /dev/null
+++ b/drivers/irqchip/.kunitconfig
@@ -0,0 +1,5 @@
+CONFIG_KUNIT=y
+CONFIG_OF=y
+CONFIG_COMPILE_TEST=y
+CONFIG_ASPEED_AST2700_INTC=y
+CONFIG_ASPEED_AST2700_INTC_TEST=y
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index e755a2a..fdf27cf 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -110,6 +110,29 @@
help
Support Amazon's Annapurna Labs Fabric Interrupt Controller.
+config ASPEED_AST2700_INTC
+ bool "ASPEED AST2700 Interrupt Controller support"
+ depends on OF
+ depends on ARCH_ASPEED || COMPILE_TEST
+ select IRQ_DOMAIN_HIERARCHY
+ help
+ Enable support for the ASPEED AST2700 interrupt controller.
+ This driver handles interrupt, routing and merged interrupt
+ sources to upstream parent interrupt controllers.
+
+ If unsure, say N.
+
+config ASPEED_AST2700_INTC_TEST
+ bool "Tests for the ASPEED AST2700 Interrupt Controller"
+ depends on ASPEED_AST2700_INTC && KUNIT=y
+ default KUNIT_ALL_TESTS
+ help
+ Enable KUnit tests for AST2700 INTC route resolution.
+ The tests exercise error handling and route selection paths.
+ This option is intended for test builds.
+
+ If unsure, say N.
+
config ATMEL_AIC_IRQ
bool
select GENERIC_IRQ_CHIP
@@ -476,7 +499,7 @@
select GENERIC_IRQ_CHIP
config QCOM_IRQ_COMBINER
- bool "QCOM IRQ combiner support"
+ bool "Qualcomm IRQ combiner support"
depends on ARCH_QCOM && ACPI
select IRQ_DOMAIN_HIERARCHY
help
@@ -509,7 +532,7 @@
for Goldfish based virtual platforms.
config QCOM_PDC
- tristate "QCOM PDC"
+ tristate "Qualcomm PDC"
depends on ARCH_QCOM
select IRQ_DOMAIN_HIERARCHY
help
@@ -517,7 +540,7 @@
IRQs for Qualcomm Technologies Inc (QTI) mobile chips.
config QCOM_MPM
- tristate "QCOM MPM"
+ tristate "Qualcomm MPM"
depends on ARCH_QCOM
depends on MAILBOX
select IRQ_DOMAIN_HIERARCHY
@@ -654,13 +677,13 @@
select IRQ_DOMAIN_HIERARCHY
select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
-config STARFIVE_JH8100_INTC
- bool "StarFive JH8100 External Interrupt Controller"
+config STARFIVE_JHB100_INTC
+ bool "StarFive JHB100 External Interrupt Controller"
depends on ARCH_STARFIVE || COMPILE_TEST
default ARCH_STARFIVE
select IRQ_DOMAIN_HIERARCHY
help
- This enables support for the INTC chip found in StarFive JH8100
+ This enables support for the INTC chip found in StarFive JHB100
SoC.
If you don't know what to do here, say Y.
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 26aa3b6..72cdcc9 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -89,8 +89,9 @@
obj-$(CONFIG_MVEBU_SEI) += irq-mvebu-sei.o
obj-$(CONFIG_LS_EXTIRQ) += irq-ls-extirq.o
obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o
+obj-$(CONFIG_ASPEED_AST2700_INTC) += irq-ast2700.o irq-ast2700-intc0.o irq-ast2700-intc1.o
+obj-$(CONFIG_ASPEED_AST2700_INTC_TEST) += irq-ast2700-intc0-test.o
obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o irq-aspeed-i2c-ic.o irq-aspeed-scu-ic.o
-obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-intc.o
obj-$(CONFIG_STM32MP_EXTI) += irq-stm32mp-exti.o
obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o
obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o
@@ -108,7 +109,7 @@
obj-$(CONFIG_RISCV_IMSIC) += irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o
obj-$(CONFIG_RISCV_RPMI_SYSMSI) += irq-riscv-rpmi-sysmsi.o
obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o
-obj-$(CONFIG_STARFIVE_JH8100_INTC) += irq-starfive-jh8100-intc.o
+obj-$(CONFIG_STARFIVE_JHB100_INTC) += irq-starfive-jhb100-intc.o
obj-$(CONFIG_ACLINT_SSWI) += irq-aclint-sswi.o
obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o
obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o
@@ -119,7 +120,7 @@
obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o
obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o
obj-$(CONFIG_TI_PRUSS_INTC) += irq-pruss-intc.o
-obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o irq-loongarch-avec.o
+obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o irq-loongarch-avec.o irq-loongarch-ir.o
obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o
obj-$(CONFIG_LOONGSON_EIOINTC) += irq-loongson-eiointc.o
obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o
diff --git a/drivers/irqchip/exynos-combiner.c b/drivers/irqchip/exynos-combiner.c
index 11d1054..d9d408c 100644
--- a/drivers/irqchip/exynos-combiner.c
+++ b/drivers/irqchip/exynos-combiner.c
@@ -24,8 +24,6 @@
#define IRQ_IN_COMBINER 8
-static DEFINE_SPINLOCK(irq_controller_lock);
-
struct combiner_chip_data {
unsigned int hwirq_offset;
unsigned int irq_mask;
@@ -72,9 +70,7 @@ static void combiner_handle_cascade_irq(struct irq_desc *desc)
chained_irq_enter(chip, desc);
- spin_lock(&irq_controller_lock);
status = readl_relaxed(chip_data->base + COMBINER_INT_STATUS);
- spin_unlock(&irq_controller_lock);
status &= chip_data->irq_mask;
if (status == 0)
diff --git a/drivers/irqchip/irq-aspeed-intc.c b/drivers/irqchip/irq-aspeed-intc.c
deleted file mode 100644
index 4fb0dd8..0000000
--- a/drivers/irqchip/irq-aspeed-intc.c
+++ /dev/null
@@ -1,139 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Aspeed Interrupt Controller.
- *
- * Copyright (C) 2023 ASPEED Technology Inc.
- */
-
-#include <linux/bitops.h>
-#include <linux/irq.h>
-#include <linux/irqchip.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/io.h>
-#include <linux/spinlock.h>
-
-#define INTC_INT_ENABLE_REG 0x00
-#define INTC_INT_STATUS_REG 0x04
-#define INTC_IRQS_PER_WORD 32
-
-struct aspeed_intc_ic {
- void __iomem *base;
- raw_spinlock_t gic_lock;
- raw_spinlock_t intc_lock;
- struct irq_domain *irq_domain;
-};
-
-static void aspeed_intc_ic_irq_handler(struct irq_desc *desc)
-{
- struct aspeed_intc_ic *intc_ic = irq_desc_get_handler_data(desc);
- struct irq_chip *chip = irq_desc_get_chip(desc);
-
- chained_irq_enter(chip, desc);
-
- scoped_guard(raw_spinlock, &intc_ic->gic_lock) {
- unsigned long bit, status;
-
- status = readl(intc_ic->base + INTC_INT_STATUS_REG);
- for_each_set_bit(bit, &status, INTC_IRQS_PER_WORD) {
- generic_handle_domain_irq(intc_ic->irq_domain, bit);
- writel(BIT(bit), intc_ic->base + INTC_INT_STATUS_REG);
- }
- }
-
- chained_irq_exit(chip, desc);
-}
-
-static void aspeed_intc_irq_mask(struct irq_data *data)
-{
- struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
- unsigned int mask = readl(intc_ic->base + INTC_INT_ENABLE_REG) & ~BIT(data->hwirq);
-
- guard(raw_spinlock)(&intc_ic->intc_lock);
- writel(mask, intc_ic->base + INTC_INT_ENABLE_REG);
-}
-
-static void aspeed_intc_irq_unmask(struct irq_data *data)
-{
- struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
- unsigned int unmask = readl(intc_ic->base + INTC_INT_ENABLE_REG) | BIT(data->hwirq);
-
- guard(raw_spinlock)(&intc_ic->intc_lock);
- writel(unmask, intc_ic->base + INTC_INT_ENABLE_REG);
-}
-
-static struct irq_chip aspeed_intc_chip = {
- .name = "ASPEED INTC",
- .irq_mask = aspeed_intc_irq_mask,
- .irq_unmask = aspeed_intc_irq_unmask,
-};
-
-static int aspeed_intc_ic_map_irq_domain(struct irq_domain *domain, unsigned int irq,
- irq_hw_number_t hwirq)
-{
- irq_set_chip_and_handler(irq, &aspeed_intc_chip, handle_level_irq);
- irq_set_chip_data(irq, domain->host_data);
-
- return 0;
-}
-
-static const struct irq_domain_ops aspeed_intc_ic_irq_domain_ops = {
- .map = aspeed_intc_ic_map_irq_domain,
-};
-
-static int __init aspeed_intc_ic_of_init(struct device_node *node,
- struct device_node *parent)
-{
- struct aspeed_intc_ic *intc_ic;
- int irq, i, ret = 0;
-
- intc_ic = kzalloc_obj(*intc_ic);
- if (!intc_ic)
- return -ENOMEM;
-
- intc_ic->base = of_iomap(node, 0);
- if (!intc_ic->base) {
- pr_err("Failed to iomap intc_ic base\n");
- ret = -ENOMEM;
- goto err_free_ic;
- }
- writel(0xffffffff, intc_ic->base + INTC_INT_STATUS_REG);
- writel(0x0, intc_ic->base + INTC_INT_ENABLE_REG);
-
- intc_ic->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), INTC_IRQS_PER_WORD,
- &aspeed_intc_ic_irq_domain_ops, intc_ic);
- if (!intc_ic->irq_domain) {
- ret = -ENOMEM;
- goto err_iounmap;
- }
-
- raw_spin_lock_init(&intc_ic->gic_lock);
- raw_spin_lock_init(&intc_ic->intc_lock);
-
- /* Check all the irq numbers valid. If not, unmaps all the base and frees the data. */
- for (i = 0; i < of_irq_count(node); i++) {
- irq = irq_of_parse_and_map(node, i);
- if (!irq) {
- pr_err("Failed to get irq number\n");
- ret = -EINVAL;
- goto err_iounmap;
- }
- }
-
- for (i = 0; i < of_irq_count(node); i++) {
- irq = irq_of_parse_and_map(node, i);
- irq_set_chained_handler_and_data(irq, aspeed_intc_ic_irq_handler, intc_ic);
- }
-
- return 0;
-
-err_iounmap:
- iounmap(intc_ic->base);
-err_free_ic:
- kfree(intc_ic);
- return ret;
-}
-
-IRQCHIP_DECLARE(ast2700_intc_ic, "aspeed,ast2700-intc-ic", aspeed_intc_ic_of_init);
diff --git a/drivers/irqchip/irq-ast2700-intc0-test.c b/drivers/irqchip/irq-ast2700-intc0-test.c
new file mode 100644
index 0000000..d497845
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc0-test.c
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Code Construct
+ */
+#include <kunit/test.h>
+
+#include "irq-ast2700.h"
+
+static void aspeed_intc0_resolve_route_bad_args(struct kunit *test)
+{
+ static const struct aspeed_intc_interrupt_range c1ranges[] = { 0 };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ const struct irq_domain c0domain = { 0 };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(NULL, 0, c1outs, 0, c1ranges, NULL);
+ KUNIT_EXPECT_EQ(test, rc, -EINVAL);
+
+ rc = aspeed_intc0_resolve_route(&c0domain, 0, c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, -ENOENT);
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ 0, c1ranges, &resolved);
+ KUNIT_EXPECT_EQ(test, rc, -ENOENT);
+}
+
+static int gicv3_fwnode_read_string_array(const struct fwnode_handle *fwnode,
+ const char *propname, const char **val, size_t nval)
+{
+ if (!propname)
+ return -EINVAL;
+
+ if (!val)
+ return 1;
+
+ if (WARN_ON(nval != 1))
+ return -EOVERFLOW;
+
+ *val = "arm,gic-v3";
+ return 1;
+}
+
+static const struct fwnode_operations arm_gicv3_fwnode_ops = {
+ .property_read_string_array = gicv3_fwnode_read_string_array,
+};
+
+static void aspeed_intc_resolve_route_invalid_c0domain(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &arm_gicv3_fwnode_ops },
+ };
+ const struct irq_domain c0domain = { .fwnode = &intc0_node.fwnode };
+ static const struct aspeed_intc_interrupt_range c1ranges[] = { 0 };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static int
+aspeed_intc0_fwnode_read_string_array(const struct fwnode_handle *fwnode_handle,
+ const char *propname, const char **val,
+ size_t nval)
+{
+ if (!propname)
+ return -EINVAL;
+
+ if (!val)
+ return 1;
+
+ if (WARN_ON(nval != 1))
+ return -EOVERFLOW;
+
+ *val = "aspeed,ast2700-intc0";
+ return nval;
+}
+
+static const struct fwnode_operations intc0_fwnode_ops = {
+ .property_read_string_array = aspeed_intc0_fwnode_read_string_array,
+};
+
+static void
+aspeed_intc0_resolve_route_c1i1o1c0i1o1_connected(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 128 }
+ }
+ }
+ };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 128,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = { .ranges = intc0_ranges, .nranges = ARRAY_SIZE(intc0_ranges), }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, 0);
+ KUNIT_EXPECT_EQ(test, resolved.start, 0);
+ KUNIT_EXPECT_EQ(test, resolved.count, 1);
+ KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 128);
+}
+
+static void
+aspeed_intc0_resolve_route_c1i1o1c0i1o1_disconnected(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 128 }
+ }
+ }
+ };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 129,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o1mc0i1o1(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 480 }
+ }
+ }
+ };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 192,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, 0);
+ KUNIT_EXPECT_EQ(test, resolved.start, 0);
+ KUNIT_EXPECT_EQ(test, resolved.count, 1);
+ KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 480);
+}
+
+static void aspeed_intc0_resolve_route_c1i2o2mc0i1o1(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 480 }
+ }
+ },
+ {
+ .start = 1,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 510 }
+ }
+ }
+ };
+ static const u32 c1outs[] = { 1 };
+ struct aspeed_intc_interrupt_range resolved;
+ static struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 208,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, 0);
+ KUNIT_EXPECT_EQ(test, resolved.start, 1);
+ KUNIT_EXPECT_EQ(test, resolved.count, 1);
+ KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 510);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o1mc0i2o1(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 510 }
+ }
+ },
+ };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ static struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 192,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = {0},
+ }
+ },
+ {
+ .start = 208,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = {0},
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, 0);
+ KUNIT_EXPECT_EQ(test, resolved.start, 0);
+ KUNIT_EXPECT_EQ(test, resolved.count, 1);
+ KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 510);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o2mc0i1o1_invalid(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 1,
+ .param = { 480 }
+ }
+ }
+ };
+ static const u32 c1outs[] = {
+ AST2700_INTC_INVALID_ROUTE, 0
+ };
+ struct aspeed_intc_interrupt_range resolved;
+ struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 192,
+ .count = 1,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_EQ(test, rc, 1);
+ KUNIT_EXPECT_EQ(test, resolved.start, 0);
+ KUNIT_EXPECT_EQ(test, resolved.count, 1);
+ KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 480);
+}
+
+static void
+aspeed_intc0_resolve_route_c1i1o1mc0i1o1_bad_range_upstream(struct kunit *test)
+{
+ struct device_node intc0_node = {
+ .fwnode = { .ops = &intc0_fwnode_ops },
+ };
+ struct aspeed_intc_interrupt_range c1ranges[] = {
+ {
+ .start = 0,
+ .count = 1,
+ .upstream = {
+ .fwnode = &intc0_node.fwnode,
+ .param_count = 0,
+ .param = { 0 }
+ }
+ }
+ };
+ static const u32 c1outs[] = { 0 };
+ struct aspeed_intc_interrupt_range resolved;
+ struct aspeed_intc_interrupt_range intc0_ranges[] = {
+ {
+ .start = 0,
+ .count = 0,
+ .upstream = {
+ .fwnode = NULL,
+ .param_count = 0,
+ .param = { 0 },
+ }
+ }
+ };
+ struct aspeed_intc0 intc0 = {
+ .ranges = {
+ .ranges = intc0_ranges,
+ .nranges = ARRAY_SIZE(intc0_ranges),
+ }
+ };
+ const struct irq_domain c0domain = {
+ .host_data = &intc0,
+ .fwnode = &intc0_node.fwnode
+ };
+ int rc;
+
+ rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+ ARRAY_SIZE(c1ranges), c1ranges,
+ &resolved);
+ KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static struct kunit_case ast2700_intc0_test_cases[] = {
+ KUNIT_CASE(aspeed_intc0_resolve_route_bad_args),
+ KUNIT_CASE(aspeed_intc_resolve_route_invalid_c0domain),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1c0i1o1_connected),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1c0i1o1_disconnected),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i1o1),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i2o2mc0i1o1),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i2o1),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o2mc0i1o1_invalid),
+ KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i1o1_bad_range_upstream),
+ {},
+};
+
+static struct kunit_suite ast2700_intc0_test_suite = {
+ .name = "ast2700-intc0",
+ .test_cases = ast2700_intc0_test_cases,
+};
+
+kunit_test_suite(ast2700_intc0_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/irqchip/irq-ast2700-intc0.c b/drivers/irqchip/irq-ast2700-intc0.c
new file mode 100644
index 0000000..14b8b88
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc0.c
@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Aspeed AST2700 Interrupt Controller.
+ *
+ * Copyright (C) 2026 ASPEED Technology Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/fwnode.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kconfig.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/overflow.h>
+#include <linux/property.h>
+#include <linux/spinlock.h>
+
+#include "irq-ast2700.h"
+
+#define INT_NUM 480
+#define INTM_NUM 50
+#define SWINT_NUM 16
+
+#define INTM_BASE (INT_NUM)
+#define SWINT_BASE (INT_NUM + INTM_NUM)
+#define INT0_NUM (INT_NUM + INTM_NUM + SWINT_NUM)
+
+#define INTC0_IN_NUM 480
+#define INTC0_ROUTE_NUM 5
+#define INTC0_INTM_NUM 50
+#define INTC0_ROUTE_BITS 3
+
+#define GIC_P2P_SPI_END 128
+#define INTC0_SWINT_OUT_BASE 144
+
+#define INTC0_SWINT_IER 0x10
+#define INTC0_SWINT_ISR 0x14
+#define INTC0_INTBANKX_IER 0x1000
+#define INTC0_INTBANK_SIZE 0x100
+#define INTC0_INTBANK_GROUPS 11
+#define INTC0_INTBANKS_PER_GRP 3
+#define INTC0_INTMX_IER 0x1b00
+#define INTC0_INTMX_ISR 0x1b04
+#define INTC0_INTMX_BANK_SIZE 0x10
+#define INTC0_INTM_BANK_NUM 3
+#define INTC0_IRQS_PER_BANK 32
+#define INTM_IRQS_PER_BANK 10
+#define INTC0_SEL_BASE 0x200
+#define INTC0_SEL_BANK_SIZE 0x4
+#define INTC0_SEL_ROUTE_SIZE 0x100
+
+static void aspeed_swint_irq_mask(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bit = data->hwirq - SWINT_BASE;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc0->intc_lock);
+ ier = readl(intc0->base + INTC0_SWINT_IER) & ~BIT(bit);
+ writel(ier, intc0->base + INTC0_SWINT_IER);
+ irq_chip_mask_parent(data);
+}
+
+static void aspeed_swint_irq_unmask(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bit = data->hwirq - SWINT_BASE;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc0->intc_lock);
+ ier = readl(intc0->base + INTC0_SWINT_IER) | BIT(bit);
+ writel(ier, intc0->base + INTC0_SWINT_IER);
+ irq_chip_unmask_parent(data);
+}
+
+static void aspeed_swint_irq_eoi(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bit = data->hwirq - SWINT_BASE;
+
+ writel(BIT(bit), intc0->base + INTC0_SWINT_ISR);
+ irq_chip_eoi_parent(data);
+}
+
+static struct irq_chip aspeed_swint_chip = {
+ .name = "ast2700-swint",
+ .irq_eoi = aspeed_swint_irq_eoi,
+ .irq_mask = aspeed_swint_irq_mask,
+ .irq_unmask = aspeed_swint_irq_unmask,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .flags = IRQCHIP_SET_TYPE_MASKED,
+};
+
+static void aspeed_intc0_irq_mask(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+ int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc0->intc_lock);
+ ier = readl(intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE) & ~BIT(bit);
+ writel(ier, intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE);
+ irq_chip_mask_parent(data);
+}
+
+static void aspeed_intc0_irq_unmask(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+ int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc0->intc_lock);
+ ier = readl(intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE) | BIT(bit);
+ writel(ier, intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE);
+ irq_chip_unmask_parent(data);
+}
+
+static void aspeed_intc0_irq_eoi(struct irq_data *data)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+ int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+
+ writel(BIT(bit), intc0->base + INTC0_INTMX_ISR + bank * INTC0_INTMX_BANK_SIZE);
+ irq_chip_eoi_parent(data);
+}
+
+static struct irq_chip aspeed_intm_chip = {
+ .name = "ast2700-intmerge",
+ .irq_eoi = aspeed_intc0_irq_eoi,
+ .irq_mask = aspeed_intc0_irq_mask,
+ .irq_unmask = aspeed_intc0_irq_unmask,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .flags = IRQCHIP_SET_TYPE_MASKED,
+};
+
+static struct irq_chip linear_intr_irq_chip = {
+ .name = "ast2700-int",
+ .irq_eoi = irq_chip_eoi_parent,
+ .irq_mask = irq_chip_mask_parent,
+ .irq_unmask = irq_chip_unmask_parent,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .flags = IRQCHIP_SET_TYPE_MASKED,
+};
+
+static const u32 aspeed_intc0_routes[INTC0_IN_NUM / INTC0_IRQS_PER_BANK][INTC0_ROUTE_NUM] = {
+ { 0, 256, 426, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+ { 32, 288, 458, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+ { 64, 320, 490, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+ { 96, 352, 522, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+ { 128, 384, 554, 160, 176 },
+ { 129, 385, 555, 161, 177 },
+ { 130, 386, 556, 162, 178 },
+ { 131, 387, 557, 163, 179 },
+ { 132, 388, 558, 164, 180 },
+ { 133, 544, 714, 165, 181 },
+ { 134, 545, 715, 166, 182 },
+ { 135, 546, 706, 167, 183 },
+ { 136, 547, 707, 168, 184 },
+ { 137, 548, 708, 169, 185 },
+ { 138, 549, 709, 170, 186 },
+};
+
+static const u32 aspeed_intc0_intm_routes[INTC0_INTM_NUM / INTM_IRQS_PER_BANK] = {
+ 192, 416, 586, 208, 224
+};
+
+static int resolve_input_from_child_ranges(const struct aspeed_intc0 *intc0,
+ const struct aspeed_intc_interrupt_range *range,
+ u32 outpin, u32 *input)
+{
+ u32 offset, base;
+
+ if (!in_range32(outpin, range->start, range->count))
+ return -ENOENT;
+
+ if (range->upstream.param_count == 0)
+ return -EINVAL;
+
+ base = range->upstream.param[ASPEED_INTC_RANGES_BASE];
+ offset = outpin - range->start;
+ if (check_add_overflow(base, offset, input)) {
+ dev_warn(intc0->dev, "%s: Arithmetic overflow for input derivation: %u + %u\n",
+ __func__, base, offset);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int resolve_parent_range_for_output(const struct aspeed_intc0 *intc0,
+ const struct fwnode_handle *parent, u32 output,
+ struct aspeed_intc_interrupt_range *resolved)
+{
+ for (size_t i = 0; i < intc0->ranges.nranges; i++) {
+ struct aspeed_intc_interrupt_range range = intc0->ranges.ranges[i];
+
+ if (!in_range32(output, range.start, range.count))
+ continue;
+
+ if (range.upstream.fwnode != parent)
+ continue;
+
+ if (resolved) {
+ resolved->start = output;
+ resolved->count = 1;
+ resolved->upstream = range.upstream;
+ resolved->upstream.param[ASPEED_INTC_RANGES_COUNT] +=
+ output - range.start;
+ }
+
+ return 0;
+ }
+
+ return -ENOENT;
+}
+
+static int resolve_parent_route_for_input(const struct aspeed_intc0 *intc0,
+ const struct fwnode_handle *parent, u32 input,
+ struct aspeed_intc_interrupt_range *resolved)
+{
+ int rc = -ENOENT;
+ u32 c0o;
+
+ if (input < INT_NUM) {
+ static_assert(INTC0_ROUTE_NUM < INT_MAX, "Broken cast");
+ for (size_t i = 0; rc == -ENOENT && i < INTC0_ROUTE_NUM; i++) {
+ c0o = aspeed_intc0_routes[input / INTC0_IRQS_PER_BANK][i];
+ if (c0o == AST2700_INTC_INVALID_ROUTE)
+ continue;
+
+ if (input < GIC_P2P_SPI_END)
+ c0o += input % INTC0_IRQS_PER_BANK;
+
+ rc = resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+ if (!rc)
+ return (int)i;
+ }
+ } else if (input < (INT_NUM + INTM_NUM)) {
+ c0o = aspeed_intc0_intm_routes[(input - INT_NUM) / INTM_IRQS_PER_BANK];
+ c0o += ((input - INT_NUM) % INTM_IRQS_PER_BANK);
+ return resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+ } else if (input < (INT_NUM + INTM_NUM + SWINT_NUM)) {
+ c0o = input - SWINT_BASE + INTC0_SWINT_OUT_BASE;
+ return resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+ } else {
+ return -ENOENT;
+ }
+
+ return rc;
+}
+
+/**
+ * aspeed_intc0_resolve_route - Determine the necessary interrupt output at intc1
+ * @c0domain: The pointer to intc0's irq_domain
+ * @nc1outs: The number of valid intc1 outputs available for the input
+ * @c1outs: The array of available intc1 output indices for the input
+ * @nc1ranges: The number of interrupt range entries for intc1
+ * @c1ranges: The array of configured intc1 interrupt ranges
+ * @resolved: The fully resolved range entry after applying the resolution
+ * algorithm
+ *
+ * Returns: The intc1 route index associated with the intc1 output identified in
+ * @resolved on success. Otherwise, a negative errno value.
+ *
+ * The AST2700 interrupt architecture allows any peripheral interrupt source
+ * to be routed to one of up to four processors running in the SoC. A processor
+ * binding a driver for a peripheral that requests an interrupt is (without
+ * further design and effort) the destination for the requested interrupt.
+ *
+ * Routing a peripheral interrupt to its destination processor requires
+ * coordination between INTC0 on the CPU die and one or more INTC1 instances.
+ * At least one INTC1 instance exists in the SoC on the IO-die, however up
+ * to two more instances may be integrated via LTPI (LVDS Tunneling Protocol
+ * & Interface).
+ *
+ * Between the multiple destinations, various route constraints, and the
+ * devicetree binding design, some information that's needed at INTC1 instances
+ * to route inbound interrupts correctly to the destination processor is only
+ * available at INTC0.
+ *
+ * aspeed_intc0_resolve_route() is to be invoked by INTC1 driver instances to
+ * perform the route resolution. The implementation in INTC0 allows INTC0 to
+ * encapsulate the information used to perform route selection, and provides it
+ * with an opportunity to apply policy as part of the selection process. Such
+ * policy may, for instance, choose to de-prioritise some interrupts destined
+ * for the PSP (Primary Service Processor) GIC.
+ */
+int aspeed_intc0_resolve_route(const struct irq_domain *c0domain, size_t nc1outs,
+ const u32 *c1outs, size_t nc1ranges,
+ const struct aspeed_intc_interrupt_range *c1ranges,
+ struct aspeed_intc_interrupt_range *resolved)
+{
+ struct fwnode_handle *parent_fwnode;
+ struct aspeed_intc0 *intc0;
+ int ret;
+
+ if (!c0domain || !resolved)
+ return -EINVAL;
+
+ if (nc1outs > INT_MAX)
+ return -EINVAL;
+
+ if (nc1outs == 0 || nc1ranges == 0)
+ return -ENOENT;
+
+ if (!IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST) &&
+ !fwnode_device_is_compatible(c0domain->fwnode, "aspeed,ast2700-intc0"))
+ return -ENODEV;
+
+ intc0 = c0domain->host_data;
+ if (!intc0)
+ return -EINVAL;
+
+ parent_fwnode = of_fwnode_handle(intc0->parent);
+
+ for (size_t i = 0; i < nc1outs; i++) {
+ u32 c1o = c1outs[i];
+
+ if (c1o == AST2700_INTC_INVALID_ROUTE)
+ continue;
+
+ for (size_t j = 0; j < nc1ranges; j++) {
+ struct aspeed_intc_interrupt_range c1r = c1ranges[j];
+ u32 input;
+
+ /*
+ * Range match for intc1 output pin
+ *
+ * Assume a failed match is still a match for the purpose of testing,
+ * saves a bunch of mess in the test fixtures
+ */
+ if (!(c0domain == c1r.domain ||
+ IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST)))
+ continue;
+
+ ret = resolve_input_from_child_ranges(intc0, &c1r, c1o, &input);
+ if (ret)
+ continue;
+
+ /*
+ * INTC1 should never request routes for peripheral interrupt sources
+ * directly attached to INTC0.
+ */
+ if (input < GIC_P2P_SPI_END)
+ continue;
+
+ ret = resolve_parent_route_for_input(intc0, parent_fwnode, input, NULL);
+ if (ret < 0)
+ continue;
+
+ /* Route resolution succeeded */
+ resolved->start = c1o;
+ resolved->count = 1;
+ resolved->upstream = c1r.upstream;
+ resolved->upstream.param[ASPEED_INTC_RANGES_BASE] = input;
+ /* Cast protected by prior test against nc1outs */
+ return (int)i;
+ }
+ }
+
+ return -ENOENT;
+}
+
+static int aspeed_intc0_irq_domain_map(struct irq_domain *domain,
+ unsigned int irq, irq_hw_number_t hwirq)
+{
+ if (hwirq < GIC_P2P_SPI_END)
+ irq_set_chip_and_handler(irq, &linear_intr_irq_chip, handle_level_irq);
+ else if (hwirq < INTM_BASE)
+ return -EINVAL;
+ else if (hwirq < SWINT_BASE)
+ irq_set_chip_and_handler(irq, &aspeed_intm_chip, handle_level_irq);
+ else if (hwirq < INT0_NUM)
+ irq_set_chip_and_handler(irq, &aspeed_swint_chip, handle_level_irq);
+ else
+ return -EINVAL;
+
+ irq_set_chip_data(irq, domain->host_data);
+ return 0;
+}
+
+static int aspeed_intc0_irq_domain_translate(struct irq_domain *domain,
+ struct irq_fwspec *fwspec,
+ unsigned long *hwirq,
+ unsigned int *type)
+{
+ if (fwspec->param_count != 1)
+ return -EINVAL;
+
+ *hwirq = fwspec->param[0];
+ *type = IRQ_TYPE_NONE;
+ return 0;
+}
+
+static int aspeed_intc0_irq_domain_alloc(struct irq_domain *domain,
+ unsigned int virq,
+ unsigned int nr_irqs, void *data)
+{
+ struct aspeed_intc0 *intc0 = domain->host_data;
+ struct aspeed_intc_interrupt_range resolved;
+ struct irq_fwspec *fwspec = data;
+ struct irq_fwspec parent_fwspec;
+ struct irq_chip *chip;
+ unsigned long hwirq;
+ unsigned int type;
+ int ret;
+
+ ret = aspeed_intc0_irq_domain_translate(domain, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ if (hwirq >= GIC_P2P_SPI_END && hwirq < INT_NUM)
+ return -EINVAL;
+
+ if (hwirq < INTM_BASE)
+ chip = &linear_intr_irq_chip;
+ else if (hwirq < SWINT_BASE)
+ chip = &aspeed_intm_chip;
+ else
+ chip = &aspeed_swint_chip;
+
+ ret = resolve_parent_route_for_input(intc0, domain->parent->fwnode,
+ (u32)hwirq, &resolved);
+ if (ret)
+ return ret;
+
+ parent_fwspec = resolved.upstream;
+ ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs,
+ &parent_fwspec);
+ if (ret)
+ return ret;
+
+ for (int i = 0; i < nr_irqs; ++i, ++hwirq, ++virq) {
+ ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip,
+ domain->host_data);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int aspeed_intc0_irq_domain_activate(struct irq_domain *domain,
+ struct irq_data *data, bool reserve)
+{
+ struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+ unsigned long hwirq = data->hwirq;
+ int route, bank, bit;
+ u32 mask;
+
+ if (hwirq >= INT0_NUM)
+ return -EINVAL;
+
+ if (in_range32(hwirq, INTM_BASE, INTM_NUM + SWINT_NUM))
+ return 0;
+
+ bank = hwirq / INTC0_IRQS_PER_BANK;
+ bit = hwirq % INTC0_IRQS_PER_BANK;
+ mask = BIT(bit);
+
+ route = resolve_parent_route_for_input(intc0, intc0->local->parent->fwnode,
+ hwirq, NULL);
+ if (route < 0)
+ return route;
+
+ guard(raw_spinlock)(&intc0->intc_lock);
+ for (int i = 0; i < INTC0_ROUTE_BITS; i++) {
+ void __iomem *sel = intc0->base + INTC0_SEL_BASE +
+ (bank * INTC0_SEL_BANK_SIZE) +
+ (INTC0_SEL_ROUTE_SIZE * i);
+ u32 reg = readl(sel);
+
+ if (route & BIT(i))
+ reg |= mask;
+ else
+ reg &= ~mask;
+
+ writel(reg, sel);
+ if (readl(sel) != reg)
+ return -EACCES;
+ }
+
+ return 0;
+}
+
+static const struct irq_domain_ops aspeed_intc0_irq_domain_ops = {
+ .translate = aspeed_intc0_irq_domain_translate,
+ .activate = aspeed_intc0_irq_domain_activate,
+ .alloc = aspeed_intc0_irq_domain_alloc,
+ .free = irq_domain_free_irqs_common,
+ .map = aspeed_intc0_irq_domain_map,
+};
+
+static void aspeed_intc0_disable_swint(struct aspeed_intc0 *intc0)
+{
+ writel(0, intc0->base + INTC0_SWINT_IER);
+}
+
+static void aspeed_intc0_disable_intbank(struct aspeed_intc0 *intc0)
+{
+ for (int i = 0; i < INTC0_INTBANK_GROUPS; i++) {
+ for (int j = 0; j < INTC0_INTBANKS_PER_GRP; j++) {
+ u32 base = INTC0_INTBANKX_IER +
+ (INTC0_INTBANK_SIZE * i) +
+ (INTC0_INTMX_BANK_SIZE * j);
+
+ writel(0, intc0->base + base);
+ }
+ }
+}
+
+static void aspeed_intc0_disable_intm(struct aspeed_intc0 *intc0)
+{
+ for (int i = 0; i < INTC0_INTM_BANK_NUM; i++)
+ writel(0, intc0->base + INTC0_INTMX_IER + (INTC0_INTMX_BANK_SIZE * i));
+}
+
+static int aspeed_intc0_probe(struct platform_device *pdev,
+ struct device_node *parent)
+{
+ struct device_node *node = pdev->dev.of_node;
+ struct irq_domain *parent_domain;
+ struct aspeed_intc0 *intc0;
+ int ret;
+
+ if (!parent) {
+ pr_err("missing parent interrupt node\n");
+ return -ENODEV;
+ }
+
+ intc0 = devm_kzalloc(&pdev->dev, sizeof(*intc0), GFP_KERNEL);
+ if (!intc0)
+ return -ENOMEM;
+
+ intc0->dev = &pdev->dev;
+ intc0->parent = parent;
+ intc0->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(intc0->base))
+ return PTR_ERR(intc0->base);
+
+ aspeed_intc0_disable_swint(intc0);
+ aspeed_intc0_disable_intbank(intc0);
+ aspeed_intc0_disable_intm(intc0);
+
+ raw_spin_lock_init(&intc0->intc_lock);
+
+ parent_domain = irq_find_host(parent);
+ if (!parent_domain) {
+ pr_err("unable to obtain parent domain\n");
+ return -ENODEV;
+ }
+
+ if (!of_device_is_compatible(parent, "arm,gic-v3"))
+ return -ENODEV;
+
+ intc0->local = irq_domain_create_hierarchy(parent_domain, 0, INT0_NUM,
+ of_fwnode_handle(node),
+ &aspeed_intc0_irq_domain_ops,
+ intc0);
+ if (!intc0->local)
+ return -ENOMEM;
+
+ ret = aspeed_intc_populate_ranges(&pdev->dev, &intc0->ranges);
+ if (ret < 0) {
+ irq_domain_remove(intc0->local);
+ return ret;
+ }
+
+ return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(ast2700_intc0)
+IRQCHIP_MATCH("aspeed,ast2700-intc0", aspeed_intc0_probe)
+IRQCHIP_PLATFORM_DRIVER_END(ast2700_intc0)
diff --git a/drivers/irqchip/irq-ast2700-intc1.c b/drivers/irqchip/irq-ast2700-intc1.c
new file mode 100644
index 0000000..59e8f0d
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc1.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Aspeed AST2700 Interrupt Controller.
+ *
+ * Copyright (C) 2026 ASPEED Technology Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/spinlock.h>
+
+#include "irq-ast2700.h"
+
+#define INTC1_IER 0x100
+#define INTC1_ISR 0x104
+#define INTC1_BANK_SIZE 0x10
+#define INTC1_SEL_BASE 0x80
+#define INTC1_SEL_BANK_SIZE 0x4
+#define INTC1_SEL_ROUTE_SIZE 0x20
+#define INTC1_IRQS_PER_BANK 32
+#define INTC1_BANK_NUM 6
+#define INTC1_ROUTE_NUM 7
+#define INTC1_IN_NUM 192
+#define INTC1_BOOTMCU_ROUTE 6
+#define INTC1_ROUTE_SELECTOR_BITS 3
+#define INTC1_ROUTE_IRQS_PER_GROUP 32
+#define INTC1_ROUTE_SHIFT 5
+
+struct aspeed_intc1 {
+ struct device *dev;
+ void __iomem *base;
+ raw_spinlock_t intc_lock;
+ struct irq_domain *local;
+ struct irq_domain *upstream;
+ struct aspeed_intc_interrupt_ranges ranges;
+};
+
+static void aspeed_intc1_disable_int(struct aspeed_intc1 *intc1)
+{
+ for (int i = 0; i < INTC1_BANK_NUM; i++)
+ writel(0, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * i));
+}
+
+static void aspeed_intc1_irq_handler(struct irq_desc *desc)
+{
+ struct aspeed_intc1 *intc1 = irq_desc_get_handler_data(desc);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ unsigned long bit, status;
+
+ chained_irq_enter(chip, desc);
+
+ for (int bank = 0; bank < INTC1_BANK_NUM; bank++) {
+ status = readl(intc1->base + INTC1_ISR + (INTC1_BANK_SIZE * bank));
+ if (!status)
+ continue;
+
+ for_each_set_bit(bit, &status, INTC1_IRQS_PER_BANK) {
+ generic_handle_domain_irq(intc1->local, (bank * INTC1_IRQS_PER_BANK) + bit);
+ writel(BIT(bit), intc1->base + INTC1_ISR + (INTC1_BANK_SIZE * bank));
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static void aspeed_intc1_irq_mask(struct irq_data *data)
+{
+ struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+ int bank = data->hwirq / INTC1_IRQS_PER_BANK;
+ int bit = data->hwirq % INTC1_IRQS_PER_BANK;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc1->intc_lock);
+ ier = readl(intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank)) & ~BIT(bit);
+ writel(ier, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank));
+}
+
+static void aspeed_intc1_irq_unmask(struct irq_data *data)
+{
+ struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+ int bank = data->hwirq / INTC1_IRQS_PER_BANK;
+ int bit = data->hwirq % INTC1_IRQS_PER_BANK;
+ u32 ier;
+
+ guard(raw_spinlock)(&intc1->intc_lock);
+ ier = readl(intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank)) | BIT(bit);
+ writel(ier, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank));
+}
+
+static struct irq_chip aspeed_intc_chip = {
+ .name = "ASPEED INTC1",
+ .irq_mask = aspeed_intc1_irq_mask,
+ .irq_unmask = aspeed_intc1_irq_unmask,
+};
+
+static int aspeed_intc1_irq_domain_translate(struct irq_domain *domain,
+ struct irq_fwspec *fwspec,
+ unsigned long *hwirq,
+ unsigned int *type)
+{
+ if (fwspec->param_count != 1)
+ return -EINVAL;
+
+ *hwirq = fwspec->param[0];
+ *type = IRQ_TYPE_LEVEL_HIGH;
+ return 0;
+}
+
+static int aspeed_intc1_map_irq_domain(struct irq_domain *domain,
+ unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_domain_set_info(domain, irq, hwirq, &aspeed_intc_chip,
+ domain->host_data, handle_level_irq, NULL, NULL);
+ return 0;
+}
+
+/*
+ * In-bound interrupts are progressively merged into one out-bound interrupt in
+ * groups of 32. Apply this fact to compress the route table in corresponding
+ * groups of 32.
+ */
+static const u32
+aspeed_intc1_routes[INTC1_IN_NUM / INTC1_ROUTE_IRQS_PER_GROUP][INTC1_ROUTE_NUM] = {
+ { 0, AST2700_INTC_INVALID_ROUTE, 10, 20, 30, 40, 50 },
+ { 1, AST2700_INTC_INVALID_ROUTE, 11, 21, 31, 41, 50 },
+ { 2, AST2700_INTC_INVALID_ROUTE, 12, 22, 32, 42, 50 },
+ { 3, AST2700_INTC_INVALID_ROUTE, 13, 23, 33, 43, 50 },
+ { 4, AST2700_INTC_INVALID_ROUTE, 14, 24, 34, 44, 50 },
+ { 5, AST2700_INTC_INVALID_ROUTE, 15, 25, 35, 45, 50 },
+};
+
+static int aspeed_intc1_irq_domain_activate(struct irq_domain *domain,
+ struct irq_data *data, bool reserve)
+{
+ struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+ struct aspeed_intc_interrupt_range resolved;
+ int rc, bank, bit;
+ u32 mask;
+
+ if (WARN_ON_ONCE((data->hwirq >> INTC1_ROUTE_SHIFT) >= ARRAY_SIZE(aspeed_intc1_routes)))
+ return -EINVAL;
+
+ /*
+ * outpin may be an error if the upstream is the BootMCU APLIC node, or
+ * anything except a valid intc0 driver instance
+ */
+ rc = aspeed_intc0_resolve_route(intc1->upstream, INTC1_ROUTE_NUM,
+ aspeed_intc1_routes[data->hwirq >> INTC1_ROUTE_SHIFT],
+ intc1->ranges.nranges,
+ intc1->ranges.ranges, &resolved);
+ if (rc < 0) {
+ if (!fwnode_device_is_compatible(intc1->upstream->fwnode, "riscv,aplic")) {
+ dev_warn(intc1->dev,
+ "Failed to resolve interrupt route for hwirq %lu in domain %s\n",
+ data->hwirq, domain->name);
+ return rc;
+ }
+ rc = INTC1_BOOTMCU_ROUTE;
+ }
+
+ bank = data->hwirq / INTC1_IRQS_PER_BANK;
+ bit = data->hwirq % INTC1_IRQS_PER_BANK;
+ mask = BIT(bit);
+
+ guard(raw_spinlock)(&intc1->intc_lock);
+ for (int i = 0; i < INTC1_ROUTE_SELECTOR_BITS; i++) {
+ void __iomem *sel = intc1->base + INTC1_SEL_BASE +
+ (bank * INTC1_SEL_BANK_SIZE) +
+ (INTC1_SEL_ROUTE_SIZE * i);
+ u32 reg = readl(sel);
+
+ if (rc & BIT(i))
+ reg |= mask;
+ else
+ reg &= ~mask;
+
+ writel(reg, sel);
+ if (readl(sel) != reg)
+ return -EACCES;
+ }
+
+ return 0;
+}
+
+static const struct irq_domain_ops aspeed_intc1_irq_domain_ops = {
+ .map = aspeed_intc1_map_irq_domain,
+ .translate = aspeed_intc1_irq_domain_translate,
+ .activate = aspeed_intc1_irq_domain_activate,
+};
+
+static void aspeed_intc1_request_interrupts(struct aspeed_intc1 *intc1)
+{
+ for (unsigned int i = 0; i < intc1->ranges.nranges; i++) {
+ struct aspeed_intc_interrupt_range *r =
+ &intc1->ranges.ranges[i];
+
+ if (intc1->upstream != r->domain)
+ continue;
+
+ for (u32 k = 0; k < r->count; k++) {
+ struct of_phandle_args parent_irq;
+ int irq;
+
+ parent_irq.np = to_of_node(r->upstream.fwnode);
+ parent_irq.args_count = 1;
+ parent_irq.args[0] =
+ intc1->ranges.ranges[i].upstream.param[ASPEED_INTC_RANGES_BASE] + k;
+
+ irq = irq_create_of_mapping(&parent_irq);
+ if (!irq)
+ continue;
+
+ irq_set_chained_handler_and_data(irq,
+ aspeed_intc1_irq_handler, intc1);
+ }
+ }
+}
+
+static int aspeed_intc1_probe(struct platform_device *pdev,
+ struct device_node *parent)
+{
+ struct device_node *node = pdev->dev.of_node;
+ struct aspeed_intc1 *intc1;
+ struct irq_domain *host;
+ int ret;
+
+ if (!parent) {
+ dev_err(&pdev->dev, "missing parent interrupt node\n");
+ return -ENODEV;
+ }
+
+ if (!of_device_is_compatible(parent, "aspeed,ast2700-intc0"))
+ return -ENODEV;
+
+ host = irq_find_host(parent);
+ if (!host)
+ return -ENODEV;
+
+ intc1 = devm_kzalloc(&pdev->dev, sizeof(*intc1), GFP_KERNEL);
+ if (!intc1)
+ return -ENOMEM;
+
+ intc1->dev = &pdev->dev;
+ intc1->upstream = host;
+ intc1->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(intc1->base))
+ return PTR_ERR(intc1->base);
+
+ aspeed_intc1_disable_int(intc1);
+
+ raw_spin_lock_init(&intc1->intc_lock);
+
+ intc1->local = irq_domain_create_linear(of_fwnode_handle(node),
+ INTC1_BANK_NUM * INTC1_IRQS_PER_BANK,
+ &aspeed_intc1_irq_domain_ops, intc1);
+ if (!intc1->local)
+ return -ENOMEM;
+
+ ret = aspeed_intc_populate_ranges(&pdev->dev, &intc1->ranges);
+ if (ret < 0) {
+ irq_domain_remove(intc1->local);
+ return ret;
+ }
+
+ aspeed_intc1_request_interrupts(intc1);
+
+ return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(ast2700_intc1)
+IRQCHIP_MATCH("aspeed,ast2700-intc1", aspeed_intc1_probe)
+IRQCHIP_PLATFORM_DRIVER_END(ast2700_intc1)
diff --git a/drivers/irqchip/irq-ast2700.c b/drivers/irqchip/irq-ast2700.c
new file mode 100644
index 0000000..1e4c4a6
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Aspeed AST2700 Interrupt Controller.
+ *
+ * Copyright (C) 2026 ASPEED Technology Inc.
+ */
+#include "irq-ast2700.h"
+
+#define ASPEED_INTC_RANGE_FIXED_CELLS 3U
+#define ASPEED_INTC_RANGE_OFF_START 0U
+#define ASPEED_INTC_RANGE_OFF_COUNT 1U
+#define ASPEED_INTC_RANGE_OFF_PHANDLE 2U
+
+/**
+ * aspeed_intc_populate_ranges
+ * @dev: Device owning the interrupt controller node.
+ * @ranges: Destination for parsed range descriptors.
+ *
+ * Return: 0 on success, negative errno on error.
+ */
+int aspeed_intc_populate_ranges(struct device *dev,
+ struct aspeed_intc_interrupt_ranges *ranges)
+{
+ struct aspeed_intc_interrupt_range *arr;
+ const __be32 *pvs, *pve;
+ struct device_node *dn;
+ int len;
+
+ if (!dev || !ranges)
+ return -EINVAL;
+
+ dn = dev->of_node;
+
+ pvs = of_get_property(dn, "aspeed,interrupt-ranges", &len);
+ if (!pvs)
+ return -EINVAL;
+
+ if (len % sizeof(__be32))
+ return -EINVAL;
+
+ /* Over-estimate the range entry count for now */
+ ranges->ranges = devm_kmalloc_array(dev,
+ len / (ASPEED_INTC_RANGE_FIXED_CELLS * sizeof(__be32)),
+ sizeof(*ranges->ranges),
+ GFP_KERNEL);
+ if (!ranges->ranges)
+ return -ENOMEM;
+
+ pve = pvs + (len / sizeof(__be32));
+ for (unsigned int i = 0; pve - pvs >= ASPEED_INTC_RANGE_FIXED_CELLS; i++) {
+ struct aspeed_intc_interrupt_range *r;
+ struct device_node *target;
+ u32 target_cells;
+
+ target = of_find_node_by_phandle(be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_PHANDLE]));
+ if (!target)
+ return -EINVAL;
+
+ if (of_property_read_u32(target, "#interrupt-cells",
+ &target_cells)) {
+ of_node_put(target);
+ return -EINVAL;
+ }
+
+ if (!target_cells || target_cells > IRQ_DOMAIN_IRQ_SPEC_PARAMS) {
+ of_node_put(target);
+ return -EINVAL;
+ }
+
+ if (pve - pvs < ASPEED_INTC_RANGE_FIXED_CELLS + target_cells) {
+ of_node_put(target);
+ return -EINVAL;
+ }
+
+ r = &ranges->ranges[i];
+ r->start = be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_START]);
+ r->count = be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_COUNT]);
+
+ {
+ struct of_phandle_args args = {
+ .np = target,
+ .args_count = target_cells,
+ };
+
+ for (u32 j = 0; j < target_cells; j++)
+ args.args[j] = be32_to_cpu(pvs[ASPEED_INTC_RANGE_FIXED_CELLS + j]);
+
+ of_phandle_args_to_fwspec(target, args.args,
+ args.args_count,
+ &r->upstream);
+ }
+
+ of_node_put(target);
+ r->domain = irq_find_matching_fwspec(&r->upstream, DOMAIN_BUS_ANY);
+ pvs += ASPEED_INTC_RANGE_FIXED_CELLS + target_cells;
+ ranges->nranges++;
+ }
+
+ /* Re-fit the range array now we know the entry count */
+ arr = devm_krealloc_array(dev, ranges->ranges, ranges->nranges,
+ sizeof(*ranges->ranges), GFP_KERNEL);
+ if (!arr)
+ return -ENOMEM;
+ ranges->ranges = arr;
+
+ return 0;
+}
diff --git a/drivers/irqchip/irq-ast2700.h b/drivers/irqchip/irq-ast2700.h
new file mode 100644
index 0000000..3182966
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Aspeed AST2700 Interrupt Controller.
+ *
+ * Copyright (C) 2026 ASPEED Technology Inc.
+ */
+#ifndef DRIVERS_IRQCHIP_AST2700
+#define DRIVERS_IRQCHIP_AST2700
+
+#include <linux/device.h>
+#include <linux/irqdomain.h>
+
+#define AST2700_INTC_INVALID_ROUTE (~0U)
+#define ASPEED_INTC_RANGES_BASE 0U
+#define ASPEED_INTC_RANGES_COUNT 1U
+
+struct aspeed_intc_interrupt_range {
+ u32 start;
+ u32 count;
+ struct irq_fwspec upstream;
+ struct irq_domain *domain;
+};
+
+struct aspeed_intc_interrupt_ranges {
+ struct aspeed_intc_interrupt_range *ranges;
+ unsigned int nranges;
+};
+
+struct aspeed_intc0 {
+ struct device *dev;
+ void __iomem *base;
+ raw_spinlock_t intc_lock;
+ struct irq_domain *local;
+ struct device_node *parent;
+ struct aspeed_intc_interrupt_ranges ranges;
+};
+
+int aspeed_intc_populate_ranges(struct device *dev,
+ struct aspeed_intc_interrupt_ranges *ranges);
+
+int aspeed_intc0_resolve_route(const struct irq_domain *c0domain,
+ size_t nc1outs,
+ const u32 *c1outs,
+ size_t nc1ranges,
+ const struct aspeed_intc_interrupt_range *c1ranges,
+ struct aspeed_intc_interrupt_range *resolved);
+
+#endif
diff --git a/drivers/irqchip/irq-ath79-cpu.c b/drivers/irqchip/irq-ath79-cpu.c
index 923e4bb..9b7273a 100644
--- a/drivers/irqchip/irq-ath79-cpu.c
+++ b/drivers/irqchip/irq-ath79-cpu.c
@@ -85,10 +85,3 @@ static int __init ar79_cpu_intc_of_init(
}
IRQCHIP_DECLARE(ar79_cpu_intc, "qca,ar7100-cpu-intc",
ar79_cpu_intc_of_init);
-
-void __init ath79_cpu_irq_init(unsigned irq_wb_chan2, unsigned irq_wb_chan3)
-{
- irq_wb_chan[2] = irq_wb_chan2;
- irq_wb_chan[3] = irq_wb_chan3;
- mips_cpu_irq_init();
-}
diff --git a/drivers/irqchip/irq-econet-en751221.c b/drivers/irqchip/irq-econet-en751221.c
index d83d5eb..2ca5d90 100644
--- a/drivers/irqchip/irq-econet-en751221.c
+++ b/drivers/irqchip/irq-econet-en751221.c
@@ -30,6 +30,8 @@
#include <linux/irqchip.h>
#include <linux/irqchip/chained_irq.h>
+#include <asm/setup.h>
+
#define IRQ_COUNT 40
#define NOT_PERCPU 0xff
@@ -41,15 +43,19 @@
#define REG_PENDING1 0x54
/**
- * @membase: Base address of the interrupt controller registers
- * @interrupt_shadows: Array of all interrupts, for each value,
- * - NOT_PERCPU: This interrupt is not per-cpu, so it has no shadow
- * - IS_SHADOW: This interrupt is a shadow of another per-cpu interrupt
- * - else: This is a per-cpu interrupt whose shadow is the value
+ * @membase: Base address of the interrupt controller registers
+ * @domain: The irq_domain for direct dispatch
+ * @ipi_domain: The irq_domain for inter-process dispatch
+ * @interrupt_shadows: Array of all interrupts, for each value,
+ * - NOT_PERCPU: This interrupt is not per-cpu, so it has no shadow
+ * - IS_SHADOW: This interrupt is a shadow of another per-cpu interrupt
+ * - else: This is a per-cpu interrupt whose shadow is the value
*/
static struct {
- void __iomem *membase;
- u8 interrupt_shadows[IRQ_COUNT];
+ void __iomem *membase;
+ struct irq_domain *domain;
+ struct irq_domain *ipi_domain;
+ u8 interrupt_shadows[IRQ_COUNT];
} econet_intc __ro_after_init;
static DEFINE_RAW_SPINLOCK(irq_lock);
@@ -150,6 +156,56 @@ static void econet_intc_from_parent(struct irq_desc *desc)
chained_irq_exit(chip, desc);
}
+/*
+ * When in VEIC mode, the CPU jumps to a handler in the vector table.
+ * The only way to know which interrupt is being triggered is from the vector table offset that
+ * has been jumped to. Reading REG_PENDING(0|1) will tell you which interrupts are currently
+ * pending in the intc, but that will not tell you which one the intc wants you to process
+ * right now. And if you are not processing the exact interrupt that the intc wants you to be
+ * processing, you might be on the wrong VPE. You can't tell which VPE any given REG_PENDING
+ * interrupt is intended for (shadow IRQ numbers are for masking only, they never flag as
+ * pending).
+ *
+ * Consequently, this little ritual of generating n handler functions and registering one per
+ * interrupt is unavoidable.
+ */
+#define X(irq) \
+ static void econet_irq_dispatch ## irq (void) \
+ { \
+ do_domain_IRQ(econet_intc.domain, irq); \
+ }
+
+ X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9)
+X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19)
+X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29)
+X(30) X(31) X(32) X(33) X(34) X(35) X(36) X(37) X(38) X(39)
+
+#undef X
+#define X(irq) econet_irq_dispatch ## irq,
+
+static void (* const econet_irq_dispatchers[])(void) = {
+ X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9)
+ X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19)
+ X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29)
+ X(30) X(31) X(32) X(33) X(34) X(35) X(36) X(37) X(38) X(39)
+};
+
+/* Likewise, we do the same for the 2 IPI IRQs so that we can route them back */
+static void econet_cpu_dispatch0(void)
+{
+ do_domain_IRQ(econet_intc.ipi_domain, 0);
+}
+
+static void econet_cpu_dispatch1(void)
+{
+ do_domain_IRQ(econet_intc.ipi_domain, 1);
+}
+
+static void (* const econet_cpu_dispatchers[])(void) = {
+ econet_cpu_dispatch0,
+ econet_cpu_dispatch1,
+};
+
static const struct irq_chip econet_irq_chip;
static int econet_intc_map(struct irq_domain *d, u32 irq, irq_hw_number_t hwirq)
@@ -174,6 +230,10 @@ static int econet_intc_map(struct irq_domain *d, u32 irq, irq_hw_number_t hwirq)
}
irq_set_chip_data(irq, NULL);
+
+ if (cpu_has_veic)
+ set_vi_handler(hwirq + 1, econet_irq_dispatchers[hwirq]);
+
return 0;
}
@@ -249,6 +309,100 @@ static int __init get_shadow_interrupts(struct device_node *node)
return 0;
}
+/**
+ * econet_cpu_init() - configure routing of CPU interrupts to the correct domain.
+ * @node: The devicetree node of this interrupt controller.
+ *
+ * Interrupts that originate from the CPU are unconditionally unmasked here and are re-routed back
+ * to the IPI irq_domain in the CPU intc. Masking still takes place but the CPU intc is in charge
+ * of it, using the mask bits of the c0_status register.
+ *
+ * Note that because IP2 ... IP7 are repurposed as Interrupt Priority Level, only the two IPI
+ * interrupts are actually supported.
+ */
+static int __init econet_cpu_init(struct device_node *node)
+{
+ const char *field = "econet,cpu-interrupt-map";
+ struct device_node *parent_intc;
+ int map_size;
+ u32 mask;
+
+ map_size = of_property_count_u32_elems(node, field);
+
+ if (map_size <= 0) {
+ return 0;
+ } else if (map_size % 2) {
+ pr_err("%pOF: %s count is odd, ignoring\n", node, field);
+ return 0;
+ }
+
+ u32 *maps __free(kfree) = kmalloc_array(map_size, sizeof(u32), GFP_KERNEL);
+ if (!maps)
+ return -ENOMEM;
+
+ if (of_property_read_u32_array(node, field, maps, map_size)) {
+ pr_err("%pOF: Failed to read %s\n", node, field);
+ return -EINVAL;
+ }
+
+ /* Validation */
+ for (int i = 0; i < map_size; i += 2) {
+ u32 receive = maps[i];
+ u32 dispatch = maps[i + 1];
+ u8 shadow;
+
+ if (receive >= IRQ_COUNT) {
+ pr_err("%pOF: Entry %d:%d in %s (%u) is out of bounds\n",
+ node, i, 0, field, receive);
+ return -EINVAL;
+ }
+
+ shadow = econet_intc.interrupt_shadows[receive];
+ if (shadow != NOT_PERCPU && shadow >= IRQ_COUNT) {
+ pr_err("%pOF: Entry %d:%d in %s (%u) has invalid shadow (%d)\n",
+ node, i, 0, field, receive, shadow);
+ return -EINVAL;
+ }
+
+ if (dispatch >= ARRAY_SIZE(econet_cpu_dispatchers)) {
+ pr_err("%pOF: Entry %d:%d in %s (%u) is out of bounds only IPI interrupts are supported\n",
+ node, i, 1, field, dispatch);
+ return -EINVAL;
+ }
+ }
+
+ parent_intc = of_irq_find_parent(node);
+ if (!parent_intc) {
+ pr_err("%pOF: Failed to find parent %s\n", node, "IRQ device");
+ return -ENODEV;
+ }
+
+ econet_intc.ipi_domain = irq_find_matching_host(parent_intc, DOMAIN_BUS_IPI);
+ if (!econet_intc.ipi_domain) {
+ pr_err("%pOF: Failed to find parent %s\n", node, "IPI domain");
+ return -ENODEV;
+ }
+
+ mask = 0;
+ for (int i = 0; i < map_size; i += 2) {
+ u32 receive = maps[i];
+ u32 dispatch = maps[i + 1];
+ u8 shadow;
+
+ set_vi_handler(receive + 1, econet_cpu_dispatchers[dispatch]);
+
+ mask |= BIT(receive);
+
+ shadow = econet_intc.interrupt_shadows[receive];
+ if (shadow != NOT_PERCPU)
+ mask |= BIT(shadow);
+ }
+
+ econet_wreg(REG_MASK0, mask, mask);
+
+ return 0;
+}
+
static int __init econet_intc_of_init(struct device_node *node, struct device_node *parent)
{
struct irq_domain *domain;
@@ -294,7 +448,23 @@ static int __init econet_intc_of_init(struct device_node *node, struct device_no
goto err_unmap;
}
- irq_set_chained_handler_and_data(irq, econet_intc_from_parent, domain);
+ /*
+ * 34K Manual (MD00534) Section 6.3.1.3 rev 1.13 page 136:
+ * In VEIC mode, IP2 ... IP7 are repurposed as Interrupt Priority Level. The controller
+ * will filter incoming interrupts whose priority is lower than the IPL number. Therefore
+ * we must not set any of these bits. We avoid setting IP2 by not actually chaining this
+ * intc to the CPU intc.
+ */
+ if (cpu_has_veic) {
+ ret = econet_cpu_init(node);
+
+ if (ret)
+ return ret;
+ } else {
+ irq_set_chained_handler_and_data(irq, econet_intc_from_parent, domain);
+ }
+
+ econet_intc.domain = domain;
return 0;
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 291d766..b57d81a 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -4784,8 +4784,7 @@ static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
struct its_node *its = data;
/* erratum 22375: only alloc 8MB table size (20 bits) */
- its->typer &= ~GITS_TYPER_DEVBITS;
- its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
+ FIELD_MODIFY(GITS_TYPER_DEVBITS, &its->typer, 20 - 1);
its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
return true;
@@ -4805,8 +4804,7 @@ static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
struct its_node *its = data;
/* On QDF2400, the size of the ITE is 16Bytes */
- its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
- its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
+ FIELD_MODIFY(GITS_TYPER_ITT_ENTRY_SIZE, &its->typer, 16 - 1);
return true;
}
@@ -4840,10 +4838,8 @@ static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
its->get_msi_base = its_irq_get_msi_base_pre_its;
ids = ilog2(pre_its_window[1]) - 2;
- if (device_ids(its) > ids) {
- its->typer &= ~GITS_TYPER_DEVBITS;
- its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
- }
+ if (device_ids(its) > ids)
+ FIELD_MODIFY(GITS_TYPER_DEVBITS, &its->typer, ids - 1);
/* the pre-ITS breaks isolation, so disable MSI remapping */
its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
@@ -5837,6 +5833,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
its_acpi_probe();
if (list_empty(&its_nodes)) {
+ rdists->has_vlpis = false;
pr_warn("ITS: No ITS available, not enabling LPIs\n");
return -ENXIO;
}
diff --git a/drivers/irqchip/irq-gic-v5-its.c b/drivers/irqchip/irq-gic-v5-its.c
index 36a8d13..28e39b0 100644
--- a/drivers/irqchip/irq-gic-v5-its.c
+++ b/drivers/irqchip/irq-gic-v5-its.c
@@ -929,14 +929,15 @@ static void gicv5_its_free_eventid(struct gicv5_its_dev *its_dev, u32 event_id_b
static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs, void *arg)
{
- u32 device_id, event_id_base, lpi;
struct gicv5_its_dev *its_dev;
+ u32 device_id, event_id_base;
msi_alloc_info_t *info = arg;
irq_hw_number_t hwirq;
struct irq_data *irqd;
int ret, i;
its_dev = info->scratchpad[0].ptr;
+ device_id = its_dev->device_id;
ret = gicv5_its_alloc_eventid(its_dev, info, nr_irqs, &event_id_base);
if (ret)
@@ -946,22 +947,11 @@ static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int vi
if (ret)
goto out_eventid;
- device_id = its_dev->device_id;
+ ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL);
+ if (ret)
+ goto out_eventid;
for (i = 0; i < nr_irqs; i++) {
- ret = gicv5_alloc_lpi();
- if (ret < 0) {
- pr_debug("Failed to find free LPI!\n");
- goto out_free_irqs;
- }
- lpi = ret;
-
- ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
- if (ret) {
- gicv5_free_lpi(lpi);
- goto out_free_irqs;
- }
-
/*
* Store eventid and deviceid into the hwirq for later use.
*
@@ -980,13 +970,6 @@ static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int vi
return 0;
-out_free_irqs:
- while (--i >= 0) {
- irqd = irq_domain_get_irq_data(domain, virq + i);
- gicv5_free_lpi(irqd->parent_data->hwirq);
- irq_domain_reset_irq_data(irqd);
- irq_domain_free_irqs_parent(domain, virq + i, 1);
- }
out_eventid:
gicv5_its_free_eventid(its_dev, event_id_base, nr_irqs);
return ret;
@@ -1009,15 +992,14 @@ static void gicv5_its_irq_domain_free(struct irq_domain *domain, unsigned int vi
bitmap_release_region(its_dev->event_map, event_id_base,
get_count_order(nr_irqs));
- /* Hierarchically free irq data */
for (i = 0; i < nr_irqs; i++) {
d = irq_domain_get_irq_data(domain, virq + i);
-
- gicv5_free_lpi(d->parent_data->hwirq);
irq_domain_reset_irq_data(d);
- irq_domain_free_irqs_parent(domain, virq + i, 1);
}
+ /* Hierarchically free irq data */
+ irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+
gicv5_its_syncr(its, its_dev);
gicv5_irs_syncr();
}
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c
index 6b0903b..c1af070 100644
--- a/drivers/irqchip/irq-gic-v5.c
+++ b/drivers/irqchip/irq-gic-v5.c
@@ -59,16 +59,6 @@ static void release_lpi(u32 lpi)
ida_free(&lpi_ida, lpi);
}
-int gicv5_alloc_lpi(void)
-{
- return alloc_lpi();
-}
-
-void gicv5_free_lpi(u32 lpi)
-{
- release_lpi(lpi);
-}
-
static void gicv5_ppi_priority_init(void)
{
write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1);
@@ -806,38 +796,64 @@ static void gicv5_lpi_config_reset(struct irq_data *d)
gicv5_lpi_irq_write_pending_state(d, false);
}
+static void gicv5_irq_lpi_domain_free(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs)
+{
+ struct irq_data *d;
+
+ for (unsigned int i = 0; i < nr_irqs; i++, virq++) {
+ d = irq_domain_get_irq_data(domain, virq);
+
+ release_lpi(d->hwirq);
+
+ irq_set_handler(virq, NULL);
+ irq_domain_reset_irq_data(d);
+ }
+}
+
static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs, void *arg)
{
irq_hw_number_t hwirq;
struct irq_data *irqd;
- u32 *lpi = arg;
+ unsigned int i;
int ret;
- if (WARN_ON_ONCE(nr_irqs != 1))
- return -EINVAL;
+ for (i = 0; i < nr_irqs; i++) {
+ ret = alloc_lpi();
+ if (ret < 0)
+ goto out_free_lpis;
+ hwirq = ret;
- hwirq = *lpi;
+ ret = gicv5_irs_iste_alloc(hwirq);
+ if (ret < 0) {
+ /* Undo partial state first, then clean up the rest */
+ release_lpi(hwirq);
+ goto out_free_lpis;
+ }
- irqd = irq_domain_get_irq_data(domain, virq);
+ irqd = irq_domain_get_irq_data(domain, virq + i);
- irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL,
- handle_fasteoi_irq, NULL, NULL);
- irqd_set_single_target(irqd);
+ irq_domain_set_info(domain, virq + i, hwirq, &gicv5_lpi_irq_chip,
+ NULL, handle_fasteoi_irq, NULL, NULL);
+ irqd_set_single_target(irqd);
- ret = gicv5_irs_iste_alloc(hwirq);
- if (ret < 0)
- return ret;
-
- gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI);
- gicv5_lpi_config_reset(irqd);
+ gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI);
+ gicv5_lpi_config_reset(irqd);
+ }
return 0;
+
+out_free_lpis:
+ if (i)
+ gicv5_irq_lpi_domain_free(domain, virq, i);
+
+ return ret;
}
static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = {
.alloc = gicv5_irq_lpi_domain_alloc,
- .free = gicv5_irq_domain_free,
+ .free = gicv5_irq_lpi_domain_free,
};
void __init gicv5_init_lpi_domain(void)
@@ -858,30 +874,21 @@ static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int vi
unsigned int nr_irqs, void *arg)
{
struct irq_data *irqd;
- int ret, i;
- u32 lpi;
+ int ret;
- for (i = 0; i < nr_irqs; i++) {
- ret = gicv5_alloc_lpi();
- if (ret < 0)
- return ret;
+ ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
+ if (ret)
+ return ret;
- lpi = ret;
+ for (unsigned int i = 0; i < nr_irqs; i++, virq++) {
+ irqd = irq_domain_get_irq_data(domain, virq);
- ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
- if (ret) {
- gicv5_free_lpi(lpi);
- return ret;
- }
-
- irqd = irq_domain_get_irq_data(domain, virq + i);
-
- irq_domain_set_hwirq_and_chip(domain, virq + i, i,
- &gicv5_ipi_irq_chip, NULL);
+ irq_domain_set_hwirq_and_chip(domain, virq, i,
+ &gicv5_ipi_irq_chip, NULL);
irqd_set_single_target(irqd);
- irq_set_handler(virq + i, handle_percpu_irq);
+ irq_set_handler(virq, handle_percpu_irq);
}
return 0;
@@ -899,12 +906,11 @@ static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int vi
if (!d)
return;
- gicv5_free_lpi(d->parent_data->hwirq);
-
irq_set_handler(virq + i, NULL);
irq_domain_reset_irq_data(d);
- irq_domain_free_irqs_parent(domain, virq + i, 1);
}
+
+ irq_domain_free_irqs_parent(domain, virq, nr_irqs);
}
static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = {
diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c
index 758262f..53d7d23 100644
--- a/drivers/irqchip/irq-loongarch-avec.c
+++ b/drivers/irqchip/irq-loongarch-avec.c
@@ -24,7 +24,6 @@
#define VECTORS_PER_REG 64
#define IRR_VECTOR_MASK 0xffUL
#define IRR_INVALID_MASK 0x80000000UL
-#define AVEC_MSG_OFFSET 0x100000
#ifdef CONFIG_SMP
struct pending_list {
@@ -47,15 +46,6 @@ struct avecintc_chip {
static struct avecintc_chip loongarch_avec;
-struct avecintc_data {
- struct list_head entry;
- unsigned int cpu;
- unsigned int vec;
- unsigned int prev_cpu;
- unsigned int prev_vec;
- unsigned int moving;
-};
-
static inline void avecintc_enable(void)
{
#ifdef CONFIG_MACH_LOONGSON64
@@ -87,7 +77,7 @@ static inline void pending_list_init(int cpu)
INIT_LIST_HEAD(&plist->head);
}
-static void avecintc_sync(struct avecintc_data *adata)
+void avecintc_sync(struct avecintc_data *adata)
{
struct pending_list *plist;
@@ -111,7 +101,7 @@ static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *de
return -EBUSY;
if (cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest))
- return 0;
+ return IRQ_SET_MASK_OK_DONE;
cpumask_and(&intersect_mask, dest, cpu_online_mask);
@@ -123,7 +113,8 @@ static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *de
adata->cpu = cpu;
adata->vec = vector;
per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data);
- avecintc_sync(adata);
+ if (!cpu_has_redirectint)
+ avecintc_sync(adata);
}
irq_data_update_effective_affinity(data, cpumask_of(cpu));
@@ -415,6 +406,9 @@ static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
static inline int __init acpi_cascade_irqdomain_init(void)
{
+ if (cpu_has_redirectint)
+ return redirect_acpi_init(loongarch_avec.domain);
+
return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
}
diff --git a/drivers/irqchip/irq-loongarch-ir.c b/drivers/irqchip/irq-loongarch-ir.c
new file mode 100644
index 0000000..21c649a8
--- /dev/null
+++ b/drivers/irqchip/irq-loongarch-ir.c
@@ -0,0 +1,537 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024-2026 Loongson Technologies, Inc.
+ */
+#define pr_fmt(fmt) "redirect: " fmt
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/irq-msi-lib.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/msi.h>
+#include <linux/spinlock.h>
+
+#include <asm/irq.h>
+#include <asm/loongarch.h>
+#include <asm/loongson.h>
+#include <asm/numa.h>
+#include <asm/setup.h>
+
+#include "irq-loongson.h"
+
+#define LOONGARCH_IOCSR_REDIRECT_CFG 0x15e0
+#define LOONGARCH_IOCSR_REDIRECT_TBR 0x15e8 /* IRT BASE REG */
+#define LOONGARCH_IOCSR_REDIRECT_CQB 0x15f0 /* IRT CACHE QUEUE BASE */
+#define LOONGARCH_IOCSR_REDIRECT_CQH 0x15f8 /* IRT CACHE QUEUE HEAD, 32bit */
+#define LOONGARCH_IOCSR_REDIRECT_CQT 0x15fc /* IRT CACHE QUEUE TAIL, 32bit */
+
+#define CQB_ADDR_MASK GENMASK_U64(47, 12)
+#define CQB_SIZE_MASK 0xf
+
+#define GPID_ADDR_MASK GENMASK_U64(47, 6)
+#define GPID_ADDR_SHIFT 6
+
+#define INVALID_INDEX 0
+#define CFG_DISABLE_IDLE 2
+
+#define MAX_IR_ENGINES 16
+
+struct redirect_entry {
+ struct {
+ u64 valid : 1,
+ res1 : 5,
+ gpid : 42,
+ res2 : 8,
+ vector : 8;
+ } lo;
+ u64 hi;
+};
+
+#define IRD_ENTRY_SIZE sizeof(struct redirect_entry)
+#define IRD_ENTRIES SZ_64K
+#define IRD_TABLE_PAGE_ORDER get_order(IRD_ENTRIES * IRD_ENTRY_SIZE)
+
+struct redirect_cmd {
+ union {
+ u64 cmd_info;
+ struct {
+ u64 res1 : 4,
+ type : 1,
+ need_notice : 1,
+ pad1 : 2,
+ index : 16,
+ pad2 : 40;
+ } index;
+ };
+ u64 notice_addr;
+};
+
+#define IRD_CMD_SIZE sizeof(struct redirect_cmd)
+#define INV_QUEUE_SIZE SZ_4K
+#define INV_QUEUE_PAGE_ORDER get_order(INV_QUEUE_SIZE * IRD_CMD_SIZE)
+
+struct redirect_gpid {
+ u64 pir[4]; /* Pending interrupt requested */
+ u8 en : 1, /* Doorbell */
+ res1 : 7;
+ u8 irqnum;
+ u16 res2;
+ u32 dstcpu;
+ u32 rsvd[6];
+};
+
+struct redirect_table {
+ struct redirect_entry *table;
+ unsigned long *bitmap;
+ raw_spinlock_t lock;
+};
+
+struct redirect_queue {
+ struct redirect_cmd *cmd_base;
+ int head;
+ int tail;
+ raw_spinlock_t lock;
+};
+
+struct redirect_desc {
+ struct redirect_table ird_table;
+ struct redirect_queue inv_queue;
+ int node;
+};
+
+struct redirect_item {
+ int index;
+ struct redirect_desc *irde;
+ struct redirect_gpid *gpid;
+};
+
+static struct irq_domain *redirect_domain;
+static struct redirect_desc redirect_descs[MAX_IR_ENGINES];
+
+static phys_addr_t msi_base_addr;
+static phys_addr_t redirect_reg_base = LOONGSON_REG_BASE;
+
+#ifdef CONFIG_32BIT
+
+#define REDIRECT_REG(reg, node) \
+ ((void __iomem *)(IO_BASE | redirect_reg_base | (reg)))
+
+#else
+
+#define REDIRECT_REG(reg, node) \
+ ((void __iomem *)(IO_BASE | redirect_reg_base | (u64)(node) << NODE_ADDRSPACE_SHIFT | (reg)))
+
+#endif
+
+static inline u32 redirect_read_reg32(u32 node, u32 reg)
+{
+ return readl(REDIRECT_REG(reg, node));
+}
+
+static inline void redirect_write_reg32(u32 node, u32 val, u32 reg)
+{
+ writel(val, REDIRECT_REG(reg, node));
+}
+
+static inline void redirect_write_reg64(u32 node, u64 val, u32 reg)
+{
+ writeq(val, REDIRECT_REG(reg, node));
+}
+
+static inline struct redirect_entry *item_get_entry(struct redirect_item *item)
+{
+ return item->irde->ird_table.table + item->index;
+}
+
+static inline bool invalid_queue_is_full(int node, u32 *tail)
+{
+ u32 head = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQH);
+
+ *tail = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQT);
+
+ return head == ((*tail + 1) % INV_QUEUE_SIZE);
+}
+
+static void invalid_enqueue(struct redirect_item *item, struct redirect_cmd *cmd)
+{
+ struct redirect_queue *inv_queue = &item->irde->inv_queue;
+ u32 tail;
+
+ guard(raw_spinlock_irqsave)(&inv_queue->lock);
+
+ while (invalid_queue_is_full(item->irde->node, &tail))
+ cpu_relax();
+
+ memcpy(&inv_queue->cmd_base[tail], cmd, sizeof(*cmd));
+
+ redirect_write_reg32(item->irde->node, (tail + 1) % INV_QUEUE_SIZE, LOONGARCH_IOCSR_REDIRECT_CQT);
+}
+
+static void irde_invalidate_entry(struct redirect_item *item)
+{
+ struct redirect_cmd cmd;
+ u64 raddr = 0;
+
+ cmd.cmd_info = 0;
+ cmd.index.type = INVALID_INDEX;
+ cmd.index.need_notice = 1;
+ cmd.index.index = item->index;
+ cmd.notice_addr = (u64)(__pa(&raddr));
+
+ invalid_enqueue(item, &cmd);
+
+ /*
+ * The CPU needs to wait here for cmd to complete, and it determines this
+ * by checking whether the invalidation queue has already written a valid value
+ * to cmd.notice_addr.
+ */
+ while (!raddr)
+ cpu_relax();
+}
+
+static inline struct avecintc_data *irq_data_get_avec_data(struct irq_data *data)
+{
+ return data->parent_data->chip_data;
+}
+
+static int redirect_table_alloc(int node, u32 nr_irqs)
+{
+ struct redirect_table *ird_table = &redirect_descs[node].ird_table;
+ int index, order = 0;
+
+ if (nr_irqs > 1) {
+ nr_irqs = __roundup_pow_of_two(nr_irqs);
+ order = ilog2(nr_irqs);
+ }
+
+ guard(raw_spinlock_irqsave)(&ird_table->lock);
+
+ index = bitmap_find_free_region(ird_table->bitmap, IRD_ENTRIES, order);
+ if (index < 0) {
+ pr_err("No redirect entry to use\n");
+ return -EINVAL;
+ }
+
+ return index;
+}
+
+static void redirect_table_free(struct redirect_item *item)
+{
+ struct redirect_table *ird_table = &item->irde->ird_table;
+ struct redirect_entry *entry = item_get_entry(item);
+
+ memset(entry, 0, sizeof(*entry));
+
+ scoped_guard(raw_spinlock_irq, &ird_table->lock)
+ clear_bit(item->index, ird_table->bitmap);
+
+ kfree(item->gpid);
+
+ irde_invalidate_entry(item);
+}
+
+static inline void redirect_domain_prepare_entry(struct redirect_item *item,
+ struct avecintc_data *adata)
+{
+ struct redirect_entry *entry = item_get_entry(item);
+
+ item->gpid->en = 1;
+ item->gpid->dstcpu = adata->cpu;
+ item->gpid->irqnum = adata->vec;
+
+ entry->lo.valid = 1;
+ entry->lo.vector = 0xff;
+ entry->lo.gpid = ((unsigned long)item->gpid & GPID_ADDR_MASK) >> GPID_ADDR_SHIFT;
+}
+
+static void redirect_free_resources(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ for (int i = 0; i < nr_irqs; i++) {
+ struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
+
+ if (irq_data && irq_data->chip_data) {
+ struct redirect_item *item = irq_data->chip_data;
+
+ redirect_table_free(item);
+ kfree(item);
+ }
+ }
+}
+
+#ifdef CONFIG_SMP
+static int redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
+{
+ struct avecintc_data *adata = irq_data_get_avec_data(data);
+ struct redirect_item *item = data->chip_data;
+ int ret;
+
+ ret = irq_chip_set_affinity_parent(data, dest, force);
+ switch (ret) {
+ case IRQ_SET_MASK_OK:
+ break;
+ case IRQ_SET_MASK_OK_DONE:
+ return ret;
+ default:
+ pr_err("IRDE: set_affinity error %d\n", ret);
+ return ret;
+ }
+
+ redirect_domain_prepare_entry(item, adata);
+ irde_invalidate_entry(item);
+ avecintc_sync(adata);
+
+ return IRQ_SET_MASK_OK;
+}
+#endif
+
+static void redirect_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
+{
+ struct redirect_item *item = irq_data_get_irq_chip_data(d);
+
+ msg->address_hi = 0x0;
+ msg->address_lo = (msi_base_addr | 1 << 2);
+ msg->data = item->index;
+}
+
+static struct irq_chip loongarch_redirect_chip = {
+ .name = "REDIRECT",
+ .irq_ack = irq_chip_ack_parent,
+ .irq_mask = irq_chip_mask_parent,
+ .irq_unmask = irq_chip_unmask_parent,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = redirect_set_affinity,
+#endif
+ .irq_compose_msi_msg = redirect_compose_msi_msg,
+};
+
+static int redirect_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *arg)
+{
+ msi_alloc_info_t *info = arg;
+ int ret, i, node, index;
+
+ node = dev_to_node(info->desc->dev);
+
+ ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
+ if (ret < 0)
+ return ret;
+
+ index = redirect_table_alloc(node, nr_irqs);
+ if (index < 0) {
+ pr_err("Alloc redirect table entry failed\n");
+ return -EINVAL;
+ }
+
+ for (i = 0; i < nr_irqs; i++) {
+ struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
+ struct redirect_item *item;
+
+ item = kzalloc(sizeof(*item), GFP_KERNEL);
+ if (!item) {
+ pr_err("Alloc redirect descriptor failed\n");
+ goto out_free_resources;
+ }
+ item->irde = &redirect_descs[node];
+
+ /*
+ * Only bits 47:6 of the GPID are passed to the controller,
+ * 64-byte alignment must be guarantee and make kzalloc can
+ * align to the respective size.
+ */
+ static_assert(sizeof(*item->gpid) == 64);
+ item->gpid = kzalloc_node(sizeof(*item->gpid), GFP_KERNEL, node);
+ if (!item->gpid) {
+ pr_err("Alloc redirect GPID failed\n");
+ goto out_free_resources;
+ }
+ item->index = index + i;
+
+ irq_data->chip_data = item;
+ irq_data->chip = &loongarch_redirect_chip;
+
+ redirect_domain_prepare_entry(item, irq_data_get_avec_data(irq_data));
+ }
+
+ return 0;
+
+out_free_resources:
+ redirect_free_resources(domain, virq, nr_irqs);
+ irq_domain_free_irqs_common(domain, virq, nr_irqs);
+
+ return -ENOMEM;
+}
+
+static void redirect_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
+{
+ redirect_free_resources(domain, virq, nr_irqs);
+ return irq_domain_free_irqs_common(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops redirect_domain_ops = {
+ .alloc = redirect_domain_alloc,
+ .free = redirect_domain_free,
+ .select = msi_lib_irq_domain_select,
+};
+
+static int redirect_table_init(struct redirect_desc *irde)
+{
+ struct redirect_table *ird_table = &irde->ird_table;
+ unsigned long *bitmap;
+ struct folio *folio;
+
+ folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, IRD_TABLE_PAGE_ORDER, irde->node);
+ if (!folio) {
+ pr_err("Node [%d] redirect table alloc pages failed!\n", irde->node);
+ return -ENOMEM;
+ }
+ ird_table->table = folio_address(folio);
+
+ bitmap = bitmap_zalloc(IRD_ENTRIES, GFP_KERNEL);
+ if (!bitmap) {
+ pr_err("Node [%d] redirect table bitmap alloc pages failed!\n", irde->node);
+ folio_put(folio);
+ ird_table->table = NULL;
+ return -ENOMEM;
+ }
+ ird_table->bitmap = bitmap;
+
+ raw_spin_lock_init(&ird_table->lock);
+
+ return 0;
+}
+
+static int redirect_queue_init(struct redirect_desc *irde)
+{
+ struct redirect_queue *inv_queue = &irde->inv_queue;
+ struct folio *folio;
+
+ folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, INV_QUEUE_PAGE_ORDER, irde->node);
+ if (!folio) {
+ pr_err("Node [%d] invalid queue alloc pages failed!\n", irde->node);
+ return -ENOMEM;
+ }
+
+ inv_queue->cmd_base = folio_address(folio);
+ inv_queue->head = 0;
+ inv_queue->tail = 0;
+ raw_spin_lock_init(&inv_queue->lock);
+
+ return 0;
+}
+
+static void redirect_irde_cfg(struct redirect_desc *irde)
+{
+ redirect_write_reg64(irde->node, CFG_DISABLE_IDLE, LOONGARCH_IOCSR_REDIRECT_CFG);
+ redirect_write_reg64(irde->node, __pa(irde->ird_table.table), LOONGARCH_IOCSR_REDIRECT_TBR);
+ redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQH);
+ redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQT);
+ redirect_write_reg64(irde->node, ((unsigned long)irde->inv_queue.cmd_base & CQB_ADDR_MASK) |
+ CQB_SIZE_MASK, LOONGARCH_IOCSR_REDIRECT_CQB);
+}
+
+static void __init redirect_irde_free(struct redirect_desc *irde)
+{
+ struct redirect_table *ird_table = &redirect_descs->ird_table;
+ struct redirect_queue *inv_queue = &redirect_descs->inv_queue;
+
+ if (ird_table->table) {
+ folio_put(virt_to_folio(ird_table->table));
+ ird_table->table = NULL;
+ }
+
+ if (ird_table->bitmap) {
+ bitmap_free(ird_table->bitmap);
+ ird_table->bitmap = NULL;
+ }
+
+ if (inv_queue->cmd_base) {
+ folio_put(virt_to_folio(inv_queue->cmd_base));
+ inv_queue->cmd_base = NULL;
+ }
+}
+
+static int __init redirect_irde_init(int node)
+{
+ struct redirect_desc *irde = &redirect_descs[node];
+ int ret;
+
+ irde->node = node;
+
+ ret = redirect_table_init(irde);
+ if (ret)
+ return ret;
+
+ ret = redirect_queue_init(irde);
+ if (ret) {
+ redirect_irde_free(irde);
+ return ret;
+ }
+
+ redirect_irde_cfg(irde);
+
+ return 0;
+}
+
+static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, const unsigned long end)
+{
+ struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
+
+ msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
+
+ return pch_msi_acpi_init_avec(redirect_domain);
+}
+
+static int __init acpi_cascade_irqdomain_init(void)
+{
+ return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
+}
+
+int __init redirect_acpi_init(struct irq_domain *parent)
+{
+ struct fwnode_handle *fwnode;
+ int ret = -EINVAL, node;
+
+ fwnode = irq_domain_alloc_named_fwnode("redirect");
+ if (!fwnode) {
+ pr_err("Unable to alloc redirect domain handle\n");
+ goto fail;
+ }
+
+ redirect_domain = irq_domain_create_hierarchy(parent, 0, IRD_ENTRIES, fwnode,
+ &redirect_domain_ops, redirect_descs);
+ if (!redirect_domain) {
+ pr_err("Unable to alloc redirect domain\n");
+ goto out_free_fwnode;
+ }
+
+ for_each_node_mask(node, node_possible_map) {
+ ret = redirect_irde_init(node);
+ if (ret)
+ goto out_clear_irde;
+ }
+
+ ret = acpi_cascade_irqdomain_init();
+ if (ret < 0) {
+ pr_err("Failed to cascade IRQ domain, ret=%d\n", ret);
+ goto out_clear_irde;
+ }
+
+ pr_info("init succeeded\n");
+
+ return 0;
+
+out_clear_irde:
+ for_each_node_mask(node, node_possible_map) {
+ redirect_irde_free(&redirect_descs[node]);
+ }
+ irq_domain_remove(redirect_domain);
+out_free_fwnode:
+ irq_domain_free_fwnode(fwnode);
+fail:
+ return ret;
+}
diff --git a/drivers/irqchip/irq-loongson.h b/drivers/irqchip/irq-loongson.h
index 11fa138..dd37cd7 100644
--- a/drivers/irqchip/irq-loongson.h
+++ b/drivers/irqchip/irq-loongson.h
@@ -6,6 +6,17 @@
#ifndef _DRIVERS_IRQCHIP_IRQ_LOONGSON_H
#define _DRIVERS_IRQCHIP_IRQ_LOONGSON_H
+#define AVEC_MSG_OFFSET 0x100000
+
+struct avecintc_data {
+ struct list_head entry;
+ unsigned int cpu;
+ unsigned int vec;
+ unsigned int prev_cpu;
+ unsigned int prev_vec;
+ unsigned int moving;
+};
+
int find_pch_pic(u32 gsi);
int liointc_acpi_init(struct irq_domain *parent,
@@ -14,6 +25,8 @@ int eiointc_acpi_init(struct irq_domain *parent,
struct acpi_madt_eio_pic *acpi_eiointc);
int avecintc_acpi_init(struct irq_domain *parent);
+int redirect_acpi_init(struct irq_domain *parent);
+
int htvec_acpi_init(struct irq_domain *parent,
struct acpi_madt_ht_pic *acpi_htvec);
int pch_lpc_acpi_init(struct irq_domain *parent,
@@ -24,4 +37,6 @@ int pch_msi_acpi_init(struct irq_domain *parent,
struct acpi_madt_msi_pic *acpi_pchmsi);
int pch_msi_acpi_init_avec(struct irq_domain *parent);
+void avecintc_sync(struct avecintc_data *adata);
+
#endif /* _DRIVERS_IRQCHIP_IRQ_LOONGSON_H */
diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c
index f722e9c..91a9c33 100644
--- a/drivers/irqchip/irq-meson-gpio.c
+++ b/drivers/irqchip/irq-meson-gpio.c
@@ -27,6 +27,10 @@
/* use for A1 like chips */
#define REG_PIN_A1_SEL 0x04
+/* use for A9 like chips */
+#define REG_A9_AO_POL 0x00
+#define REG_A9_AO_EDGE 0x30
+
/*
* Note: The S905X3 datasheet reports that BOTH_EDGE is controlled by
* bits 24 to 31. Tests on the actual HW show that these bits are
@@ -53,6 +57,8 @@ static void meson_a1_gpio_irq_sel_pin(struct meson_gpio_irq_controller *ctl,
static void meson_a1_gpio_irq_init(struct meson_gpio_irq_controller *ctl);
static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
unsigned int type, u32 *channel_hwirq);
+static int meson_a9_ao_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
+ unsigned int type, u32 *channel_hwirq);
static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
unsigned int type, u32 *channel_hwirq);
@@ -116,6 +122,18 @@ struct meson_gpio_irq_params {
.pin_sel_mask = 0xff, \
.nr_channels = 2, \
+#define INIT_MESON_A9_AO_COMMON_DATA(irqs) \
+ INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init, \
+ meson_a1_gpio_irq_sel_pin, \
+ meson_a9_ao_gpio_irq_set_type) \
+ .support_edge_both = true, \
+ .edge_both_offset = 0, \
+ .edge_single_offset = 0, \
+ .edge_pol_reg = 0x2c, \
+ .pol_low_offset = 0, \
+ .pin_sel_mask = 0xff, \
+ .nr_channels = 20, \
+
#define INIT_MESON_S4_COMMON_DATA(irqs) \
INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init, \
meson_a1_gpio_irq_sel_pin, \
@@ -170,6 +188,14 @@ static const struct meson_gpio_irq_params a5_params = {
INIT_MESON_S4_COMMON_DATA(99)
};
+static const struct meson_gpio_irq_params a9_params = {
+ INIT_MESON_S4_COMMON_DATA(96)
+};
+
+static const struct meson_gpio_irq_params a9_ao_params = {
+ INIT_MESON_A9_AO_COMMON_DATA(39)
+};
+
static const struct meson_gpio_irq_params s4_params = {
INIT_MESON_S4_COMMON_DATA(82)
};
@@ -203,6 +229,8 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = {
{ .compatible = "amlogic,a4-gpio-ao-intc", .data = &a4_ao_params },
{ .compatible = "amlogic,a4-gpio-intc", .data = &a4_params },
{ .compatible = "amlogic,a5-gpio-intc", .data = &a5_params },
+ { .compatible = "amlogic,a9-gpio-ao-intc", .data = &a9_ao_params },
+ { .compatible = "amlogic,a9-gpio-intc", .data = &a9_params },
{ .compatible = "amlogic,s6-gpio-intc", .data = &s6_params },
{ .compatible = "amlogic,s7-gpio-intc", .data = &s7_params },
{ .compatible = "amlogic,s7d-gpio-intc", .data = &s7_params },
@@ -376,6 +404,55 @@ static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
}
/*
+ * gpio irq relative registers for a9_ao
+ * -PADCTRL_GPIO_IRQ_CTRL0
+ * bit[31]: enable/disable all the irq lines
+ * bit[0-19]: polarity trigger
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[X]
+ * bit[0-5]: 6 bits to choose gpio source for irq line 2*[X] - 2
+ * bit[16-21]:6 bits to choose gpio source for irq line 2*[X] - 1
+ * where X = 1-10
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[11]
+ * bit[0-19]: both edge trigger
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[12]
+ * bit[0-19]: single edge trigger
+ */
+static int meson_a9_ao_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
+ unsigned int type, u32 *channel_hwirq)
+{
+ const struct meson_gpio_irq_params *params = ctl->params;
+ unsigned int idx;
+ u32 val;
+
+ idx = meson_gpio_irq_get_channel_idx(ctl, channel_hwirq);
+
+ type &= IRQ_TYPE_SENSE_MASK;
+
+ meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, BIT(idx), 0);
+
+ if (type == IRQ_TYPE_EDGE_BOTH) {
+ val = BIT(ctl->params->edge_both_offset + idx);
+ meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, val, val);
+ return 0;
+ }
+
+ val = 0;
+ if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING))
+ val = BIT(idx);
+ meson_gpio_irq_update_bits(ctl, REG_A9_AO_POL, BIT(idx), val);
+
+ val = 0;
+ if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING))
+ val = BIT(idx);
+ meson_gpio_irq_update_bits(ctl, REG_A9_AO_EDGE, BIT(idx), val);
+
+ return 0;
+};
+
+/*
* gpio irq relative registers for s4
* -PADCTRL_GPIO_IRQ_CTRL0
* bit[31]: enable/disable all the irq lines
@@ -415,8 +492,7 @@ static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING))
val |= BIT(ctl->params->edge_single_offset + idx);
- meson_gpio_irq_update_bits(ctl, params->edge_pol_reg,
- BIT(idx) | BIT(12 + idx), val);
+ meson_gpio_irq_update_bits(ctl, REG_EDGE_POL, BIT(idx) | BIT(12 + idx), val);
return 0;
};
diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c
index 942c1f8..2ae3be7 100644
--- a/drivers/irqchip/irq-realtek-rtl.c
+++ b/drivers/irqchip/irq-realtek-rtl.c
@@ -23,10 +23,10 @@
#define RTL_ICTL_NUM_INPUTS 32
-#define REG(x) (realtek_ictl_base + x)
+#define REG(cpu, x) (realtek_ictl_base[cpu] + x)
static DEFINE_RAW_SPINLOCK(irq_lock);
-static void __iomem *realtek_ictl_base;
+static void __iomem *realtek_ictl_base[NR_CPUS];
/*
* IRR0-IRR3 store 4 bits per interrupt, but Realtek uses inverted numbering,
@@ -37,10 +37,29 @@ static void __iomem *realtek_ictl_base;
#define IRR_OFFSET(idx) (4 * (3 - (idx * 4) / 32))
#define IRR_SHIFT(idx) ((idx * 4) % 32)
-static void write_irr(void __iomem *irr0, int idx, u32 value)
+static inline void enable_gimr(unsigned int cpu, unsigned int hw_irq)
{
- unsigned int offset = IRR_OFFSET(idx);
- unsigned int shift = IRR_SHIFT(idx);
+ u32 gimr;
+
+ gimr = readl(REG(cpu, RTL_ICTL_GIMR));
+ gimr |= BIT(hw_irq);
+ writel(gimr, REG(cpu, RTL_ICTL_GIMR));
+}
+
+static inline void disable_gimr(unsigned int cpu, unsigned int hw_irq)
+{
+ u32 gimr;
+
+ gimr = readl(REG(cpu, RTL_ICTL_GIMR));
+ gimr &= ~BIT(hw_irq);
+ writel(gimr, REG(cpu, RTL_ICTL_GIMR));
+}
+
+static void write_irr(unsigned int cpu, int hw_irq, u32 value)
+{
+ void __iomem *irr0 = REG(cpu, RTL_ICTL_IRR0);
+ unsigned int offset = IRR_OFFSET(hw_irq);
+ unsigned int shift = IRR_SHIFT(hw_irq);
u32 irr;
irr = readl(irr0 + offset) & ~(0xf << shift);
@@ -50,47 +69,51 @@ static void write_irr(void __iomem *irr0, int idx, u32 value)
static void realtek_ictl_unmask_irq(struct irq_data *i)
{
- unsigned long flags;
- u32 value;
+ unsigned int cpu;
- raw_spin_lock_irqsave(&irq_lock, flags);
-
- value = readl(REG(RTL_ICTL_GIMR));
- value |= BIT(i->hwirq);
- writel(value, REG(RTL_ICTL_GIMR));
-
- raw_spin_unlock_irqrestore(&irq_lock, flags);
+ guard(raw_spinlock)(&irq_lock);
+ for_each_cpu(cpu, irq_data_get_effective_affinity_mask(i))
+ enable_gimr(cpu, i->hwirq);
}
static void realtek_ictl_mask_irq(struct irq_data *i)
{
- unsigned long flags;
- u32 value;
+ unsigned int cpu;
- raw_spin_lock_irqsave(&irq_lock, flags);
+ guard(raw_spinlock)(&irq_lock);
+ for_each_cpu(cpu, irq_data_get_effective_affinity_mask(i))
+ disable_gimr(cpu, i->hwirq);
+}
- value = readl(REG(RTL_ICTL_GIMR));
- value &= ~BIT(i->hwirq);
- writel(value, REG(RTL_ICTL_GIMR));
+static int realtek_ictl_irq_affinity(struct irq_data *i, const struct cpumask *dest, bool force)
+{
+ if (!irqd_irq_masked(i))
+ realtek_ictl_mask_irq(i);
- raw_spin_unlock_irqrestore(&irq_lock, flags);
+ irq_data_update_effective_affinity(i, dest);
+
+ if (!irqd_irq_masked(i))
+ realtek_ictl_unmask_irq(i);
+
+ return IRQ_SET_MASK_OK;
}
static struct irq_chip realtek_ictl_irq = {
- .name = "realtek-rtl-intc",
- .irq_mask = realtek_ictl_mask_irq,
- .irq_unmask = realtek_ictl_unmask_irq,
+ .name = "realtek-rtl-intc",
+ .irq_mask = realtek_ictl_mask_irq,
+ .irq_unmask = realtek_ictl_unmask_irq,
+ .irq_set_affinity = realtek_ictl_irq_affinity,
};
static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
{
- unsigned long flags;
+ unsigned int cpu;
irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq);
- raw_spin_lock_irqsave(&irq_lock, flags);
- write_irr(REG(RTL_ICTL_IRR0), hw, 1);
- raw_spin_unlock_irqrestore(&irq_lock, flags);
+ guard(raw_spinlock_irqsave)(&irq_lock);
+ for_each_present_cpu(cpu)
+ write_irr(cpu, hw, 1);
return 0;
}
@@ -103,12 +126,13 @@ static const struct irq_domain_ops irq_domain_ops = {
static void realtek_irq_dispatch(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
+ unsigned int cpu = smp_processor_id();
struct irq_domain *domain;
unsigned long pending;
unsigned int soc_int;
chained_irq_enter(chip, desc);
- pending = readl(REG(RTL_ICTL_GIMR)) & readl(REG(RTL_ICTL_GISR));
+ pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR));
if (unlikely(!pending)) {
spurious_interrupt();
@@ -116,7 +140,7 @@ static void realtek_irq_dispatch(struct irq_desc *desc)
}
domain = irq_desc_get_handler_data(desc);
- for_each_set_bit(soc_int, &pending, 32)
+ for_each_set_bit(soc_int, &pending, RTL_ICTL_NUM_INPUTS)
generic_handle_domain_irq(domain, soc_int);
out:
@@ -127,17 +151,19 @@ static int __init realtek_rtl_of_init(struct device_node *node, struct device_no
{
struct of_phandle_args oirq;
struct irq_domain *domain;
- unsigned int soc_irq;
- int parent_irq;
+ int cpu, parent_irq;
- realtek_ictl_base = of_iomap(node, 0);
- if (!realtek_ictl_base)
- return -ENXIO;
+ for_each_present_cpu(cpu) {
+ realtek_ictl_base[cpu] = of_iomap(node, cpu);
+ if (!realtek_ictl_base[cpu])
+ return -ENXIO;
- /* Disable all cascaded interrupts and clear routing */
- writel(0, REG(RTL_ICTL_GIMR));
- for (soc_irq = 0; soc_irq < RTL_ICTL_NUM_INPUTS; soc_irq++)
- write_irr(REG(RTL_ICTL_IRR0), soc_irq, 0);
+ /* Disable all cascaded interrupts and clear routing */
+ for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) {
+ disable_gimr(cpu, hw_irq);
+ write_irr(cpu, hw_irq, 0);
+ }
+ }
if (WARN_ON(!of_irq_count(node))) {
/*
diff --git a/drivers/irqchip/irq-renesas-rzt2h.c b/drivers/irqchip/irq-renesas-rzt2h.c
index 53cf80e..e06264a 100644
--- a/drivers/irqchip/irq-renesas-rzt2h.c
+++ b/drivers/irqchip/irq-renesas-rzt2h.c
@@ -2,6 +2,7 @@
#include <linux/bitfield.h>
#include <linux/err.h>
+#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irqchip.h>
#include <linux/irqchip/irq-renesas-rzt2h.h>
@@ -30,16 +31,44 @@
RZT2H_ICU_IRQ_S_COUNT)
#define RZT2H_ICU_SEI_COUNT 1
+#define RZT2H_ICU_CA55_ERR_START (RZT2H_ICU_SEI_START + \
+ RZT2H_ICU_SEI_COUNT)
+#define RZT2H_ICU_CA55_ERR_COUNT 2
+
+#define RZT2H_ICU_CR52_ERR_START (RZT2H_ICU_CA55_ERR_START + \
+ RZT2H_ICU_CA55_ERR_COUNT)
+#define RZT2H_ICU_CR52_ERR_COUNT 4
+
+#define RZT2H_ICU_PERI_ERR_START (RZT2H_ICU_CR52_ERR_START + \
+ RZT2H_ICU_CR52_ERR_COUNT)
+#define RZT2H_ICU_PERI_ERR_COUNT 2
+
+#define RZT2H_ICU_DSMIF_ERR_START (RZT2H_ICU_PERI_ERR_START + \
+ RZT2H_ICU_PERI_ERR_COUNT)
+#define RZT2H_ICU_DSMIF_ERR_COUNT 2
+
+#define RZT2H_ICU_ENCIF_ERR_START (RZT2H_ICU_DSMIF_ERR_START + \
+ RZT2H_ICU_DSMIF_ERR_COUNT)
+#define RZT2H_ICU_ENCIF_ERR_COUNT 2
+
#define RZT2H_ICU_NUM_IRQ (RZT2H_ICU_INTCPU_NS_COUNT + \
RZT2H_ICU_INTCPU_S_COUNT + \
RZT2H_ICU_IRQ_NS_COUNT + \
RZT2H_ICU_IRQ_S_COUNT + \
- RZT2H_ICU_SEI_COUNT)
+ RZT2H_ICU_SEI_COUNT + \
+ RZT2H_ICU_CA55_ERR_COUNT + \
+ RZT2H_ICU_CR52_ERR_COUNT + \
+ RZT2H_ICU_PERI_ERR_COUNT + \
+ RZT2H_ICU_DSMIF_ERR_COUNT + \
+ RZT2H_ICU_ENCIF_ERR_COUNT)
#define RZT2H_ICU_IRQ_IN_RANGE(n, type) \
((n) >= RZT2H_ICU_##type##_START && \
(n) < RZT2H_ICU_##type##_START + RZT2H_ICU_##type##_COUNT)
+#define RZT2H_ICU_SWINT 0x0
+#define RZT2H_ICU_SWINT_IC_MASK(i) BIT(i)
+
#define RZT2H_ICU_PORTNF_MD 0xc
#define RZT2H_ICU_PORTNF_MDi_MASK(i) (GENMASK(1, 0) << ((i) * 2))
#define RZT2H_ICU_PORTNF_MDi_PREP(i, val) (FIELD_PREP(GENMASK(1, 0), val) << ((i) * 2))
@@ -49,6 +78,29 @@
#define RZT2H_ICU_MD_RISING_EDGE 0b10
#define RZT2H_ICU_MD_BOTH_EDGES 0b11
+#define RZT2H_ICU_CA55ERR_E0MSK 0x50
+#define RZT2H_ICU_CA55ERR_CLR 0x60
+#define RZT2H_ICU_CA55ERR_STAT 0x64
+#define RZT2H_ICU_CA55ERR_MASK GENMASK(12, 0)
+
+#define RZT2H_ICU_PERIERR_E0MSKn(n) (0x98 + 0x4 * (n))
+#define RZT2H_ICU_PERIERR_CLRn(n) (0xc8 + 0x4 * (n))
+#define RZT2H_ICU_PERIERR_STAT 0xd4
+#define RZT2H_ICU_PERIERR_NUM 3
+#define RZT2H_ICU_PERIERR_MASK GENMASK(31, 0)
+
+#define RZT2H_ICU_DSMIFERR_E0MSKn(n) (0xe0 + 0x4 * (n))
+#define RZT2H_ICU_DSMIFERR_CLRn(n) (0x1a0 + 0x4 * (n))
+#define RZT2H_ICU_DSMIFERR_STAT 0x1d0
+#define RZT2H_ICU_DSMIFERR_NUM 12
+#define RZT2H_ICU_DSMIFERR_MASK GENMASK(31, 0)
+
+#define RZT2H_ICU_ENCIFERR_E0MSKn(n) (0x200 + 0x4 * (n))
+#define RZT2H_ICU_ENCIFERR_CLRn(n) (0x250 + 0x4 * (n))
+#define RZT2H_ICU_ENCIFERR_STAT 0x264
+#define RZT2H_ICU_ENCIFERR_NUM 5
+#define RZT2H_ICU_ENCIFERR_MASK GENMASK(31, 0)
+
#define RZT2H_ICU_DMACn_RSSELi(n, i) (0x7d0 + 0x18 * (n) + 0x4 * (i))
#define RZT2H_ICU_DMAC_REQ_SELx_MASK(x) (GENMASK(9, 0) << ((x) * 10))
#define RZT2H_ICU_DMAC_REQ_SELx_PREP(x, val) (FIELD_PREP(GENMASK(9, 0), val) << ((x) * 10))
@@ -99,6 +151,12 @@ static inline int rzt2h_icu_irq_to_offset(struct irq_data *d, void __iomem **bas
} else if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, IRQ_S) || RZT2H_ICU_IRQ_IN_RANGE(hwirq, SEI)) {
*offset = hwirq - RZT2H_ICU_IRQ_S_START;
*base = priv->base_s;
+ } else if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, INTCPU_NS)) {
+ *offset = hwirq - RZT2H_ICU_INTCPU_NS_START;
+ *base = priv->base_ns;
+ } else if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, INTCPU_S)) {
+ *offset = hwirq - RZT2H_ICU_INTCPU_S_START;
+ *base = priv->base_s;
} else {
return -EINVAL;
}
@@ -164,6 +222,28 @@ static int rzt2h_icu_set_type(struct irq_data *d, unsigned int type)
return irq_chip_set_type_parent(d, IRQ_TYPE_EDGE_RISING);
}
+static int rzt2h_icu_intcpu_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which,
+ bool state)
+{
+ unsigned int offset;
+ void __iomem *base;
+ int ret;
+
+ if (which != IRQCHIP_STATE_PENDING)
+ return irq_chip_set_parent_state(d, which, state);
+
+ if (!state)
+ return 0;
+
+ ret = rzt2h_icu_irq_to_offset(d, &base, &offset);
+ if (ret)
+ return ret;
+
+ writel_relaxed(RZT2H_ICU_SWINT_IC_MASK(offset), base + RZT2H_ICU_SWINT);
+
+ return 0;
+}
+
static const struct irq_chip rzt2h_icu_chip = {
.name = "rzt2h-icu",
.irq_mask = irq_chip_mask_parent,
@@ -180,10 +260,27 @@ static const struct irq_chip rzt2h_icu_chip = {
IRQCHIP_SKIP_SET_WAKE,
};
+static const struct irq_chip rzt2h_icu_intcpu_chip = {
+ .name = "rzt2h-icu",
+ .irq_mask = irq_chip_mask_parent,
+ .irq_unmask = irq_chip_unmask_parent,
+ .irq_eoi = irq_chip_eoi_parent,
+ .irq_set_type = irq_chip_set_type_parent,
+ .irq_set_wake = irq_chip_set_wake_parent,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .irq_retrigger = irq_chip_retrigger_hierarchy,
+ .irq_get_irqchip_state = irq_chip_get_parent_state,
+ .irq_set_irqchip_state = rzt2h_icu_intcpu_set_irqchip_state,
+ .flags = IRQCHIP_MASK_ON_SUSPEND |
+ IRQCHIP_SET_TYPE_MASKED |
+ IRQCHIP_SKIP_SET_WAKE,
+};
+
static int rzt2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
void *arg)
{
struct rzt2h_icu_priv *priv = domain->host_data;
+ const struct irq_chip *chip;
irq_hw_number_t hwirq;
unsigned int type;
int ret;
@@ -192,7 +289,12 @@ static int rzt2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigne
if (ret)
return ret;
- ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &rzt2h_icu_chip, NULL);
+ if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, INTCPU_NS) || RZT2H_ICU_IRQ_IN_RANGE(hwirq, INTCPU_S))
+ chip = &rzt2h_icu_intcpu_chip;
+ else
+ chip = &rzt2h_icu_chip;
+
+ ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, NULL);
if (ret)
return ret;
@@ -222,6 +324,155 @@ static int rzt2h_icu_parse_interrupts(struct rzt2h_icu_priv *priv, struct device
return 0;
}
+static irqreturn_t rzt2h_icu_intcpu_irq(int irq, void *data)
+{
+ unsigned int intcpu = (uintptr_t)data;
+
+ pr_info("INTCPU%u software interrupt\n", intcpu);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t rzt2h_icu_err_irq(struct rzt2h_icu_priv *priv, const char *name,
+ unsigned int num, u32 stat_base, u32 clr_base)
+{
+ bool handled = false;
+
+ for (unsigned int n = 0; n < num; n++) {
+ u32 stat = readl(priv->base_ns + stat_base + n * 0x4);
+
+ if (!stat)
+ continue;
+
+ handled = true;
+
+ pr_err("rzt2h-icu: %s error n=%u status=0x%08x\n", name, n, stat);
+
+ writel_relaxed(stat, priv->base_ns + clr_base + n * 0x4);
+ }
+
+ return handled ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static irqreturn_t rzt2h_icu_ca55_err_irq(int irq, void *data)
+{
+ return rzt2h_icu_err_irq(data, "CA55", 1, RZT2H_ICU_CA55ERR_STAT, RZT2H_ICU_CA55ERR_CLR);
+}
+
+static irqreturn_t rzt2h_icu_peri_err_irq(int irq, void *data)
+{
+ return rzt2h_icu_err_irq(data, "peripheral", RZT2H_ICU_PERIERR_NUM, RZT2H_ICU_PERIERR_STAT,
+ RZT2H_ICU_PERIERR_CLRn(0));
+}
+
+static irqreturn_t rzt2h_icu_dsmif_err_irq(int irq, void *data)
+{
+ return rzt2h_icu_err_irq(data, "DSMIF", RZT2H_ICU_DSMIFERR_NUM, RZT2H_ICU_DSMIFERR_STAT,
+ RZT2H_ICU_DSMIFERR_CLRn(0));
+}
+
+static irqreturn_t rzt2h_icu_encif_err_irq(int irq, void *data)
+{
+ return rzt2h_icu_err_irq(data, "ENCIF", RZT2H_ICU_ENCIFERR_NUM, RZT2H_ICU_ENCIFERR_STAT,
+ RZT2H_ICU_ENCIFERR_CLRn(0));
+}
+
+static int rzt2h_icu_request_irqs(struct platform_device *pdev, struct irq_domain *irq_domain,
+ unsigned int start, unsigned int count, irq_handler_t handler,
+ void *data)
+{
+ struct device *dev = &pdev->dev;
+ unsigned int offset, virq;
+ struct irq_fwspec fwspec;
+ int ret;
+
+ for (offset = start; offset < start + count; offset++) {
+ fwspec.fwnode = irq_domain->fwnode;
+ fwspec.param_count = 2;
+ fwspec.param[0] = offset;
+ fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
+
+ virq = irq_create_fwspec_mapping(&fwspec);
+ if (!virq)
+ return dev_err_probe(dev, -EINVAL, "Failed to create IRQ %u mapping\n", offset);
+
+ ret = devm_request_irq(dev, virq, handler, 0, dev_name(dev),
+ data ?: (void *)(uintptr_t)offset);
+ if (ret)
+ return dev_err_probe(dev, ret, "Failed to request IRQ %u\n", offset);
+ }
+
+ return 0;
+}
+
+static int rzt2h_icu_setup_irqs(struct platform_device *pdev, struct irq_domain *irq_domain)
+{
+ struct rzt2h_icu_priv *priv = platform_get_drvdata(pdev);
+ unsigned int n;
+ int ret;
+
+ if (IS_ENABLED(CONFIG_GENERIC_IRQ_INJECTION)) {
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_INTCPU_NS_START,
+ RZT2H_ICU_INTCPU_NS_COUNT, rzt2h_icu_intcpu_irq, NULL);
+ if (ret)
+ return ret;
+
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_INTCPU_S_START,
+ RZT2H_ICU_INTCPU_S_COUNT, rzt2h_icu_intcpu_irq, NULL);
+ if (ret)
+ return ret;
+ }
+
+ /*
+ * There are two error interrupts and two error masks that can be used
+ * separately for each error type. It would not be very useful to
+ * receive two interrupts for the same error, so use only the first one.
+ */
+
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_CA55_ERR_START, 1,
+ rzt2h_icu_ca55_err_irq, priv);
+ if (ret)
+ return ret;
+
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_PERI_ERR_START, 1,
+ rzt2h_icu_peri_err_irq, priv);
+ if (ret)
+ return ret;
+
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_DSMIF_ERR_START, 1,
+ rzt2h_icu_dsmif_err_irq, priv);
+ if (ret)
+ return ret;
+
+ ret = rzt2h_icu_request_irqs(pdev, irq_domain, RZT2H_ICU_ENCIF_ERR_START, 1,
+ rzt2h_icu_encif_err_irq, priv);
+ if (ret)
+ return ret;
+
+ /* Clear and unmask CA55 error events */
+ writel_relaxed(RZT2H_ICU_CA55ERR_MASK, priv->base_ns + RZT2H_ICU_CA55ERR_CLR);
+ writel_relaxed(0, priv->base_ns + RZT2H_ICU_CA55ERR_E0MSK);
+
+ /* Clear and unmask peripheral error events */
+ for (n = 0; n < RZT2H_ICU_PERIERR_NUM; n++) {
+ writel_relaxed(RZT2H_ICU_PERIERR_MASK, priv->base_ns + RZT2H_ICU_PERIERR_CLRn(n));
+ writel_relaxed(0, priv->base_ns + RZT2H_ICU_PERIERR_E0MSKn(n));
+ }
+
+ /* Clear and unmask DSMIF error events */
+ for (n = 0; n < RZT2H_ICU_DSMIFERR_NUM; n++) {
+ writel_relaxed(RZT2H_ICU_DSMIFERR_MASK, priv->base_ns + RZT2H_ICU_DSMIFERR_CLRn(n));
+ writel_relaxed(0, priv->base_ns + RZT2H_ICU_DSMIFERR_E0MSKn(n));
+ }
+
+ /* Clear and unmask ENCIF error events */
+ for (n = 0; n < RZT2H_ICU_ENCIFERR_NUM; n++) {
+ writel_relaxed(RZT2H_ICU_ENCIFERR_MASK, priv->base_ns + RZT2H_ICU_ENCIFERR_CLRn(n));
+ writel_relaxed(0, priv->base_ns + RZT2H_ICU_ENCIFERR_E0MSKn(n));
+ }
+
+ return 0;
+}
+
static int rzt2h_icu_init(struct platform_device *pdev, struct device_node *parent)
{
struct irq_domain *irq_domain, *parent_domain;
@@ -265,11 +516,20 @@ static int rzt2h_icu_init(struct platform_device *pdev, struct device_node *pare
irq_domain = irq_domain_create_hierarchy(parent_domain, 0, RZT2H_ICU_NUM_IRQ,
dev_fwnode(dev), &rzt2h_icu_domain_ops, priv);
if (!irq_domain) {
- pm_runtime_put(dev);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto err_pm_put;
}
+ ret = rzt2h_icu_setup_irqs(pdev, irq_domain);
+ if (ret)
+ goto err_irq_domain_free;
return 0;
+
+err_irq_domain_free:
+ irq_domain_remove(irq_domain);
+err_pm_put:
+ pm_runtime_put_sync(dev);
+ return ret;
}
IRQCHIP_PLATFORM_DRIVER_BEGIN(rzt2h_icu)
diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c
index ba903fa..a7a1852 100644
--- a/drivers/irqchip/irq-riscv-imsic-early.c
+++ b/drivers/irqchip/irq-riscv-imsic-early.c
@@ -158,6 +158,8 @@ static int imsic_dying_cpu(unsigned int cpu)
/* Cleanup IPIs */
imsic_ipi_dying_cpu();
+ imsic_local_sync_all(false);
+
/* Mark per-CPU IMSIC state as offline */
imsic_state_offline();
diff --git a/drivers/irqchip/irq-starfive-jh8100-intc.c b/drivers/irqchip/irq-starfive-jh8100-intc.c
deleted file mode 100644
index bb62ef3..0000000
--- a/drivers/irqchip/irq-starfive-jh8100-intc.c
+++ /dev/null
@@ -1,207 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * StarFive JH8100 External Interrupt Controller driver
- *
- * Copyright (C) 2023 StarFive Technology Co., Ltd.
- *
- * Author: Changhuang Liang <changhuang.liang@starfivetech.com>
- */
-
-#define pr_fmt(fmt) "irq-starfive-jh8100: " fmt
-
-#include <linux/bitops.h>
-#include <linux/clk.h>
-#include <linux/irq.h>
-#include <linux/irqchip.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/reset.h>
-#include <linux/spinlock.h>
-
-#define STARFIVE_INTC_SRC0_CLEAR 0x10
-#define STARFIVE_INTC_SRC0_MASK 0x14
-#define STARFIVE_INTC_SRC0_INT 0x1c
-
-#define STARFIVE_INTC_SRC_IRQ_NUM 32
-
-struct starfive_irq_chip {
- void __iomem *base;
- struct irq_domain *domain;
- raw_spinlock_t lock;
-};
-
-static void starfive_intc_bit_set(struct starfive_irq_chip *irqc,
- u32 reg, u32 bit_mask)
-{
- u32 value;
-
- value = ioread32(irqc->base + reg);
- value |= bit_mask;
- iowrite32(value, irqc->base + reg);
-}
-
-static void starfive_intc_bit_clear(struct starfive_irq_chip *irqc,
- u32 reg, u32 bit_mask)
-{
- u32 value;
-
- value = ioread32(irqc->base + reg);
- value &= ~bit_mask;
- iowrite32(value, irqc->base + reg);
-}
-
-static void starfive_intc_unmask(struct irq_data *d)
-{
- struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
-
- raw_spin_lock(&irqc->lock);
- starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq));
- raw_spin_unlock(&irqc->lock);
-}
-
-static void starfive_intc_mask(struct irq_data *d)
-{
- struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
-
- raw_spin_lock(&irqc->lock);
- starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq));
- raw_spin_unlock(&irqc->lock);
-}
-
-static struct irq_chip intc_dev = {
- .name = "StarFive JH8100 INTC",
- .irq_unmask = starfive_intc_unmask,
- .irq_mask = starfive_intc_mask,
-};
-
-static int starfive_intc_map(struct irq_domain *d, unsigned int irq,
- irq_hw_number_t hwirq)
-{
- irq_domain_set_info(d, irq, hwirq, &intc_dev, d->host_data,
- handle_level_irq, NULL, NULL);
-
- return 0;
-}
-
-static const struct irq_domain_ops starfive_intc_domain_ops = {
- .xlate = irq_domain_xlate_onecell,
- .map = starfive_intc_map,
-};
-
-static void starfive_intc_irq_handler(struct irq_desc *desc)
-{
- struct starfive_irq_chip *irqc = irq_data_get_irq_handler_data(&desc->irq_data);
- struct irq_chip *chip = irq_desc_get_chip(desc);
- unsigned long value;
- int hwirq;
-
- chained_irq_enter(chip, desc);
-
- value = ioread32(irqc->base + STARFIVE_INTC_SRC0_INT);
- while (value) {
- hwirq = ffs(value) - 1;
-
- generic_handle_domain_irq(irqc->domain, hwirq);
-
- starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq));
- starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq));
-
- __clear_bit(hwirq, &value);
- }
-
- chained_irq_exit(chip, desc);
-}
-
-static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent)
-{
- struct device_node *intc = pdev->dev.of_node;
- struct starfive_irq_chip *irqc;
- struct reset_control *rst;
- struct clk *clk;
- int parent_irq;
- int ret;
-
- irqc = kzalloc_obj(*irqc);
- if (!irqc)
- return -ENOMEM;
-
- irqc->base = of_iomap(intc, 0);
- if (!irqc->base) {
- pr_err("Unable to map registers\n");
- ret = -ENXIO;
- goto err_free;
- }
-
- rst = of_reset_control_get_exclusive(intc, NULL);
- if (IS_ERR(rst)) {
- pr_err("Unable to get reset control %pe\n", rst);
- ret = PTR_ERR(rst);
- goto err_unmap;
- }
-
- clk = of_clk_get(intc, 0);
- if (IS_ERR(clk)) {
- pr_err("Unable to get clock %pe\n", clk);
- ret = PTR_ERR(clk);
- goto err_reset_put;
- }
-
- ret = reset_control_deassert(rst);
- if (ret)
- goto err_clk_put;
-
- ret = clk_prepare_enable(clk);
- if (ret)
- goto err_reset_assert;
-
- raw_spin_lock_init(&irqc->lock);
-
- irqc->domain = irq_domain_create_linear(of_fwnode_handle(intc), STARFIVE_INTC_SRC_IRQ_NUM,
- &starfive_intc_domain_ops, irqc);
- if (!irqc->domain) {
- pr_err("Unable to create IRQ domain\n");
- ret = -EINVAL;
- goto err_clk_disable;
- }
-
- parent_irq = of_irq_get(intc, 0);
- if (parent_irq < 0) {
- pr_err("Failed to get main IRQ: %d\n", parent_irq);
- ret = parent_irq;
- goto err_remove_domain;
- }
-
- irq_set_chained_handler_and_data(parent_irq, starfive_intc_irq_handler,
- irqc);
-
- pr_info("Interrupt controller register, nr_irqs %d\n",
- STARFIVE_INTC_SRC_IRQ_NUM);
-
- return 0;
-
-err_remove_domain:
- irq_domain_remove(irqc->domain);
-err_clk_disable:
- clk_disable_unprepare(clk);
-err_reset_assert:
- reset_control_assert(rst);
-err_clk_put:
- clk_put(clk);
-err_reset_put:
- reset_control_put(rst);
-err_unmap:
- iounmap(irqc->base);
-err_free:
- kfree(irqc);
- return ret;
-}
-
-IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc)
-IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_probe)
-IRQCHIP_PLATFORM_DRIVER_END(starfive_intc)
-
-MODULE_DESCRIPTION("StarFive JH8100 External Interrupt Controller");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>");
diff --git a/drivers/irqchip/irq-starfive-jhb100-intc.c b/drivers/irqchip/irq-starfive-jhb100-intc.c
new file mode 100644
index 0000000..838885b
--- /dev/null
+++ b/drivers/irqchip/irq-starfive-jhb100-intc.c
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * StarFive JHB100 External Interrupt Controller driver
+ *
+ * Copyright (C) 2023 StarFive Technology Co., Ltd.
+ *
+ * Author: Changhuang Liang <changhuang.liang@starfivetech.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+
+#define STARFIVE_INTC_SRC_TYPE(n) (0x04 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_CLEAR(n) (0x10 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_MASK(n) (0x14 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_INT(n) (0x1c + ((n) * 0x20))
+
+#define STARFIVE_INTC_TRIGGER_MASK 0x3
+#define STARFIVE_INTC_TRIGGER_HIGH 0
+#define STARFIVE_INTC_TRIGGER_LOW 1
+#define STARFIVE_INTC_TRIGGER_POSEDGE 2
+#define STARFIVE_INTC_TRIGGER_NEGEDGE 3
+
+#define STARFIVE_INTC_NUM 2
+#define STARFIVE_INTC_SRC_IRQ_NUM 32
+#define STARFIVE_INTC_TYPE_NUM 16
+
+struct starfive_irq_chip {
+ void __iomem *base;
+ struct irq_domain *domain;
+ raw_spinlock_t lock;
+};
+
+static void starfive_intc_mod(struct starfive_irq_chip *irqc, u32 reg, u32 mask, u32 data)
+{
+ u32 value;
+
+ value = ioread32(irqc->base + reg) & ~mask;
+ data &= mask;
+ data |= value;
+ iowrite32(data, irqc->base + reg);
+}
+
+static void starfive_intc_bit_set(struct starfive_irq_chip *irqc,
+ u32 reg, u32 bit_mask)
+{
+ u32 value;
+
+ value = ioread32(irqc->base + reg);
+ value |= bit_mask;
+ iowrite32(value, irqc->base + reg);
+}
+
+static void starfive_intc_bit_clear(struct starfive_irq_chip *irqc,
+ u32 reg, u32 bit_mask)
+{
+ u32 value;
+
+ value = ioread32(irqc->base + reg);
+ value &= ~bit_mask;
+ iowrite32(value, irqc->base + reg);
+}
+
+static void starfive_intc_unmask(struct irq_data *d)
+{
+ struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+ int i, bitpos;
+
+ i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+ bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+
+ guard(raw_spinlock)(&irqc->lock);
+ starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_MASK(i), BIT(bitpos));
+}
+
+static void starfive_intc_mask(struct irq_data *d)
+{
+ struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+ int i, bitpos;
+
+ i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+ bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+
+ guard(raw_spinlock)(&irqc->lock);
+ starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_MASK(i), BIT(bitpos));
+}
+
+static void starfive_intc_ack(struct irq_data *d)
+{
+ /* for handle_edge_irq, nothing to do */
+}
+
+static int starfive_intc_set_type(struct irq_data *d, unsigned int type)
+{
+ struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+ u32 i, bitpos, ty_pos, ty_shift, trigger, typeval;
+ irq_flow_handler_t handler;
+
+ i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+ bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+ ty_pos = bitpos / STARFIVE_INTC_TYPE_NUM;
+ ty_shift = (bitpos % STARFIVE_INTC_TYPE_NUM) * 2;
+
+ switch (type) {
+ case IRQF_TRIGGER_LOW:
+ trigger = STARFIVE_INTC_TRIGGER_LOW;
+ handler = handle_level_irq;
+ break;
+ case IRQF_TRIGGER_HIGH:
+ trigger = STARFIVE_INTC_TRIGGER_HIGH;
+ handler = handle_level_irq;
+ break;
+ case IRQF_TRIGGER_FALLING:
+ trigger = STARFIVE_INTC_TRIGGER_NEGEDGE;
+ handler = handle_edge_irq;
+ break;
+ case IRQF_TRIGGER_RISING:
+ trigger = STARFIVE_INTC_TRIGGER_POSEDGE;
+ handler = handle_edge_irq;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ irq_set_handler_locked(d, handler);
+ typeval = trigger << ty_shift;
+
+ guard(raw_spinlock)(&irqc->lock);
+
+ starfive_intc_mod(irqc, STARFIVE_INTC_SRC_TYPE(i) + 4 * ty_pos,
+ STARFIVE_INTC_TRIGGER_MASK << ty_shift, typeval);
+
+ /* Once the type is updated, clear interrupt can help to reset the type value */
+ starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(bitpos));
+ starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(bitpos));
+
+ return 0;
+}
+
+static struct irq_chip intc_dev = {
+ .name = "StarFive JHB100 INTC",
+ .irq_unmask = starfive_intc_unmask,
+ .irq_mask = starfive_intc_mask,
+ .irq_ack = starfive_intc_ack,
+ .irq_set_type = starfive_intc_set_type,
+};
+
+static int starfive_intc_map(struct irq_domain *d, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_domain_set_info(d, irq, hwirq, &intc_dev, d->host_data,
+ handle_level_irq, NULL, NULL);
+
+ return 0;
+}
+
+static const struct irq_domain_ops starfive_intc_domain_ops = {
+ .xlate = irq_domain_xlate_onecell,
+ .map = starfive_intc_map,
+};
+
+static void starfive_intc_irq_handler(struct irq_desc *desc)
+{
+ struct starfive_irq_chip *irqc = irq_data_get_irq_handler_data(&desc->irq_data);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ unsigned long value;
+ int hwirq;
+
+ chained_irq_enter(chip, desc);
+
+ for (int i = 0; i < STARFIVE_INTC_NUM; i++) {
+ value = ioread32(irqc->base + STARFIVE_INTC_SRC_INT(i));
+ while (value) {
+ hwirq = ffs(value) - 1;
+
+ generic_handle_domain_irq(irqc->domain,
+ hwirq + i * STARFIVE_INTC_SRC_IRQ_NUM);
+
+ starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(hwirq));
+ starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(hwirq));
+
+ __clear_bit(hwirq, &value);
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent)
+{
+ struct device_node *intc = pdev->dev.of_node;
+ struct reset_control *rst;
+ struct clk *clk;
+ int parent_irq;
+
+ struct starfive_irq_chip *irqc __free(kfree) = kzalloc_obj(*irqc);
+ if (!irqc)
+ return -ENOMEM;
+
+ irqc->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(irqc->base))
+ return dev_err_probe(&pdev->dev, PTR_ERR(irqc->base), "unable to map registers\n");
+
+ rst = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, NULL);
+ if (IS_ERR(rst))
+ return dev_err_probe(&pdev->dev, PTR_ERR(rst),
+ "Unable to get and deassert reset control\n");
+
+ clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
+ if (IS_ERR(clk))
+ return dev_err_probe(&pdev->dev, PTR_ERR(clk), "Unable to get and enable clock\n");
+
+
+ raw_spin_lock_init(&irqc->lock);
+
+ irqc->domain = irq_domain_create_linear(of_fwnode_handle(intc),
+ STARFIVE_INTC_SRC_IRQ_NUM * STARFIVE_INTC_NUM,
+ &starfive_intc_domain_ops, irqc);
+ if (!irqc->domain)
+ return dev_err_probe(&pdev->dev, -EINVAL, "Unable to create IRQ domain\n");
+
+ parent_irq = of_irq_get(intc, 0);
+ if (parent_irq < 0) {
+ irq_domain_remove(irqc->domain);
+ return dev_err_probe(&pdev->dev, parent_irq, "Failed to get main IRQ\n");
+ }
+
+ irq_set_chained_handler_and_data(parent_irq, starfive_intc_irq_handler,
+ irqc);
+
+ dev_info(&pdev->dev, "Interrupt controller register, nr_irqs %d\n",
+ STARFIVE_INTC_SRC_IRQ_NUM * STARFIVE_INTC_NUM);
+
+ retain_and_null_ptr(irqc);
+ return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc)
+IRQCHIP_MATCH("starfive,jhb100-intc", starfive_intc_probe)
+IRQCHIP_PLATFORM_DRIVER_END(starfive_intc)
+
+MODULE_DESCRIPTION("StarFive JHB100 External Interrupt Controller");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>");
diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c
index 32b77fa..2014dbb 100644
--- a/drivers/irqchip/qcom-pdc.c
+++ b/drivers/irqchip/qcom-pdc.c
@@ -3,6 +3,7 @@
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
*/
+#include <linux/bitfield.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/interrupt.h>
@@ -21,22 +22,30 @@
#include <linux/types.h>
#define PDC_MAX_GPIO_IRQS 256
-#define PDC_DRV_OFFSET 0x10000
+#define PDC_DRV_SIZE 0x10000
/* Valid only on HW version < 3.2 */
#define IRQ_ENABLE_BANK 0x10
#define IRQ_ENABLE_BANK_MAX (IRQ_ENABLE_BANK + BITS_TO_BYTES(PDC_MAX_GPIO_IRQS))
+#define IRQ_ENABLE_BANK_INDEX_MASK GENMASK(31, 5)
+#define IRQ_ENABLE_BANK_BIT_MASK GENMASK(4, 0)
#define IRQ_i_CFG 0x110
/* Valid only on HW version >= 3.2 */
#define IRQ_i_CFG_IRQ_ENABLE 3
-#define IRQ_i_CFG_TYPE_MASK GENMASK(2, 0)
+#define IRQ_i_CFG_TYPE_MASK GENMASK(2, 0)
-#define PDC_VERSION_REG 0x1000
+#define PDC_VERSION_REG 0x1000
+#define PDC_VERSION_MAJOR GENMASK(23, 16)
+#define PDC_VERSION_MINOR GENMASK(15, 8)
+#define PDC_VERSION_STEP GENMASK(7, 0)
+#define PDC_VERSION(maj, min, step) (FIELD_PREP(PDC_VERSION_MAJOR, (maj)) | \
+ FIELD_PREP(PDC_VERSION_MINOR, (min)) | \
+ FIELD_PREP(PDC_VERSION_STEP, (step)))
/* Notable PDC versions */
-#define PDC_VERSION_3_2 0x30200
+#define PDC_VERSION_3_2 PDC_VERSION(3, 2, 0)
struct pdc_pin_region {
u32 pin_base;
@@ -97,28 +106,37 @@ static void pdc_x1e_irq_enable_write(u32 bank, u32 enable)
pdc_base_reg_write(base, IRQ_ENABLE_BANK, bank, enable);
}
-static void __pdc_enable_intr(int pin_out, bool on)
+static void pdc_enable_intr_bank(int pin_out, bool on)
{
unsigned long enable;
+ u32 index, mask;
- if (pdc_version < PDC_VERSION_3_2) {
- u32 index, mask;
+ index = FIELD_GET(IRQ_ENABLE_BANK_INDEX_MASK, pin_out);
+ mask = FIELD_GET(IRQ_ENABLE_BANK_BIT_MASK, pin_out);
- index = pin_out / 32;
- mask = pin_out % 32;
+ enable = pdc_reg_read(IRQ_ENABLE_BANK, index);
+ __assign_bit(mask, &enable, on);
- enable = pdc_reg_read(IRQ_ENABLE_BANK, index);
- __assign_bit(mask, &enable, on);
+ if (pdc_x1e_quirk)
+ pdc_x1e_irq_enable_write(index, enable);
+ else
+ pdc_reg_write(IRQ_ENABLE_BANK, index, enable);
+}
- if (pdc_x1e_quirk)
- pdc_x1e_irq_enable_write(index, enable);
- else
- pdc_reg_write(IRQ_ENABLE_BANK, index, enable);
- } else {
- enable = pdc_reg_read(IRQ_i_CFG, pin_out);
- __assign_bit(IRQ_i_CFG_IRQ_ENABLE, &enable, on);
- pdc_reg_write(IRQ_i_CFG, pin_out, enable);
- }
+static void pdc_enable_intr_cfg(int pin_out, bool on)
+{
+ unsigned long enable = pdc_reg_read(IRQ_i_CFG, pin_out);
+
+ __assign_bit(IRQ_i_CFG_IRQ_ENABLE, &enable, on);
+ pdc_reg_write(IRQ_i_CFG, pin_out, enable);
+}
+
+static void __pdc_enable_intr(int pin_out, bool on)
+{
+ if (pdc_version < PDC_VERSION_3_2)
+ pdc_enable_intr_bank(pin_out, on);
+ else
+ pdc_enable_intr_cfg(pin_out, on);
}
static void pdc_enable_intr(struct irq_data *d, bool on)
@@ -348,7 +366,6 @@ static int pdc_setup_pin_mapping(struct device_node *np)
return 0;
}
-#define QCOM_PDC_SIZE 0x30000
static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *parent)
{
@@ -362,7 +379,7 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare
if (of_address_to_resource(node, 0, &res))
return -EINVAL;
- res_size = max_t(resource_size_t, resource_size(&res), QCOM_PDC_SIZE);
+ res_size = max_t(resource_size_t, resource_size(&res), PDC_DRV_SIZE);
if (res_size > resource_size(&res))
pr_warn("%pOF: invalid reg size, please fix DT\n", node);
@@ -375,7 +392,7 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare
* region with the expected offset to preserve support for old DTs.
*/
if (of_device_is_compatible(node, "qcom,x1e80100-pdc")) {
- pdc_prev_base = ioremap(res.start - PDC_DRV_OFFSET, IRQ_ENABLE_BANK_MAX);
+ pdc_prev_base = ioremap(res.start - PDC_DRV_SIZE, IRQ_ENABLE_BANK_MAX);
if (!pdc_prev_base) {
pr_err("%pOF: unable to map previous PDC DRV region\n", node);
return -ENXIO;
diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c
index 8a1e2c0..26cb931 100644
--- a/drivers/macintosh/rack-meter.c
+++ b/drivers/macintosh/rack-meter.c
@@ -87,7 +87,7 @@ static inline u64 get_cpu_idle_time(unsigned int cpu)
kcpustat->cpustat[CPUTIME_IOWAIT];
if (rackmeter_ignore_nice)
- retval += kcpustat_field(kcpustat, CPUTIME_NICE, cpu);
+ retval += kcpustat_field(CPUTIME_NICE, cpu);
return retval;
}
diff --git a/drivers/mailbox/cix-mailbox.c b/drivers/mailbox/cix-mailbox.c
index 443620e..43c76cd 100644
--- a/drivers/mailbox/cix-mailbox.c
+++ b/drivers/mailbox/cix-mailbox.c
@@ -12,8 +12,6 @@
#include <linux/module.h>
#include <linux/platform_device.h>
-#include "mailbox.h"
-
/*
* The maximum transmission size is 32 words or 128 bytes.
*/
@@ -405,7 +403,7 @@ static int cix_mbox_startup(struct mbox_chan *chan)
int index = cp->index, ret;
u32 val;
- ret = request_irq(priv->irq, cix_mbox_isr, 0,
+ ret = request_irq(priv->irq, cix_mbox_isr, IRQF_NO_SUSPEND,
dev_name(priv->dev), chan);
if (ret) {
dev_err(priv->dev, "Unable to acquire IRQ %d\n", priv->irq);
@@ -415,7 +413,7 @@ static int cix_mbox_startup(struct mbox_chan *chan)
switch (cp->type) {
case CIX_MBOX_TYPE_DB:
/* Overwrite txdone_method for DB channel */
- chan->txdone_method = TXDONE_BY_ACK;
+ chan->txdone_method = MBOX_TXDONE_BY_ACK;
fallthrough;
case CIX_MBOX_TYPE_REG:
if (priv->dir == CIX_MBOX_TX) {
diff --git a/drivers/mailbox/exynos-mailbox.c b/drivers/mailbox/exynos-mailbox.c
index 5f2d3b8..d2355b1 100644
--- a/drivers/mailbox/exynos-mailbox.c
+++ b/drivers/mailbox/exynos-mailbox.c
@@ -99,7 +99,6 @@ static int exynos_mbox_probe(struct platform_device *pdev)
struct mbox_controller *mbox;
struct mbox_chan *chans;
struct clk *pclk;
- int i;
exynos_mbox = devm_kzalloc(dev, sizeof(*exynos_mbox), GFP_KERNEL);
if (!exynos_mbox)
@@ -129,9 +128,6 @@ static int exynos_mbox_probe(struct platform_device *pdev)
mbox->ops = &exynos_mbox_chan_ops;
mbox->of_xlate = exynos_mbox_of_xlate;
- for (i = 0; i < EXYNOS_MBOX_CHAN_COUNT; i++)
- chans[i].mbox = mbox;
-
exynos_mbox->mbox = mbox;
platform_set_drvdata(pdev, exynos_mbox);
diff --git a/drivers/mailbox/hi3660-mailbox.c b/drivers/mailbox/hi3660-mailbox.c
index 17c29e9..9b727a2 100644
--- a/drivers/mailbox/hi3660-mailbox.c
+++ b/drivers/mailbox/hi3660-mailbox.c
@@ -15,8 +15,6 @@
#include <linux/platform_device.h>
#include <linux/slab.h>
-#include "mailbox.h"
-
#define MBOX_CHAN_MAX 32
#define MBOX_RX 0x0
diff --git a/drivers/mailbox/hi6220-mailbox.c b/drivers/mailbox/hi6220-mailbox.c
index f77741c..69d15b6 100644
--- a/drivers/mailbox/hi6220-mailbox.c
+++ b/drivers/mailbox/hi6220-mailbox.c
@@ -79,12 +79,12 @@ struct hi6220_mbox {
/* region for mailbox */
void __iomem *base;
- unsigned int chan_num;
- struct hi6220_mbox_chan *mchan;
-
void *irq_map_chan[MBOX_CHAN_MAX];
struct mbox_chan *chan;
struct mbox_controller controller;
+
+ unsigned int chan_num;
+ struct hi6220_mbox_chan mchan[] __counted_by(chan_num);
};
static void mbox_set_state(struct hi6220_mbox *mbox,
@@ -267,16 +267,12 @@ static int hi6220_mbox_probe(struct platform_device *pdev)
struct hi6220_mbox *mbox;
int i, err;
- mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
+ mbox = devm_kzalloc(dev, struct_size(mbox, mchan, MBOX_CHAN_MAX), GFP_KERNEL);
if (!mbox)
return -ENOMEM;
- mbox->dev = dev;
mbox->chan_num = MBOX_CHAN_MAX;
- mbox->mchan = devm_kcalloc(dev,
- mbox->chan_num, sizeof(*mbox->mchan), GFP_KERNEL);
- if (!mbox->mchan)
- return -ENOMEM;
+ mbox->dev = dev;
mbox->chan = devm_kcalloc(dev,
mbox->chan_num, sizeof(*mbox->chan), GFP_KERNEL);
diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c
index 003f923..246a9a9 100644
--- a/drivers/mailbox/imx-mailbox.c
+++ b/drivers/mailbox/imx-mailbox.c
@@ -23,8 +23,6 @@
#include <linux/slab.h>
#include <linux/workqueue.h>
-#include "mailbox.h"
-
#define IMX_MU_CHANS 24
/* TX0/RX0/RXDB[0-3] */
#define IMX_MU_SCU_CHANS 6
@@ -734,7 +732,7 @@ static struct mbox_chan * imx_mu_xlate(struct mbox_controller *mbox,
p_chan = &mbox->chans[chan];
if (type == IMX_MU_TYPE_TXDB_V2)
- p_chan->txdone_method = TXDONE_BY_ACK;
+ p_chan->txdone_method = MBOX_TXDONE_BY_ACK;
return p_chan;
}
diff --git a/drivers/mailbox/mailbox-sti.c b/drivers/mailbox/mailbox-sti.c
index b4b5bdd..b6c9ecb 100644
--- a/drivers/mailbox/mailbox-sti.c
+++ b/drivers/mailbox/mailbox-sti.c
@@ -21,8 +21,6 @@
#include <linux/property.h>
#include <linux/slab.h>
-#include "mailbox.h"
-
#define STI_MBOX_INST_MAX 4 /* RAM saving: Max supported instances */
#define STI_MBOX_CHAN_MAX 20 /* RAM saving: Max supported channels */
diff --git a/drivers/mailbox/mailbox-test.c b/drivers/mailbox/mailbox-test.c
index 3a28ab5..7b6ef03 100644
--- a/drivers/mailbox/mailbox-test.c
+++ b/drivers/mailbox/mailbox-test.c
@@ -28,8 +28,6 @@
#define MBOX_HEXDUMP_MAX_LEN (MBOX_HEXDUMP_LINE_LEN * \
(MBOX_MAX_MSG_LEN / MBOX_BYTES_PER_LINE))
-static bool mbox_data_ready;
-
struct mbox_test_device {
struct device *dev;
void __iomem *tx_mmio;
@@ -42,6 +40,7 @@ struct mbox_test_device {
spinlock_t lock;
struct mutex mutex;
wait_queue_head_t waitq;
+ bool data_ready;
struct fasync_struct *async_queue;
struct dentry *root_debugfs_dir;
};
@@ -162,7 +161,7 @@ static bool mbox_test_message_data_ready(struct mbox_test_device *tdev)
unsigned long flags;
spin_lock_irqsave(&tdev->lock, flags);
- data_ready = mbox_data_ready;
+ data_ready = tdev->data_ready;
spin_unlock_irqrestore(&tdev->lock, flags);
return data_ready;
@@ -227,7 +226,7 @@ static ssize_t mbox_test_message_read(struct file *filp, char __user *userbuf,
*(touser + l) = '\0';
memset(tdev->rx_buffer, 0, MBOX_MAX_MSG_LEN);
- mbox_data_ready = false;
+ tdev->data_ready = false;
spin_unlock_irqrestore(&tdev->lock, flags);
@@ -297,7 +296,7 @@ static void mbox_test_receive_message(struct mbox_client *client, void *message)
message, MBOX_MAX_MSG_LEN);
memcpy(tdev->rx_buffer, message, MBOX_MAX_MSG_LEN);
}
- mbox_data_ready = true;
+ tdev->data_ready = true;
spin_unlock_irqrestore(&tdev->lock, flags);
wake_up_interruptible(&tdev->waitq);
@@ -336,7 +335,7 @@ mbox_test_request_channel(struct platform_device *pdev, const char *name)
client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
if (!client)
- return ERR_PTR(-ENOMEM);
+ return NULL;
client->dev = &pdev->dev;
client->rx_callback = mbox_test_receive_message;
@@ -355,67 +354,80 @@ mbox_test_request_channel(struct platform_device *pdev, const char *name)
return channel;
}
+static void __iomem *mbox_test_ioremap(struct platform_device *pdev, unsigned int res_num)
+{
+ struct resource *res;
+ void __iomem *mmio;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, res_num);
+ if (!res)
+ return NULL;
+
+ mmio = devm_ioremap_resource(&pdev->dev, res);
+ if (PTR_ERR(mmio) == -EBUSY) {
+ dev_info(&pdev->dev, "trying workaround with plain ioremap\n");
+ return devm_ioremap(&pdev->dev, res->start, resource_size(res));
+ }
+
+ return IS_ERR(mmio) ? NULL : mmio;
+}
+
static int mbox_test_probe(struct platform_device *pdev)
{
struct mbox_test_device *tdev;
- struct resource *res;
- resource_size_t size;
int ret;
tdev = devm_kzalloc(&pdev->dev, sizeof(*tdev), GFP_KERNEL);
if (!tdev)
return -ENOMEM;
+ tdev->dev = &pdev->dev;
+ spin_lock_init(&tdev->lock);
+ mutex_init(&tdev->mutex);
+ init_waitqueue_head(&tdev->waitq);
+ platform_set_drvdata(pdev, tdev);
+
/* It's okay for MMIO to be NULL */
- tdev->tx_mmio = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
- if (PTR_ERR(tdev->tx_mmio) == -EBUSY) {
- /* if reserved area in SRAM, try just ioremap */
- size = resource_size(res);
- tdev->tx_mmio = devm_ioremap(&pdev->dev, res->start, size);
- } else if (IS_ERR(tdev->tx_mmio)) {
- tdev->tx_mmio = NULL;
- }
+ tdev->tx_mmio = mbox_test_ioremap(pdev, 0);
/* If specified, second reg entry is Rx MMIO */
- tdev->rx_mmio = devm_platform_get_and_ioremap_resource(pdev, 1, &res);
- if (PTR_ERR(tdev->rx_mmio) == -EBUSY) {
- size = resource_size(res);
- tdev->rx_mmio = devm_ioremap(&pdev->dev, res->start, size);
- } else if (IS_ERR(tdev->rx_mmio)) {
+ tdev->rx_mmio = mbox_test_ioremap(pdev, 1);
+ if (!tdev->rx_mmio)
tdev->rx_mmio = tdev->tx_mmio;
- }
tdev->tx_channel = mbox_test_request_channel(pdev, "tx");
tdev->rx_channel = mbox_test_request_channel(pdev, "rx");
- if (IS_ERR_OR_NULL(tdev->tx_channel) && IS_ERR_OR_NULL(tdev->rx_channel))
+ if (!tdev->tx_channel && !tdev->rx_channel)
return -EPROBE_DEFER;
/* If Rx is not specified but has Rx MMIO, then Rx = Tx */
if (!tdev->rx_channel && (tdev->rx_mmio != tdev->tx_mmio))
tdev->rx_channel = tdev->tx_channel;
- tdev->dev = &pdev->dev;
- platform_set_drvdata(pdev, tdev);
-
- spin_lock_init(&tdev->lock);
- mutex_init(&tdev->mutex);
-
if (tdev->rx_channel) {
tdev->rx_buffer = devm_kzalloc(&pdev->dev,
MBOX_MAX_MSG_LEN, GFP_KERNEL);
- if (!tdev->rx_buffer)
- return -ENOMEM;
+ if (!tdev->rx_buffer) {
+ ret = -ENOMEM;
+ goto err_free_chans;
+ }
}
ret = mbox_test_add_debugfs(pdev, tdev);
if (ret)
- return ret;
+ goto err_free_chans;
- init_waitqueue_head(&tdev->waitq);
dev_info(&pdev->dev, "Successfully registered\n");
return 0;
+
+err_free_chans:
+ if (tdev->tx_channel)
+ mbox_free_channel(tdev->tx_channel);
+ if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel)
+ mbox_free_channel(tdev->rx_channel);
+ return ret;
}
static void mbox_test_remove(struct platform_device *pdev)
@@ -426,7 +438,7 @@ static void mbox_test_remove(struct platform_device *pdev)
if (tdev->tx_channel)
mbox_free_channel(tdev->tx_channel);
- if (tdev->rx_channel)
+ if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel)
mbox_free_channel(tdev->rx_channel);
}
diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c
index 617ba50..bbc9fd7 100644
--- a/drivers/mailbox/mailbox.c
+++ b/drivers/mailbox/mailbox.c
@@ -18,8 +18,6 @@
#include <linux/property.h>
#include <linux/spinlock.h>
-#include "mailbox.h"
-
static LIST_HEAD(mbox_cons);
static DEFINE_MUTEX(con_mutex);
@@ -52,7 +50,7 @@ static void msg_submit(struct mbox_chan *chan)
int err = -EBUSY;
scoped_guard(spinlock_irqsave, &chan->lock) {
- if (!chan->msg_count || chan->active_req)
+ if (!chan->msg_count || chan->active_req != MBOX_NO_MSG)
break;
count = chan->msg_count;
@@ -74,7 +72,7 @@ static void msg_submit(struct mbox_chan *chan)
}
}
- if (!err && (chan->txdone_method & TXDONE_BY_POLL)) {
+ if (!err && (chan->txdone_method & MBOX_TXDONE_BY_POLL)) {
/* kick start the timer immediately to avoid delays */
scoped_guard(spinlock_irqsave, &chan->mbox->poll_hrt_lock)
hrtimer_start(&chan->mbox->poll_hrt, 0, HRTIMER_MODE_REL);
@@ -87,13 +85,13 @@ static void tx_tick(struct mbox_chan *chan, int r)
scoped_guard(spinlock_irqsave, &chan->lock) {
mssg = chan->active_req;
- chan->active_req = NULL;
+ chan->active_req = MBOX_NO_MSG;
}
/* Submit next message */
msg_submit(chan);
- if (!mssg)
+ if (mssg == MBOX_NO_MSG)
return;
/* Notify the client */
@@ -114,7 +112,7 @@ static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer)
for (i = 0; i < mbox->num_chans; i++) {
struct mbox_chan *chan = &mbox->chans[i];
- if (chan->active_req && chan->cl) {
+ if (chan->active_req != MBOX_NO_MSG && chan->cl) {
txdone = chan->mbox->ops->last_tx_done(chan);
if (txdone)
tx_tick(chan, 0);
@@ -164,7 +162,7 @@ EXPORT_SYMBOL_GPL(mbox_chan_received_data);
*/
void mbox_chan_txdone(struct mbox_chan *chan, int r)
{
- if (unlikely(!(chan->txdone_method & TXDONE_BY_IRQ))) {
+ if (unlikely(!(chan->txdone_method & MBOX_TXDONE_BY_IRQ))) {
dev_err(chan->mbox->dev,
"Controller can't run the TX ticker\n");
return;
@@ -185,7 +183,7 @@ EXPORT_SYMBOL_GPL(mbox_chan_txdone);
*/
void mbox_client_txdone(struct mbox_chan *chan, int r)
{
- if (unlikely(!(chan->txdone_method & TXDONE_BY_ACK))) {
+ if (unlikely(!(chan->txdone_method & MBOX_TXDONE_BY_ACK))) {
dev_err(chan->mbox->dev, "Client can't run the TX ticker\n");
return;
}
@@ -219,6 +217,29 @@ bool mbox_client_peek_data(struct mbox_chan *chan)
EXPORT_SYMBOL_GPL(mbox_client_peek_data);
/**
+ * mbox_chan_tx_slots_available - Query the number of available TX queue slots.
+ * @chan: Mailbox channel to query.
+ *
+ * Clients may call this to check how many messages can be queued via
+ * mbox_send_message() before the channel's TX queue is full. This helps
+ * clients avoid the -ENOBUFS error without needing to increase
+ * MBOX_TX_QUEUE_LEN.
+ * This can be called from atomic context.
+ *
+ * Return: Number of available slots in the channel's TX queue.
+ */
+unsigned int mbox_chan_tx_slots_available(struct mbox_chan *chan)
+{
+ unsigned int ret;
+
+ guard(spinlock_irqsave)(&chan->lock);
+ ret = MBOX_TX_QUEUE_LEN - chan->msg_count;
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(mbox_chan_tx_slots_available);
+
+/**
* mbox_send_message - For client to submit a message to be
* sent to the remote.
* @chan: Mailbox channel assigned to this client.
@@ -246,7 +267,7 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg)
{
int t;
- if (!chan || !chan->cl)
+ if (!chan || !chan->cl || mssg == MBOX_NO_MSG)
return -EINVAL;
t = add_to_rbuf(chan, mssg);
@@ -319,12 +340,12 @@ static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl)
scoped_guard(spinlock_irqsave, &chan->lock) {
chan->msg_free = 0;
chan->msg_count = 0;
- chan->active_req = NULL;
+ chan->active_req = MBOX_NO_MSG;
chan->cl = cl;
init_completion(&chan->tx_complete);
- if (chan->txdone_method == TXDONE_BY_POLL && cl->knows_txdone)
- chan->txdone_method = TXDONE_BY_ACK;
+ if (chan->txdone_method == MBOX_TXDONE_BY_POLL && cl->knows_txdone)
+ chan->txdone_method = MBOX_TXDONE_BY_ACK;
}
if (chan->mbox->ops->startup) {
@@ -341,7 +362,7 @@ static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl)
}
/**
- * mbox_bind_client - Request a mailbox channel.
+ * mbox_bind_client - Bind client to a mailbox channel.
* @chan: The mailbox channel to bind the client to.
* @cl: Identity of the client requesting the channel.
*
@@ -477,9 +498,9 @@ void mbox_free_channel(struct mbox_chan *chan)
/* The queued TX requests are simply aborted, no callbacks are made */
scoped_guard(spinlock_irqsave, &chan->lock) {
chan->cl = NULL;
- chan->active_req = NULL;
- if (chan->txdone_method == TXDONE_BY_ACK)
- chan->txdone_method = TXDONE_BY_POLL;
+ chan->active_req = MBOX_NO_MSG;
+ if (chan->txdone_method == MBOX_TXDONE_BY_ACK)
+ chan->txdone_method = MBOX_TXDONE_BY_POLL;
}
module_put(chan->mbox->dev->driver->owner);
@@ -505,18 +526,17 @@ int mbox_controller_register(struct mbox_controller *mbox)
{
int i, txdone;
- /* Sanity check */
- if (!mbox || !mbox->dev || !mbox->ops || !mbox->num_chans)
+ if (!mbox || !mbox->dev || !mbox->ops || !mbox->chans || !mbox->num_chans)
return -EINVAL;
if (mbox->txdone_irq)
- txdone = TXDONE_BY_IRQ;
+ txdone = MBOX_TXDONE_BY_IRQ;
else if (mbox->txdone_poll)
- txdone = TXDONE_BY_POLL;
+ txdone = MBOX_TXDONE_BY_POLL;
else /* It has to be ACK then */
- txdone = TXDONE_BY_ACK;
+ txdone = MBOX_TXDONE_BY_ACK;
- if (txdone == TXDONE_BY_POLL) {
+ if (txdone == MBOX_TXDONE_BY_POLL) {
if (!mbox->ops->last_tx_done) {
dev_err(mbox->dev, "last_tx_done method is absent\n");
@@ -532,6 +552,7 @@ int mbox_controller_register(struct mbox_controller *mbox)
chan->cl = NULL;
chan->mbox = mbox;
+ chan->active_req = MBOX_NO_MSG;
chan->txdone_method = txdone;
spin_lock_init(&chan->lock);
}
diff --git a/drivers/mailbox/mailbox.h b/drivers/mailbox/mailbox.h
deleted file mode 100644
index e1ec4ef..0000000
--- a/drivers/mailbox/mailbox.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-
-#ifndef __MAILBOX_H
-#define __MAILBOX_H
-
-#include <linux/bits.h>
-
-#define TXDONE_BY_IRQ BIT(0) /* controller has remote RTR irq */
-#define TXDONE_BY_POLL BIT(1) /* controller can read status of last TX */
-#define TXDONE_BY_ACK BIT(2) /* S/W ACK received by Client ticks the TX */
-
-#endif /* __MAILBOX_H */
diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c
index d7c6b38..e523c84 100644
--- a/drivers/mailbox/mtk-cmdq-mailbox.c
+++ b/drivers/mailbox/mtk-cmdq-mailbox.c
@@ -493,14 +493,14 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data)
if (curr_pa == end_pa - CMDQ_INST_SIZE ||
curr_pa == end_pa) {
/* set to this task directly */
- writel(task->pa_base >> cmdq->pdata->shift,
- thread->base + CMDQ_THR_CURR_ADDR);
+ gce_addr = cmdq_convert_gce_addr(task->pa_base, cmdq->pdata);
+ writel(gce_addr, thread->base + CMDQ_THR_CURR_ADDR);
} else {
cmdq_task_insert_into_thread(task);
smp_mb(); /* modify jump before enable thread */
}
- writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->pdata->shift,
- thread->base + CMDQ_THR_END_ADDR);
+ gce_addr = cmdq_convert_gce_addr(task->pa_base + pkt->cmd_buf_size, cmdq->pdata);
+ writel(gce_addr, thread->base + CMDQ_THR_END_ADDR);
cmdq_thread_resume(thread);
}
list_move_tail(&task->list_entry, &thread->task_busy_list);
@@ -728,7 +728,7 @@ static int cmdq_probe(struct platform_device *pdev)
cmdq->mbox.ops = &cmdq_mbox_chan_ops;
cmdq->mbox.of_xlate = cmdq_xlate;
- /* make use of TXDONE_BY_ACK */
+ /* make use of MBOX_TXDONE_BY_ACK */
cmdq->mbox.txdone_irq = false;
cmdq->mbox.txdone_poll = false;
diff --git a/drivers/mailbox/mtk-vcp-mailbox.c b/drivers/mailbox/mtk-vcp-mailbox.c
index cedad57..1b291b8 100644
--- a/drivers/mailbox/mtk-vcp-mailbox.c
+++ b/drivers/mailbox/mtk-vcp-mailbox.c
@@ -50,7 +50,7 @@ static struct mbox_chan *mtk_vcp_mbox_xlate(struct mbox_controller *mbox,
const struct of_phandle_args *sp)
{
if (sp->args_count)
- return NULL;
+ return ERR_PTR(-EINVAL);
return &mbox->chans[0];
}
diff --git a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c
index d9f100c..535ca80 100644
--- a/drivers/mailbox/omap-mailbox.c
+++ b/drivers/mailbox/omap-mailbox.c
@@ -22,8 +22,6 @@
#include <linux/pm_runtime.h>
#include <linux/mailbox_controller.h>
-#include "mailbox.h"
-
#define MAILBOX_REVISION 0x000
#define MAILBOX_MESSAGE(m) (0x040 + 4 * (m))
#define MAILBOX_FIFOSTATUS(m) (0x080 + 4 * (m))
@@ -240,7 +238,7 @@ static int omap_mbox_startup(struct omap_mbox *mbox)
}
if (mbox->send_no_irq)
- mbox->chan->txdone_method = TXDONE_BY_ACK;
+ mbox->chan->txdone_method = MBOX_TXDONE_BY_ACK;
omap_mbox_enable_irq(mbox, IRQ_RX);
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index 22e70af..636879a 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -59,8 +59,6 @@
#include <linux/io-64-nonatomic-lo-hi.h>
#include <acpi/pcc.h>
-#include "mailbox.h"
-
#define MBOX_IRQ_NAME "pcc-mbox"
/**
diff --git a/drivers/mailbox/rockchip-mailbox.c b/drivers/mailbox/rockchip-mailbox.c
index 4d966cb..a1a7dee 100644
--- a/drivers/mailbox/rockchip-mailbox.c
+++ b/drivers/mailbox/rockchip-mailbox.c
@@ -46,7 +46,7 @@ struct rockchip_mbox {
/* The maximum size of buf for each channel */
u32 buf_size;
- struct rockchip_mbox_chan *chans;
+ struct rockchip_mbox_chan chans[];
};
static int rockchip_mbox_send_data(struct mbox_chan *chan, void *data)
@@ -173,15 +173,10 @@ static int rockchip_mbox_probe(struct platform_device *pdev)
drv_data = (const struct rockchip_mbox_data *) device_get_match_data(&pdev->dev);
- mb = devm_kzalloc(&pdev->dev, sizeof(*mb), GFP_KERNEL);
+ mb = devm_kzalloc(&pdev->dev, struct_size(mb, chans, drv_data->num_chans), GFP_KERNEL);
if (!mb)
return -ENOMEM;
- mb->chans = devm_kcalloc(&pdev->dev, drv_data->num_chans,
- sizeof(*mb->chans), GFP_KERNEL);
- if (!mb->chans)
- return -ENOMEM;
-
mb->mbox.chans = devm_kcalloc(&pdev->dev, drv_data->num_chans,
sizeof(*mb->mbox.chans), GFP_KERNEL);
if (!mb->mbox.chans)
diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c
index ed9a0bb..500fa77 100644
--- a/drivers/mailbox/tegra-hsp.c
+++ b/drivers/mailbox/tegra-hsp.c
@@ -16,8 +16,6 @@
#include <dt-bindings/mailbox/tegra186-hsp.h>
-#include "mailbox.h"
-
#define HSP_INT_IE(x) (0x100 + ((x) * 4))
#define HSP_INT_IV 0x300
#define HSP_INT_IR 0x304
@@ -497,7 +495,7 @@ static int tegra_hsp_mailbox_flush(struct mbox_chan *chan,
mbox_chan_txdone(chan, 0);
/* Wait until channel is empty */
- if (chan->active_req != NULL)
+ if (chan->active_req != MBOX_NO_MSG)
continue;
return 0;
@@ -516,7 +514,7 @@ static int tegra_hsp_mailbox_startup(struct mbox_chan *chan)
struct tegra_hsp *hsp = mb->channel.hsp;
unsigned long flags;
- chan->txdone_method = TXDONE_BY_IRQ;
+ chan->txdone_method = MBOX_TXDONE_BY_IRQ;
/*
* Shared mailboxes start out as consumers by default. FULL and EMPTY
diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c
index dd77a93..1ae304c 100644
--- a/drivers/md/dm-cache-policy-smq.c
+++ b/drivers/md/dm-cache-policy-smq.c
@@ -1590,18 +1590,22 @@ static int smq_invalidate_mapping(struct dm_cache_policy *p, dm_cblock_t cblock)
struct smq_policy *mq = to_smq_policy(p);
struct entry *e = get_entry(&mq->cache_alloc, from_cblock(cblock));
unsigned long flags;
-
- if (!e->allocated)
- return -ENODATA;
+ int r = 0;
spin_lock_irqsave(&mq->lock, flags);
+ if (!e->allocated) {
+ r = -ENODATA;
+ goto out;
+ }
// FIXME: what if this block has pending background work?
del_queue(mq, e);
h_remove(&mq->table, e);
free_entry(&mq->cache_alloc, e);
+
+out:
spin_unlock_irqrestore(&mq->lock, flags);
- return 0;
+ return r;
}
static uint32_t smq_get_hint(struct dm_cache_policy *p, dm_cblock_t cblock)
diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c
index 7bec241..d0d4e02 100644
--- a/drivers/md/dm-vdo/vdo.c
+++ b/drivers/md/dm-vdo/vdo.c
@@ -965,7 +965,7 @@ static int __must_check clear_partition(struct vdo *vdo, enum partition_id id)
return blkdev_issue_zeroout(vdo_get_backing_device(vdo),
partition->offset * VDO_SECTORS_PER_BLOCK,
partition->count * VDO_SECTORS_PER_BLOCK,
- GFP_NOWAIT, 0);
+ GFP_NOIO, 0);
}
int vdo_clear_layout(struct vdo *vdo)
@@ -976,7 +976,7 @@ int vdo_clear_layout(struct vdo *vdo)
result = blkdev_issue_zeroout(vdo_get_backing_device(vdo),
VDO_SECTORS_PER_BLOCK,
VDO_SECTORS_PER_BLOCK,
- GFP_NOWAIT, 0);
+ GFP_NOIO, 0);
if (result != VDO_SUCCESS)
return result;
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index 83378c0..028b9ca 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -216,6 +216,7 @@ struct bitmap {
};
static struct workqueue_struct *md_bitmap_wq;
+static struct attribute_group md_bitmap_internal_group;
static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,
int chunksize, bool init);
@@ -2580,6 +2581,30 @@ static int bitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize)
return __bitmap_resize(bitmap, blocks, chunksize, false);
}
+static bool bitmap_none_enabled(void *data, bool flush)
+{
+ return false;
+}
+
+static int bitmap_none_create(struct mddev *mddev)
+{
+ return 0;
+}
+
+static int bitmap_none_load(struct mddev *mddev)
+{
+ return 0;
+}
+
+static void bitmap_none_destroy(struct mddev *mddev)
+{
+}
+
+static int bitmap_none_get_stats(void *data, struct md_bitmap_stats *stats)
+{
+ return -ENOENT;
+}
+
static ssize_t
location_show(struct mddev *mddev, char *page)
{
@@ -2618,7 +2643,11 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
goto out;
}
- bitmap_destroy(mddev);
+ sysfs_unmerge_group(&mddev->kobj, &md_bitmap_internal_group);
+ md_bitmap_destroy_nosysfs(mddev);
+ mddev->bitmap_id = ID_BITMAP_NONE;
+ if (!mddev_set_bitmap_ops_nosysfs(mddev))
+ goto none_err;
mddev->bitmap_info.offset = 0;
if (mddev->bitmap_info.file) {
struct file *f = mddev->bitmap_info.file;
@@ -2654,16 +2683,25 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
}
mddev->bitmap_info.offset = offset;
- rv = bitmap_create(mddev);
- if (rv)
- goto out;
+ md_bitmap_destroy_nosysfs(mddev);
+ mddev->bitmap_id = ID_BITMAP;
+ if (!mddev_set_bitmap_ops_nosysfs(mddev))
+ goto bitmap_err;
- rv = bitmap_load(mddev);
+ rv = md_bitmap_create_nosysfs(mddev);
+ if (rv)
+ goto create_err;
+
+ rv = mddev->bitmap_ops->load(mddev);
if (rv) {
mddev->bitmap_info.offset = 0;
- bitmap_destroy(mddev);
- goto out;
+ goto load_err;
}
+
+ rv = sysfs_merge_group(&mddev->kobj,
+ &md_bitmap_internal_group);
+ if (rv)
+ goto merge_err;
}
}
if (!mddev->external) {
@@ -2679,6 +2717,22 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
if (rv)
return rv;
return len;
+
+merge_err:
+ mddev->bitmap_info.offset = 0;
+load_err:
+ md_bitmap_destroy_nosysfs(mddev);
+create_err:
+ mddev->bitmap_info.offset = 0;
+ mddev->bitmap_id = ID_BITMAP_NONE;
+ if (!mddev_set_bitmap_ops_nosysfs(mddev))
+ rv = -ENOENT;
+ goto out;
+bitmap_err:
+ rv = -ENOENT;
+none_err:
+ mddev->bitmap_info.offset = 0;
+ goto out;
}
static struct md_sysfs_entry bitmap_location =
@@ -2955,8 +3009,12 @@ static struct md_sysfs_entry max_backlog_used =
__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
behind_writes_used_show, behind_writes_used_reset);
-static struct attribute *md_bitmap_attrs[] = {
+static struct attribute *md_bitmap_common_attrs[] = {
&bitmap_location.attr,
+ NULL
+};
+
+static struct attribute *md_bitmap_internal_attrs[] = {
&bitmap_space.attr,
&bitmap_timeout.attr,
&bitmap_backlog.attr,
@@ -2967,9 +3025,41 @@ static struct attribute *md_bitmap_attrs[] = {
NULL
};
-static struct attribute_group md_bitmap_group = {
+static struct attribute_group md_bitmap_common_group = {
.name = "bitmap",
- .attrs = md_bitmap_attrs,
+ .attrs = md_bitmap_common_attrs,
+};
+
+static struct attribute_group md_bitmap_internal_group = {
+ .name = "bitmap",
+ .attrs = md_bitmap_internal_attrs,
+};
+
+static const struct attribute_group *bitmap_groups[] = {
+ &md_bitmap_common_group,
+ &md_bitmap_internal_group,
+ NULL,
+};
+
+static const struct attribute_group *bitmap_none_groups[] = {
+ &md_bitmap_common_group,
+ NULL,
+};
+
+static struct bitmap_operations bitmap_none_ops = {
+ .head = {
+ .type = MD_BITMAP,
+ .id = ID_BITMAP_NONE,
+ .name = "none",
+ },
+
+ .enabled = bitmap_none_enabled,
+ .create = bitmap_none_create,
+ .load = bitmap_none_load,
+ .destroy = bitmap_none_destroy,
+ .get_stats = bitmap_none_get_stats,
+
+ .groups = bitmap_none_groups,
};
static struct bitmap_operations bitmap_ops = {
@@ -3013,21 +3103,38 @@ static struct bitmap_operations bitmap_ops = {
.set_pages = bitmap_set_pages,
.free = md_bitmap_free,
- .group = &md_bitmap_group,
+ .groups = bitmap_groups,
};
int md_bitmap_init(void)
{
+ int err;
+
md_bitmap_wq = alloc_workqueue("md_bitmap", WQ_MEM_RECLAIM | WQ_UNBOUND,
0);
if (!md_bitmap_wq)
return -ENOMEM;
- return register_md_submodule(&bitmap_ops.head);
+ err = register_md_submodule(&bitmap_none_ops.head);
+ if (err)
+ goto err_wq;
+
+ err = register_md_submodule(&bitmap_ops.head);
+ if (err)
+ goto err_none;
+
+ return 0;
+
+err_none:
+ unregister_md_submodule(&bitmap_none_ops.head);
+err_wq:
+ destroy_workqueue(md_bitmap_wq);
+ return err;
}
void md_bitmap_exit(void)
{
- destroy_workqueue(md_bitmap_wq);
unregister_md_submodule(&bitmap_ops.head);
+ unregister_md_submodule(&bitmap_none_ops.head);
+ destroy_workqueue(md_bitmap_wq);
}
diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h
index b42a28f..214f623 100644
--- a/drivers/md/md-bitmap.h
+++ b/drivers/md/md-bitmap.h
@@ -125,7 +125,7 @@ struct bitmap_operations {
void (*set_pages)(void *data, unsigned long pages);
void (*free)(void *data);
- struct attribute_group *group;
+ const struct attribute_group **groups;
};
/* the bitmap API */
diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c
index 9e7e6b1..1adc5b1 100644
--- a/drivers/md/md-llbitmap.c
+++ b/drivers/md/md-llbitmap.c
@@ -1738,6 +1738,11 @@ static struct attribute_group md_llbitmap_group = {
.attrs = md_llbitmap_attrs,
};
+static const struct attribute_group *md_llbitmap_groups[] = {
+ &md_llbitmap_group,
+ NULL,
+};
+
static struct bitmap_operations llbitmap_ops = {
.head = {
.type = MD_BITMAP,
@@ -1774,7 +1779,7 @@ static struct bitmap_operations llbitmap_ops = {
.dirty_bits = llbitmap_dirty_bits,
.write_all = llbitmap_write_all,
- .group = &md_llbitmap_group,
+ .groups = md_llbitmap_groups,
};
int md_llbitmap_init(void)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 5fb5ae8..8b568ee 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -396,27 +396,19 @@ bool md_handle_request(struct mddev *mddev, struct bio *bio)
{
check_suspended:
if (is_suspended(mddev, bio)) {
- DEFINE_WAIT(__wait);
/* Bail out if REQ_NOWAIT is set for the bio */
if (bio->bi_opf & REQ_NOWAIT) {
bio_wouldblock_error(bio);
return true;
}
- for (;;) {
- prepare_to_wait(&mddev->sb_wait, &__wait,
- TASK_UNINTERRUPTIBLE);
- if (!is_suspended(mddev, bio))
- break;
- schedule();
- }
- finish_wait(&mddev->sb_wait, &__wait);
+ wait_event(mddev->sb_wait, !is_suspended(mddev, bio));
}
if (!percpu_ref_tryget_live(&mddev->active_io))
goto check_suspended;
if (!mddev->pers->make_request(mddev, bio)) {
percpu_ref_put(&mddev->active_io);
- if (!mddev->gendisk && mddev->pers->prepare_suspend)
+ if (mddev_is_dm(mddev) && mddev->pers->prepare_suspend)
return false;
goto check_suspended;
}
@@ -687,13 +679,38 @@ static void active_io_release(struct percpu_ref *ref)
static void no_op(struct percpu_ref *r) {}
-static bool mddev_set_bitmap_ops(struct mddev *mddev)
+static void md_bitmap_sysfs_add(struct mddev *mddev)
{
- struct bitmap_operations *old = mddev->bitmap_ops;
+ if (sysfs_update_groups(&mddev->kobj, mddev->bitmap_ops->groups))
+ pr_warn("md: cannot register extra bitmap attributes for %s\n",
+ mdname(mddev));
+ else
+ /*
+ * Inform user with KOBJ_CHANGE about new bitmap
+ * attributes.
+ */
+ kobject_uevent(&mddev->kobj, KOBJ_CHANGE);
+}
+
+static void md_bitmap_sysfs_del(struct mddev *mddev)
+{
+ int nr_groups = 0;
+
+ for (nr_groups = 0; mddev->bitmap_ops->groups[nr_groups]; nr_groups++)
+ ;
+
+ while (--nr_groups >= 1)
+ sysfs_unmerge_group(&mddev->kobj,
+ mddev->bitmap_ops->groups[nr_groups]);
+ sysfs_remove_group(&mddev->kobj, mddev->bitmap_ops->groups[0]);
+}
+
+bool mddev_set_bitmap_ops_nosysfs(struct mddev *mddev)
+{
struct md_submodule_head *head;
- if (mddev->bitmap_id == ID_BITMAP_NONE ||
- (old && old->head.id == mddev->bitmap_id))
+ if (mddev->bitmap_ops &&
+ mddev->bitmap_ops->head.id == mddev->bitmap_id)
return true;
xa_lock(&md_submodule);
@@ -711,18 +728,6 @@ static bool mddev_set_bitmap_ops(struct mddev *mddev)
mddev->bitmap_ops = (void *)head;
xa_unlock(&md_submodule);
-
- if (!mddev_is_dm(mddev) && mddev->bitmap_ops->group) {
- if (sysfs_create_group(&mddev->kobj, mddev->bitmap_ops->group))
- pr_warn("md: cannot register extra bitmap attributes for %s\n",
- mdname(mddev));
- else
- /*
- * Inform user with KOBJ_CHANGE about new bitmap
- * attributes.
- */
- kobject_uevent(&mddev->kobj, KOBJ_CHANGE);
- }
return true;
err:
@@ -730,15 +735,6 @@ static bool mddev_set_bitmap_ops(struct mddev *mddev)
return false;
}
-static void mddev_clear_bitmap_ops(struct mddev *mddev)
-{
- if (!mddev_is_dm(mddev) && mddev->bitmap_ops &&
- mddev->bitmap_ops->group)
- sysfs_remove_group(&mddev->kobj, mddev->bitmap_ops->group);
-
- mddev->bitmap_ops = NULL;
-}
-
int mddev_init(struct mddev *mddev)
{
int err = 0;
@@ -4279,7 +4275,7 @@ bitmap_type_show(struct mddev *mddev, char *page)
xa_lock(&md_submodule);
xa_for_each(&md_submodule, i, head) {
- if (head->type != MD_BITMAP)
+ if (head->type != MD_BITMAP || head->id == ID_BITMAP_NONE)
continue;
if (mddev->bitmap_id == head->id)
@@ -6059,10 +6055,7 @@ static struct attribute *md_default_attrs[] = {
&md_logical_block_size.attr,
NULL,
};
-
-static const struct attribute_group md_default_group = {
- .attrs = md_default_attrs,
-};
+ATTRIBUTE_GROUPS(md_default);
static struct attribute *md_redundancy_attrs[] = {
&md_scan_mode.attr,
@@ -6087,11 +6080,6 @@ static const struct attribute_group md_redundancy_group = {
.attrs = md_redundancy_attrs,
};
-static const struct attribute_group *md_attr_groups[] = {
- &md_default_group,
- NULL,
-};
-
static ssize_t
md_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
@@ -6174,7 +6162,7 @@ static const struct sysfs_ops md_sysfs_ops = {
static const struct kobj_type md_ktype = {
.release = md_kobj_release,
.sysfs_ops = &md_sysfs_ops,
- .default_groups = md_attr_groups,
+ .default_groups = md_default_groups,
};
int mdp_major = 0;
@@ -6539,7 +6527,7 @@ static enum md_submodule_id md_bitmap_get_id_from_sb(struct mddev *mddev)
return id;
}
-static int md_bitmap_create(struct mddev *mddev)
+int md_bitmap_create_nosysfs(struct mddev *mddev)
{
enum md_submodule_id orig_id = mddev->bitmap_id;
enum md_submodule_id sb_id;
@@ -6548,8 +6536,10 @@ static int md_bitmap_create(struct mddev *mddev)
if (mddev->bitmap_id == ID_BITMAP_NONE)
return -EINVAL;
- if (!mddev_set_bitmap_ops(mddev))
+ if (!mddev_set_bitmap_ops_nosysfs(mddev)) {
+ mddev->bitmap_id = orig_id;
return -ENOENT;
+ }
err = mddev->bitmap_ops->create(mddev);
if (!err)
@@ -6560,37 +6550,72 @@ static int md_bitmap_create(struct mddev *mddev)
* doesn't match, and mdadm is not the latest version to set
* bitmap_type, set bitmap_ops based on the disk version.
*/
- mddev_clear_bitmap_ops(mddev);
+ mddev->bitmap_ops = NULL;
sb_id = md_bitmap_get_id_from_sb(mddev);
- if (sb_id == ID_BITMAP_NONE || sb_id == orig_id)
+ if (sb_id == ID_BITMAP_NONE || sb_id == orig_id) {
+ mddev->bitmap_id = orig_id;
return err;
+ }
pr_info("md: %s: bitmap version mismatch, switching from %d to %d\n",
mdname(mddev), orig_id, sb_id);
mddev->bitmap_id = sb_id;
- if (!mddev_set_bitmap_ops(mddev)) {
+ if (!mddev_set_bitmap_ops_nosysfs(mddev)) {
mddev->bitmap_id = orig_id;
return -ENOENT;
}
err = mddev->bitmap_ops->create(mddev);
if (err) {
- mddev_clear_bitmap_ops(mddev);
+ mddev->bitmap_ops = NULL;
mddev->bitmap_id = orig_id;
}
return err;
}
-static void md_bitmap_destroy(struct mddev *mddev)
+static int md_bitmap_create(struct mddev *mddev)
+{
+ int err;
+
+ err = md_bitmap_create_nosysfs(mddev);
+ if (err)
+ return err;
+
+ if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups)
+ md_bitmap_sysfs_add(mddev);
+
+ return 0;
+}
+
+void md_bitmap_destroy_nosysfs(struct mddev *mddev)
{
if (!md_bitmap_registered(mddev))
return;
mddev->bitmap_ops->destroy(mddev);
- mddev_clear_bitmap_ops(mddev);
+ mddev->bitmap_ops = NULL;
+}
+
+static void md_bitmap_destroy(struct mddev *mddev)
+{
+ if (!mddev_is_dm(mddev) && mddev->bitmap_ops &&
+ mddev->bitmap_ops->groups)
+ md_bitmap_sysfs_del(mddev);
+
+ md_bitmap_destroy_nosysfs(mddev);
+}
+
+static void md_bitmap_set_none(struct mddev *mddev)
+{
+ mddev->bitmap_id = ID_BITMAP_NONE;
+ if (!mddev_set_bitmap_ops_nosysfs(mddev))
+ return;
+
+ if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups)
+ md_bitmap_sysfs_add(mddev);
}
int md_run(struct mddev *mddev)
@@ -6713,7 +6738,7 @@ int md_run(struct mddev *mddev)
}
/* dm-raid expect sync_thread to be frozen until resume */
- if (mddev->gendisk)
+ if (!mddev_is_dm(mddev))
mddev->recovery = 0;
/* may be over-ridden by personality */
@@ -6802,6 +6827,10 @@ int md_run(struct mddev *mddev)
if (mddev->sb_flags)
md_update_sb(mddev, 0);
+ if (IS_ENABLED(CONFIG_MD_BITMAP) && !mddev->bitmap_info.file &&
+ !mddev->bitmap_info.offset)
+ md_bitmap_set_none(mddev);
+
md_new_event();
return 0;
@@ -7747,7 +7776,8 @@ static int set_bitmap_file(struct mddev *mddev, int fd)
{
int err = 0;
- if (!md_bitmap_registered(mddev))
+ if (!md_bitmap_registered(mddev) ||
+ mddev->bitmap_id == ID_BITMAP_NONE)
return -EINVAL;
if (mddev->pers) {
@@ -7812,10 +7842,12 @@ static int set_bitmap_file(struct mddev *mddev, int fd)
if (err) {
md_bitmap_destroy(mddev);
+ md_bitmap_set_none(mddev);
fd = -1;
}
} else if (fd < 0) {
md_bitmap_destroy(mddev);
+ md_bitmap_set_none(mddev);
}
}
@@ -8122,12 +8154,16 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
mddev->bitmap_info.default_offset;
mddev->bitmap_info.space =
mddev->bitmap_info.default_space;
+ mddev->bitmap_id = ID_BITMAP;
rv = md_bitmap_create(mddev);
if (!rv)
rv = mddev->bitmap_ops->load(mddev);
- if (rv)
+ if (rv) {
md_bitmap_destroy(mddev);
+ mddev->bitmap_info.offset = 0;
+ md_bitmap_set_none(mddev);
+ }
} else {
struct md_bitmap_stats stats;
@@ -8155,6 +8191,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
}
md_bitmap_destroy(mddev);
mddev->bitmap_info.offset = 0;
+ md_bitmap_set_none(mddev);
}
}
md_update_sb(mddev, 1);
@@ -9341,9 +9378,11 @@ static void md_bitmap_end(struct mddev *mddev, struct md_io_clone *md_io_clone)
static void md_end_clone_io(struct bio *bio)
{
- struct md_io_clone *md_io_clone = bio->bi_private;
+ struct md_io_clone *md_io_clone = container_of(bio, struct md_io_clone,
+ bio_clone);
struct bio *orig_bio = md_io_clone->orig_bio;
struct mddev *mddev = md_io_clone->mddev;
+ struct completion *reshape_completion = bio->bi_private;
if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false))
md_bitmap_end(mddev, md_io_clone);
@@ -9355,7 +9394,10 @@ static void md_end_clone_io(struct bio *bio)
bio_end_io_acct(orig_bio, md_io_clone->start_time);
bio_put(bio);
- bio_endio(orig_bio);
+ if (unlikely(reshape_completion))
+ complete(reshape_completion);
+ else
+ bio_endio(orig_bio);
percpu_ref_put(&mddev->active_io);
}
@@ -9380,7 +9422,7 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio)
}
clone->bi_end_io = md_end_clone_io;
- clone->bi_private = md_io_clone;
+ clone->bi_private = NULL;
*bio = clone;
}
@@ -9391,26 +9433,6 @@ void md_account_bio(struct mddev *mddev, struct bio **bio)
}
EXPORT_SYMBOL_GPL(md_account_bio);
-void md_free_cloned_bio(struct bio *bio)
-{
- struct md_io_clone *md_io_clone = bio->bi_private;
- struct bio *orig_bio = md_io_clone->orig_bio;
- struct mddev *mddev = md_io_clone->mddev;
-
- if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false))
- md_bitmap_end(mddev, md_io_clone);
-
- if (bio->bi_status && !orig_bio->bi_status)
- orig_bio->bi_status = bio->bi_status;
-
- if (md_io_clone->start_time)
- bio_end_io_acct(orig_bio, md_io_clone->start_time);
-
- bio_put(bio);
- percpu_ref_put(&mddev->active_io);
-}
-EXPORT_SYMBOL_GPL(md_free_cloned_bio);
-
/* md_allow_write(mddev)
* Calling this ensures that the array is marked 'active' so that writes
* may proceed without blocking. It is important to call this before
diff --git a/drivers/md/md.h b/drivers/md/md.h
index d6f5482..52c3780 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -920,7 +920,6 @@ extern void md_finish_reshape(struct mddev *mddev);
void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
struct bio *bio, sector_t start, sector_t size);
void md_account_bio(struct mddev *mddev, struct bio **bio);
-void md_free_cloned_bio(struct bio *bio);
extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio);
void md_write_metadata(struct mddev *mddev, struct md_rdev *rdev,
@@ -935,6 +934,9 @@ extern void md_allow_write(struct mddev *mddev);
extern void md_wait_for_blocked_rdev(struct md_rdev *rdev, struct mddev *mddev);
extern void md_set_array_sectors(struct mddev *mddev, sector_t array_sectors);
extern int md_check_no_bitmap(struct mddev *mddev);
+bool mddev_set_bitmap_ops_nosysfs(struct mddev *mddev);
+int md_bitmap_create_nosysfs(struct mddev *mddev);
+void md_bitmap_destroy_nosysfs(struct mddev *mddev);
extern int md_integrity_register(struct mddev *mddev);
extern int strict_strtoul_scaled(const char *cp, unsigned long *res, int scale);
@@ -1015,7 +1017,7 @@ static inline int mddev_suspend_and_lock(struct mddev *mddev)
static inline void mddev_suspend_and_lock_nointr(struct mddev *mddev)
{
mddev_suspend(mddev, false);
- mutex_lock(&mddev->reconfig_mutex);
+ mddev_lock_nointr(mddev);
}
static inline void mddev_unlock_and_resume(struct mddev *mddev)
diff --git a/drivers/md/persistent-data/dm-btree-remove.c b/drivers/md/persistent-data/dm-btree-remove.c
index 942cd47..aeec5b9 100644
--- a/drivers/md/persistent-data/dm-btree-remove.c
+++ b/drivers/md/persistent-data/dm-btree-remove.c
@@ -490,12 +490,20 @@ static int rebalance_children(struct shadow_spine *s,
if (le32_to_cpu(n->header.nr_entries) == 1) {
struct dm_block *child;
+ int is_shared;
dm_block_t b = value64(n, 0);
+ r = dm_tm_block_is_shared(info->tm, b, &is_shared);
+ if (r)
+ return r;
+
r = dm_tm_read_lock(info->tm, b, &btree_node_validator, &child);
if (r)
return r;
+ if (is_shared)
+ inc_children(info->tm, dm_block_data(child), vt);
+
memcpy(n, dm_block_data(child),
dm_bm_block_size(dm_tm_get_bm(info->tm)));
diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c
index c330999..56a56a4 100644
--- a/drivers/md/raid1-10.c
+++ b/drivers/md/raid1-10.c
@@ -293,8 +293,13 @@ static inline bool raid1_should_read_first(struct mddev *mddev,
* bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is
* submitted to the underlying disks, hence don't record badblocks or retry
* in this case.
+ *
+ * BLK_STS_INVAL means the bio was not valid for the underlying device. This
+ * is a user error, not a device failure, so retrying or recording bad blocks
+ * would be wrong.
*/
static inline bool raid1_should_handle_error(struct bio *bio)
{
- return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT));
+ return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) &&
+ bio->bi_status != BLK_STS_INVAL;
}
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index ba91f7e..64d970e 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1510,21 +1510,14 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
mddev->cluster_ops->area_resyncing(mddev, WRITE,
bio->bi_iter.bi_sector, bio_end_sector(bio))) {
- DEFINE_WAIT(w);
if (bio->bi_opf & REQ_NOWAIT) {
bio_wouldblock_error(bio);
return;
}
- for (;;) {
- prepare_to_wait(&conf->wait_barrier,
- &w, TASK_IDLE);
- if (!mddev->cluster_ops->area_resyncing(mddev, WRITE,
- bio->bi_iter.bi_sector,
- bio_end_sector(bio)))
- break;
- schedule();
- }
- finish_wait(&conf->wait_barrier, &w);
+ wait_event_idle(conf->wait_barrier,
+ !mddev->cluster_ops->area_resyncing(mddev, WRITE,
+ bio->bi_iter.bi_sector,
+ bio_end_sector(bio)));
}
/*
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 4901ebe..39085e7 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -3791,6 +3791,8 @@ static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
nc = layout & 255;
fc = (layout >> 8) & 255;
fo = layout & (1<<16);
+ if (!nc || !fc)
+ return -1;
geo->raid_disks = disks;
geo->near_copies = nc;
geo->far_copies = fc;
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 6e79829..0d76e82 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -6217,7 +6217,12 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
mempool_free(ctx, conf->ctx_pool);
if (res == STRIPE_WAIT_RESHAPE) {
- md_free_cloned_bio(bi);
+ DECLARE_COMPLETION_ONSTACK(done);
+ WRITE_ONCE(bi->bi_private, &done);
+
+ bio_endio(bi);
+
+ wait_for_completion(&done);
return false;
}
diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen3.c b/drivers/media/platform/qcom/camss/camss-csid-gen3.c
index 664245c..bd05924 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-gen3.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-gen3.c
@@ -48,9 +48,9 @@
#define IS_CSID_690(csid) ((csid->camss->res->version == CAMSS_8775P) \
|| (csid->camss->res->version == CAMSS_8300))
#define CSID_BUF_DONE_IRQ_STATUS 0x8C
-#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ?\
- 1 : (IS_CSID_690(csid) ?\
- 13 : 14))
+#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ? \
+ ((IS_CSID_690(csid) ? 0 : 1)) : \
+ ((IS_CSID_690(csid) ? 13 : 14)))
#define CSID_BUF_DONE_IRQ_MASK 0x90
#define CSID_BUF_DONE_IRQ_CLEAR 0x94
#define CSID_BUF_DONE_IRQ_SET 0x98
diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c
index 6262339..78a1b56 100644
--- a/drivers/media/platform/qcom/camss/camss-csiphy.c
+++ b/drivers/media/platform/qcom/camss/camss-csiphy.c
@@ -558,12 +558,16 @@ static int csiphy_init_formats(struct v4l2_subdev *sd,
return csiphy_set_format(sd, fh ? fh->state : NULL, &format);
}
-static bool csiphy_match_clock_name(const char *clock_name, const char *format,
- int index)
+static bool __printf(2, 3)
+csiphy_match_clock_name(const char *clock_name, const char *format, ...)
{
char name[16]; /* csiphyXXX_timer\0 */
+ va_list args;
- snprintf(name, sizeof(name), format, index);
+ va_start(args, format);
+ vsnprintf(name, sizeof(name), format, args);
+ va_end(args);
+
return !strcmp(clock_name, name);
}
diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c
index 00b87fd..9335636 100644
--- a/drivers/media/platform/qcom/camss/camss.c
+++ b/drivers/media/platform/qcom/camss/camss.c
@@ -3598,12 +3598,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = {
/* CSID2 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
- "vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" },
.clock_rate = {
- { 0, 0, 400000000, 400000000, 0},
- { 0, 0, 400000000, 480000000, 0}
+ { 400000000, 480000000 },
+ { 400000000, 480000000 }
},
.reg = { "csid_lite0" },
.interrupt = { "csid_lite0" },
@@ -3617,12 +3615,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = {
/* CSID3 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
- "vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" },
.clock_rate = {
- { 0, 0, 400000000, 400000000, 0},
- { 0, 0, 400000000, 480000000, 0}
+ { 400000000, 480000000 },
+ { 400000000, 480000000 }
},
.reg = { "csid_lite1" },
.interrupt = { "csid_lite1" },
@@ -3636,12 +3632,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = {
/* CSID4 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
- "vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" },
.clock_rate = {
- { 0, 0, 400000000, 400000000, 0},
- { 0, 0, 400000000, 480000000, 0}
+ { 400000000, 480000000 },
+ { 400000000, 480000000 }
},
.reg = { "csid_lite2" },
.interrupt = { "csid_lite2" },
@@ -3655,12 +3649,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = {
/* CSID5 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
- "vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" },
.clock_rate = {
- { 0, 0, 400000000, 400000000, 0},
- { 0, 0, 400000000, 480000000, 0}
+ { 400000000, 480000000 },
+ { 400000000, 480000000 }
},
.reg = { "csid_lite3" },
.interrupt = { "csid_lite3" },
@@ -3674,12 +3666,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = {
/* CSID6 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
- "vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" },
.clock_rate = {
- { 0, 0, 400000000, 400000000, 0},
- { 0, 0, 400000000, 480000000, 0}
+ { 400000000, 480000000 },
+ { 400000000, 480000000 }
},
.reg = { "csid_lite4" },
.interrupt = { "csid_lite4" },
@@ -3752,15 +3742,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = {
/* VFE2 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
+ .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb",
"vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ "vfe_lite", "camnoc_axi"},
.clock_rate = {
- { 0, 0, 0, 0 },
+ { 0 },
+ { 0 },
{ 300000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 480000000, 600000000, 600000000, 600000000 },
+ { 400000000 },
},
.reg = { "vfe_lite0" },
.interrupt = { "vfe_lite0" },
@@ -3775,15 +3767,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = {
/* VFE3 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
+ .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb",
"vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ "vfe_lite", "camnoc_axi"},
.clock_rate = {
- { 0, 0, 0, 0 },
+ { 0 },
+ { 0 },
{ 300000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 480000000, 600000000, 600000000, 600000000 },
+ { 400000000 },
},
.reg = { "vfe_lite1" },
.interrupt = { "vfe_lite1" },
@@ -3798,15 +3792,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = {
/* VFE4 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
+ .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb",
"vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ "vfe_lite", "camnoc_axi"},
.clock_rate = {
- { 0, 0, 0, 0 },
+ { 0 },
+ { 0 },
{ 300000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 480000000, 600000000, 600000000, 600000000 },
+ { 400000000 },
},
.reg = { "vfe_lite2" },
.interrupt = { "vfe_lite2" },
@@ -3821,15 +3817,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = {
/* VFE5 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
+ .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb",
"vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ "vfe_lite", "camnoc_axi"},
.clock_rate = {
- { 0, 0, 0, 0 },
+ { 0 },
+ { 0 },
{ 300000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 480000000, 600000000, 600000000, 600000000 },
+ { 400000000 },
},
.reg = { "vfe_lite3" },
.interrupt = { "vfe_lite3" },
@@ -3844,15 +3842,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = {
/* VFE6 (lite) */
{
.regulators = {},
- .clock = { "cpas_vfe_lite", "vfe_lite_ahb",
+ .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb",
"vfe_lite_csid", "vfe_lite_cphy_rx",
- "vfe_lite"},
+ "vfe_lite", "camnoc_axi"},
.clock_rate = {
- { 0, 0, 0, 0 },
+ { 0 },
+ { 0 },
{ 300000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 400000000, 400000000, 400000000, 400000000 },
{ 480000000, 600000000, 600000000, 600000000 },
+ { 400000000 },
},
.reg = { "vfe_lite4" },
.interrupt = { "vfe_lite4" },
diff --git a/drivers/media/platform/qcom/iris/Kconfig b/drivers/media/platform/qcom/iris/Kconfig
index 3c803a0..5498f48 100644
--- a/drivers/media/platform/qcom/iris/Kconfig
+++ b/drivers/media/platform/qcom/iris/Kconfig
@@ -3,7 +3,7 @@
depends on VIDEO_DEV
depends on ARCH_QCOM || COMPILE_TEST
select V4L2_MEM2MEM_DEV
- select QCOM_MDT_LOADER if ARCH_QCOM
+ select QCOM_MDT_LOADER
select QCOM_SCM
select VIDEOBUF2_DMA_CONTIG
help
diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c
index 9151f43..1d53c74 100644
--- a/drivers/media/platform/qcom/iris/iris_buffer.c
+++ b/drivers/media/platform/qcom/iris/iris_buffer.c
@@ -582,10 +582,12 @@ static int iris_release_internal_buffers(struct iris_inst *inst,
continue;
if (!(buffer->attr & BUF_ATTR_QUEUED))
continue;
- ret = hfi_ops->session_release_buf(inst, buffer);
- if (ret)
- return ret;
buffer->attr |= BUF_ATTR_PENDING_RELEASE;
+ ret = hfi_ops->session_release_buf(inst, buffer);
+ if (ret) {
+ buffer->attr &= ~BUF_ATTR_PENDING_RELEASE;
+ return ret;
+ }
}
return 0;
diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c
index 8406c48..dbaac01 100644
--- a/drivers/media/platform/qcom/iris/iris_core.c
+++ b/drivers/media/platform/qcom/iris/iris_core.c
@@ -75,6 +75,10 @@ int iris_core_init(struct iris_core *core)
if (ret)
goto error_unload_fw;
+ ret = iris_vpu_switch_to_hwmode(core);
+ if (ret)
+ goto error_unload_fw;
+
ret = iris_hfi_core_init(core);
if (ret)
goto error_unload_fw;
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_common.c b/drivers/media/platform/qcom/iris/iris_hfi_common.c
index 92112eb..621c665 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_common.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_common.c
@@ -159,6 +159,10 @@ int iris_hfi_pm_resume(struct iris_core *core)
if (ret)
goto err_suspend_hw;
+ ret = iris_vpu_switch_to_hwmode(core);
+ if (ret)
+ goto err_suspend_hw;
+
ret = ops->sys_interframe_powercollapse(core);
if (ret)
goto err_suspend_hw;
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_queue.c b/drivers/media/platform/qcom/iris/iris_hfi_queue.c
index b3ed062..bf6db23 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_queue.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_queue.c
@@ -263,7 +263,7 @@ int iris_hfi_queues_init(struct iris_core *core)
GFP_KERNEL, DMA_ATTR_WRITE_COMBINE);
if (!core->sfr_vaddr) {
dev_err(core->dev, "sfr alloc and map failed\n");
- dma_free_attrs(core->dev, sizeof(*q_tbl_hdr), core->iface_q_table_vaddr,
+ dma_free_attrs(core->dev, queue_size, core->iface_q_table_vaddr,
core->iface_q_table_daddr, DMA_ATTR_WRITE_COMBINE);
return -ENOMEM;
}
diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c
index 7192173..99d544e 100644
--- a/drivers/media/platform/qcom/iris/iris_vdec.c
+++ b/drivers/media/platform/qcom/iris/iris_vdec.c
@@ -61,12 +61,6 @@ int iris_vdec_inst_init(struct iris_inst *inst)
return iris_ctrls_init(inst);
}
-void iris_vdec_inst_deinit(struct iris_inst *inst)
-{
- kfree(inst->fmt_dst);
- kfree(inst->fmt_src);
-}
-
static const struct iris_fmt iris_vdec_formats_cap[] = {
[IRIS_FMT_NV12] = {
.pixfmt = V4L2_PIX_FMT_NV12,
diff --git a/drivers/media/platform/qcom/iris/iris_vdec.h b/drivers/media/platform/qcom/iris/iris_vdec.h
index ec1ce55..5123d2a 100644
--- a/drivers/media/platform/qcom/iris/iris_vdec.h
+++ b/drivers/media/platform/qcom/iris/iris_vdec.h
@@ -9,7 +9,6 @@
struct iris_inst;
int iris_vdec_inst_init(struct iris_inst *inst);
-void iris_vdec_inst_deinit(struct iris_inst *inst);
int iris_vdec_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f);
int iris_vdec_try_fmt(struct iris_inst *inst, struct v4l2_format *f);
int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f);
diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c
index aa27b22..4d88676 100644
--- a/drivers/media/platform/qcom/iris/iris_venc.c
+++ b/drivers/media/platform/qcom/iris/iris_venc.c
@@ -79,12 +79,6 @@ int iris_venc_inst_init(struct iris_inst *inst)
return iris_ctrls_init(inst);
}
-void iris_venc_inst_deinit(struct iris_inst *inst)
-{
- kfree(inst->fmt_dst);
- kfree(inst->fmt_src);
-}
-
static const struct iris_fmt iris_venc_formats_cap[] = {
[IRIS_FMT_H264] = {
.pixfmt = V4L2_PIX_FMT_H264,
diff --git a/drivers/media/platform/qcom/iris/iris_venc.h b/drivers/media/platform/qcom/iris/iris_venc.h
index c4db743..00c1716b 100644
--- a/drivers/media/platform/qcom/iris/iris_venc.h
+++ b/drivers/media/platform/qcom/iris/iris_venc.h
@@ -9,7 +9,6 @@
struct iris_inst;
int iris_venc_inst_init(struct iris_inst *inst);
-void iris_venc_inst_deinit(struct iris_inst *inst);
int iris_venc_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f);
int iris_venc_try_fmt(struct iris_inst *inst, struct v4l2_format *f);
int iris_venc_s_fmt(struct iris_inst *inst, struct v4l2_format *f);
diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c
index bd38d84..5eb1786 100644
--- a/drivers/media/platform/qcom/iris/iris_vidc.c
+++ b/drivers/media/platform/qcom/iris/iris_vidc.c
@@ -289,10 +289,6 @@ int iris_close(struct file *filp)
v4l2_m2m_ctx_release(inst->m2m_ctx);
v4l2_m2m_release(inst->m2m_dev);
mutex_lock(&inst->lock);
- if (inst->domain == DECODER)
- iris_vdec_inst_deinit(inst);
- else if (inst->domain == ENCODER)
- iris_venc_inst_deinit(inst);
iris_session_close(inst);
iris_inst_change_state(inst, IRIS_INST_DEINIT);
iris_v4l2_fh_deinit(inst, filp);
@@ -304,6 +300,8 @@ int iris_close(struct file *filp)
mutex_unlock(&inst->lock);
mutex_destroy(&inst->ctx_q_lock);
mutex_destroy(&inst->lock);
+ kfree(inst->fmt_src);
+ kfree(inst->fmt_dst);
kfree(inst);
return 0;
diff --git a/drivers/media/platform/qcom/iris/iris_vpu2.c b/drivers/media/platform/qcom/iris/iris_vpu2.c
index 9c103a2e..01ef40f 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu2.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu2.c
@@ -44,4 +44,5 @@ const struct vpu_ops iris_vpu2_ops = {
.power_off_controller = iris_vpu_power_off_controller,
.power_on_controller = iris_vpu_power_on_controller,
.calc_freq = iris_vpu2_calc_freq,
+ .set_hwmode = iris_vpu_set_hwmode,
};
diff --git a/drivers/media/platform/qcom/iris/iris_vpu3x.c b/drivers/media/platform/qcom/iris/iris_vpu3x.c
index fe4423b..3dad47b 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu3x.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu3x.c
@@ -234,14 +234,8 @@ static int iris_vpu35_power_on_hw(struct iris_core *core)
if (ret)
goto err_disable_hw_free_clk;
- ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true);
- if (ret)
- goto err_disable_hw_clk;
-
return 0;
-err_disable_hw_clk:
- iris_disable_unprepare_clock(core, IRIS_HW_CLK);
err_disable_hw_free_clk:
iris_disable_unprepare_clock(core, IRIS_HW_FREERUN_CLK);
err_disable_axi_clk:
@@ -266,6 +260,7 @@ const struct vpu_ops iris_vpu3_ops = {
.power_off_controller = iris_vpu_power_off_controller,
.power_on_controller = iris_vpu_power_on_controller,
.calc_freq = iris_vpu3x_vpu4x_calculate_frequency,
+ .set_hwmode = iris_vpu_set_hwmode,
};
const struct vpu_ops iris_vpu33_ops = {
@@ -274,6 +269,7 @@ const struct vpu_ops iris_vpu33_ops = {
.power_off_controller = iris_vpu33_power_off_controller,
.power_on_controller = iris_vpu_power_on_controller,
.calc_freq = iris_vpu3x_vpu4x_calculate_frequency,
+ .set_hwmode = iris_vpu_set_hwmode,
};
const struct vpu_ops iris_vpu35_ops = {
@@ -283,4 +279,5 @@ const struct vpu_ops iris_vpu35_ops = {
.power_on_controller = iris_vpu35_vpu4x_power_on_controller,
.program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers,
.calc_freq = iris_vpu3x_vpu4x_calculate_frequency,
+ .set_hwmode = iris_vpu_set_hwmode,
};
diff --git a/drivers/media/platform/qcom/iris/iris_vpu4x.c b/drivers/media/platform/qcom/iris/iris_vpu4x.c
index a8db02c..02e100a 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu4x.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu4x.c
@@ -252,21 +252,10 @@ static int iris_vpu4x_power_on_hardware(struct iris_core *core)
ret = iris_vpu4x_power_on_apv(core);
if (ret)
goto disable_hw_clocks;
-
- iris_vpu4x_ahb_sync_reset_apv(core);
}
- iris_vpu4x_ahb_sync_reset_hardware(core);
-
- ret = iris_vpu4x_genpd_set_hwmode(core, true, efuse_value);
- if (ret)
- goto disable_apv_power_domain;
-
return 0;
-disable_apv_power_domain:
- if (!(efuse_value & DISABLE_VIDEO_APV_BIT))
- iris_vpu4x_power_off_apv(core);
disable_hw_clocks:
iris_vpu4x_disable_hardware_clocks(core, efuse_value);
disable_vpp1_power_domain:
@@ -359,6 +348,18 @@ static void iris_vpu4x_power_off_hardware(struct iris_core *core)
iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]);
}
+static int iris_vpu4x_set_hwmode(struct iris_core *core)
+{
+ u32 efuse_value = readl(core->reg_base + WRAPPER_EFUSE_MONITOR);
+
+ if (!(efuse_value & DISABLE_VIDEO_APV_BIT))
+ iris_vpu4x_ahb_sync_reset_apv(core);
+
+ iris_vpu4x_ahb_sync_reset_hardware(core);
+
+ return iris_vpu4x_genpd_set_hwmode(core, true, efuse_value);
+}
+
const struct vpu_ops iris_vpu4x_ops = {
.power_off_hw = iris_vpu4x_power_off_hardware,
.power_on_hw = iris_vpu4x_power_on_hardware,
@@ -366,4 +367,5 @@ const struct vpu_ops iris_vpu4x_ops = {
.power_on_controller = iris_vpu35_vpu4x_power_on_controller,
.program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers,
.calc_freq = iris_vpu3x_vpu4x_calculate_frequency,
+ .set_hwmode = iris_vpu4x_set_hwmode,
};
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h
index 12640eb..8c0d6b7 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h
+++ b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h
@@ -67,7 +67,7 @@ struct iris_inst;
#define SIZE_DOLBY_RPU_METADATA (41 * 1024)
#define H264_CABAC_HDR_RATIO_HD_TOT 1
#define H264_CABAC_RES_RATIO_HD_TOT 3
-#define H265D_MAX_SLICE 1200
+#define H265D_MAX_SLICE 3600
#define SIZE_H265D_HW_PIC_T SIZE_H264D_HW_PIC_T
#define H265_CABAC_HDR_RATIO_HD_TOT 2
#define H265_CABAC_RES_RATIO_HD_TOT 2
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c
index 548e5f1..69e6126 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_common.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c
@@ -292,14 +292,8 @@ int iris_vpu_power_on_hw(struct iris_core *core)
if (ret && ret != -ENOENT)
goto err_disable_hw_clock;
- ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true);
- if (ret)
- goto err_disable_hw_ahb_clock;
-
return 0;
-err_disable_hw_ahb_clock:
- iris_disable_unprepare_clock(core, IRIS_HW_AHB_CLK);
err_disable_hw_clock:
iris_disable_unprepare_clock(core, IRIS_HW_CLK);
err_disable_power:
@@ -308,6 +302,16 @@ int iris_vpu_power_on_hw(struct iris_core *core)
return ret;
}
+int iris_vpu_set_hwmode(struct iris_core *core)
+{
+ return dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true);
+}
+
+int iris_vpu_switch_to_hwmode(struct iris_core *core)
+{
+ return core->iris_platform_data->vpu_ops->set_hwmode(core);
+}
+
int iris_vpu35_vpu4x_power_off_controller(struct iris_core *core)
{
u32 clk_rst_tbl_size = core->iris_platform_data->clk_rst_tbl_size;
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.h b/drivers/media/platform/qcom/iris/iris_vpu_common.h
index f6dffc6..dee3b13 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_common.h
+++ b/drivers/media/platform/qcom/iris/iris_vpu_common.h
@@ -21,6 +21,7 @@ struct vpu_ops {
int (*power_on_controller)(struct iris_core *core);
void (*program_bootup_registers)(struct iris_core *core);
u64 (*calc_freq)(struct iris_inst *inst, size_t data_size);
+ int (*set_hwmode)(struct iris_core *core);
};
int iris_vpu_boot_firmware(struct iris_core *core);
@@ -30,6 +31,8 @@ int iris_vpu_watchdog(struct iris_core *core, u32 intr_status);
int iris_vpu_prepare_pc(struct iris_core *core);
int iris_vpu_power_on_controller(struct iris_core *core);
int iris_vpu_power_on_hw(struct iris_core *core);
+int iris_vpu_set_hwmode(struct iris_core *core);
+int iris_vpu_switch_to_hwmode(struct iris_core *core);
int iris_vpu_power_on(struct iris_core *core);
int iris_vpu_power_off_controller(struct iris_core *core);
void iris_vpu_power_off_hw(struct iris_core *core);
diff --git a/drivers/media/platform/qcom/venus/Kconfig b/drivers/media/platform/qcom/venus/Kconfig
index ffb731ec..63ee8c7 100644
--- a/drivers/media/platform/qcom/venus/Kconfig
+++ b/drivers/media/platform/qcom/venus/Kconfig
@@ -4,7 +4,7 @@
depends on VIDEO_DEV && QCOM_SMEM
depends on (ARCH_QCOM && ARM64 && IOMMU_API) || COMPILE_TEST
select OF_DYNAMIC if ARCH_QCOM
- select QCOM_MDT_LOADER if ARCH_QCOM
+ select QCOM_MDT_LOADER
select QCOM_SCM
select VIDEOBUF2_DMA_CONTIG
select V4L2_MEM2MEM_DEV
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_brx.c b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
index b1a2c68e..9d93cb8 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_brx.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
@@ -156,20 +156,14 @@ static int brx_set_format(struct v4l2_subdev *subdev,
compose->height = format->height;
}
- /*
- * Propagate the format code to all pads, and the whole format to the
- * source pad.
- */
+ /* Propagate the format code to all pads. */
if (fmt->pad == BRX_PAD_SINK(0)) {
unsigned int i;
- for (i = 0; i < brx->entity.source_pad; ++i) {
+ for (i = 0; i <= brx->entity.source_pad; ++i) {
format = v4l2_subdev_state_get_format(state, i);
format->code = fmt->format.code;
}
-
- format = v4l2_subdev_state_get_format(state, i);
- *format = fmt->format;
}
done:
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.c b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
index 1dad958..839b75b 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
@@ -380,7 +380,7 @@ static int vsp1_entity_init_state(struct v4l2_subdev *subdev,
unsigned int pad;
/* Initialize all pad formats with default values. */
- for (pad = 0; pad < subdev->entity.num_pads; ++pad) {
+ for (pad = 0; pad < subdev->entity.num_pads - 1; ++pad) {
struct v4l2_subdev_format format = {
.pad = pad,
.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY
diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c
index 3e10f6f..b5117ee 100644
--- a/drivers/media/rc/igorplugusb.c
+++ b/drivers/media/rc/igorplugusb.c
@@ -184,7 +184,7 @@ static int igorplugusb_probe(struct usb_interface *intf,
if (!ir->buf_in)
goto fail;
usb_fill_control_urb(ir->urb, udev,
- usb_rcvctrlpipe(udev, 0), (uint8_t *)&ir->request,
+ usb_rcvctrlpipe(udev, 0), (uint8_t *)ir->request,
ir->buf_in, MAX_PACKET, igorplugusb_callback, ir);
usb_make_path(udev, ir->phys, sizeof(ir->phys));
diff --git a/drivers/media/rc/ttusbir.c b/drivers/media/rc/ttusbir.c
index 3848ad3..db2f669 100644
--- a/drivers/media/rc/ttusbir.c
+++ b/drivers/media/rc/ttusbir.c
@@ -191,7 +191,7 @@ static int ttusbir_probe(struct usb_interface *intf,
tt = kzalloc_obj(*tt);
buffer = kzalloc(5, GFP_KERNEL);
rc = rc_allocate_device(RC_DRIVER_IR_RAW);
- if (!tt || !rc || buffer) {
+ if (!tt || !rc || !buffer) {
ret = -ENOMEM;
goto out;
}
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index 8db970d..1e8e8ab 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -628,10 +628,11 @@ static __maybe_unused int atmel_ebi_resume(struct device *dev)
static SIMPLE_DEV_PM_OPS(atmel_ebi_pm_ops, NULL, atmel_ebi_resume);
static struct platform_driver atmel_ebi_driver = {
+ .probe = atmel_ebi_probe,
.driver = {
.name = "atmel-ebi",
.of_match_table = atmel_ebi_id_table,
.pm = &atmel_ebi_pm_ops,
},
};
-builtin_platform_driver_probe(atmel_ebi_driver, atmel_ebi_probe);
+builtin_platform_driver(atmel_ebi_driver);
diff --git a/drivers/misc/ntsync.c b/drivers/misc/ntsync.c
index 30af282..02c9d11 100644
--- a/drivers/misc/ntsync.c
+++ b/drivers/misc/ntsync.c
@@ -19,6 +19,7 @@
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/time_namespace.h>
#include <uapi/linux/ntsync.h>
#define NTSYNC_NAME "ntsync"
@@ -836,6 +837,8 @@ static int ntsync_schedule(const struct ntsync_q *q, const struct ntsync_wait_ar
if (args->flags & NTSYNC_WAIT_REALTIME)
clock = CLOCK_REALTIME;
+ else
+ timeout = timens_ktime_to_host(clock, timeout);
do {
if (signal_pending(current)) {
diff --git a/drivers/misc/rp1/rp1_pci.c b/drivers/misc/rp1/rp1_pci.c
index d210da8..81685e3 100644
--- a/drivers/misc/rp1/rp1_pci.c
+++ b/drivers/misc/rp1/rp1_pci.c
@@ -143,6 +143,7 @@ static int rp1_irq_activate(struct irq_domain *d, struct irq_data *irqd,
struct rp1_dev *rp1 = d->host_data;
msix_cfg_set(rp1, (unsigned int)irqd->hwirq, MSIX_CFG_ENABLE);
+ msix_cfg_set(rp1, (unsigned int)irqd->hwirq, MSIX_CFG_IACK);
return 0;
}
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 8846550..05444ec 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -1371,7 +1371,9 @@ static void mmc_select_driver_type(struct mmc_card *card)
card->drive_strength = drive_strength;
- if (drv_type)
+ if (fixed_drv_type >= 0 && drive_strength)
+ mmc_set_driver_type(card->host, drive_strength);
+ else if (drv_type)
mmc_set_driver_type(card->host, drv_type);
}
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index c6eece4..75c82ff 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -441,6 +441,22 @@ static int dw_mci_common_parse_dt(struct dw_mci *host)
return 0;
}
+static int dw_mci_rk2928_parse_dt(struct dw_mci *host)
+{
+ struct dw_mci_rockchip_priv_data *priv;
+ int err;
+
+ err = dw_mci_common_parse_dt(host);
+ if (err)
+ return err;
+
+ priv = host->priv;
+
+ priv->internal_phase = false;
+
+ return 0;
+}
+
static int dw_mci_rk3288_parse_dt(struct dw_mci *host)
{
struct dw_mci_rockchip_priv_data *priv;
@@ -514,6 +530,7 @@ static int dw_mci_rockchip_init(struct dw_mci *host)
static const struct dw_mci_drv_data rk2928_drv_data = {
.init = dw_mci_rockchip_init,
+ .parse_dt = dw_mci_rk2928_parse_dt,
};
static const struct dw_mci_drv_data rk3288_drv_data = {
diff --git a/drivers/mmc/host/litex_mmc.c b/drivers/mmc/host/litex_mmc.c
index d2f19c2..3655542 100644
--- a/drivers/mmc/host/litex_mmc.c
+++ b/drivers/mmc/host/litex_mmc.c
@@ -16,6 +16,7 @@
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/litex.h>
+#include <linux/math.h>
#include <linux/mod_devicetable.h>
#include <linux/module.h>
#include <linux/platform_device.h>
@@ -68,6 +69,9 @@
#define SD_SLEEP_US 5
#define SD_TIMEOUT_US 20000
+#define SD_INIT_DELAY_US 1000
+#define SD_INIT_CLK_HZ 400000
+
#define SDIRQ_CARD_DETECT 1
#define SDIRQ_SD_TO_MEM_DONE 2
#define SDIRQ_MEM_TO_SD_DONE 4
@@ -436,11 +440,10 @@ static void litex_mmc_setclk(struct litex_mmc_host *host, unsigned int freq)
struct device *dev = mmc_dev(host->mmc);
u32 div;
- div = freq ? host->ref_clk / freq : 256U;
- div = roundup_pow_of_two(div);
+ div = freq ? DIV_ROUND_UP(host->ref_clk, freq) : 256U;
div = clamp(div, 2U, 256U);
dev_dbg(dev, "sd_clk_freq=%d: set to %d via div=%d\n",
- freq, host->ref_clk / div, div);
+ freq, host->ref_clk / ((div + 1) & ~1U), div);
litex_write16(host->sdphy + LITEX_PHY_CLOCKERDIV, div);
host->sd_clk = freq;
}
@@ -450,6 +453,17 @@ static void litex_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
struct litex_mmc_host *host = mmc_priv(mmc);
/*
+ * The SD specification requires at least 74 idle clocks before CMD0.
+ * These dummy cycles is generated by writing LITEX_PHY_INITIALIZE.
+ */
+ if (ios->chip_select == MMC_CS_HIGH) {
+ litex_mmc_setclk(host, SD_INIT_CLK_HZ);
+ litex_write8(host->sdphy + LITEX_PHY_INITIALIZE, 1);
+ fsleep(SD_INIT_DELAY_US);
+ return;
+ }
+
+ /*
* NOTE: Ignore any ios->bus_width updates; they occur right after
* the mmc core sends its own acmd6 bus-width change notification,
* which is redundant since we snoop on the command flow and inject
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index f6ebb7b..838248b 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -279,6 +279,7 @@ static const struct renesas_sdhi_of_data_with_quirks of_rza2_compatible = {
static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = {
{ .compatible = "renesas,sdhi-r7s9210", .data = &of_rza2_compatible, },
{ .compatible = "renesas,sdhi-mmc-r8a77470", .data = &of_rcar_gen3_compatible, },
+ { .compatible = "renesas,sdhi-r8a774e1", .data = &of_r8a7795_compatible, },
{ .compatible = "renesas,sdhi-r8a7795", .data = &of_r8a7795_compatible, },
{ .compatible = "renesas,sdhi-r8a77961", .data = &of_r8a77961_compatible, },
{ .compatible = "renesas,sdhi-r8a77965", .data = &of_r8a77965_compatible, },
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 633462c..0882ce7 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -1918,13 +1918,13 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host,
return 0;
ice = devm_of_qcom_ice_get(dev);
- if (ice == ERR_PTR(-EOPNOTSUPP)) {
- dev_warn(dev, "Disabling inline encryption support\n");
- ice = NULL;
- }
+ if (IS_ERR(ice)) {
+ if (ice != ERR_PTR(-EOPNOTSUPP))
+ return PTR_ERR(ice);
- if (IS_ERR_OR_NULL(ice))
- return PTR_ERR_OR_ZERO(ice);
+ dev_warn(dev, "Disabling inline encryption support\n");
+ return 0;
+ }
msm_host->ice = ice;
diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c
index 0b2158a..b9ecd91 100644
--- a/drivers/mmc/host/sdhci-of-dwcmshc.c
+++ b/drivers/mmc/host/sdhci-of-dwcmshc.c
@@ -277,6 +277,7 @@
#define PHY_DELAY_CODE_MAX 0x7f
#define PHY_DELAY_CODE_EMMC 0x17
#define PHY_DELAY_CODE_SD 0x55
+#define PHY_DELAY_CODE_SDIO 0x29
struct rk35xx_priv {
struct reset_control *reset;
@@ -1433,10 +1434,7 @@ static void sdhci_eic7700_set_clock(struct sdhci_host *host, unsigned int clock)
clk_set_rate(pltfm_host->clk, clock);
clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
- clk |= SDHCI_CLOCK_INT_EN;
- sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
-
- dwcmshc_enable_card_clk(host);
+ sdhci_enable_clk(host, clk);
}
static void sdhci_eic7700_config_phy_delay(struct sdhci_host *host, int delay)
@@ -1497,7 +1495,7 @@ static void sdhci_eic7700_config_phy(struct sdhci_host *host)
static void sdhci_eic7700_reset(struct sdhci_host *host, u8 mask)
{
- sdhci_reset(host, mask);
+ dwcmshc_reset(host, mask);
/* after reset all, the phy's config will be clear */
if (mask == SDHCI_RESET_ALL)
@@ -1594,18 +1592,17 @@ static int sdhci_eic7700_phase_code_tuning(struct sdhci_host *host, u32 opcode)
{
struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
- u32 sd_caps = MMC_CAP2_NO_MMC | MMC_CAP2_NO_SDIO;
+ u32 emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO;
int phase_code = -1;
int code_range = -1;
- bool is_sd = false;
int code_min = -1;
int code_max = -1;
int cmd_error = 0;
+ bool is_emmc;
int ret = 0;
int i = 0;
- if ((host->mmc->caps2 & sd_caps) == sd_caps)
- is_sd = true;
+ is_emmc = (host->mmc->caps2 & emmc_caps) == emmc_caps;
for (i = 0; i <= MAX_PHASE_CODE; i++) {
/* Centered Phase code */
@@ -1614,8 +1611,8 @@ static int sdhci_eic7700_phase_code_tuning(struct sdhci_host *host, u32 opcode)
host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
if (ret) {
- /* SD specific range tracking */
- if (is_sd && code_min != -1 && code_max != -1) {
+ /* SD/SDIO specific range tracking */
+ if (!is_emmc && code_min != -1 && code_max != -1) {
if (code_max - code_min > code_range) {
code_range = code_max - code_min;
phase_code = (code_min + code_max) / 2;
@@ -1626,17 +1623,17 @@ static int sdhci_eic7700_phase_code_tuning(struct sdhci_host *host, u32 opcode)
code_max = -1;
}
/* EMMC breaks after first valid range */
- if (!is_sd && code_min != -1 && code_max != -1)
+ if (is_emmc && code_min != -1 && code_max != -1)
break;
} else {
/* Track valid phase code range */
if (code_min == -1) {
code_min = i;
- if (!is_sd)
+ if (is_emmc)
continue;
}
code_max = i;
- if (is_sd && i == MAX_PHASE_CODE) {
+ if (!is_emmc && i == MAX_PHASE_CODE) {
if (code_max - code_min > code_range) {
code_range = code_max - code_min;
phase_code = (code_min + code_max) / 2;
@@ -1646,19 +1643,19 @@ static int sdhci_eic7700_phase_code_tuning(struct sdhci_host *host, u32 opcode)
}
/* Handle tuning failure case */
- if ((is_sd && phase_code == -1) ||
- (!is_sd && code_min == -1 && code_max == -1)) {
+ if ((!is_emmc && phase_code == -1) ||
+ (is_emmc && code_min == -1 && code_max == -1)) {
pr_err("%s: phase code tuning failed!\n", mmc_hostname(host->mmc));
sdhci_writew(host, 0, priv->vendor_specific_area1 + DWCMSHC_AT_STAT);
return -EIO;
}
- if (!is_sd)
+ if (is_emmc)
phase_code = (code_min + code_max) / 2;
sdhci_writew(host, phase_code, priv->vendor_specific_area1 + DWCMSHC_AT_STAT);
- /* SD specific final verification */
- if (is_sd) {
+ /* SD/SDIO specific final verification */
+ if (!is_emmc) {
ret = mmc_send_tuning(host->mmc, opcode, &cmd_error);
host->ops->reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
if (ret) {
@@ -1756,9 +1753,9 @@ static void sdhci_eic7700_set_uhs_signaling(struct sdhci_host *host, unsigned in
static void sdhci_eic7700_set_uhs_wrapper(struct sdhci_host *host, unsigned int timing)
{
- u32 sd_caps = MMC_CAP2_NO_MMC | MMC_CAP2_NO_SDIO;
+ u32 emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO;
- if ((host->mmc->caps2 & sd_caps) == sd_caps)
+ if ((host->mmc->caps2 & emmc_caps) != emmc_caps)
sdhci_set_uhs_signaling(host, timing);
else
sdhci_eic7700_set_uhs_signaling(host, timing);
@@ -1767,6 +1764,7 @@ static void sdhci_eic7700_set_uhs_wrapper(struct sdhci_host *host, unsigned int
static int eic7700_init(struct device *dev, struct sdhci_host *host, struct dwcmshc_priv *dwc_priv)
{
u32 emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO;
+ u32 sd_caps = MMC_CAP2_NO_MMC | MMC_CAP2_NO_SDIO;
unsigned int val, hsp_int_status, hsp_pwr_ctrl;
static const char * const clk_ids[] = {"axi"};
struct of_phandle_args args;
@@ -1821,8 +1819,10 @@ static int eic7700_init(struct device *dev, struct sdhci_host *host, struct dwcm
if ((host->mmc->caps2 & emmc_caps) == emmc_caps)
dwc_priv->delay_line = PHY_DELAY_CODE_EMMC;
- else
+ else if ((host->mmc->caps2 & sd_caps) == sd_caps)
dwc_priv->delay_line = PHY_DELAY_CODE_SD;
+ else
+ dwc_priv->delay_line = PHY_DELAY_CODE_SDIO;
if (!of_property_read_u32(dev->of_node, "eswin,drive-impedance-ohms", &val))
priv->drive_impedance = eic7700_convert_drive_impedance_ohm(dev, val);
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 605be55..e3bf901 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -3836,6 +3836,7 @@ int sdhci_resume_host(struct sdhci_host *host)
host->pwr = 0;
host->clock = 0;
host->reinit_uhs = true;
+ mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
mmc->ops->set_ios(mmc, &mmc->ios);
} else {
sdhci_init(host, (mmc->pm_flags & MMC_PM_KEEP_POWER));
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index 8aa3753..0b07679 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -100,6 +100,17 @@ spinand_fill_page_read_op(struct spinand_device *spinand, u64 addr)
return op;
}
+static struct spi_mem_op
+spinand_fill_page_read_packed_op(struct spinand_device *spinand, u64 addr)
+{
+ struct spi_mem_op op = spinand->op_templates->page_read;
+
+ op.cmd.opcode |= addr >> 16;
+ op.addr.val = addr & 0xFFFF;
+
+ return op;
+}
+
struct spi_mem_op
spinand_fill_prog_exec_op(struct spinand_device *spinand, u64 addr)
{
@@ -453,7 +464,10 @@ static int spinand_load_page_op(struct spinand_device *spinand,
{
struct nand_device *nand = spinand_to_nand(spinand);
unsigned int row = nanddev_pos_to_row(nand, &req->pos);
- struct spi_mem_op op = SPINAND_OP(spinand, page_read, row);
+ bool packed = spinand->flags & SPINAND_ODTR_PACKED_PAGE_READ;
+ struct spi_mem_op op = packed ?
+ SPINAND_OP(spinand, page_read_packed, row) :
+ SPINAND_OP(spinand, page_read, row);
return spi_mem_exec_op(spinand->spimem, &op);
}
@@ -1489,9 +1503,13 @@ static int spinand_init_odtr_instruction_set(struct spinand_device *spinand)
if (!spi_mem_supports_op(spinand->spimem, &tmpl->blk_erase))
return -EOPNOTSUPP;
- tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_8D_8D_0_OP(0);
- if (!spi_mem_supports_op(spinand->spimem, &tmpl->page_read))
+ if (spinand->flags & SPINAND_ODTR_PACKED_PAGE_READ)
+ tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_PACKED_8D_8D_0_OP(0);
+ else
+ tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_8D_8D_0_OP(0);
+ if (!spi_mem_supports_op(spinand->spimem, &tmpl->page_read)) {
return -EOPNOTSUPP;
+ }
tmpl->prog_exec = (struct spi_mem_op)SPINAND_PROG_EXEC_8D_8D_0_OP(0);
if (!spi_mem_supports_op(spinand->spimem, &tmpl->prog_exec))
diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c
index ad22774..7cc0f00 100644
--- a/drivers/mtd/nand/spi/winbond.c
+++ b/drivers/mtd/nand/spi/winbond.c
@@ -99,7 +99,7 @@ static SPINAND_OP_VARIANTS(update_cache_variants,
#define SPINAND_WINBOND_WRITE_VCR_8D_8D_8D(reg, buf) \
SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x81, 8), \
- SPI_MEM_DTR_OP_ADDR(4, reg, 8), \
+ SPI_MEM_DTR_OP_ADDR(4, reg << 8, 8), \
SPI_MEM_OP_NO_DUMMY, \
SPI_MEM_DTR_OP_DATA_OUT(2, buf, 8))
@@ -518,7 +518,7 @@ static const struct spinand_info winbond_spinand_table[] = {
SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants,
&write_cache_octal_variants,
&update_cache_octal_variants),
- 0,
+ SPINAND_ODTR_PACKED_PAGE_READ,
SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL),
SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)),
@@ -529,7 +529,7 @@ static const struct spinand_info winbond_spinand_table[] = {
SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants,
&write_cache_octal_variants,
&update_cache_octal_variants),
- 0,
+ SPINAND_ODTR_PACKED_PAGE_READ,
SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL),
SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)),
diff --git a/drivers/mtd/spi-nor/debugfs.c b/drivers/mtd/spi-nor/debugfs.c
index fa69561..14ba168 100644
--- a/drivers/mtd/spi-nor/debugfs.c
+++ b/drivers/mtd/spi-nor/debugfs.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
+#include <linux/array_size.h>
#include <linux/debugfs.h>
#include <linux/mtd/spi-nor.h>
#include <linux/spi/spi.h>
@@ -92,7 +93,8 @@ static int spi_nor_params_show(struct seq_file *s, void *data)
seq_printf(s, "address nbytes\t%u\n", nor->addr_nbytes);
seq_puts(s, "flags\t\t");
- spi_nor_print_flags(s, nor->flags, snor_f_names, sizeof(snor_f_names));
+ spi_nor_print_flags(s, nor->flags, snor_f_names,
+ ARRAY_SIZE(snor_f_names));
seq_puts(s, "\n");
seq_puts(s, "\nopcodes\n");
diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c
index 0df3208..da5866b 100644
--- a/drivers/net/bareudp.c
+++ b/drivers/net/bareudp.c
@@ -529,6 +529,9 @@ static int bareudp_fill_metadata_dst(struct net_device *dev,
struct in6_addr saddr;
struct socket *sock = rcu_dereference(bareudp->sock);
+ if (!sock)
+ return -ESHUTDOWN;
+
dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sock,
0, &saddr, &info->key,
sport, bareudp->port, info->key.tos,
diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c
index af7f74c..985ef66 100644
--- a/drivers/net/bonding/bond_3ad.c
+++ b/drivers/net/bonding/bond_3ad.c
@@ -1029,6 +1029,7 @@ static void ad_cond_set_peer_notif(struct port *port)
static void ad_mux_machine(struct port *port, bool *update_slave_arr)
{
struct bonding *bond = __get_bond_by_port(port);
+ struct aggregator *aggregator;
mux_states_t last_state;
/* keep current State Machine state to compare later if it was
@@ -1036,6 +1037,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
*/
last_state = port->sm_mux_state;
+ aggregator = rcu_dereference(port->aggregator);
if (port->sm_vars & AD_PORT_BEGIN) {
port->sm_mux_state = AD_MUX_DETACHED;
} else {
@@ -1055,7 +1057,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
* cycle to update ready variable, we check
* READY_N and update READY here
*/
- __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator));
+ __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator));
port->sm_mux_state = AD_MUX_DETACHED;
break;
}
@@ -1070,7 +1072,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
* update ready variable, we check READY_N and update
* READY here
*/
- __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator));
+ __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator));
/* if the wait_while_timer expired, and the port is
* in READY state, move to ATTACHED state
@@ -1086,7 +1088,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
if ((port->sm_vars & AD_PORT_SELECTED) &&
(port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) &&
!__check_agg_selection_timer(port)) {
- if (port->aggregator->is_active) {
+ if (aggregator->is_active) {
int state = AD_MUX_COLLECTING_DISTRIBUTING;
if (!bond->params.coupled_control)
@@ -1102,9 +1104,9 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
* cycle to update ready variable, we check
* READY_N and update READY here
*/
- __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator));
+ __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator));
port->sm_mux_state = AD_MUX_DETACHED;
- } else if (port->aggregator->is_active) {
+ } else if (aggregator->is_active) {
port->actor_oper_port_state |=
LACP_STATE_SYNCHRONIZATION;
}
@@ -1115,7 +1117,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
* sure that a collecting distributing
* port in an active aggregator is enabled
*/
- if (port->aggregator->is_active &&
+ if (aggregator->is_active &&
!__port_is_collecting_distributing(port)) {
__enable_port(port);
*update_slave_arr = true;
@@ -1134,7 +1136,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
*/
struct slave *slave = port->slave;
- if (port->aggregator->is_active &&
+ if (aggregator->is_active &&
bond_is_slave_rx_disabled(slave)) {
ad_enable_collecting(port);
*update_slave_arr = true;
@@ -1154,8 +1156,8 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
* sure that a collecting distributing
* port in an active aggregator is enabled
*/
- if (port->aggregator &&
- port->aggregator->is_active &&
+ if (aggregator &&
+ aggregator->is_active &&
!__port_is_collecting_distributing(port)) {
__enable_port(port);
*update_slave_arr = true;
@@ -1187,7 +1189,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr)
port->sm_mux_timer_counter = __ad_timer_to_ticks(AD_WAIT_WHILE_TIMER, 0);
break;
case AD_MUX_ATTACHED:
- if (port->aggregator->is_active)
+ if (aggregator->is_active)
port->actor_oper_port_state |=
LACP_STATE_SYNCHRONIZATION;
else
@@ -1384,8 +1386,8 @@ static void ad_churn_machine(struct port *port)
{
if (port->sm_vars & AD_PORT_CHURNED) {
port->sm_vars &= ~AD_PORT_CHURNED;
- port->sm_churn_actor_state = AD_CHURN_MONITOR;
- port->sm_churn_partner_state = AD_CHURN_MONITOR;
+ WRITE_ONCE(port->sm_churn_actor_state, AD_CHURN_MONITOR);
+ WRITE_ONCE(port->sm_churn_partner_state, AD_CHURN_MONITOR);
port->sm_churn_actor_timer_counter =
__ad_timer_to_ticks(AD_ACTOR_CHURN_TIMER, 0);
port->sm_churn_partner_timer_counter =
@@ -1396,20 +1398,22 @@ static void ad_churn_machine(struct port *port)
!(--port->sm_churn_actor_timer_counter) &&
port->sm_churn_actor_state == AD_CHURN_MONITOR) {
if (port->actor_oper_port_state & LACP_STATE_SYNCHRONIZATION) {
- port->sm_churn_actor_state = AD_NO_CHURN;
+ WRITE_ONCE(port->sm_churn_actor_state, AD_NO_CHURN);
} else {
- port->churn_actor_count++;
- port->sm_churn_actor_state = AD_CHURN;
+ WRITE_ONCE(port->churn_actor_count,
+ port->churn_actor_count + 1);
+ WRITE_ONCE(port->sm_churn_actor_state, AD_CHURN);
}
}
if (port->sm_churn_partner_timer_counter &&
!(--port->sm_churn_partner_timer_counter) &&
port->sm_churn_partner_state == AD_CHURN_MONITOR) {
if (port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) {
- port->sm_churn_partner_state = AD_NO_CHURN;
+ WRITE_ONCE(port->sm_churn_partner_state, AD_NO_CHURN);
} else {
- port->churn_partner_count++;
- port->sm_churn_partner_state = AD_CHURN;
+ WRITE_ONCE(port->churn_partner_count,
+ port->churn_partner_count + 1);
+ WRITE_ONCE(port->sm_churn_partner_state, AD_CHURN);
}
}
}
@@ -1561,9 +1565,9 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
bond = __get_bond_by_port(port);
/* if the port is connected to other aggregator, detach it */
- if (port->aggregator) {
+ temp_aggregator = rcu_dereference(port->aggregator);
+ if (temp_aggregator) {
/* detach the port from its former aggregator */
- temp_aggregator = port->aggregator;
for (curr_port = temp_aggregator->lag_ports; curr_port;
last_port = curr_port,
curr_port = curr_port->next_port_in_aggregator) {
@@ -1586,7 +1590,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
/* clear the port's relations to this
* aggregator
*/
- port->aggregator = NULL;
+ RCU_INIT_POINTER(port->aggregator, NULL);
port->next_port_in_aggregator = NULL;
port->actor_port_aggregator_identifier = 0;
@@ -1609,7 +1613,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
port->slave->bond->dev->name,
port->slave->dev->name,
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ temp_aggregator->aggregator_identifier);
}
}
/* search on all aggregators for a suitable aggregator for this port */
@@ -1633,15 +1637,15 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
)
) {
/* attach to the founded aggregator */
- port->aggregator = aggregator;
+ rcu_assign_pointer(port->aggregator, aggregator);
port->actor_port_aggregator_identifier =
- port->aggregator->aggregator_identifier;
+ aggregator->aggregator_identifier;
port->next_port_in_aggregator = aggregator->lag_ports;
- port->aggregator->num_of_ports++;
+ aggregator->num_of_ports++;
aggregator->lag_ports = port;
slave_dbg(bond->dev, slave->dev, "Port %d joined LAG %d (existing LAG)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ aggregator->aggregator_identifier);
/* mark this port as selected */
port->sm_vars |= AD_PORT_SELECTED;
@@ -1656,39 +1660,40 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
if (!found) {
if (free_aggregator) {
/* assign port a new aggregator */
- port->aggregator = free_aggregator;
port->actor_port_aggregator_identifier =
- port->aggregator->aggregator_identifier;
+ free_aggregator->aggregator_identifier;
/* update the new aggregator's parameters
* if port was responsed from the end-user
*/
if (port->actor_oper_port_key & AD_DUPLEX_KEY_MASKS)
/* if port is full duplex */
- port->aggregator->is_individual = false;
+ free_aggregator->is_individual = false;
else
- port->aggregator->is_individual = true;
+ free_aggregator->is_individual = true;
- port->aggregator->actor_admin_aggregator_key =
+ free_aggregator->actor_admin_aggregator_key =
port->actor_admin_port_key;
- port->aggregator->actor_oper_aggregator_key =
+ free_aggregator->actor_oper_aggregator_key =
port->actor_oper_port_key;
- port->aggregator->partner_system =
+ free_aggregator->partner_system =
port->partner_oper.system;
- port->aggregator->partner_system_priority =
+ free_aggregator->partner_system_priority =
port->partner_oper.system_priority;
- port->aggregator->partner_oper_aggregator_key = port->partner_oper.key;
- port->aggregator->receive_state = 1;
- port->aggregator->transmit_state = 1;
- port->aggregator->lag_ports = port;
- port->aggregator->num_of_ports++;
+ free_aggregator->partner_oper_aggregator_key = port->partner_oper.key;
+ free_aggregator->receive_state = 1;
+ free_aggregator->transmit_state = 1;
+ free_aggregator->lag_ports = port;
+ free_aggregator->num_of_ports++;
+
+ rcu_assign_pointer(port->aggregator, free_aggregator);
/* mark this port as selected */
port->sm_vars |= AD_PORT_SELECTED;
slave_dbg(bond->dev, port->slave->dev, "Port %d joined LAG %d (new LAG)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ free_aggregator->aggregator_identifier);
} else {
slave_err(bond->dev, port->slave->dev,
"Port %d did not find a suitable aggregator\n",
@@ -1700,13 +1705,12 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr)
* in all aggregator's ports, else set ready=FALSE in all
* aggregator's ports
*/
- __set_agg_ports_ready(port->aggregator,
- __agg_ports_are_ready(port->aggregator));
+ aggregator = rcu_dereference(port->aggregator);
+ __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator));
- aggregator = __get_first_agg(port);
- ad_agg_selection_logic(aggregator, update_slave_arr);
+ ad_agg_selection_logic(__get_first_agg(port), update_slave_arr);
- if (!port->aggregator->is_active)
+ if (!aggregator->is_active)
port->actor_oper_port_state &= ~LACP_STATE_SYNCHRONIZATION;
}
@@ -2075,13 +2079,15 @@ static void ad_initialize_port(struct port *port, const struct bond_params *bond
*/
static void ad_enable_collecting(struct port *port)
{
- if (port->aggregator->is_active) {
+ struct aggregator *aggregator = rcu_dereference(port->aggregator);
+
+ if (aggregator->is_active) {
struct slave *slave = port->slave;
slave_dbg(slave->bond->dev, slave->dev,
"Enabling collecting on port %d (LAG %d)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ aggregator->aggregator_identifier);
__enable_collecting_port(port);
}
}
@@ -2093,11 +2099,13 @@ static void ad_enable_collecting(struct port *port)
*/
static void ad_disable_distributing(struct port *port, bool *update_slave_arr)
{
- if (port->aggregator && __agg_has_partner(port->aggregator)) {
+ struct aggregator *aggregator = rcu_dereference(port->aggregator);
+
+ if (aggregator && __agg_has_partner(aggregator)) {
slave_dbg(port->slave->bond->dev, port->slave->dev,
"Disabling distributing on port %d (LAG %d)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ aggregator->aggregator_identifier);
__disable_distributing_port(port);
/* Slave array needs an update */
*update_slave_arr = true;
@@ -2114,11 +2122,13 @@ static void ad_disable_distributing(struct port *port, bool *update_slave_arr)
static void ad_enable_collecting_distributing(struct port *port,
bool *update_slave_arr)
{
- if (port->aggregator->is_active) {
+ struct aggregator *aggregator = rcu_dereference(port->aggregator);
+
+ if (aggregator->is_active) {
slave_dbg(port->slave->bond->dev, port->slave->dev,
"Enabling port %d (LAG %d)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ aggregator->aggregator_identifier);
__enable_port(port);
/* Slave array needs update */
*update_slave_arr = true;
@@ -2135,11 +2145,13 @@ static void ad_enable_collecting_distributing(struct port *port,
static void ad_disable_collecting_distributing(struct port *port,
bool *update_slave_arr)
{
- if (port->aggregator && __agg_has_partner(port->aggregator)) {
+ struct aggregator *aggregator = rcu_dereference(port->aggregator);
+
+ if (aggregator && __agg_has_partner(aggregator)) {
slave_dbg(port->slave->bond->dev, port->slave->dev,
"Disabling port %d (LAG %d)\n",
port->actor_port_number,
- port->aggregator->aggregator_identifier);
+ aggregator->aggregator_identifier);
__disable_port(port);
/* Slave array needs an update */
*update_slave_arr = true;
@@ -2379,7 +2391,7 @@ void bond_3ad_unbind_slave(struct slave *slave)
*/
for (temp_port = aggregator->lag_ports; temp_port;
temp_port = temp_port->next_port_in_aggregator) {
- temp_port->aggregator = new_aggregator;
+ rcu_assign_pointer(temp_port->aggregator, new_aggregator);
temp_port->actor_port_aggregator_identifier = new_aggregator->aggregator_identifier;
}
@@ -2848,15 +2860,16 @@ int bond_3ad_set_carrier(struct bonding *bond)
int __bond_3ad_get_active_agg_info(struct bonding *bond,
struct ad_info *ad_info)
{
- struct aggregator *aggregator = NULL;
+ struct aggregator *aggregator = NULL, *tmp;
struct list_head *iter;
struct slave *slave;
struct port *port;
bond_for_each_slave_rcu(bond, slave, iter) {
port = &(SLAVE_AD_INFO(slave)->port);
- if (port->aggregator && port->aggregator->is_active) {
- aggregator = port->aggregator;
+ tmp = rcu_dereference(port->aggregator);
+ if (tmp && tmp->is_active) {
+ aggregator = tmp;
break;
}
}
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index c7baa5c..8e75453 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1433,7 +1433,7 @@ static void bond_poll_controller(struct net_device *bond_dev)
if (BOND_MODE(bond) == BOND_MODE_8023AD) {
struct aggregator *agg =
- SLAVE_AD_INFO(slave)->port.aggregator;
+ rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator);
if (agg &&
agg->aggregator_identifier != ad_info.aggregator_id)
@@ -1890,6 +1890,12 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
struct sockaddr_storage ss;
int res = 0, i;
+ if (slave_dev->type == ARPHRD_CAN) {
+ BOND_NL_ERR(bond_dev, extack,
+ "CAN devices cannot be enslaved");
+ return -EPERM;
+ }
+
if (slave_dev->flags & IFF_MASTER &&
!netif_is_bond_master(slave_dev)) {
BOND_NL_ERR(bond_dev, extack,
@@ -4615,11 +4621,11 @@ static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmd
slave_dev = __dev_get_by_name(net, ifr->ifr_slave);
- slave_dbg(bond_dev, slave_dev, "slave_dev=%p:\n", slave_dev);
-
if (!slave_dev)
return -ENODEV;
+ slave_dbg(bond_dev, slave_dev, "slave_dev=%p:\n", slave_dev);
+
switch (cmd) {
case SIOCBONDENSLAVE:
res = bond_enslave(bond_dev, slave_dev, NULL);
@@ -5179,15 +5185,16 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave)
spin_unlock_bh(&bond->mode_lock);
agg_id = ad_info.aggregator_id;
}
+ rcu_read_lock();
bond_for_each_slave(bond, slave, iter) {
if (skipslave == slave)
continue;
all_slaves->arr[all_slaves->count++] = slave;
if (BOND_MODE(bond) == BOND_MODE_8023AD) {
- struct aggregator *agg;
+ const struct aggregator *agg;
- agg = SLAVE_AD_INFO(slave)->port.aggregator;
+ agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator);
if (!agg || agg->aggregator_identifier != agg_id)
continue;
}
@@ -5199,6 +5206,7 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave)
usable_slaves->arr[usable_slaves->count++] = slave;
}
+ rcu_read_unlock();
bond_set_slave_arr(bond, usable_slaves, all_slaves);
return ret;
diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c
index ea1a80e..90365d3 100644
--- a/drivers/net/bonding/bond_netlink.c
+++ b/drivers/net/bonding/bond_netlink.c
@@ -66,27 +66,29 @@ static int bond_fill_slave_info(struct sk_buff *skb,
const struct port *ad_port;
ad_port = &SLAVE_AD_INFO(slave)->port;
- agg = SLAVE_AD_INFO(slave)->port.aggregator;
+ rcu_read_lock();
+ agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator);
if (agg) {
if (nla_put_u16(skb, IFLA_BOND_SLAVE_AD_AGGREGATOR_ID,
agg->aggregator_identifier))
- goto nla_put_failure;
+ goto nla_put_failure_rcu;
if (nla_put_u8(skb,
IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE,
ad_port->actor_oper_port_state))
- goto nla_put_failure;
+ goto nla_put_failure_rcu;
if (nla_put_u16(skb,
IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE,
ad_port->partner_oper.port_state))
- goto nla_put_failure;
+ goto nla_put_failure_rcu;
if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE,
- ad_port->sm_churn_actor_state))
- goto nla_put_failure;
+ READ_ONCE(ad_port->sm_churn_actor_state)))
+ goto nla_put_failure_rcu;
if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE,
- ad_port->sm_churn_partner_state))
- goto nla_put_failure;
+ READ_ONCE(ad_port->sm_churn_partner_state)))
+ goto nla_put_failure_rcu;
}
+ rcu_read_unlock();
if (nla_put_u16(skb, IFLA_BOND_SLAVE_ACTOR_PORT_PRIO,
SLAVE_AD_INFO(slave)->port_priority))
@@ -95,6 +97,8 @@ static int bond_fill_slave_info(struct sk_buff *skb,
return 0;
+nla_put_failure_rcu:
+ rcu_read_unlock();
nla_put_failure:
return -EMSGSIZE;
}
diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c
index e34f803..3607b62f 100644
--- a/drivers/net/bonding/bond_procfs.c
+++ b/drivers/net/bonding/bond_procfs.c
@@ -188,6 +188,7 @@ static void bond_info_show_master(struct seq_file *seq)
}
}
+/* Note: runs under rcu_read_lock() */
static void bond_info_show_slave(struct seq_file *seq,
const struct slave *slave)
{
@@ -214,19 +215,19 @@ static void bond_info_show_slave(struct seq_file *seq,
if (BOND_MODE(bond) == BOND_MODE_8023AD) {
const struct port *port = &SLAVE_AD_INFO(slave)->port;
- const struct aggregator *agg = port->aggregator;
+ const struct aggregator *agg = rcu_dereference(port->aggregator);
if (agg) {
seq_printf(seq, "Aggregator ID: %d\n",
agg->aggregator_identifier);
seq_printf(seq, "Actor Churn State: %s\n",
- bond_3ad_churn_desc(port->sm_churn_actor_state));
+ bond_3ad_churn_desc(READ_ONCE(port->sm_churn_actor_state)));
seq_printf(seq, "Partner Churn State: %s\n",
- bond_3ad_churn_desc(port->sm_churn_partner_state));
+ bond_3ad_churn_desc(READ_ONCE(port->sm_churn_partner_state)));
seq_printf(seq, "Actor Churned Count: %d\n",
- port->churn_actor_count);
+ READ_ONCE(port->churn_actor_count));
seq_printf(seq, "Partner Churned Count: %d\n",
- port->churn_partner_count);
+ READ_ONCE(port->churn_partner_count));
if (capable(CAP_NET_ADMIN)) {
seq_puts(seq, "details actor lacp pdu:\n");
diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c
index 36d0e84..fc6fe71 100644
--- a/drivers/net/bonding/bond_sysfs_slave.c
+++ b/drivers/net/bonding/bond_sysfs_slave.c
@@ -62,10 +62,15 @@ static ssize_t ad_aggregator_id_show(struct slave *slave, char *buf)
const struct aggregator *agg;
if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
- agg = SLAVE_AD_INFO(slave)->port.aggregator;
- if (agg)
- return sysfs_emit(buf, "%d\n",
- agg->aggregator_identifier);
+ rcu_read_lock();
+ agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator);
+ if (agg) {
+ ssize_t res = sysfs_emit(buf, "%d\n",
+ agg->aggregator_identifier);
+ rcu_read_unlock();
+ return res;
+ }
+ rcu_read_unlock();
}
return sysfs_emit(buf, "N/A\n");
@@ -78,7 +83,7 @@ static ssize_t ad_actor_oper_port_state_show(struct slave *slave, char *buf)
if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
ad_port = &SLAVE_AD_INFO(slave)->port;
- if (ad_port->aggregator)
+ if (rcu_access_pointer(ad_port->aggregator))
return sysfs_emit(buf, "%u\n",
ad_port->actor_oper_port_state);
}
@@ -93,7 +98,7 @@ static ssize_t ad_partner_oper_port_state_show(struct slave *slave, char *buf)
if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
ad_port = &SLAVE_AD_INFO(slave)->port;
- if (ad_port->aggregator)
+ if (rcu_access_pointer(ad_port->aggregator))
return sysfs_emit(buf, "%u\n",
ad_port->partner_oper.port_state);
}
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index b942338..3c2a302 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -25,6 +25,9 @@
#include "mt7530.h"
+#define MT7530_STATS_POLL_INTERVAL (1 * HZ)
+#define MT7530_STATS_RATE_LIMIT (HZ / 10)
+
static struct mt753x_pcs *pcs_to_mt753x_pcs(struct phylink_pcs *pcs)
{
return container_of(pcs, struct mt753x_pcs, pcs);
@@ -906,10 +909,9 @@ static void mt7530_get_rmon_stats(struct dsa_switch *ds, int port,
*ranges = mt7530_rmon_ranges;
}
-static void mt7530_get_stats64(struct dsa_switch *ds, int port,
- struct rtnl_link_stats64 *storage)
+static void mt7530_read_port_stats64(struct mt7530_priv *priv, int port,
+ struct rtnl_link_stats64 *storage)
{
- struct mt7530_priv *priv = ds->priv;
uint64_t data;
/* MIB counter doesn't provide a FramesTransmittedOK but instead
@@ -951,6 +953,54 @@ static void mt7530_get_stats64(struct dsa_switch *ds, int port,
&storage->rx_crc_errors);
}
+static void mt7530_stats_refresh(struct mt7530_priv *priv)
+{
+ struct rtnl_link_stats64 stats = {};
+ struct dsa_port *dp;
+ int port;
+
+ dsa_switch_for_each_user_port(dp, priv->ds) {
+ port = dp->index;
+
+ mt7530_read_port_stats64(priv, port, &stats);
+
+ spin_lock_bh(&priv->stats_lock);
+ priv->ports[port].stats = stats;
+ priv->stats_last = jiffies;
+ spin_unlock_bh(&priv->stats_lock);
+ }
+}
+
+static void mt7530_stats_poll(struct work_struct *work)
+{
+ struct mt7530_priv *priv = container_of(work, struct mt7530_priv,
+ stats_work.work);
+
+ mt7530_stats_refresh(priv);
+ schedule_delayed_work(&priv->stats_work,
+ MT7530_STATS_POLL_INTERVAL);
+}
+
+static void mt7530_get_stats64(struct dsa_switch *ds, int port,
+ struct rtnl_link_stats64 *storage)
+{
+ struct mt7530_priv *priv = ds->priv;
+ bool refresh;
+
+ if (priv->bus) {
+ spin_lock_bh(&priv->stats_lock);
+ *storage = priv->ports[port].stats;
+ refresh = time_after(jiffies, priv->stats_last +
+ MT7530_STATS_RATE_LIMIT);
+ spin_unlock_bh(&priv->stats_lock);
+ if (refresh)
+ mod_delayed_work(system_percpu_wq,
+ &priv->stats_work, 0);
+ } else {
+ mt7530_read_port_stats64(priv, port, storage);
+ }
+}
+
static void mt7530_get_eth_ctrl_stats(struct dsa_switch *ds, int port,
struct ethtool_eth_ctrl_stats *ctrl_stats)
{
@@ -973,12 +1023,16 @@ mt7530_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
unsigned int age_count;
unsigned int age_unit;
- /* Applied timer is (AGE_CNT + 1) * (AGE_UNIT + 1) seconds */
- if (secs < 1 || secs > (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1))
- return -ERANGE;
-
- /* iterate through all possible age_count to find the closest pair */
- for (tmp_age_count = 0; tmp_age_count <= AGE_CNT_MAX; ++tmp_age_count) {
+ /* Applied timer is (AGE_CNT + 1) * (AGE_UNIT + 1) seconds.
+ * The DSA core has already validated the range using
+ * ds->ageing_time_min and ds->ageing_time_max.
+ *
+ * Iterate through all possible age_count values to find the closest
+ * pair. Start from 1 because the per-entry aging counter is
+ * initialized to AGE_CNT and a value of 0 means the entry will
+ * never be aged out.
+ */
+ for (tmp_age_count = 1; tmp_age_count <= AGE_CNT_MAX; ++tmp_age_count) {
unsigned int tmp_age_unit = secs / (tmp_age_count + 1) - 1;
if (tmp_age_unit <= AGE_UNIT_MAX) {
@@ -1246,37 +1300,40 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
static void
mt753x_trap_frames(struct mt7530_priv *priv)
{
- /* Trap 802.1X PAE frames and BPDUs to the CPU port(s) and egress them
- * VLAN-untagged.
+ /* Trap 802.1X PAE frames and BPDUs to the CPU port(s) and egress
+ * them with the EG_TAG attribute set to disabled (system default)
+ * so that any VLAN tags in the frame are not modified by the
+ * switch egress VLAN tag processing. This preserves VLAN tags
+ * for reception on VLAN sub-interfaces.
*/
mt7530_rmw(priv, MT753X_BPC,
PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
- PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
- BPDU_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
TO_CPU_FW_CPU_ONLY);
- /* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and egress
- * them VLAN-untagged.
+ /* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and
+ * egress them with EG_TAG disabled.
*/
mt7530_rmw(priv, MT753X_RGAC1,
R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
- R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
- R01_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
TO_CPU_FW_CPU_ONLY);
- /* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and egress
- * them VLAN-untagged.
+ /* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and
+ * egress them with EG_TAG disabled.
*/
mt7530_rmw(priv, MT753X_RGAC2,
R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
- R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
- R03_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+ R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
TO_CPU_FW_CPU_ONLY);
}
@@ -1566,6 +1623,49 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port,
return 0;
}
+static int
+mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
+{
+ struct mt7530_dummy_poll p;
+ u32 val;
+ int ret;
+
+ val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
+ mt7530_write(priv, MT7530_VTCR, val);
+
+ INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
+ ret = readx_poll_timeout(_mt7530_read, &p, val,
+ !(val & VTCR_BUSY), 20, 20000);
+ if (ret < 0) {
+ dev_err(priv->dev, "poll timeout\n");
+ return ret;
+ }
+
+ val = mt7530_read(priv, MT7530_VTCR);
+ if (val & VTCR_INVALID) {
+ dev_err(priv->dev, "read VTCR invalid\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+mt7530_setup_vlan0(struct mt7530_priv *priv)
+{
+ u32 val;
+
+ /* Validate the entry with independent learning, keep the original
+ * ingress tag attribute.
+ */
+ val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
+ VLAN_VALID;
+ mt7530_write(priv, MT7530_VAWD1, val);
+ mt7530_write(priv, MT7530_VAWD2, 0);
+
+ return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
+}
+
static void
mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
{
@@ -1591,6 +1691,8 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
G0_PORT_VID_DEF);
for (i = 0; i < priv->ds->num_ports; i++) {
+ if (i == port)
+ continue;
if (dsa_is_user_port(ds, i) &&
dsa_port_is_vlan_filtering(dsa_to_port(ds, i))) {
all_user_ports_removed = false;
@@ -1602,13 +1704,9 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
* the CPU port get out of VLAN filtering mode.
*/
if (all_user_ports_removed) {
- struct dsa_port *dp = dsa_to_port(ds, port);
- struct dsa_port *cpu_dp = dp->cpu_dp;
-
- mt7530_write(priv, MT7530_PCR_P(cpu_dp->index),
- PCR_MATRIX(dsa_user_ports(priv->ds)));
- mt7530_write(priv, MT7530_PVC_P(cpu_dp->index), PORT_SPEC_TAG
- | PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+ mutex_lock(&priv->reg_mutex);
+ mt7530_setup_vlan0(priv);
+ mutex_unlock(&priv->reg_mutex);
}
}
@@ -1797,33 +1895,6 @@ mt7530_port_mdb_del(struct dsa_switch *ds, int port,
}
static int
-mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
-{
- struct mt7530_dummy_poll p;
- u32 val;
- int ret;
-
- val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
- mt7530_write(priv, MT7530_VTCR, val);
-
- INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
- ret = readx_poll_timeout(_mt7530_read, &p, val,
- !(val & VTCR_BUSY), 20, 20000);
- if (ret < 0) {
- dev_err(priv->dev, "poll timeout\n");
- return ret;
- }
-
- val = mt7530_read(priv, MT7530_VTCR);
- if (val & VTCR_INVALID) {
- dev_err(priv->dev, "read VTCR invalid\n");
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int
mt7530_port_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering,
struct netlink_ext_ack *extack)
{
@@ -1928,21 +1999,6 @@ mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
}
static int
-mt7530_setup_vlan0(struct mt7530_priv *priv)
-{
- u32 val;
-
- /* Validate the entry with independent learning, keep the original
- * ingress tag attribute.
- */
- val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
- VLAN_VALID;
- mt7530_write(priv, MT7530_VAWD1, val);
-
- return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
-}
-
-static int
mt7530_port_vlan_add(struct dsa_switch *ds, int port,
const struct switchdev_obj_port_vlan *vlan,
struct netlink_ext_ack *extack)
@@ -1954,9 +2010,18 @@ mt7530_port_vlan_add(struct dsa_switch *ds, int port,
mutex_lock(&priv->reg_mutex);
+ /* VID 0 is managed exclusively by mt7530_setup_vlan0() for
+ * VLAN-unaware bridge operation. Don't let the bridge overwrite
+ * its EG_CON flag with VTAG_EN and corrupt PORT_MEM.
+ */
+ if (vlan->vid == 0)
+ goto skip_vlan_table;
+
mt7530_hw_vlan_entry_init(&new_entry, port, untagged);
mt7530_hw_vlan_update(priv, vlan->vid, &new_entry, mt7530_hw_vlan_add);
+skip_vlan_table:
+
if (pvid) {
priv->ports[port].pvid = vlan->vid;
@@ -1996,10 +2061,15 @@ mt7530_port_vlan_del(struct dsa_switch *ds, int port,
mutex_lock(&priv->reg_mutex);
+ /* VID 0 is managed exclusively by mt7530_setup_vlan0(). */
+ if (vlan->vid == 0)
+ goto skip_vlan_table;
+
mt7530_hw_vlan_entry_init(&target_entry, port, 0);
mt7530_hw_vlan_update(priv, vlan->vid, &target_entry,
mt7530_hw_vlan_del);
+skip_vlan_table:
/* PVID is being restored to the default whenever the PVID port
* is being removed from the VLAN.
*/
@@ -2377,7 +2447,10 @@ mt7530_setup(struct dsa_switch *ds)
}
ds->assisted_learning_on_cpu_port = true;
+ ds->untag_vlan_aware_bridge_pvid = true;
ds->mtu_enforcement_ingress = true;
+ ds->ageing_time_min = 2 * 1000;
+ ds->ageing_time_max = (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1) * 1000;
if (priv->id == ID_MT7530) {
regulator_set_voltage(priv->core_pwr, 1000000, 1000000);
@@ -2566,7 +2639,10 @@ mt7531_setup_common(struct dsa_switch *ds)
int ret, i;
ds->assisted_learning_on_cpu_port = true;
+ ds->untag_vlan_aware_bridge_pvid = true;
ds->mtu_enforcement_ingress = true;
+ ds->ageing_time_min = 2 * 1000;
+ ds->ageing_time_max = (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1) * 1000;
mt753x_trap_frames(priv);
@@ -3137,9 +3213,24 @@ mt753x_setup(struct dsa_switch *ds)
if (ret && priv->irq_domain)
mt7530_free_mdio_irq(priv);
+ if (!ret && priv->bus) {
+ mt7530_stats_refresh(priv);
+ schedule_delayed_work(&priv->stats_work,
+ MT7530_STATS_POLL_INTERVAL);
+ }
+
return ret;
}
+static void
+mt753x_teardown(struct dsa_switch *ds)
+{
+ struct mt7530_priv *priv = ds->priv;
+
+ if (priv->bus)
+ cancel_delayed_work_sync(&priv->stats_work);
+}
+
static int mt753x_set_mac_eee(struct dsa_switch *ds, int port,
struct ethtool_keee *e)
{
@@ -3257,6 +3348,7 @@ static int mt7988_setup(struct dsa_switch *ds)
static const struct dsa_switch_ops mt7530_switch_ops = {
.get_tag_protocol = mtk_get_tag_protocol,
.setup = mt753x_setup,
+ .teardown = mt753x_teardown,
.preferred_default_local_cpu_port = mt753x_preferred_default_local_cpu_port,
.get_strings = mt7530_get_strings,
.get_ethtool_stats = mt7530_get_ethtool_stats,
@@ -3395,6 +3487,9 @@ mt7530_probe_common(struct mt7530_priv *priv)
priv->ds->ops = &mt7530_switch_ops;
priv->ds->phylink_mac_ops = &mt753x_phylink_mac_ops;
mutex_init(&priv->reg_mutex);
+ spin_lock_init(&priv->stats_lock);
+ INIT_DELAYED_WORK(&priv->stats_work, mt7530_stats_poll);
+
dev_set_drvdata(dev, priv);
return 0;
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index 3e0090b..dd33b0d 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -796,6 +796,7 @@ struct mt7530_fdb {
* @pvid: The VLAN specified is to be considered a PVID at ingress. Any
* untagged frames will be assigned to the related VLAN.
* @sgmii_pcs: Pointer to PCS instance for SerDes ports
+ * @stats: Cached port statistics for MDIO-connected switches
*/
struct mt7530_port {
bool enable;
@@ -803,6 +804,7 @@ struct mt7530_port {
u32 pm;
u16 pvid;
struct phylink_pcs *sgmii_pcs;
+ struct rtnl_link_stats64 stats;
};
/* Port 5 mode definitions of the MT7530 switch */
@@ -875,6 +877,9 @@ struct mt753x_info {
* @create_sgmii: Pointer to function creating SGMII PCS instance(s)
* @active_cpu_ports: Holding the active CPU ports
* @mdiodev: The pointer to the MDIO device structure
+ * @stats_lock: Protects cached per-port stats from concurrent access
+ * @stats_work: Delayed work for polling MIB counters on MDIO switches
+ * @stats_last: Jiffies timestamp of last MIB counter poll
*/
struct mt7530_priv {
struct device *dev;
@@ -900,6 +905,9 @@ struct mt7530_priv {
int (*create_sgmii)(struct mt7530_priv *priv);
u8 active_cpu_ports;
struct mdio_device *mdiodev;
+ spinlock_t stats_lock; /* protects cached stats counters */
+ struct delayed_work stats_work;
+ unsigned long stats_last;
};
struct mt7530_hw_vlan_entry {
diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c
index c72c2bf..2697073 100644
--- a/drivers/net/dsa/sja1105/sja1105_main.c
+++ b/drivers/net/dsa/sja1105/sja1105_main.c
@@ -2310,10 +2310,10 @@ int sja1105_static_config_reload(struct sja1105_private *priv,
goto out;
}
- t1 = timespec64_to_ns(&ptp_sts_before.pre_ts);
- t2 = timespec64_to_ns(&ptp_sts_before.post_ts);
- t3 = timespec64_to_ns(&ptp_sts_after.pre_ts);
- t4 = timespec64_to_ns(&ptp_sts_after.post_ts);
+ t1 = ktime_to_ns(ptp_sts_before.pre_sts.systime);
+ t2 = ktime_to_ns(ptp_sts_before.post_sts.systime);
+ t3 = ktime_to_ns(ptp_sts_after.pre_sts.systime);
+ t4 = ktime_to_ns(ptp_sts_after.post_sts.systime);
/* Mid point, corresponds to pre-reset PTPCLKVAL */
t12 = t1 + (t2 - t1) / 2;
/* Mid point, corresponds to post-reset PTPCLKVAL, aka 0 */
diff --git a/drivers/net/ethernet/3com/3c509.c b/drivers/net/ethernet/3com/3c509.c
new file mode 100644
index 0000000..f23be74
--- /dev/null
+++ b/drivers/net/ethernet/3com/3c509.c
@@ -0,0 +1,1543 @@
+// SPDX-License-Identifier: GPL-2.0
+/* 3c509.c: A 3c509 EtherLink3 ethernet driver for linux. */
+/*
+ * Written 1993-2000 by Donald Becker.
+ *
+ * Copyright 1994-2000 by Donald Becker.
+ * Copyright 1993 United States Government as represented by the
+ * Director, National Security Agency. This software may be used and
+ * distributed according to the terms of the GNU General Public License,
+ * incorporated herein by reference.
+ *
+ * This driver is for the 3Com EtherLinkIII series.
+ *
+ * The author may be reached as becker@scyld.com, or C/O
+ * Scyld Computing Corporation
+ * 410 Severn Ave., Suite 210
+ * Annapolis MD 21403
+ *
+ * Known limitations:
+ * Because of the way 3c509 ISA detection works it's difficult to predict
+ * a priori which of several ISA-mode cards will be detected first.
+ *
+ * This driver does not use predictive interrupt mode, resulting in higher
+ * packet latency but lower overhead. If interrupts are disabled for an
+ * unusually long time it could also result in missed packets, but in
+ * practice this rarely happens.
+ *
+ *
+ * FIXES:
+ * Alan Cox: Removed the 'Unexpected interrupt' bug.
+ * Michael Meskes: Upgraded to Donald Becker's version 1.07.
+ * Alan Cox: Increased the eeprom delay. Regardless of
+ * what the docs say some people definitely
+ * get problems with lower (but in card spec)
+ * delays.
+ * v1.10 4/21/97 Fixed module code so that multiple cards may be
+ * detected, other cleanups. -djb
+ * Andrea Arcangeli: Upgraded to Donald Becker's version 1.12.
+ * Rick Payne: Fixed SMP race condition.
+ * v1.13 9/8/97 Made 'max_interrupt_work' an insmod-settable
+ * variable. -djb
+ * v1.14 10/15/97 Avoided waiting..discard message for fast
+ * machines. -djb
+ * v1.15 1/31/98 Faster recovery for Tx errors. -djb
+ * v1.16 2/3/98 Different ID port handling to avoid sound
+ * cards. -djb
+ * v1.18 12Mar2001 Andrew Morton
+ * - Avoid bogus detect of 3c590's (Andrzej Krzysztofowicz)
+ * - Reviewed against 1.18 from scyld.com
+ * v1.18a 17Nov2001 Jeff Garzik <jgarzik@pobox.com>
+ * - ethtool support.
+ * v1.18b 1Mar2002 Zwane Mwaikambo <zwane@commfireservices.com>
+ * - Power Management support.
+ * v1.18c 1Mar2002 David Ruggiero <jdr@farfalle.com>
+ * - Full duplex support.
+ * v1.19 16Oct2002 Zwane Mwaikambo <zwane@linuxpower.ca>
+ * - Additional ethtool features.
+ * v1.19a 28Oct2002 David Ruggiero <jdr@farfalle.com>
+ * - Increase *read_eeprom udelay to workaround oops with
+ * 2 cards.
+ * v1.19b 08Nov2002 Marc Zyngier <maz@wild-wind.fr.eu.org>
+ * - Introduce driver model for EISA cards.
+ * v1.20 04Feb2008 Ondrej Zary <linux@rainbow-software.org>
+ * - convert to isa_driver and pnp_driver and some
+ * cleanups.
+ */
+
+#define DRV_NAME "3c509"
+
+/* A few values that may be tweaked. */
+
+/* Time in jiffies before concluding the transmitter is hung. */
+#define TX_TIMEOUT (400 * HZ / 1000)
+
+#include <linux/bitops.h>
+#include <linux/delay.h> /* for udelay() */
+#include <linux/device.h>
+#include <linux/eisa.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/in.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/isa.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/pm.h>
+#include <linux/pnp.h>
+#include <linux/skbuff.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/uaccess.h>
+
+#include <asm/irq.h>
+
+#ifdef EL3_DEBUG
+static int el3_debug = EL3_DEBUG;
+#else
+static int el3_debug = 2;
+#endif
+
+/* Used to do a global count of all the cards in the system. Must be
+ * a global variable so that the eisa probe routines can increment it.
+ */
+static int el3_cards;
+#define EL3_MAX_CARDS 8
+
+/* To minimize the size of the driver source I only define operating
+ * constants if they are used several times. You'll need the manual
+ * anyway if you want to understand driver details.
+ */
+/* Offsets from base I/O address. */
+#define EL3_DATA 0x00
+#define EL3_CMD 0x0e
+#define EL3_STATUS 0x0e
+#define EEPROM_READ 0x80
+
+#define EL3_IO_EXTENT 16
+
+#define EL3WINDOW(win_num) outw(SELECT_WINDOW + (win_num), ioaddr + EL3_CMD)
+
+/* The top five bits written to EL3_CMD are a command, the lower
+ * 11 bits are the parameter, if applicable.
+ */
+enum c509cmd {
+ TOTAL_RESET = 0 << 11,
+ SELECT_WINDOW = 1 << 11,
+ START_COAX = 2 << 11,
+ RX_DISABLE = 3 << 11,
+ RX_ENABLE = 4 << 11,
+ RX_RESET = 5 << 11,
+ RX_DISCARD = 8 << 11,
+ TX_ENABLE = 9 << 11,
+ TX_DISABLE = 10 << 11,
+ TX_RESET = 11 << 11,
+ FAKE_INTR = 12 << 11,
+ ACK_INTR = 13 << 11,
+ SET_INTR_ENB = 14 << 11,
+ SET_STATUS_ENB = 15 << 11,
+ SET_RX_FILTER = 16 << 11,
+ SET_RX_THRESHOLD = 17 << 11,
+ SET_TX_THRESHOLD = 18 << 11,
+ SET_TX_START = 19 << 11,
+ STATS_ENABLE = 21 << 11,
+ STATS_DISABLE = 22 << 11,
+ STOP_COAX = 23 << 11,
+ POWER_UP = 27 << 11,
+ POWER_DOWN = 28 << 11,
+ POWER_AUTO = 29 << 11,
+};
+
+enum c509status {
+ INT_LATCH = 0x0001,
+ ADAPTER_FAILURE = 0x0002,
+ TX_COMPLETE = 0x0004,
+ TX_AVAILABLE = 0x0008,
+ RX_COMPLETE = 0x0010,
+ RX_EARLY = 0x0020,
+ INT_REQ = 0x0040,
+ STATS_FULL = 0x0080,
+ CMD_BUSY = 0x1000,
+};
+
+/* The SET_RX_FILTER command accepts the following classes: */
+enum rx_filter {
+ RX_STATION = 1,
+ RX_MULTICAST = 2,
+ RX_BROADCAST = 4,
+ RX_PROM = 8,
+};
+
+/* Register window 1 offsets, the window used in normal operation. */
+#define TX_FIFO 0x00
+#define RX_FIFO 0x00
+#define RX_STATUS 0x08
+#define TX_STATUS 0x0B
+#define TX_FREE 0x0C /* Remaining free bytes in Tx buffer. */
+
+#define WN0_CONF_CTRL 0x04 /* Window 0: Configuration control register. */
+#define WN0_ADDR_CONF 0x06 /* Window 0: Address configuration register. */
+#define WN0_IRQ 0x08 /* Window 0: Set IRQ line in bits 12-15. */
+#define WN4_MEDIA 0x0A /* Window 4: Various transcvr/media bits. */
+#define MEDIA_TP 0x00C0 /* Enable link beat and jabber for 10baseT. */
+#define WN4_NETDIAG 0x06 /* Window 4: Net diagnostic. */
+#define FD_ENABLE 0x8000 /* Enable full-duplex ("external loopback"). */
+
+/*
+ * Must be a power of two (we use a binary and in the
+ * circular queue).
+ */
+#define SKB_QUEUE_SIZE 64
+
+enum el3_cardtype { EL3_ISA, EL3_PNP, EL3_EISA };
+
+struct el3_private {
+ /* for device access */
+ spinlock_t lock;
+ /* skb send-queue */
+ int head, size;
+ struct sk_buff *queue[SKB_QUEUE_SIZE];
+ enum el3_cardtype type;
+};
+
+static int id_port;
+static int current_tag;
+static struct net_device *el3_devs[EL3_MAX_CARDS];
+
+/* Parameters that may be passed into the module. */
+static int debug = -1;
+static int irq[] = {-1, -1, -1, -1, -1, -1, -1, -1};
+/* Maximum events (Rx packets, etc.) to handle at each interrupt. */
+static int max_interrupt_work = 10;
+#ifdef CONFIG_PNP
+static int nopnp;
+#endif
+
+static int el3_common_init(struct net_device *dev);
+static void el3_common_remove(struct net_device *dev);
+static ushort id_read_eeprom(int index);
+static ushort read_eeprom(int ioaddr, int index);
+static int el3_open(struct net_device *dev);
+static netdev_tx_t el3_start_xmit(struct sk_buff *skb, struct net_device *dev);
+static irqreturn_t el3_interrupt(int irq, void *dev_id);
+static void update_stats(struct net_device *dev);
+static struct net_device_stats *el3_get_stats(struct net_device *dev);
+static int el3_rx(struct net_device *dev);
+static int el3_close(struct net_device *dev);
+static void set_multicast_list(struct net_device *dev);
+static void el3_tx_timeout(struct net_device *dev, unsigned int txqueue);
+static void el3_down(struct net_device *dev);
+static void el3_up(struct net_device *dev);
+static const struct ethtool_ops ethtool_ops;
+#ifdef CONFIG_PM
+static int el3_suspend(struct device *, pm_message_t);
+static int el3_resume(struct device *);
+#else
+#define el3_suspend NULL
+#define el3_resume NULL
+#endif
+
+/* Generic device remove for all device types. */
+static int el3_device_remove(struct device *device);
+#ifdef CONFIG_NET_POLL_CONTROLLER
+static void el3_poll_controller(struct net_device *dev);
+#endif
+
+/* Return 0 on success, 1 on error, 2 when found already detected PnP card. */
+static int el3_isa_id_sequence(__be16 *phys_addr)
+{
+ short lrs_state = 0xff;
+ int i;
+
+ /* ISA boards are detected by sending the ID sequence to the
+ * ID_PORT. We find cards past the first by setting the 'current_tag'
+ * on cards as they are found. Cards with their tag set will not
+ * respond to subsequent ID sequences.
+ */
+ outb(0x00, id_port);
+ outb(0x00, id_port);
+ for (i = 0; i < 255; i++) {
+ outb(lrs_state, id_port);
+ lrs_state <<= 1;
+ lrs_state = lrs_state & 0x100 ? lrs_state ^ 0xcf : lrs_state;
+ }
+ /* For the first probe, clear all board's tag registers. */
+ if (current_tag == 0)
+ outb(0xd0, id_port);
+ else /* Otherwise kill off already-found boards. */
+ outb(0xd8, id_port);
+ if (id_read_eeprom(7) != 0x6d50)
+ return 1;
+ /* Read in EEPROM data, which does contention-select.
+ * Only the lowest address board will stay "on-line".
+ * 3Com got the byte order backwards.
+ */
+ for (i = 0; i < 3; i++)
+ phys_addr[i] = htons(id_read_eeprom(i));
+#ifdef CONFIG_PNP
+ if (!nopnp) {
+ /* The ISA PnP 3c509 cards respond to the ID sequence too.
+ * This check is needed in order not to register them twice.
+ */
+ for (i = 0; i < el3_cards; i++) {
+ struct el3_private *lp = netdev_priv(el3_devs[i]);
+
+ if (lp->type == EL3_PNP &&
+ ether_addr_equal((u8 *)phys_addr,
+ el3_devs[i]->dev_addr)) {
+ if (el3_debug > 3)
+ pr_debug("3c509 with address %02x %02x %02x %02x %02x %02x was found by ISAPnP\n",
+ phys_addr[0] & 0xff,
+ phys_addr[0] >> 8,
+ phys_addr[1] & 0xff,
+ phys_addr[1] >> 8,
+ phys_addr[2] & 0xff,
+ phys_addr[2] >> 8);
+ /* Set the adaptor tag so that the next card
+ * can be found.
+ */
+ outb(0xd0 + ++current_tag, id_port);
+ return 2;
+ }
+ }
+ }
+#endif /* CONFIG_PNP */
+ return 0;
+}
+
+static void el3_dev_fill(struct net_device *dev, __be16 *phys_addr, int ioaddr,
+ int irq, int if_port, enum el3_cardtype type)
+{
+ struct el3_private *lp = netdev_priv(dev);
+
+ eth_hw_addr_set(dev, (u8 *)phys_addr);
+ dev->base_addr = ioaddr;
+ dev->irq = irq;
+ dev->if_port = if_port;
+ lp->type = type;
+}
+
+static int el3_isa_match(struct device *pdev, unsigned int ndev)
+{
+ int ioaddr, isa_irq, if_port, err;
+ struct net_device *dev;
+ unsigned int iobase;
+ __be16 phys_addr[3];
+
+ while ((err = el3_isa_id_sequence(phys_addr)) == 2)
+ ; /* Skip to next card when PnP card found */
+ if (err == 1)
+ return 0;
+
+ iobase = id_read_eeprom(8);
+ if_port = iobase >> 14;
+ ioaddr = 0x200 + ((iobase & 0x1f) << 4);
+ if (irq[el3_cards] > 1 && irq[el3_cards] < 16)
+ isa_irq = irq[el3_cards];
+ else
+ isa_irq = id_read_eeprom(9) >> 12;
+
+ dev = alloc_etherdev(sizeof(struct el3_private));
+ if (!dev)
+ return -ENOMEM;
+
+ SET_NETDEV_DEV(dev, pdev);
+
+ if (!request_region(ioaddr, EL3_IO_EXTENT, "3c509-isa")) {
+ free_netdev(dev);
+ return 0;
+ }
+
+ /* Set the adaptor tag so that the next card can be found. */
+ outb(0xd0 + ++current_tag, id_port);
+
+ /* Activate the adaptor at the EEPROM location. */
+ outb((ioaddr >> 4) | 0xe0, id_port);
+
+ EL3WINDOW(0);
+ if (inw(ioaddr) != 0x6d50) {
+ free_netdev(dev);
+ return 0;
+ }
+
+ /* Free the interrupt so that some other card can use it. */
+ outw(0x0f00, ioaddr + WN0_IRQ);
+
+ el3_dev_fill(dev, phys_addr, ioaddr, isa_irq, if_port, EL3_ISA);
+ dev_set_drvdata(pdev, dev);
+ if (el3_common_init(dev)) {
+ free_netdev(dev);
+ return 0;
+ }
+
+ el3_devs[el3_cards++] = dev;
+ return 1;
+}
+
+static void el3_isa_remove(struct device *pdev, unsigned int ndev)
+{
+ el3_device_remove(pdev);
+ dev_set_drvdata(pdev, NULL);
+}
+
+#ifdef CONFIG_PM
+static int el3_isa_suspend(struct device *dev, unsigned int n,
+ pm_message_t state)
+{
+ current_tag = 0;
+ return el3_suspend(dev, state);
+}
+
+static int el3_isa_resume(struct device *dev, unsigned int n)
+{
+ struct net_device *ndev = dev_get_drvdata(dev);
+ int ioaddr = ndev->base_addr, err;
+ __be16 phys_addr[3];
+
+ while ((err = el3_isa_id_sequence(phys_addr)) == 2)
+ ; /* Skip to next card when PnP card found */
+ if (err == 1)
+ return 0;
+ /* Set the adaptor tag so that the next card can be found. */
+ outb(0xd0 + ++current_tag, id_port);
+ /* Enable the card */
+ outb((ioaddr >> 4) | 0xe0, id_port);
+ EL3WINDOW(0);
+ if (inw(ioaddr) != 0x6d50)
+ return 1;
+ /* Free the interrupt so that some other card can use it. */
+ outw(0x0f00, ioaddr + WN0_IRQ);
+ return el3_resume(dev);
+}
+#endif
+
+static struct isa_driver el3_isa_driver = {
+ .match = el3_isa_match,
+ .remove = el3_isa_remove,
+#ifdef CONFIG_PM
+ .suspend = el3_isa_suspend,
+ .resume = el3_isa_resume,
+#endif
+ .driver = {
+ .name = "3c509"
+ },
+};
+
+static int isa_registered;
+
+#ifdef CONFIG_PNP
+static const struct pnp_device_id el3_pnp_ids[] = {
+ { .id = "TCM5090" }, /* 3Com Etherlink III (TP) */
+ { .id = "TCM5091" }, /* 3Com Etherlink III */
+ { .id = "TCM5094" }, /* 3Com Etherlink III (combo) */
+ { .id = "TCM5095" }, /* 3Com Etherlink III (TPO) */
+ { .id = "TCM5098" }, /* 3Com Etherlink III (TPC) */
+ { .id = "PNP80f7" }, /* 3Com Etherlink III compatible */
+ { .id = "PNP80f8" }, /* 3Com Etherlink III compatible */
+ { .id = "" }
+};
+MODULE_DEVICE_TABLE(pnp, el3_pnp_ids);
+
+static int el3_pnp_probe(struct pnp_dev *pdev, const struct pnp_device_id *id)
+{
+ struct net_device *dev = NULL;
+ int ioaddr, irq, if_port;
+ __be16 phys_addr[3];
+ short i;
+ int err;
+
+ ioaddr = pnp_port_start(pdev, 0);
+ if (!request_region(ioaddr, EL3_IO_EXTENT, "3c509-pnp"))
+ return -EBUSY;
+ irq = pnp_irq(pdev, 0);
+ EL3WINDOW(0);
+ for (i = 0; i < 3; i++)
+ phys_addr[i] = htons(read_eeprom(ioaddr, i));
+ if_port = read_eeprom(ioaddr, 8) >> 14;
+ dev = alloc_etherdev(sizeof(struct el3_private));
+ if (!dev) {
+ release_region(ioaddr, EL3_IO_EXTENT);
+ return -ENOMEM;
+ }
+ SET_NETDEV_DEV(dev, &pdev->dev);
+
+ el3_dev_fill(dev, phys_addr, ioaddr, irq, if_port, EL3_PNP);
+ pnp_set_drvdata(pdev, dev);
+ err = el3_common_init(dev);
+
+ if (err) {
+ pnp_set_drvdata(pdev, NULL);
+ free_netdev(dev);
+ return err;
+ }
+
+ el3_devs[el3_cards++] = dev;
+ return 0;
+}
+
+static void el3_pnp_remove(struct pnp_dev *pdev)
+{
+ el3_common_remove(pnp_get_drvdata(pdev));
+ pnp_set_drvdata(pdev, NULL);
+}
+
+#ifdef CONFIG_PM
+static int el3_pnp_suspend(struct pnp_dev *pdev, pm_message_t state)
+{
+ return el3_suspend(&pdev->dev, state);
+}
+
+static int el3_pnp_resume(struct pnp_dev *pdev)
+{
+ return el3_resume(&pdev->dev);
+}
+#endif
+
+static struct pnp_driver el3_pnp_driver = {
+ .name = "3c509",
+ .id_table = el3_pnp_ids,
+ .probe = el3_pnp_probe,
+ .remove = el3_pnp_remove,
+#ifdef CONFIG_PM
+ .suspend = el3_pnp_suspend,
+ .resume = el3_pnp_resume,
+#endif
+};
+
+static int pnp_registered;
+#endif /* CONFIG_PNP */
+
+#ifdef CONFIG_EISA
+static const struct eisa_device_id el3_eisa_ids[] = {
+ { "TCM5090" },
+ { "TCM5091" },
+ { "TCM5092" },
+ { "TCM5093" },
+ { "TCM5094" },
+ { "TCM5095" },
+ { "TCM5098" },
+ { "" }
+};
+MODULE_DEVICE_TABLE(eisa, el3_eisa_ids);
+
+static int el3_eisa_probe(struct device *device);
+
+static struct eisa_driver el3_eisa_driver = {
+ .id_table = el3_eisa_ids,
+ .driver = {
+ .name = "3c579",
+ .probe = el3_eisa_probe,
+ .remove = el3_device_remove,
+ .suspend = el3_suspend,
+ .resume = el3_resume,
+ }
+};
+
+static int eisa_registered;
+#endif
+
+static const struct net_device_ops netdev_ops = {
+ .ndo_open = el3_open,
+ .ndo_stop = el3_close,
+ .ndo_start_xmit = el3_start_xmit,
+ .ndo_get_stats = el3_get_stats,
+ .ndo_set_rx_mode = set_multicast_list,
+ .ndo_tx_timeout = el3_tx_timeout,
+ .ndo_set_mac_address = eth_mac_addr,
+ .ndo_validate_addr = eth_validate_addr,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+ .ndo_poll_controller = el3_poll_controller,
+#endif
+};
+
+static int el3_common_init(struct net_device *dev)
+{
+ static const char *const if_names[] = {
+ "10baseT", "AUI", "undefined", "BNC"
+ };
+ struct el3_private *lp = netdev_priv(dev);
+ int err;
+
+ spin_lock_init(&lp->lock);
+
+ if (dev->mem_start & 0x05) { /* xcvr codes 1/3/4/12 */
+ dev->if_port = (dev->mem_start & 0x0f);
+ } else { /* xcvr codes 0/8 */
+ /* use eeprom value, but save user's full-duplex selection */
+ dev->if_port |= (dev->mem_start & 0x08);
+ }
+
+ /* The EL3-specific entries in the device structure. */
+ dev->netdev_ops = &netdev_ops;
+ dev->watchdog_timeo = TX_TIMEOUT;
+ dev->ethtool_ops = ðtool_ops;
+
+ err = register_netdev(dev);
+ if (err) {
+ pr_err("Failed to register 3c5x9 at %#3.3lx, IRQ %d.\n",
+ dev->base_addr, dev->irq);
+ release_region(dev->base_addr, EL3_IO_EXTENT);
+ return err;
+ }
+
+ pr_info("%s: 3c5x9 found at %#3.3lx, %s port, address %pM, IRQ %d.\n",
+ dev->name, dev->base_addr, if_names[(dev->if_port & 0x03)],
+ dev->dev_addr, dev->irq);
+
+ return 0;
+}
+
+static void el3_common_remove(struct net_device *dev)
+{
+ unregister_netdev(dev);
+ release_region(dev->base_addr, EL3_IO_EXTENT);
+ free_netdev(dev);
+}
+
+#ifdef CONFIG_EISA
+static int el3_eisa_probe(struct device *device)
+{
+ struct net_device *dev = NULL;
+ struct eisa_device *edev;
+ int ioaddr, irq, if_port;
+ __be16 phys_addr[3];
+ short i;
+ int err;
+
+ /* Yeepee, The driver framework is calling us ! */
+ edev = to_eisa_device(device);
+ ioaddr = edev->base_addr;
+
+ if (!request_region(ioaddr, EL3_IO_EXTENT, "3c579-eisa"))
+ return -EBUSY;
+
+ /* Change the register set to the configuration window 0. */
+ outw(SELECT_WINDOW | 0, ioaddr + 0xC80 + EL3_CMD);
+
+ irq = inw(ioaddr + WN0_IRQ) >> 12;
+ if_port = inw(ioaddr + 6) >> 14;
+ for (i = 0; i < 3; i++)
+ phys_addr[i] = htons(read_eeprom(ioaddr, i));
+
+ /* Restore the "Product ID" to the EEPROM read register. */
+ read_eeprom(ioaddr, 3);
+
+ dev = alloc_etherdev(sizeof(struct el3_private));
+ if (!dev) {
+ release_region(ioaddr, EL3_IO_EXTENT);
+ return -ENOMEM;
+ }
+
+ SET_NETDEV_DEV(dev, device);
+
+ el3_dev_fill(dev, phys_addr, ioaddr, irq, if_port, EL3_EISA);
+ eisa_set_drvdata(edev, dev);
+ err = el3_common_init(dev);
+
+ if (err) {
+ eisa_set_drvdata(edev, NULL);
+ free_netdev(dev);
+ return err;
+ }
+
+ el3_devs[el3_cards++] = dev;
+ return 0;
+}
+#endif
+
+/* This remove works for all device types.
+ *
+ * The net dev must be stored in the driver data field.
+ */
+static int el3_device_remove(struct device *device)
+{
+ struct net_device *dev;
+
+ dev = dev_get_drvdata(device);
+
+ el3_common_remove(dev);
+ return 0;
+}
+
+/* Read a word from the EEPROM using the regular EEPROM access register.
+ * Assume that we are in register window zero.
+ */
+static ushort read_eeprom(int ioaddr, int index)
+{
+ outw(EEPROM_READ + index, ioaddr + 10);
+ /* Pause for at least 162 us for the read to take place.
+ * Some chips seem to require much longer.
+ */
+ mdelay(2);
+ return inw(ioaddr + 12);
+}
+
+/* Read a word from the EEPROM when in the ISA ID probe state. */
+static ushort id_read_eeprom(int index)
+{
+ int bit, word = 0;
+
+ /* Issue read command, and pause for at least 162 us for it to
+ * complete. Assume extra-fast 16MHz bus.
+ */
+ outb(EEPROM_READ + index, id_port);
+
+ /* Pause for at least 162 us for the read to take place.
+ * Some chips seem to require much longer.
+ */
+ mdelay(4);
+
+ for (bit = 15; bit >= 0; bit--)
+ word = (word << 1) + (inb(id_port) & 0x01);
+
+ if (el3_debug > 3)
+ pr_debug(" 3c509 EEPROM word %d %#4.4x.\n", index, word);
+
+ return word;
+}
+
+static int el3_open(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+ int i;
+
+ outw(TX_RESET, ioaddr + EL3_CMD);
+ outw(RX_RESET, ioaddr + EL3_CMD);
+ outw(SET_STATUS_ENB | 0x00, ioaddr + EL3_CMD);
+
+ i = request_irq(dev->irq, el3_interrupt, 0, dev->name, dev);
+ if (i)
+ return i;
+
+ EL3WINDOW(0);
+ if (el3_debug > 3)
+ pr_debug("%s: Opening, IRQ %d status@%x %4.4x.\n",
+ dev->name, dev->irq,
+ ioaddr + EL3_STATUS, inw(ioaddr + EL3_STATUS));
+
+ el3_up(dev);
+
+ if (el3_debug > 3)
+ pr_debug("%s: Opened 3c509 IRQ %d status %4.4x.\n",
+ dev->name, dev->irq, inw(ioaddr + EL3_STATUS));
+
+ return 0;
+}
+
+static void el3_tx_timeout(struct net_device *dev, unsigned int txqueue)
+{
+ int ioaddr = dev->base_addr;
+
+ /* Transmitter timeout, serious problems. */
+ pr_warn("%s: transmit timed out, Tx_status %2.2x status %4.4x Tx FIFO room %d\n",
+ dev->name, inb(ioaddr + TX_STATUS), inw(ioaddr + EL3_STATUS),
+ inw(ioaddr + TX_FREE));
+ dev->stats.tx_errors++;
+ netif_trans_update(dev); /* prevent tx timeout */
+ /* Issue TX_RESET and TX_START commands. */
+ outw(TX_RESET, ioaddr + EL3_CMD);
+ outw(TX_ENABLE, ioaddr + EL3_CMD);
+ netif_wake_queue(dev);
+}
+
+static netdev_tx_t el3_start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ int ioaddr = dev->base_addr;
+ unsigned long flags;
+
+ netif_stop_queue(dev);
+
+ dev->stats.tx_bytes += skb->len;
+
+ if (el3_debug > 4) {
+ pr_debug("%s: el3_start_xmit(length = %u) called, status %4.4x.\n",
+ dev->name, skb->len, inw(ioaddr + EL3_STATUS));
+ }
+ /*
+ * We lock the driver against other processors. Note
+ * we don't need to lock versus the IRQ as we suspended
+ * that. This means that we lose the ability to take
+ * an RX during a TX upload. That sucks a bit with SMP
+ * on an original 3c509 (2K buffer).
+ *
+ * Using disable_irq stops us crapping on other
+ * time sensitive devices.
+ */
+
+ spin_lock_irqsave(&lp->lock, flags);
+
+ /* Put out the doubleword header... */
+ outw(skb->len, ioaddr + TX_FIFO);
+ outw(0x00, ioaddr + TX_FIFO);
+ /* ... and the packet rounded to a doubleword. */
+ outsl(ioaddr + TX_FIFO, skb->data, (skb->len + 3) >> 2);
+
+ if (inw(ioaddr + TX_FREE) > 1536) {
+ netif_start_queue(dev);
+ } else {
+ /* Interrupt us when the FIFO has room for max-sized packet. */
+ outw(SET_TX_THRESHOLD + 1536, ioaddr + EL3_CMD);
+ }
+
+ spin_unlock_irqrestore(&lp->lock, flags);
+
+ dev_consume_skb_any(skb);
+
+ /* Clear the Tx status stack. */
+ {
+ short tx_status;
+ int i = 4;
+
+ while (--i > 0 && (tx_status = inb(ioaddr + TX_STATUS)) > 0) {
+ if (tx_status & 0x38)
+ dev->stats.tx_aborted_errors++;
+ if (tx_status & 0x30)
+ outw(TX_RESET, ioaddr + EL3_CMD);
+ if (tx_status & 0x3C)
+ outw(TX_ENABLE, ioaddr + EL3_CMD);
+ /* Pop the status stack. */
+ outb(0x00, ioaddr + TX_STATUS);
+ }
+ }
+ return NETDEV_TX_OK;
+}
+
+/* The EL3 interrupt handler. */
+static irqreturn_t el3_interrupt(int irq, void *dev_id)
+{
+ struct net_device *dev = dev_id;
+ int i = max_interrupt_work;
+ struct el3_private *lp;
+ int ioaddr, status;
+
+ lp = netdev_priv(dev);
+ spin_lock(&lp->lock);
+
+ ioaddr = dev->base_addr;
+
+ if (el3_debug > 4) {
+ status = inw(ioaddr + EL3_STATUS);
+ pr_debug("%s: interrupt, status %4.4x.\n", dev->name, status);
+ }
+
+ while ((status = inw(ioaddr + EL3_STATUS)) &
+ (INT_LATCH | RX_COMPLETE | STATS_FULL)) {
+
+ if (status & RX_COMPLETE)
+ el3_rx(dev);
+
+ if (status & TX_AVAILABLE) {
+ if (el3_debug > 5)
+ pr_debug(" TX room bit was handled.\n");
+ /* There's room in the FIFO for a full-sized packet. */
+ outw(ACK_INTR | TX_AVAILABLE, ioaddr + EL3_CMD);
+ netif_wake_queue(dev);
+ }
+ if (status &
+ (ADAPTER_FAILURE | RX_EARLY | STATS_FULL | TX_COMPLETE)) {
+ /* Handle all uncommon interrupts. */
+ if (status & STATS_FULL) {
+ /* Empty statistics. */
+ update_stats(dev);
+ }
+ if (status & RX_EARLY) {
+ /* Rx early is unused. */
+ el3_rx(dev);
+ outw(ACK_INTR | RX_EARLY, ioaddr + EL3_CMD);
+ }
+ if (status & TX_COMPLETE) {
+ /* Really Tx error. */
+ short tx_status;
+ int i = 4;
+
+ while (--i > 0 &&
+ ((tx_status = inb(ioaddr + TX_STATUS))
+ > 0)) {
+ if (tx_status & 0x38)
+ dev->stats.tx_aborted_errors++;
+ if (tx_status & 0x30)
+ outw(TX_RESET,
+ ioaddr + EL3_CMD);
+ if (tx_status & 0x3C)
+ outw(TX_ENABLE,
+ ioaddr + EL3_CMD);
+ /* Pop the status stack. */
+ outb(0x00, ioaddr + TX_STATUS);
+ }
+ }
+ if (status & ADAPTER_FAILURE) {
+ /* Adapter failure requires Rx reset
+ * and reinit.
+ */
+ outw(RX_RESET, ioaddr + EL3_CMD);
+ /* Set the Rx filter to the current state. */
+ outw((SET_RX_FILTER | RX_STATION |
+ RX_BROADCAST |
+ (dev->flags & IFF_ALLMULTI ?
+ RX_MULTICAST : 0) |
+ (dev->flags & IFF_PROMISC ?
+ RX_PROM : 0)),
+ ioaddr + EL3_CMD);
+ /* Re-enable the receiver. */
+ outw(RX_ENABLE, ioaddr + EL3_CMD);
+ outw(ACK_INTR | ADAPTER_FAILURE,
+ ioaddr + EL3_CMD);
+ }
+ }
+
+ if (--i < 0) {
+ pr_err("%s: Infinite loop in interrupt, status %4.4x.\n",
+ dev->name, status);
+ /* Clear all interrupts. */
+ outw(ACK_INTR | 0xFF, ioaddr + EL3_CMD);
+ break;
+ }
+ /* Acknowledge the IRQ. */
+ outw(ACK_INTR | INT_REQ | INT_LATCH, ioaddr + EL3_CMD);
+ }
+
+ if (el3_debug > 4) {
+ pr_debug("%s: exiting interrupt, status %4.4x.\n", dev->name,
+ inw(ioaddr + EL3_STATUS));
+ }
+ spin_unlock(&lp->lock);
+ return IRQ_HANDLED;
+}
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+/*
+ * Polling receive - used by netconsole and other diagnostic tools
+ * to allow network i/o with interrupts disabled.
+ */
+static void el3_poll_controller(struct net_device *dev)
+{
+ disable_irq(dev->irq);
+ el3_interrupt(dev->irq, dev);
+ enable_irq(dev->irq);
+}
+#endif
+
+static struct net_device_stats *el3_get_stats(struct net_device *dev)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ unsigned long flags;
+
+ /* This is fast enough not to bother with disable IRQ stuff. */
+ spin_lock_irqsave(&lp->lock, flags);
+ update_stats(dev);
+ spin_unlock_irqrestore(&lp->lock, flags);
+ return &dev->stats;
+}
+
+/* Update statistics. We change to register window 6, so this should be run
+ * single-threaded if the device is active. This is expected to be a rare
+ * operation, and it's simpler for the rest of the driver to assume that
+ * window 1 is always valid rather than use a special window-state variable.
+ */
+static void update_stats(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+
+ if (el3_debug > 5)
+ pr_debug(" Updating the statistics.\n");
+ /* Turn off statistics updates while reading. */
+ outw(STATS_DISABLE, ioaddr + EL3_CMD);
+ /* Switch to the stats window, and read everything. */
+ EL3WINDOW(6);
+ dev->stats.tx_carrier_errors += inb(ioaddr + 0);
+ dev->stats.tx_heartbeat_errors += inb(ioaddr + 1);
+ /* Multiple collisions. */ inb(ioaddr + 2);
+ dev->stats.collisions += inb(ioaddr + 3);
+ dev->stats.tx_window_errors += inb(ioaddr + 4);
+ dev->stats.rx_fifo_errors += inb(ioaddr + 5);
+ dev->stats.tx_packets += inb(ioaddr + 6);
+ /* Rx packets */ inb(ioaddr + 7);
+ /* Tx deferrals */ inb(ioaddr + 8);
+ inw(ioaddr + 10); /* Total Rx and Tx octets. */
+ inw(ioaddr + 12);
+
+ /* Back to window 1, and turn statistics back on. */
+ EL3WINDOW(1);
+ outw(STATS_ENABLE, ioaddr + EL3_CMD);
+}
+
+static int el3_rx(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+ short rx_status;
+
+ if (el3_debug > 5)
+ pr_debug(" In rx_packet(), status %4.4x, rx_status %4.4x.\n",
+ inw(ioaddr + EL3_STATUS), inw(ioaddr + RX_STATUS));
+ while ((rx_status = inw(ioaddr + RX_STATUS)) > 0) {
+ if (rx_status & 0x4000) {
+ /* Error, update stats. */
+ short error = rx_status & 0x3800;
+
+ outw(RX_DISCARD, ioaddr + EL3_CMD);
+ dev->stats.rx_errors++;
+ switch (error) {
+ case 0x0000:
+ dev->stats.rx_over_errors++;
+ break;
+ case 0x0800:
+ dev->stats.rx_length_errors++;
+ break;
+ case 0x1000:
+ dev->stats.rx_frame_errors++;
+ break;
+ case 0x1800:
+ dev->stats.rx_length_errors++;
+ break;
+ case 0x2000:
+ dev->stats.rx_frame_errors++;
+ break;
+ case 0x2800:
+ dev->stats.rx_crc_errors++; break;
+ }
+ } else {
+ short pkt_len = rx_status & 0x7ff;
+ struct sk_buff *skb;
+
+ skb = netdev_alloc_skb(dev, pkt_len + 5);
+ if (el3_debug > 4)
+ pr_debug("Receiving packet size %d status %4.4x.\n",
+ pkt_len, rx_status);
+ if (skb) {
+ /* Align IP on 16 byte. */
+ skb_reserve(skb, 2);
+
+ /* 'skb->data' points to the start of sk_buff
+ * data area.
+ */
+ insl(ioaddr + RX_FIFO, skb_put(skb, pkt_len),
+ (pkt_len + 3) >> 2);
+
+ /* Pop top Rx packet. */
+ outw(RX_DISCARD, ioaddr + EL3_CMD);
+ skb->protocol = eth_type_trans(skb, dev);
+ netif_rx(skb);
+ dev->stats.rx_bytes += pkt_len;
+ dev->stats.rx_packets++;
+ continue;
+ }
+ outw(RX_DISCARD, ioaddr + EL3_CMD);
+ dev->stats.rx_dropped++;
+ if (el3_debug)
+ pr_debug("%s: Couldn't allocate a sk_buff of size %d.\n",
+ dev->name, pkt_len);
+ }
+ inw(ioaddr + EL3_STATUS); /* Delay. */
+ while (inw(ioaddr + EL3_STATUS) & 0x1000)
+ pr_debug(" Waiting for 3c509 to discard packet, status %x.\n",
+ inw(ioaddr + EL3_STATUS));
+ }
+
+ return 0;
+}
+
+/* Set or clear the multicast filter for this adaptor. */
+static void set_multicast_list(struct net_device *dev)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ int ioaddr = dev->base_addr;
+ int mc_count = netdev_mc_count(dev);
+ unsigned long flags;
+
+ if (el3_debug > 1) {
+ static int old;
+
+ if (old != mc_count) {
+ old = mc_count;
+ pr_debug("%s: Setting Rx mode to %d addresses.\n",
+ dev->name, mc_count);
+ }
+ }
+ spin_lock_irqsave(&lp->lock, flags);
+ if (dev->flags & IFF_PROMISC) {
+ outw((SET_RX_FILTER | RX_STATION | RX_MULTICAST |
+ RX_BROADCAST | RX_PROM),
+ ioaddr + EL3_CMD);
+ } else if (mc_count || (dev->flags & IFF_ALLMULTI)) {
+ outw(SET_RX_FILTER | RX_STATION | RX_MULTICAST | RX_BROADCAST,
+ ioaddr + EL3_CMD);
+ } else {
+ outw(SET_RX_FILTER | RX_STATION | RX_BROADCAST,
+ ioaddr + EL3_CMD);
+ }
+ spin_unlock_irqrestore(&lp->lock, flags);
+}
+
+static int el3_close(struct net_device *dev)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ int ioaddr = dev->base_addr;
+
+ if (el3_debug > 2)
+ pr_debug("%s: Shutting down ethercard.\n", dev->name);
+
+ el3_down(dev);
+
+ free_irq(dev->irq, dev);
+ /* Switching back to window 0 disables the IRQ. */
+ EL3WINDOW(0);
+ if (lp->type != EL3_EISA) {
+ /* But we explicitly zero the IRQ line select anyway. Don't do
+ * it on EISA cards, it prevents the module from getting an
+ * IRQ after unload+reload...
+ */
+ outw(0x0f00, ioaddr + WN0_IRQ);
+ }
+
+ return 0;
+}
+
+static int el3_link_ok(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+ u16 tmp;
+
+ EL3WINDOW(4);
+ tmp = inw(ioaddr + WN4_MEDIA);
+ EL3WINDOW(1);
+ return tmp & (1 << 11);
+}
+
+static void el3_netdev_get_ecmd(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
+{
+ int ioaddr = dev->base_addr;
+ u32 supported;
+ u16 tmp;
+
+ EL3WINDOW(0);
+ /* Obtain current transceiver via WN4_MEDIA? */
+ tmp = inw(ioaddr + WN0_ADDR_CONF);
+ switch (tmp >> 14) {
+ case 0:
+ cmd->base.port = PORT_TP;
+ break;
+ case 1:
+ cmd->base.port = PORT_AUI;
+ break;
+ case 3:
+ cmd->base.port = PORT_BNC;
+ break;
+ default:
+ break;
+ }
+
+ cmd->base.duplex = DUPLEX_HALF;
+ supported = 0;
+ tmp = inw(ioaddr + WN0_CONF_CTRL);
+ if (tmp & (1 << 13))
+ supported |= SUPPORTED_AUI;
+ if (tmp & (1 << 12))
+ supported |= SUPPORTED_BNC;
+ if (tmp & (1 << 9)) {
+ supported |= SUPPORTED_TP | SUPPORTED_10baseT_Half |
+ SUPPORTED_10baseT_Full; /* hmm... */
+ EL3WINDOW(4);
+ tmp = inw(ioaddr + WN4_NETDIAG);
+ if (tmp & FD_ENABLE)
+ cmd->base.duplex = DUPLEX_FULL;
+ }
+
+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+ supported);
+ cmd->base.speed = SPEED_10;
+ EL3WINDOW(1);
+}
+
+static int el3_netdev_set_ecmd(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
+{
+ int ioaddr = dev->base_addr;
+ u16 tmp;
+
+ if (cmd->base.speed != SPEED_10)
+ return -EINVAL;
+ if (cmd->base.duplex != DUPLEX_HALF && cmd->base.duplex != DUPLEX_FULL)
+ return -EINVAL;
+
+ /* change XCVR type */
+ EL3WINDOW(0);
+ tmp = inw(ioaddr + WN0_ADDR_CONF);
+ switch (cmd->base.port) {
+ case PORT_TP:
+ tmp &= ~(3 << 14);
+ dev->if_port = 0;
+ break;
+ case PORT_AUI:
+ tmp &= ~(3 << 14);
+ tmp |= 1 << 14;
+ dev->if_port = 1;
+ break;
+ case PORT_BNC:
+ tmp |= 3 << 14;
+ dev->if_port = 3;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ outw(tmp, ioaddr + WN0_ADDR_CONF);
+ if (dev->if_port == 3) {
+ /* Fire up the DC-DC converter if BNC gets enabled. */
+ tmp = inw(ioaddr + WN0_ADDR_CONF);
+ if (tmp & (3 << 14)) {
+ outw(START_COAX, ioaddr + EL3_CMD);
+ udelay(800);
+ } else {
+ return -EIO;
+ }
+ }
+
+ EL3WINDOW(4);
+ tmp = inw(ioaddr + WN4_NETDIAG);
+ if (cmd->base.duplex == DUPLEX_FULL)
+ tmp |= FD_ENABLE;
+ else
+ tmp &= ~FD_ENABLE;
+ outw(tmp, ioaddr + WN4_NETDIAG);
+ EL3WINDOW(1);
+
+ return 0;
+}
+
+static void el3_get_drvinfo(struct net_device *dev,
+ struct ethtool_drvinfo *info)
+{
+ strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+}
+
+static int el3_get_link_ksettings(struct net_device *dev,
+ struct ethtool_link_ksettings *cmd)
+{
+ struct el3_private *lp = netdev_priv(dev);
+
+ spin_lock_irq(&lp->lock);
+ el3_netdev_get_ecmd(dev, cmd);
+ spin_unlock_irq(&lp->lock);
+ return 0;
+}
+
+static int el3_set_link_ksettings(struct net_device *dev,
+ const struct ethtool_link_ksettings *cmd)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ int ret;
+
+ spin_lock_irq(&lp->lock);
+ ret = el3_netdev_set_ecmd(dev, cmd);
+ spin_unlock_irq(&lp->lock);
+ return ret;
+}
+
+static u32 el3_get_link(struct net_device *dev)
+{
+ struct el3_private *lp = netdev_priv(dev);
+ u32 ret;
+
+ spin_lock_irq(&lp->lock);
+ ret = el3_link_ok(dev);
+ spin_unlock_irq(&lp->lock);
+ return ret;
+}
+
+static u32 el3_get_msglevel(struct net_device *dev)
+{
+ return el3_debug;
+}
+
+static void el3_set_msglevel(struct net_device *dev, u32 v)
+{
+ el3_debug = v;
+}
+
+static const struct ethtool_ops ethtool_ops = {
+ .get_drvinfo = el3_get_drvinfo,
+ .get_link = el3_get_link,
+ .get_msglevel = el3_get_msglevel,
+ .set_msglevel = el3_set_msglevel,
+ .get_link_ksettings = el3_get_link_ksettings,
+ .set_link_ksettings = el3_set_link_ksettings,
+};
+
+static void el3_down(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+
+ netif_stop_queue(dev);
+
+ /* Turn off statistics ASAP. We update lp->stats below. */
+ outw(STATS_DISABLE, ioaddr + EL3_CMD);
+
+ /* Disable the receiver and transmitter. */
+ outw(RX_DISABLE, ioaddr + EL3_CMD);
+ outw(TX_DISABLE, ioaddr + EL3_CMD);
+
+ if (dev->if_port == 3) {
+ /* Turn off thinnet power. Green! */
+ outw(STOP_COAX, ioaddr + EL3_CMD);
+ } else if (dev->if_port == 0) {
+ /* Disable link beat and jabber, if_port may change here next
+ * open().
+ */
+ EL3WINDOW(4);
+ outw(inw(ioaddr + WN4_MEDIA) & ~MEDIA_TP, ioaddr + WN4_MEDIA);
+ }
+
+ outw(SET_INTR_ENB | 0x0000, ioaddr + EL3_CMD);
+
+ update_stats(dev);
+}
+
+static void el3_up(struct net_device *dev)
+{
+ int ioaddr = dev->base_addr;
+ int i, sw_info, net_diag;
+
+ /* Activating the board required and does no harm otherwise. */
+ outw(0x0001, ioaddr + 4);
+
+ /* Set the IRQ line. */
+ outw((dev->irq << 12) | 0x0f00, ioaddr + WN0_IRQ);
+
+ /* Set the station address in window 2 each time opened. */
+ EL3WINDOW(2);
+
+ for (i = 0; i < 6; i++)
+ outb(dev->dev_addr[i], ioaddr + i);
+
+ if ((dev->if_port & 0x03) == 3) {
+ /* BNC interface */
+
+ /* Start the thinnet transceiver. We should really wait
+ * 50ms...
+ */
+ outw(START_COAX, ioaddr + EL3_CMD);
+ } else if ((dev->if_port & 0x03) == 0) {
+ /* 10baseT interface */
+
+ /* Combine secondary sw_info word (the adapter level) and
+ * primary sw_info word (duplex setting plus other useless
+ * bits).
+ */
+ EL3WINDOW(0);
+ sw_info = (read_eeprom(ioaddr, 0x14) & 0x400f) |
+ (read_eeprom(ioaddr, 0x0d) & 0xBff0);
+
+ EL3WINDOW(4);
+ net_diag = inw(ioaddr + WN4_NETDIAG);
+ /* Temporarily assume full-duplex will be set. */
+ net_diag = (net_diag | FD_ENABLE);
+ pr_info("%s: ", dev->name);
+ switch (dev->if_port & 0x0c) {
+ case 12:
+ /* Force full-duplex mode if 3c5x9b. */
+ if (sw_info & 0x000f) {
+ pr_cont("Forcing 3c5x9b full-duplex mode");
+ break;
+ }
+ fallthrough;
+ case 8:
+ /* Set full-duplex mode based on eeprom config
+ * setting.
+ */
+ if ((sw_info & 0x000f) && (sw_info & 0x8000)) {
+ pr_cont("Setting 3c5x9b full-duplex mode (from EEPROM configuration bit)");
+ break;
+ }
+ fallthrough;
+ default:
+ /* xcvr = (0 || 4) OR user has an old 3c5x9 non "B"
+ * model.
+ */
+ pr_cont("Setting 3c5x9/3c5x9B half-duplex mode");
+ /* Disable full duplex. */
+ net_diag = (net_diag & ~FD_ENABLE);
+ }
+
+ outw(net_diag, ioaddr + WN4_NETDIAG);
+ pr_cont(" if_port: %d, sw_info: %4.4x\n",
+ dev->if_port, sw_info);
+ if (el3_debug > 3)
+ pr_debug("%s: 3c5x9 net diag word is now: %4.4x.\n",
+ dev->name, net_diag);
+ /* Enable link beat and jabber check. */
+ outw(inw(ioaddr + WN4_MEDIA) | MEDIA_TP, ioaddr + WN4_MEDIA);
+ }
+
+ /* Switch to the stats window, and clear all stats by reading. */
+ outw(STATS_DISABLE, ioaddr + EL3_CMD);
+ EL3WINDOW(6);
+ for (i = 0; i < 9; i++)
+ inb(ioaddr + i);
+ inw(ioaddr + 10);
+ inw(ioaddr + 12);
+
+ /* Switch to register set 1 for normal use. */
+ EL3WINDOW(1);
+
+ /* Accept b-case and phys addr only. */
+ outw(SET_RX_FILTER | RX_STATION | RX_BROADCAST, ioaddr + EL3_CMD);
+ /* Turn on statistics. */
+ outw(STATS_ENABLE, ioaddr + EL3_CMD);
+
+ /* Enable the receiver. */
+ outw(RX_ENABLE, ioaddr + EL3_CMD);
+ /* Enable transmitter. */
+ outw(TX_ENABLE, ioaddr + EL3_CMD);
+ /* Allow status bits to be seen. */
+ outw(SET_STATUS_ENB | 0xff, ioaddr + EL3_CMD);
+ /* Ack all pending events, and set active indicator mask. */
+ outw(ACK_INTR | INT_LATCH | TX_AVAILABLE | RX_EARLY | INT_REQ,
+ ioaddr + EL3_CMD);
+ outw((SET_INTR_ENB | INT_LATCH | TX_AVAILABLE | TX_COMPLETE |
+ RX_COMPLETE | STATS_FULL),
+ ioaddr + EL3_CMD);
+
+ netif_start_queue(dev);
+}
+
+/* Power Management support functions */
+#ifdef CONFIG_PM
+
+static int el3_suspend(struct device *pdev, pm_message_t state)
+{
+ struct net_device *dev;
+ struct el3_private *lp;
+ unsigned long flags;
+ int ioaddr;
+
+ dev = dev_get_drvdata(pdev);
+ lp = netdev_priv(dev);
+ ioaddr = dev->base_addr;
+
+ spin_lock_irqsave(&lp->lock, flags);
+
+ if (netif_running(dev))
+ netif_device_detach(dev);
+
+ el3_down(dev);
+ outw(POWER_DOWN, ioaddr + EL3_CMD);
+
+ spin_unlock_irqrestore(&lp->lock, flags);
+ return 0;
+}
+
+static int el3_resume(struct device *pdev)
+{
+ struct net_device *dev;
+ struct el3_private *lp;
+ unsigned long flags;
+ int ioaddr;
+
+ dev = dev_get_drvdata(pdev);
+ lp = netdev_priv(dev);
+ ioaddr = dev->base_addr;
+
+ spin_lock_irqsave(&lp->lock, flags);
+
+ outw(POWER_UP, ioaddr + EL3_CMD);
+ EL3WINDOW(0);
+ el3_up(dev);
+
+ if (netif_running(dev))
+ netif_device_attach(dev);
+
+ spin_unlock_irqrestore(&lp->lock, flags);
+ return 0;
+}
+
+#endif /* CONFIG_PM */
+
+module_param(debug, int, 0);
+module_param_hw_array(irq, int, irq, NULL, 0);
+module_param(max_interrupt_work, int, 0);
+MODULE_PARM_DESC(debug, "debug level (0-6)");
+MODULE_PARM_DESC(irq, "IRQ number(s) (assigned)");
+MODULE_PARM_DESC(max_interrupt_work, "maximum events handled per interrupt");
+#ifdef CONFIG_PNP
+module_param(nopnp, int, 0);
+MODULE_PARM_DESC(nopnp, "disable ISA PnP support (0-1)");
+#endif /* CONFIG_PNP */
+MODULE_DESCRIPTION("3Com Etherlink III (3c509, 3c509B, 3c529, 3c579) ethernet driver");
+MODULE_LICENSE("GPL");
+
+static int __init el3_init_module(void)
+{
+ int ret = 0;
+
+ if (debug >= 0)
+ el3_debug = debug;
+
+#ifdef CONFIG_PNP
+ if (!nopnp) {
+ ret = pnp_register_driver(&el3_pnp_driver);
+ if (!ret)
+ pnp_registered = 1;
+ }
+#endif
+ /* Select an open I/O location at 0x1*0 to do ISA contention select. */
+ /* Start with 0x110 to avoid some sound cards.*/
+ for (id_port = 0x110; id_port < 0x200; id_port += 0x10) {
+ if (!request_region(id_port, 1, "3c509-control"))
+ continue;
+ outb(0x00, id_port);
+ outb(0xff, id_port);
+ if (inb(id_port) & 0x01)
+ break;
+ release_region(id_port, 1);
+ }
+ if (id_port >= 0x200) {
+ id_port = 0;
+ pr_err("No I/O port available for 3c509 activation.\n");
+ } else {
+ ret = isa_register_driver(&el3_isa_driver, EL3_MAX_CARDS);
+ if (!ret)
+ isa_registered = 1;
+ }
+#ifdef CONFIG_EISA
+ ret = eisa_driver_register(&el3_eisa_driver);
+ if (!ret)
+ eisa_registered = 1;
+#endif
+
+#ifdef CONFIG_PNP
+ if (pnp_registered)
+ ret = 0;
+#endif
+ if (isa_registered)
+ ret = 0;
+#ifdef CONFIG_EISA
+ if (eisa_registered)
+ ret = 0;
+#endif
+ return ret;
+}
+
+static void __exit el3_cleanup_module(void)
+{
+#ifdef CONFIG_PNP
+ if (pnp_registered)
+ pnp_unregister_driver(&el3_pnp_driver);
+#endif
+ if (isa_registered)
+ isa_unregister_driver(&el3_isa_driver);
+ if (id_port)
+ release_region(id_port, 1);
+#ifdef CONFIG_EISA
+ if (eisa_registered)
+ eisa_driver_unregister(&el3_eisa_driver);
+#endif
+}
+
+module_init(el3_init_module);
+module_exit(el3_cleanup_module);
diff --git a/drivers/net/ethernet/3com/Kconfig b/drivers/net/ethernet/3com/Kconfig
index 399cb6c..81db167 100644
--- a/drivers/net/ethernet/3com/Kconfig
+++ b/drivers/net/ethernet/3com/Kconfig
@@ -17,6 +17,20 @@
if NET_VENDOR_3COM
+config EL3
+ tristate "3c509/3c579 \"EtherLink III\" support"
+ depends on (ISA || EISA)
+ help
+ If you have a network (Ethernet) card belonging to the 3Com
+ EtherLinkIII series, say Y here.
+
+ If your card is not working you may need to use the DOS
+ setup disk to disable Plug & Play mode, and to select the default
+ media type.
+
+ To compile this driver as a module, choose M here. The module
+ will be called 3c509.
+
config VORTEX
tristate "3c590/3c900 series (592/595/597) \"Vortex/Boomerang\" support"
depends on (PCI || EISA) && HAS_IOPORT_MAP
diff --git a/drivers/net/ethernet/3com/Makefile b/drivers/net/ethernet/3com/Makefile
index 5c4d07f..2c65e47 100644
--- a/drivers/net/ethernet/3com/Makefile
+++ b/drivers/net/ethernet/3com/Makefile
@@ -3,5 +3,6 @@
# Makefile for the 3Com Ethernet device drivers
#
+obj-$(CONFIG_EL3) += 3c509.o
obj-$(CONFIG_VORTEX) += 3c59x.o
obj-$(CONFIG_TYPHOON) += typhoon.o
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index 2bb0a3f..eab6a98 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -847,13 +847,24 @@ static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
{
struct airoha_qdma *qdma = q->qdma;
struct airoha_eth *eth = qdma->eth;
- int i;
+ int i, qid = q - &qdma->q_tx[0];
for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
struct airoha_gdm_port *port = eth->ports[i];
+ int j;
- if (port && port->qdma == qdma)
- netif_tx_wake_all_queues(port->dev);
+ if (!port)
+ continue;
+
+ if (port->qdma != qdma)
+ continue;
+
+ for (j = 0; j < port->dev->num_tx_queues; j++) {
+ if (airoha_qdma_get_txq(qdma, j) != qid)
+ continue;
+
+ netif_wake_subqueue(port->dev, j);
+ }
}
q->txq_stopped = false;
}
@@ -929,10 +940,9 @@ static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
q->queued--;
if (skb) {
- u16 queue = skb_get_queue_mapping(skb);
struct netdev_queue *txq;
- txq = netdev_get_tx_queue(skb->dev, queue);
+ txq = skb_get_tx_queue(skb->dev, skb);
netdev_tx_completed_queue(txq, 1, skb->len);
dev_kfree_skb_any(skb);
}
@@ -1737,14 +1747,11 @@ static int airoha_dev_stop(struct net_device *dev)
{
struct airoha_gdm_port *port = netdev_priv(dev);
struct airoha_qdma *qdma = port->qdma;
- int i, err;
+ int i;
netif_tx_disable(dev);
- err = airoha_set_vip_for_gdm_port(port, false);
- if (err)
- return err;
-
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++)
+ airoha_set_vip_for_gdm_port(port, false);
+ for (i = 0; i < dev->num_tx_queues; i++)
netdev_tx_reset_subqueue(dev, i);
airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id),
@@ -1786,11 +1793,8 @@ static int airoha_set_gdm2_loopback(struct airoha_gdm_port *port)
u32 val, pse_port, chan;
int i, src_port;
- /* Forward the traffic to the proper GDM port */
- pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3
- : FE_PSE_PORT_GDM4;
airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
- pse_port);
+ FE_PSE_PORT_DROP);
airoha_fe_clear(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
GDM_STRIP_CRC_MASK);
@@ -1808,6 +1812,11 @@ static int airoha_set_gdm2_loopback(struct airoha_gdm_port *port)
GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_MTU));
+ /* Forward the traffic to the proper GDM port */
+ pse_port = port->id == AIROHA_GDM3_IDX ? FE_PSE_PORT_GDM3
+ : FE_PSE_PORT_GDM4;
+ airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
+ pse_port);
/* Disable VIP and IFC for GDM2 */
airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX));
@@ -1997,12 +2006,12 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
struct netdev_queue *txq;
struct airoha_queue *q;
LIST_HEAD(tx_list);
+ int i = 0, qid;
void *data;
- int i, qid;
u16 index;
u8 fport;
- qid = skb_get_queue_mapping(skb) % ARRAY_SIZE(qdma->q_tx);
+ qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb));
tag = airoha_get_dsa_tag(skb, dev);
msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
@@ -2039,7 +2048,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
spin_lock_bh(&q->lock);
- txq = netdev_get_tx_queue(dev, qid);
+ txq = skb_get_tx_queue(dev, skb);
nr_frags = 1 + skb_shinfo(skb)->nr_frags;
if (q->queued + nr_frags >= q->ndesc) {
@@ -2057,7 +2066,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
list);
index = e - q->entry;
- for (i = 0; i < nr_frags; i++) {
+ while (true) {
struct airoha_qdma_desc *desc = &q->desc[index];
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
dma_addr_t addr;
@@ -2069,7 +2078,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
goto error_unmap;
list_move_tail(&e->list, &tx_list);
- e->skb = i ? NULL : skb;
+ e->skb = i == nr_frags - 1 ? skb : NULL;
e->dma_addr = addr;
e->dma_len = len;
@@ -2088,6 +2097,9 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
+ if (++i == nr_frags)
+ break;
+
data = skb_frag_address(frag);
len = skb_frag_size(frag);
}
@@ -2095,30 +2107,27 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
skb_tx_timestamp(skb);
netdev_tx_sent_queue(txq, skb->len);
+ if (q->ndesc - q->queued < q->free_thr) {
+ netif_tx_stop_queue(txq);
+ q->txq_stopped = true;
+ }
if (netif_xmit_stopped(txq) || !netdev_xmit_more())
airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid),
TX_RING_CPU_IDX_MASK,
FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
- if (q->ndesc - q->queued < q->free_thr) {
- netif_tx_stop_queue(txq);
- q->txq_stopped = true;
- }
-
spin_unlock_bh(&q->lock);
return NETDEV_TX_OK;
error_unmap:
- while (!list_empty(&tx_list)) {
- e = list_first_entry(&tx_list, struct airoha_queue_entry,
- list);
+ list_for_each_entry(e, &tx_list, list) {
dma_unmap_single(dev->dev.parent, e->dma_addr, e->dma_len,
DMA_TO_DEVICE);
e->dma_addr = 0;
- list_move_tail(&e->list, &q->tx_list);
}
+ list_splice(&tx_list, &q->tx_list);
spin_unlock_bh(&q->lock);
error:
@@ -2927,7 +2936,7 @@ static void airoha_metadata_dst_free(struct airoha_gdm_port *port)
if (!port->dsa_meta[i])
continue;
- metadata_dst_free(port->dsa_meta[i]);
+ dst_release(&port->dsa_meta[i]->dst);
}
}
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index e389d2f..4fad3ac 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -631,6 +631,11 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
#define airoha_qdma_clear(qdma, offset, val) \
airoha_rmw((qdma)->regs, (offset), (val), 0)
+static inline u16 airoha_qdma_get_txq(struct airoha_qdma *qdma, u16 qid)
+{
+ return qid % ARRAY_SIZE(qdma->q_tx);
+}
+
static inline bool airoha_is_lan_gdm_port(struct airoha_gdm_port *port)
{
/* GDM1 port on EN7581 SoC is connected to the lan dsa switch.
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c
index e67b592..8c86789 100644
--- a/drivers/net/ethernet/amazon/ena/ena_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_com.c
@@ -1782,20 +1782,23 @@ void ena_com_phc_destroy(struct ena_com_dev *ena_dev)
int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp)
{
- volatile struct ena_admin_phc_resp *resp = ena_dev->phc.virt_addr;
const ktime_t zero_system_time = ktime_set(0, 0);
struct ena_com_phc_info *phc = &ena_dev->phc;
+ volatile struct ena_admin_phc_resp *resp;
ktime_t expire_time;
ktime_t block_time;
unsigned long flags = 0;
int ret = 0;
+ spin_lock_irqsave(&phc->lock, flags);
+
if (!phc->active) {
+ spin_unlock_irqrestore(&phc->lock, flags);
netdev_err(ena_dev->net_device, "PHC feature is not active in the device\n");
return -EOPNOTSUPP;
}
- spin_lock_irqsave(&phc->lock, flags);
+ resp = ena_dev->phc.virt_addr;
/* Check if PHC is in blocked state */
if (unlikely(ktime_compare(phc->system_time, zero_system_time))) {
diff --git a/drivers/net/ethernet/amazon/ena/ena_phc.c b/drivers/net/ethernet/amazon/ena/ena_phc.c
index 7867e89..c2a3ff1 100644
--- a/drivers/net/ethernet/amazon/ena/ena_phc.c
+++ b/drivers/net/ethernet/amazon/ena/ena_phc.c
@@ -46,9 +46,12 @@ static int ena_phc_gettimex64(struct ptp_clock_info *clock_info,
spin_unlock_irqrestore(&phc_info->lock, flags);
+ if (rc)
+ return rc;
+
*ts = ns_to_timespec64(timestamp_nsec);
- return rc;
+ return 0;
}
static int ena_phc_settime64(struct ptp_clock_info *clock_info,
diff --git a/drivers/net/ethernet/amd/pcnet32.c b/drivers/net/ethernet/amd/pcnet32.c
index 911808a..4f3076d 100644
--- a/drivers/net/ethernet/amd/pcnet32.c
+++ b/drivers/net/ethernet/amd/pcnet32.c
@@ -1407,8 +1407,10 @@ static int pcnet32_poll(struct napi_struct *napi, int budget)
pcnet32_restart(dev, CSR0_START);
netif_wake_queue(dev);
}
+ spin_unlock_irqrestore(&lp->lock, flags);
if (work_done < budget && napi_complete_done(napi, work_done)) {
+ spin_lock_irqsave(&lp->lock, flags);
/* clear interrupt masks */
val = lp->a->read_csr(ioaddr, CSR3);
val &= 0x00ff;
@@ -1416,9 +1418,9 @@ static int pcnet32_poll(struct napi_struct *napi, int budget)
/* Set interrupt enable. */
lp->a->write_csr(ioaddr, CSR0, CSR0_INTEN);
+ spin_unlock_irqrestore(&lp->lock, flags);
}
- spin_unlock_irqrestore(&lp->lock, flags);
return work_done;
}
diff --git a/drivers/net/ethernet/amd/pds_core/debugfs.c b/drivers/net/ethernet/amd/pds_core/debugfs.c
index 04c5e3a..810a0cd 100644
--- a/drivers/net/ethernet/amd/pds_core/debugfs.c
+++ b/drivers/net/ethernet/amd/pds_core/debugfs.c
@@ -64,9 +64,14 @@ DEFINE_SHOW_ATTRIBUTE(identity);
void pdsc_debugfs_add_ident(struct pdsc *pdsc)
{
+ struct dentry *dentry;
+
/* This file will already exist in the reset flow */
- if (debugfs_lookup("identity", pdsc->dentry))
+ dentry = debugfs_lookup("identity", pdsc->dentry);
+ if (!IS_ERR_OR_NULL(dentry)) {
+ dput(dentry);
return;
+ }
debugfs_create_file("identity", 0400, pdsc->dentry,
pdsc, &identity_fops);
diff --git a/drivers/net/ethernet/amd/pds_core/dev.c b/drivers/net/ethernet/amd/pds_core/dev.c
index 2e1d0d0..bded6b3 100644
--- a/drivers/net/ethernet/amd/pds_core/dev.c
+++ b/drivers/net/ethernet/amd/pds_core/dev.c
@@ -162,12 +162,19 @@ static int pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds)
dev_dbg(dev, "DEVCMD %d %s after %ld secs\n",
opcode, pdsc_devcmd_str(opcode), duration / HZ);
- if ((!done || timeout) && running) {
+ if (!running) {
+ dev_err(dev, "DEVCMD %d %s fw not running\n",
+ opcode, pdsc_devcmd_str(opcode));
+ pdsc_devcmd_clean(pdsc);
+ return -ENXIO;
+ }
+
+ if (!done || timeout) {
dev_err(dev, "DEVCMD %d %s timeout, done %d timeout %d max_seconds=%d\n",
opcode, pdsc_devcmd_str(opcode), done, timeout,
max_seconds);
- err = -ETIMEDOUT;
pdsc_devcmd_clean(pdsc);
+ return -ETIMEDOUT;
}
status = pdsc_devcmd_status(pdsc);
diff --git a/drivers/net/ethernet/amd/pds_core/devlink.c b/drivers/net/ethernet/amd/pds_core/devlink.c
index b576be6..3f0e56b 100644
--- a/drivers/net/ethernet/amd/pds_core/devlink.c
+++ b/drivers/net/ethernet/amd/pds_core/devlink.c
@@ -122,12 +122,14 @@ int pdsc_dl_info_get(struct devlink *dl, struct devlink_info_req *req,
listlen = min(fw_list.num_fw_slots, ARRAY_SIZE(fw_list.fw_names));
for (i = 0; i < listlen; i++) {
+ char *fw_ver = fw_list.fw_names[i].fw_version;
+
if (i < ARRAY_SIZE(fw_slotnames))
strscpy(buf, fw_slotnames[i], sizeof(buf));
else
snprintf(buf, sizeof(buf), "fw.slot_%d", i);
- err = devlink_info_version_stored_put(req, buf,
- fw_list.fw_names[i].fw_version);
+ fw_ver[sizeof(fw_list.fw_names[i].fw_version) - 1] = '\0';
+ err = devlink_info_version_stored_put(req, buf, fw_ver);
if (err)
return err;
}
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h
index 60b7e53..3d3b090 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe.h
@@ -135,11 +135,11 @@
*/
#define XGBE_TSTAMP_SSINC 20
#define XGBE_TSTAMP_SNSINC 0
-#define XGBE_PTP_ACT_CLK_FREQ 500000000
+#define XGBE_PTP_ACT_CLK_FREQ (NSEC_PER_SEC / XGBE_TSTAMP_SSINC)
#define XGBE_V2_TSTAMP_SSINC 0xA
#define XGBE_V2_TSTAMP_SNSINC 0
-#define XGBE_V2_PTP_ACT_CLK_FREQ 1000000000
+#define XGBE_V2_PTP_ACT_CLK_FREQ (NSEC_PER_SEC / XGBE_V2_TSTAMP_SSINC)
/* Define maximum supported values */
#define XGBE_MAX_PPS_OUT 4
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
index b854b6b..2926e1e 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
@@ -910,7 +910,9 @@ static int xgene_mdiobus_register(struct xgene_enet_pdata *pdata,
return -ENXIO;
}
- return of_mdiobus_register(mdio, mdio_np);
+ ret = of_mdiobus_register(mdio, mdio_np);
+ of_node_put(mdio_np);
+ return ret;
}
/* Mask out all PHYs from auto probing. */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
index e9e38af..39e1b60 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
@@ -371,7 +371,7 @@ static void aq_pci_shutdown(struct pci_dev *pdev)
pci_disable_device(pdev);
if (system_state == SYSTEM_POWER_OFF) {
- pci_wake_from_d3(pdev, false);
+ pci_wake_from_d3(pdev, self->aq_hw->aq_nic_cfg->wol);
pci_set_power_state(pdev, PCI_D3hot);
}
}
diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c
index a5ab994..4e4794c 100644
--- a/drivers/net/ethernet/atheros/ag71xx.c
+++ b/drivers/net/ethernet/atheros/ag71xx.c
@@ -1856,6 +1856,9 @@ static int ag71xx_probe(struct platform_device *pdev)
ag71xx_int_disable(ag, AG71XX_INT_POLL);
ndev->irq = platform_get_irq(pdev, 0);
+ if (ndev->irq < 0)
+ return ndev->irq;
+
err = devm_request_irq(&pdev->dev, ndev->irq, ag71xx_interrupt,
0x0, dev_name(&pdev->dev), ndev);
if (err) {
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 8c55874..35e1f8f 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -3825,7 +3825,10 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp)
if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
if (!bp->max_tpa_v2)
return 0;
- bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
+ bp->max_tpa = min_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
+ /* Older P5 FW sets max_tpa_v2 low by mistake except NPAR */
+ if (bp->max_tpa <= 32 && BNXT_CHIP_P5(bp) && !BNXT_NPAR(bp))
+ bp->max_tpa = MAX_TPA_P5;
}
for (i = 0; i < bp->rx_nr_rings; i++) {
@@ -14385,13 +14388,28 @@ static void bnxt_unlock_sp(struct bnxt *bp)
netdev_unlock(bp->dev);
}
+/* Same as bnxt_lock_sp() with additional rtnl_lock */
+static void bnxt_rtnl_lock_sp(struct bnxt *bp)
+{
+ clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
+ rtnl_lock();
+ netdev_lock(bp->dev);
+}
+
+static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
+{
+ set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
+ netdev_unlock(bp->dev);
+ rtnl_unlock();
+}
+
/* Only called from bnxt_sp_task() */
static void bnxt_reset(struct bnxt *bp, bool silent)
{
- bnxt_lock_sp(bp);
+ bnxt_rtnl_lock_sp(bp);
if (test_bit(BNXT_STATE_OPEN, &bp->state))
bnxt_reset_task(bp, silent);
- bnxt_unlock_sp(bp);
+ bnxt_rtnl_unlock_sp(bp);
}
/* Only called from bnxt_sp_task() */
@@ -14399,9 +14417,9 @@ static void bnxt_rx_ring_reset(struct bnxt *bp)
{
int i;
- bnxt_lock_sp(bp);
+ bnxt_rtnl_lock_sp(bp);
if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
- bnxt_unlock_sp(bp);
+ bnxt_rtnl_unlock_sp(bp);
return;
}
/* Disable and flush TPA before resetting the RX ring */
@@ -14440,7 +14458,7 @@ static void bnxt_rx_ring_reset(struct bnxt *bp)
}
if (bp->flags & BNXT_FLAG_TPA)
bnxt_set_tpa(bp, true);
- bnxt_unlock_sp(bp);
+ bnxt_rtnl_unlock_sp(bp);
}
static void bnxt_fw_fatal_close(struct bnxt *bp)
@@ -15355,15 +15373,17 @@ static void bnxt_fw_reset_task(struct work_struct *work)
bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
fallthrough;
case BNXT_FW_RESET_STATE_OPENING:
- while (!netdev_trylock(bp->dev)) {
+ while (!rtnl_trylock()) {
bnxt_queue_fw_reset_work(bp, HZ / 10);
return;
}
+ netdev_lock(bp->dev);
rc = bnxt_open(bp->dev);
if (rc) {
netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
bnxt_fw_reset_abort(bp, rc);
netdev_unlock(bp->dev);
+ rtnl_unlock();
goto ulp_start;
}
@@ -15383,6 +15403,7 @@ static void bnxt_fw_reset_task(struct work_struct *work)
bnxt_dl_health_fw_status_update(bp, true);
}
netdev_unlock(bp->dev);
+ rtnl_unlock();
bnxt_ulp_start(bp);
bnxt_reenable_sriov(bp);
netdev_lock(bp->dev);
@@ -16376,7 +16397,7 @@ static int bnxt_queue_start(struct net_device *dev,
rc);
napi_enable_locked(&bnapi->napi);
bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
- bnxt_reset_task(bp, true);
+ netif_close(dev);
return rc;
}
@@ -17227,6 +17248,7 @@ static int bnxt_resume(struct device *device)
struct bnxt *bp = netdev_priv(dev);
int rc = 0;
+ rtnl_lock();
netdev_lock(dev);
rc = pci_enable_device(bp->pdev);
if (rc) {
@@ -17271,6 +17293,7 @@ static int bnxt_resume(struct device *device)
resume_exit:
netdev_unlock(bp->dev);
+ rtnl_unlock();
if (!rc) {
bnxt_ulp_start(bp);
bnxt_reenable_sriov(bp);
@@ -17360,9 +17383,14 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
netdev_info(bp->dev, "PCI Slot Reset\n");
- if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
- test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state))
- msleep(900);
+ if (test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) {
+ /* After DPC, the chip should return CRS when the vendor ID
+ * config register is read until it is ready. On all chips,
+ * this is not happening reliably so add a 5-second delay as a
+ * workaround.
+ */
+ msleep(5000);
+ }
netdev_lock(netdev);
@@ -17437,6 +17465,7 @@ static void bnxt_io_resume(struct pci_dev *pdev)
int err;
netdev_info(bp->dev, "PCI Slot Resume\n");
+ rtnl_lock();
netdev_lock(netdev);
err = bnxt_hwrm_func_qcaps(bp);
@@ -17454,6 +17483,7 @@ static void bnxt_io_resume(struct pci_dev *pdev)
netif_device_attach(netdev);
netdev_unlock(netdev);
+ rtnl_unlock();
if (!err) {
bnxt_ulp_start(bp);
bnxt_reenable_sriov(bp);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
index 53f336d..5d41dc1 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
@@ -419,31 +419,13 @@ void bnxt_ptp_reapply_pps(struct bnxt *bp)
}
}
-static int bnxt_get_target_cycles(struct bnxt_ptp_cfg *ptp, u64 target_ns,
- u64 *cycles_delta)
-{
- u64 cycles_now;
- u64 nsec_now, nsec_delta;
- int rc;
-
- rc = bnxt_refclk_read(ptp->bp, NULL, &cycles_now);
- if (rc)
- return rc;
-
- nsec_now = bnxt_timecounter_cyc2time(ptp, cycles_now);
-
- nsec_delta = target_ns - nsec_now;
- *cycles_delta = div64_u64(nsec_delta << ptp->cc.shift, ptp->cc.mult);
- return 0;
-}
-
static int bnxt_ptp_perout_cfg(struct bnxt_ptp_cfg *ptp,
struct ptp_clock_request *rq)
{
struct hwrm_func_ptp_cfg_input *req;
struct bnxt *bp = ptp->bp;
struct timespec64 ts;
- u64 target_ns, delta;
+ u64 target_ns;
u16 enables;
int rc;
@@ -451,10 +433,6 @@ static int bnxt_ptp_perout_cfg(struct bnxt_ptp_cfg *ptp,
ts.tv_nsec = rq->perout.start.nsec;
target_ns = timespec64_to_ns(&ts);
- rc = bnxt_get_target_cycles(ptp, target_ns, &delta);
- if (rc)
- return rc;
-
rc = hwrm_req_init(bp, req, HWRM_FUNC_PTP_CFG);
if (rc)
return rc;
@@ -468,7 +446,10 @@ static int bnxt_ptp_perout_cfg(struct bnxt_ptp_cfg *ptp,
req->ptp_freq_adj_dll_phase = 0;
req->ptp_freq_adj_ext_period = cpu_to_le32(NSEC_PER_SEC);
req->ptp_freq_adj_ext_up = 0;
- req->ptp_freq_adj_ext_phase_lower = cpu_to_le32(delta);
+ req->ptp_freq_adj_ext_phase_lower =
+ cpu_to_le32(lower_32_bits(target_ns));
+ req->ptp_freq_adj_ext_phase_upper =
+ cpu_to_le32(upper_32_bits(target_ns));
return hwrm_req_send(bp, req);
}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
index 052bf69..5c75193 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
@@ -175,8 +175,14 @@ int bnxt_register_dev(struct bnxt_en_dev *edev,
ulp->handle = handle;
rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
- if (test_bit(BNXT_STATE_OPEN, &bp->state))
- bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[BNXT_VNIC_DEFAULT]);
+ if (test_bit(BNXT_STATE_OPEN, &bp->state)) {
+ rc = bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[BNXT_VNIC_DEFAULT]);
+ if (rc) {
+ netdev_err(dev, "Failed to configure dual VNIC mode\n");
+ RCU_INIT_POINTER(ulp->ulp_ops, NULL);
+ goto exit;
+ }
+ }
edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp);
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index 54f71b1..7c11cf9 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -1368,13 +1368,12 @@ void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
bcmgenet_writel(reg, priv->base + off);
- /* Do the same for thing for RBUF */
+ /* RBUF EEE/PM can break the RX path on GENET. Keep it disabled. */
reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
- if (enable)
- reg |= RBUF_EEE_EN | RBUF_PM_EN;
- else
+ if (reg & (RBUF_EEE_EN | RBUF_PM_EN)) {
reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
- bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
+ bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
+ }
if (!enable && priv->clk_eee_enabled) {
clk_disable_unprepare(priv->clk_eee);
diff --git a/drivers/net/ethernet/cirrus/cs89x0.c b/drivers/net/ethernet/cirrus/cs89x0.c
index fa58579..b4bfd6c 100644
--- a/drivers/net/ethernet/cirrus/cs89x0.c
+++ b/drivers/net/ethernet/cirrus/cs89x0.c
@@ -1271,7 +1271,6 @@ static const struct net_device_ops net_ops = {
static void __init reset_chip(struct net_device *dev)
{
-#if !defined(CONFIG_MACH_MX31ADS)
struct net_local *lp = netdev_priv(dev);
unsigned long reset_start_time;
@@ -1298,7 +1297,6 @@ static void __init reset_chip(struct net_device *dev)
while ((readreg(dev, PP_SelfST) & INIT_DONE) == 0 &&
time_before(jiffies, reset_start_time + 2))
;
-#endif /* !CONFIG_MACH_MX31ADS */
}
/* This is the real probe routine.
diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c
index 4824232..4c76222 100644
--- a/drivers/net/ethernet/cortina/gemini.c
+++ b/drivers/net/ethernet/cortina/gemini.c
@@ -122,6 +122,9 @@ struct gemini_ethernet_port {
struct napi_struct napi;
struct hrtimer rx_coalesce_timer;
unsigned int rx_coalesce_nsecs;
+ struct sk_buff *rx_skb;
+ unsigned int rx_frag_nr;
+
unsigned int freeq_refill;
struct gmac_txq txq[TX_QUEUE_NUM];
unsigned int txq_order;
@@ -1442,10 +1445,11 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
unsigned short m = (1 << port->rxq_order) - 1;
struct gemini_ethernet *geth = port->geth;
void __iomem *ptr_reg = port->rxq_rwptr;
+ unsigned int frag_nr = port->rx_frag_nr;
+ struct sk_buff *skb = port->rx_skb;
unsigned int frame_len, frag_len;
struct gmac_rxdesc *rx = NULL;
struct gmac_queue_page *gpage;
- static struct sk_buff *skb;
union gmac_rxdesc_0 word0;
union gmac_rxdesc_1 word1;
union gmac_rxdesc_3 word3;
@@ -1455,7 +1459,6 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
unsigned short r, w;
union dma_rwptr rw;
dma_addr_t mapping;
- int frag_nr = 0;
spin_lock_irqsave(&geth->irq_lock, flags);
rw.bits32 = readl(ptr_reg);
@@ -1491,6 +1494,12 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
if (!gpage) {
dev_err(geth->dev, "could not find mapping\n");
+ port->stats.rx_dropped++;
+ if (skb) {
+ napi_free_frags(&port->napi);
+ skb = NULL;
+ frag_nr = 0;
+ }
continue;
}
page = gpage->page;
@@ -1499,6 +1508,8 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
if (skb) {
napi_free_frags(&port->napi);
port->stats.rx_dropped++;
+ skb = NULL;
+ frag_nr = 0;
}
skb = gmac_skb_if_good_frame(port, word0, frame_len);
@@ -1533,6 +1544,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
if (word3.bits32 & EOF_BIT) {
napi_gro_frags(&port->napi);
skb = NULL;
+ frag_nr = 0;
--budget;
}
continue;
@@ -1541,6 +1553,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
if (skb) {
napi_free_frags(&port->napi);
skb = NULL;
+ frag_nr = 0;
}
if (mapping)
@@ -1549,6 +1562,8 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
port->stats.rx_dropped++;
}
+ port->rx_skb = skb;
+ port->rx_frag_nr = frag_nr;
writew(r, ptr_reg);
return budget;
}
@@ -1876,6 +1891,8 @@ static int gmac_stop(struct net_device *netdev)
gmac_disable_tx_rx(netdev);
gmac_stop_dma(port);
napi_disable(&port->napi);
+ port->rx_skb = NULL;
+ port->rx_frag_nr = 0;
gmac_enable_irq(netdev, 0);
gmac_cleanup_rxq(netdev);
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h
index e663bb5..e691144 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc.h
@@ -330,6 +330,7 @@ struct enetc_si {
struct workqueue_struct *workqueue;
struct work_struct rx_mode_task;
struct dentry *debugfs_root;
+ struct enetc_msg_swbd msg; /* Only valid for VSI */
};
#define ENETC_SI_ALIGN 32
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
index 662e4fb..e58cc81 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
@@ -56,11 +56,21 @@ static inline u32 enetc_vsi_set_msize(u32 size)
}
#define ENETC_PSIMSGRR 0x204
-#define ENETC_PSIMSGRR_MR_MASK GENMASK(2, 1)
-#define ENETC_PSIMSGRR_MR(n) BIT((n) + 1) /* n = VSI index */
#define ENETC_PSIVMSGRCVAR0(n) (0x210 + (n) * 0x8) /* n = VSI index */
#define ENETC_PSIVMSGRCVAR1(n) (0x214 + (n) * 0x8)
+/* Message received mask, n is the active number of VSIs.
+ * It is available for ENETC_PSIMSGRR, ENETC_PSIIER, and
+ * ENETC_PSIIDR registers.
+ */
+#define ENETC_PSIMR_MASK(n) \
+ ({ typeof(n) _n = (n); (_n) ? GENMASK((_n), 1) : 0; })
+
+/* Message received bit, n is VSI index. It is available for
+ * ENETC_PSIMSGRR, ENETC_PSIIER, and ENETC_PSIIDR registers.
+ */
+#define ENETC_PSIMR_BIT(n) BIT((n) + 1)
+
#define ENETC_VSIMSGSR 0x204 /* RO */
#define ENETC_VSIMSGSR_MB BIT(0)
#define ENETC_VSIMSGSR_MS BIT(1)
@@ -94,7 +104,6 @@ static inline u32 enetc_vsi_set_msize(u32 size)
#define ENETC_SICAPR1 0x904
#define ENETC_PSIIER 0xa00
-#define ENETC_PSIIER_MR_MASK GENMASK(2, 1)
#define ENETC_PSIIDR 0xa08
#define ENETC_SITXIDR 0xa18
#define ENETC_SIRXIDR 0xa28
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_msg.c b/drivers/net/ethernet/freescale/enetc/enetc_msg.c
index 40d22eb..c09635e 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_msg.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_msg.c
@@ -3,18 +3,25 @@
#include "enetc_pf.h"
-static void enetc_msg_disable_mr_int(struct enetc_hw *hw)
+static void enetc_msg_disable_mr_int(struct enetc_pf *pf)
{
- u32 psiier = enetc_rd(hw, ENETC_PSIIER);
+ struct enetc_hw *hw = &pf->si->hw;
+ u32 psiier;
+
+ psiier = enetc_rd(hw, ENETC_PSIIER) & ~ENETC_PSIMR_MASK(pf->num_vfs);
+
/* disable MR int source(s) */
- enetc_wr(hw, ENETC_PSIIER, psiier & ~ENETC_PSIIER_MR_MASK);
+ enetc_wr(hw, ENETC_PSIIER, psiier);
}
-static void enetc_msg_enable_mr_int(struct enetc_hw *hw)
+static void enetc_msg_enable_mr_int(struct enetc_pf *pf)
{
- u32 psiier = enetc_rd(hw, ENETC_PSIIER);
+ struct enetc_hw *hw = &pf->si->hw;
+ u32 psiier;
- enetc_wr(hw, ENETC_PSIIER, psiier | ENETC_PSIIER_MR_MASK);
+ psiier = enetc_rd(hw, ENETC_PSIIER) | ENETC_PSIMR_MASK(pf->num_vfs);
+
+ enetc_wr(hw, ENETC_PSIIER, psiier);
}
static irqreturn_t enetc_msg_psi_msix(int irq, void *data)
@@ -22,7 +29,7 @@ static irqreturn_t enetc_msg_psi_msix(int irq, void *data)
struct enetc_si *si = (struct enetc_si *)data;
struct enetc_pf *pf = enetc_si_priv(si);
- enetc_msg_disable_mr_int(&si->hw);
+ enetc_msg_disable_mr_int(pf);
schedule_work(&pf->msg_task);
return IRQ_HANDLED;
@@ -31,33 +38,35 @@ static irqreturn_t enetc_msg_psi_msix(int irq, void *data)
static void enetc_msg_task(struct work_struct *work)
{
struct enetc_pf *pf = container_of(work, struct enetc_pf, msg_task);
+ u32 mr_mask = ENETC_PSIMR_MASK(pf->num_vfs);
struct enetc_hw *hw = &pf->si->hw;
- unsigned long mr_mask;
+ u32 mr_status;
int i;
- for (;;) {
- mr_mask = enetc_rd(hw, ENETC_PSIMSGRR) & ENETC_PSIMSGRR_MR_MASK;
- if (!mr_mask) {
- /* re-arm MR interrupts, w1c the IDR reg */
- enetc_wr(hw, ENETC_PSIIDR, ENETC_PSIIER_MR_MASK);
- enetc_msg_enable_mr_int(hw);
- return;
- }
+ mr_status = (enetc_rd(hw, ENETC_PSIMSGRR) & mr_mask) |
+ (enetc_rd(hw, ENETC_PSIIDR) & mr_mask);
+ if (!mr_status)
+ goto out;
- for (i = 0; i < pf->num_vfs; i++) {
- u32 psimsgrr;
- u16 msg_code;
+ for (i = 0; i < pf->num_vfs; i++) {
+ u32 psimsgrr;
+ u16 msg_code;
- if (!(ENETC_PSIMSGRR_MR(i) & mr_mask))
- continue;
+ if (!(ENETC_PSIMR_BIT(i) & mr_status))
+ continue;
- enetc_msg_handle_rxmsg(pf, i, &msg_code);
+ enetc_msg_handle_rxmsg(pf, i, &msg_code);
- psimsgrr = ENETC_SIMSGSR_SET_MC(msg_code);
- psimsgrr |= ENETC_PSIMSGRR_MR(i); /* w1c */
- enetc_wr(hw, ENETC_PSIMSGRR, psimsgrr);
- }
+ /* w1c to clear the corresponding VF MR bit */
+ enetc_wr(hw, ENETC_PSIIDR, ENETC_PSIMR_BIT(i));
+
+ psimsgrr = ENETC_SIMSGSR_SET_MC(msg_code);
+ psimsgrr |= ENETC_PSIMR_BIT(i); /* w1c */
+ enetc_wr(hw, ENETC_PSIMSGRR, psimsgrr);
}
+
+out:
+ enetc_msg_enable_mr_int(pf);
}
/* Init */
@@ -96,12 +105,12 @@ static void enetc_msg_free_mbx(struct enetc_si *si, int idx)
struct enetc_hw *hw = &si->hw;
struct enetc_msg_swbd *msg;
+ enetc_wr(hw, ENETC_PSIVMSGRCVAR0(idx), 0);
+ enetc_wr(hw, ENETC_PSIVMSGRCVAR1(idx), 0);
+
msg = &pf->rxmsg[idx];
dma_free_coherent(&si->pdev->dev, msg->size, msg->vaddr, msg->dma);
memset(msg, 0, sizeof(*msg));
-
- enetc_wr(hw, ENETC_PSIVMSGRCVAR0(idx), 0);
- enetc_wr(hw, ENETC_PSIVMSGRCVAR1(idx), 0);
}
int enetc_msg_psi_init(struct enetc_pf *pf)
@@ -109,6 +118,15 @@ int enetc_msg_psi_init(struct enetc_pf *pf)
struct enetc_si *si = pf->si;
int vector, i, err;
+ for (i = 0; i < pf->num_vfs; i++) {
+ err = enetc_msg_alloc_mbx(si, i);
+ if (err)
+ goto free_mbx;
+ }
+
+ /* initialize PSI mailbox */
+ INIT_WORK(&pf->msg_task, enetc_msg_task);
+
/* register message passing interrupt handler */
snprintf(pf->msg_int_name, sizeof(pf->msg_int_name), "%s-vfmsg",
si->ndev->name);
@@ -117,32 +135,21 @@ int enetc_msg_psi_init(struct enetc_pf *pf)
if (err) {
dev_err(&si->pdev->dev,
"PSI messaging: request_irq() failed!\n");
- return err;
+ goto free_mbx;
}
/* set one IRQ entry for PSI message receive notification (SI int) */
enetc_wr(&si->hw, ENETC_SIMSIVR, ENETC_SI_INT_IDX);
- /* initialize PSI mailbox */
- INIT_WORK(&pf->msg_task, enetc_msg_task);
-
- for (i = 0; i < pf->num_vfs; i++) {
- err = enetc_msg_alloc_mbx(si, i);
- if (err)
- goto err_init_mbx;
- }
-
/* enable MR interrupts */
- enetc_msg_enable_mr_int(&si->hw);
+ enetc_msg_enable_mr_int(pf);
return 0;
-err_init_mbx:
+free_mbx:
for (i--; i >= 0; i--)
enetc_msg_free_mbx(si, i);
- free_irq(vector, si);
-
return err;
}
@@ -151,14 +158,17 @@ void enetc_msg_psi_free(struct enetc_pf *pf)
struct enetc_si *si = pf->si;
int i;
- cancel_work_sync(&pf->msg_task);
-
/* disable MR interrupts */
- enetc_msg_disable_mr_int(&si->hw);
-
- for (i = 0; i < pf->num_vfs; i++)
- enetc_msg_free_mbx(si, i);
+ enetc_msg_disable_mr_int(pf);
/* de-register message passing interrupt handler */
free_irq(pci_irq_vector(si->pdev, ENETC_SI_INT_IDX), si);
+
+ cancel_work_sync(&pf->msg_task);
+
+ /* MR interrupts may be re-enabled by workqueue */
+ enetc_msg_disable_mr_int(pf);
+
+ for (i = 0; i < pf->num_vfs; i++)
+ enetc_msg_free_mbx(si, i);
}
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
index a12fd54..3206b3d 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
@@ -252,8 +252,12 @@ static int enetc_pf_set_vf_mac(struct net_device *ndev, int vf, u8 *mac)
return -EADDRNOTAVAIL;
vf_state = &pf->vf_state[vf];
+
+ mutex_lock(&vf_state->lock);
vf_state->flags |= ENETC_VF_FLAG_PF_SET_MAC;
enetc_pf_set_primary_mac_addr(&priv->si->hw, vf + 1, mac);
+ mutex_unlock(&vf_state->lock);
+
return 0;
}
@@ -478,49 +482,77 @@ static void enetc_configure_port(struct enetc_pf *pf)
/* Messaging */
static u16 enetc_msg_pf_set_vf_primary_mac_addr(struct enetc_pf *pf,
- int vf_id)
+ int vf_id, void *msg)
{
struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
- struct enetc_msg_swbd *msg = &pf->rxmsg[vf_id];
- struct enetc_msg_cmd_set_primary_mac *cmd;
+ struct enetc_msg_cmd_set_primary_mac *cmd = msg;
struct device *dev = &pf->si->pdev->dev;
- u16 cmd_id;
+ u16 cmd_id = cmd->header.id;
char *addr;
- cmd = (struct enetc_msg_cmd_set_primary_mac *)msg->vaddr;
- cmd_id = cmd->header.id;
if (cmd_id != ENETC_MSG_CMD_MNG_ADD)
return ENETC_MSG_CMD_STATUS_FAIL;
addr = cmd->mac.sa_data;
- if (vf_state->flags & ENETC_VF_FLAG_PF_SET_MAC)
- dev_warn(dev, "Attempt to override PF set mac addr for VF%d\n",
- vf_id);
- else
- enetc_pf_set_primary_mac_addr(&pf->si->hw, vf_id + 1, addr);
+ if (!is_valid_ether_addr(addr)) {
+ dev_err_ratelimited(dev, "VF%d attempted to set invalid MAC\n",
+ vf_id);
+ return ENETC_MSG_CMD_STATUS_FAIL;
+ }
+
+ mutex_lock(&vf_state->lock);
+ if (vf_state->flags & ENETC_VF_FLAG_PF_SET_MAC) {
+ mutex_unlock(&vf_state->lock);
+ dev_err_ratelimited(dev,
+ "VF%d attempted to override PF set MAC\n",
+ vf_id);
+ return ENETC_MSG_CMD_STATUS_FAIL;
+ }
+
+ enetc_pf_set_primary_mac_addr(&pf->si->hw, vf_id + 1, addr);
+ mutex_unlock(&vf_state->lock);
return ENETC_MSG_CMD_STATUS_OK;
}
void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id, u16 *status)
{
- struct enetc_msg_swbd *msg = &pf->rxmsg[vf_id];
+ struct enetc_msg_swbd *msg_swbd = &pf->rxmsg[vf_id];
struct device *dev = &pf->si->pdev->dev;
struct enetc_msg_cmd_header *cmd_hdr;
u16 cmd_type;
+ u8 *msg;
- *status = ENETC_MSG_CMD_STATUS_OK;
- cmd_hdr = (struct enetc_msg_cmd_header *)msg->vaddr;
+ msg = kzalloc_objs(*msg, msg_swbd->size);
+ if (!msg) {
+ dev_err_ratelimited(dev,
+ "Failed to allocate message buffer\n");
+ *status = ENETC_MSG_CMD_STATUS_FAIL;
+ return;
+ }
+
+ /* Currently, only ENETC_MSG_CMD_MNG_MAC command is supported, so
+ * only sizeof(struct enetc_msg_cmd_set_primary_mac) bytes need to
+ * be copied. This data already includes the cmd_type field, so it
+ * can correctly return an error code.
+ */
+ memcpy(msg, msg_swbd->vaddr,
+ sizeof(struct enetc_msg_cmd_set_primary_mac));
+ cmd_hdr = (struct enetc_msg_cmd_header *)msg;
cmd_type = cmd_hdr->type;
switch (cmd_type) {
case ENETC_MSG_CMD_MNG_MAC:
- *status = enetc_msg_pf_set_vf_primary_mac_addr(pf, vf_id);
+ *status = enetc_msg_pf_set_vf_primary_mac_addr(pf, vf_id, msg);
break;
default:
- dev_err(dev, "command not supported (cmd_type: 0x%x)\n",
- cmd_type);
+ *status = ENETC_MSG_CMD_STATUS_FAIL;
+ dev_err_ratelimited(dev,
+ "command not supported (cmd_type: 0x%x)\n",
+ cmd_type);
}
+
+ kfree(msg);
}
#ifdef CONFIG_PCI_IOV
@@ -531,9 +563,9 @@ static int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
int err;
if (!num_vfs) {
+ pci_disable_sriov(pdev);
enetc_msg_psi_free(pf);
pf->num_vfs = 0;
- pci_disable_sriov(pdev);
} else {
pf->num_vfs = num_vfs;
@@ -960,8 +992,13 @@ static int enetc_pf_probe(struct pci_dev *pdev,
if (pf->total_vfs) {
pf->vf_state = kzalloc_objs(struct enetc_vf_state,
pf->total_vfs);
- if (!pf->vf_state)
+ if (!pf->vf_state) {
+ err = -ENOMEM;
goto err_alloc_vf_state;
+ }
+
+ for (int i = 0; i < pf->total_vfs; i++)
+ mutex_init(&pf->vf_state[i].lock);
}
err = enetc_setup_mac_addresses(node, pf);
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.h b/drivers/net/ethernet/freescale/enetc/enetc_pf.h
index ae407e9..35d4848 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.h
@@ -14,6 +14,7 @@ enum enetc_vf_flags {
};
struct enetc_vf_state {
+ struct mutex lock; /* Prevent concurrent access */
enum enetc_vf_flags flags;
};
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_vf.c b/drivers/net/ethernet/freescale/enetc/enetc_vf.c
index 6c4b374..df8e95c 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_vf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_vf.c
@@ -17,11 +17,36 @@ static void enetc_msg_vsi_write_msg(struct enetc_hw *hw,
enetc_wr(hw, ENETC_VSIMSGSNDAR0, val);
}
+static void enetc_msg_dma_free(struct device *dev, struct enetc_msg_swbd *msg)
+{
+ if (msg->vaddr) {
+ dma_free_coherent(dev, msg->size, msg->vaddr, msg->dma);
+ msg->vaddr = NULL;
+ }
+}
+
static int enetc_msg_vsi_send(struct enetc_si *si, struct enetc_msg_swbd *msg)
{
+ struct device *dev = &si->pdev->dev;
int timeout = 100;
u32 vsimsgsr;
+ /* The VSI mailbox may be busy if last message was not yet processed
+ * by PSI. So need to check the mailbox status before sending.
+ */
+ vsimsgsr = enetc_rd(&si->hw, ENETC_VSIMSGSR);
+ if (vsimsgsr & ENETC_VSIMSGSR_MB) {
+ /* It is safe to free the DMA buffer here, the caller does
+ * not access the DMA buffer if enetc_msg_vsi_send() fails.
+ */
+ enetc_msg_dma_free(dev, msg);
+ dev_err(dev, "VSI mailbox is busy\n");
+ return -EIO;
+ }
+
+ /* Free the DMA buffer of the last message */
+ enetc_msg_dma_free(dev, &si->msg);
+ si->msg = *msg;
enetc_msg_vsi_write_msg(&si->hw, msg);
do {
@@ -32,12 +57,15 @@ static int enetc_msg_vsi_send(struct enetc_si *si, struct enetc_msg_swbd *msg)
usleep_range(1000, 2000);
} while (--timeout);
- if (!timeout)
+ if (!timeout) {
+ dev_err(dev, "VSI mailbox timeout\n");
+
return -ETIMEDOUT;
+ }
/* check for message delivery error */
if (vsimsgsr & ENETC_VSIMSGSR_MS) {
- dev_err(&si->pdev->dev, "VSI command execute error: %d\n",
+ dev_err(dev, "VSI command execute error: %d\n",
ENETC_SIMSGSR_GET_MC(vsimsgsr));
return -EIO;
}
@@ -50,7 +78,6 @@ static int enetc_msg_vsi_set_primary_mac_addr(struct enetc_ndev_priv *priv,
{
struct enetc_msg_cmd_set_primary_mac *cmd;
struct enetc_msg_swbd msg;
- int err;
msg.size = ALIGN(sizeof(struct enetc_msg_cmd_set_primary_mac), 64);
msg.vaddr = dma_alloc_coherent(priv->dev, msg.size, &msg.dma,
@@ -67,11 +94,7 @@ static int enetc_msg_vsi_set_primary_mac_addr(struct enetc_ndev_priv *priv,
memcpy(&cmd->mac, saddr, sizeof(struct sockaddr));
/* send the command and wait */
- err = enetc_msg_vsi_send(priv->si, &msg);
-
- dma_free_coherent(priv->dev, msg.size, msg.vaddr, msg.dma);
-
- return err;
+ return enetc_msg_vsi_send(priv->si, &msg);
}
static int enetc_vf_set_mac_addr(struct net_device *ndev, void *addr)
@@ -259,6 +282,7 @@ static void enetc_vf_remove(struct pci_dev *pdev)
{
struct enetc_si *si = pci_get_drvdata(pdev);
struct enetc_ndev_priv *priv;
+ struct enetc_msg_swbd msg;
priv = netdev_priv(si->ndev);
unregister_netdev(si->ndev);
@@ -270,7 +294,9 @@ static void enetc_vf_remove(struct pci_dev *pdev)
free_netdev(si->ndev);
+ msg = si->msg;
enetc_pci_remove(pdev);
+ enetc_msg_dma_free(&pdev->dev, &msg);
}
static const struct pci_device_id enetc_vf_id_table[] = {
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index f89aa94..6ebde65 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -5594,6 +5594,7 @@ static int fec_resume(struct device *dev)
if (fep->rpm_active)
pm_runtime_force_resume(dev);
+ pinctrl_pm_select_default_state(&fep->pdev->dev);
ret = fec_enet_clk_enable(ndev, true);
if (ret) {
rtnl_unlock();
@@ -5610,8 +5611,6 @@ static int fec_resume(struct device *dev)
val &= ~(FEC_ECR_MAGICEN | FEC_ECR_SLEEP);
writel(val, fep->hwp + FEC_ECNTRL);
fep->wol_flag &= ~FEC_WOL_FLAG_SLEEP_ON;
- } else {
- pinctrl_pm_select_default_state(&fep->pdev->dev);
}
fec_restart(ndev);
netif_tx_lock_bh(ndev);
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
index 068da2f..f721e98 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
@@ -420,6 +420,9 @@ static int hbg_pci_init(struct pci_dev *pdev)
return -ENOMEM;
pci_set_master(pdev);
+ pcie_capability_clear_word(pdev, PCI_EXP_DEVCTL,
+ PCI_EXP_DEVCTL_RELAX_EN);
+ pci_save_state(pdev);
return 0;
}
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
index a4ea92c..0ae3149 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
@@ -452,12 +452,12 @@ static bool hbg_sync_data_from_hw(struct hbg_priv *priv,
{
struct hbg_rx_desc *rx_desc;
- /* make sure HW write desc complete */
- dma_rmb();
-
dma_sync_single_for_cpu(&priv->pdev->dev, buffer->page_dma,
buffer->page_size, DMA_FROM_DEVICE);
+ /* make sure HW write desc complete */
+ dma_rmb();
+
rx_desc = (struct hbg_rx_desc *)buffer->page_addr;
return FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2) != 0;
}
diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
index 58cc314..73e051d 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -1756,6 +1756,27 @@ static int ibmveth_set_mac_addr(struct net_device *dev, void *p)
return 0;
}
+static netdev_features_t ibmveth_features_check(struct sk_buff *skb,
+ struct net_device *dev,
+ netdev_features_t features)
+{
+ /* Some physical adapters do not support segmentation offload with
+ * MSS < 224. Disable GSO for such packets to avoid adapter freeze.
+ * Note: Single-segment packets (gso_segs == 1) don't need this check
+ * as they bypass the LSO path and are transmitted without segmentation.
+ */
+ if (skb_is_gso(skb)) {
+ if (skb_shinfo(skb)->gso_size < IBMVETH_MIN_LSO_MSS) {
+ netdev_warn_once(dev,
+ "MSS %u too small for LSO, disabling GSO\n",
+ skb_shinfo(skb)->gso_size);
+ features &= ~NETIF_F_GSO_MASK;
+ }
+ }
+
+ return vlan_features_check(skb, features);
+}
+
static const struct net_device_ops ibmveth_netdev_ops = {
.ndo_open = ibmveth_open,
.ndo_stop = ibmveth_close,
@@ -1767,6 +1788,7 @@ static const struct net_device_ops ibmveth_netdev_ops = {
.ndo_set_features = ibmveth_set_features,
.ndo_validate_addr = eth_validate_addr,
.ndo_set_mac_address = ibmveth_set_mac_addr,
+ .ndo_features_check = ibmveth_features_check,
#ifdef CONFIG_NET_POLL_CONTROLLER
.ndo_poll_controller = ibmveth_poll_controller,
#endif
diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h
index 068f99d..d877136 100644
--- a/drivers/net/ethernet/ibm/ibmveth.h
+++ b/drivers/net/ethernet/ibm/ibmveth.h
@@ -37,6 +37,7 @@
#define IBMVETH_ILLAN_IPV4_TCP_CSUM 0x0000000000000002UL
#define IBMVETH_ILLAN_ACTIVE_TRUNK 0x0000000000000001UL
+#define IBMVETH_MIN_LSO_MSS 224 /* Minimum MSS for LSO */
/* hcall macros */
#define h_register_logical_lan(ua, buflst, rxq, fltlst, mac) \
plpar_hcall_norets(H_REGISTER_LOGICAL_LAN, ua, buflst, rxq, fltlst, mac)
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index dcb50c2..83e7809 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -1318,6 +1318,7 @@ void i40e_ptp_restore_hw_time(struct i40e_pf *pf);
void i40e_ptp_init(struct i40e_pf *pf);
void i40e_ptp_stop(struct i40e_pf *pf);
int i40e_ptp_alloc_pins(struct i40e_pf *pf);
+void i40e_ptp_free_pins(struct i40e_pf *pf);
int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset);
int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi);
int i40e_get_partition_bw_setting(struct i40e_pf *pf);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 028bd50..6d4f921 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -16108,9 +16108,11 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
/* Unwind what we've done if something failed in the setup */
err_vsis:
set_bit(__I40E_DOWN, pf->state);
+ i40e_ptp_stop(pf);
i40e_clear_interrupt_scheme(pf);
kfree(pf->vsi);
err_switch_setup:
+ i40e_ptp_free_pins(pf);
i40e_reset_interrupt_capability(pf);
timer_shutdown_sync(&pf->service_timer);
err_mac_addr:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 404a716..7d07c38 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -940,12 +940,13 @@ int i40e_ptp_hwtstamp_get(struct net_device *netdev,
*
* Release memory allocated for PTP pins.
**/
-static void i40e_ptp_free_pins(struct i40e_pf *pf)
+void i40e_ptp_free_pins(struct i40e_pf *pf)
{
if (i40e_is_ptp_pin_dev(&pf->hw)) {
kfree(pf->ptp_pins);
kfree(pf->ptp_caps.pin_config);
pf->ptp_pins = NULL;
+ pf->ptp_caps.pin_config = NULL;
}
}
diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h
index e9fb0a0..050f824 100644
--- a/drivers/net/ethernet/intel/iavf/iavf.h
+++ b/drivers/net/ethernet/intel/iavf/iavf.h
@@ -158,11 +158,10 @@ struct iavf_vlan {
enum iavf_vlan_state_t {
IAVF_VLAN_INVALID,
IAVF_VLAN_ADD, /* filter needs to be added */
- IAVF_VLAN_IS_NEW, /* filter is new, wait for PF answer */
- IAVF_VLAN_ACTIVE, /* filter is accepted by PF */
- IAVF_VLAN_DISABLE, /* filter needs to be deleted by PF, then marked INACTIVE */
- IAVF_VLAN_INACTIVE, /* filter is inactive, we are in IFF_DOWN */
- IAVF_VLAN_REMOVE, /* filter needs to be removed from list */
+ IAVF_VLAN_ADDING, /* ADD sent to PF, waiting for response */
+ IAVF_VLAN_ACTIVE, /* PF confirmed, filter is in HW */
+ IAVF_VLAN_REMOVE, /* filter queued for DEL from PF */
+ IAVF_VLAN_REMOVING, /* DEL sent to PF, waiting for response */
};
struct iavf_vlan_filter {
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index 3c1465c..d2914c5 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -757,10 +757,10 @@ iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
adapter->num_vlan_filters++;
iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
} else if (f->state == IAVF_VLAN_REMOVE) {
- /* Re-add the filter since we cannot tell whether the
- * pending delete has already been processed by the PF.
- * A duplicate add is harmless.
- */
+ /* DEL not yet sent to PF, cancel it */
+ f->state = IAVF_VLAN_ACTIVE;
+ } else if (f->state == IAVF_VLAN_REMOVING) {
+ /* DEL already sent to PF, re-add after completion */
f->state = IAVF_VLAN_ADD;
iavf_schedule_aq_request(adapter,
IAVF_FLAG_AQ_ADD_VLAN_FILTER);
@@ -791,37 +791,19 @@ static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
list_del(&f->list);
kfree(f);
adapter->num_vlan_filters--;
- } else {
+ } else if (f->state != IAVF_VLAN_REMOVING) {
f->state = IAVF_VLAN_REMOVE;
iavf_schedule_aq_request(adapter,
IAVF_FLAG_AQ_DEL_VLAN_FILTER);
}
+ /* If REMOVING, DEL is already sent to PF; completion
+ * handler will free the filter when PF confirms.
+ */
}
spin_unlock_bh(&adapter->mac_vlan_list_lock);
}
-/**
- * iavf_restore_filters
- * @adapter: board private structure
- *
- * Restore existing non MAC filters when VF netdev comes back up
- **/
-static void iavf_restore_filters(struct iavf_adapter *adapter)
-{
- struct iavf_vlan_filter *f;
-
- /* re-add all VLAN filters */
- spin_lock_bh(&adapter->mac_vlan_list_lock);
-
- list_for_each_entry(f, &adapter->vlan_filter_list, list) {
- if (f->state == IAVF_VLAN_INACTIVE)
- f->state = IAVF_VLAN_ADD;
- }
-
- spin_unlock_bh(&adapter->mac_vlan_list_lock);
- adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
-}
/**
* iavf_get_num_vlans_added - get number of VLANs added
@@ -1246,13 +1228,12 @@ static void iavf_up_complete(struct iavf_adapter *adapter)
}
/**
- * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
- * yet and mark other to be removed.
+ * iavf_clear_mac_filters - Remove MAC filters not sent to PF yet and mark
+ * others to be removed.
* @adapter: board private structure
**/
-static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
+static void iavf_clear_mac_filters(struct iavf_adapter *adapter)
{
- struct iavf_vlan_filter *vlf, *vlftmp;
struct iavf_mac_filter *f, *ftmp;
spin_lock_bh(&adapter->mac_vlan_list_lock);
@@ -1271,11 +1252,6 @@ static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
}
}
- /* disable all VLAN filters */
- list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
- list)
- vlf->state = IAVF_VLAN_DISABLE;
-
spin_unlock_bh(&adapter->mac_vlan_list_lock);
}
@@ -1371,7 +1347,7 @@ void iavf_down(struct iavf_adapter *adapter)
iavf_napi_disable_all(adapter);
iavf_irq_disable(adapter);
- iavf_clear_mac_vlan_filters(adapter);
+ iavf_clear_mac_filters(adapter);
iavf_clear_cloud_filters(adapter);
iavf_clear_fdir_filters(adapter);
iavf_clear_adv_rss_conf(adapter);
@@ -1388,8 +1364,6 @@ void iavf_down(struct iavf_adapter *adapter)
*/
if (!list_empty(&adapter->mac_filter_list))
adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
- if (!list_empty(&adapter->vlan_filter_list))
- adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
if (!list_empty(&adapter->cloud_filter_list))
adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
if (!list_empty(&adapter->fdir_list_head))
@@ -4494,8 +4468,6 @@ static int iavf_open(struct net_device *netdev)
iavf_add_filter(adapter, adapter->hw.mac.addr);
spin_unlock_bh(&adapter->mac_vlan_list_lock);
- /* Restore filters that were removed with IFF_DOWN */
- iavf_restore_filters(adapter);
iavf_restore_fdir_filters(adapter);
iavf_configure(adapter);
diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
index a52c100..4f2defd 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
@@ -746,7 +746,7 @@ static void iavf_vlan_add_reject(struct iavf_adapter *adapter)
spin_lock_bh(&adapter->mac_vlan_list_lock);
list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
- if (f->state == IAVF_VLAN_IS_NEW) {
+ if (f->state == IAVF_VLAN_ADDING) {
list_del(&f->list);
kfree(f);
adapter->num_vlan_filters--;
@@ -812,7 +812,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter)
if (f->state == IAVF_VLAN_ADD) {
vvfl->vlan_id[i] = f->vlan.vid;
i++;
- f->state = IAVF_VLAN_IS_NEW;
+ f->state = IAVF_VLAN_ADDING;
if (i == count)
break;
}
@@ -874,7 +874,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter)
vlan->tpid = f->vlan.tpid;
i++;
- f->state = IAVF_VLAN_IS_NEW;
+ f->state = IAVF_VLAN_ADDING;
}
}
@@ -911,22 +911,12 @@ void iavf_del_vlans(struct iavf_adapter *adapter)
spin_lock_bh(&adapter->mac_vlan_list_lock);
list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
- /* since VLAN capabilities are not allowed, we dont want to send
- * a VLAN delete request because it will most likely fail and
- * create unnecessary errors/noise, so just free the VLAN
- * filters marked for removal to enable bailing out before
- * sending a virtchnl message
- */
if (f->state == IAVF_VLAN_REMOVE &&
!VLAN_FILTERING_ALLOWED(adapter)) {
list_del(&f->list);
kfree(f);
adapter->num_vlan_filters--;
- } else if (f->state == IAVF_VLAN_DISABLE &&
- !VLAN_FILTERING_ALLOWED(adapter)) {
- f->state = IAVF_VLAN_INACTIVE;
- } else if (f->state == IAVF_VLAN_REMOVE ||
- f->state == IAVF_VLAN_DISABLE) {
+ } else if (f->state == IAVF_VLAN_REMOVE) {
count++;
}
}
@@ -958,18 +948,10 @@ void iavf_del_vlans(struct iavf_adapter *adapter)
vvfl->vsi_id = adapter->vsi_res->vsi_id;
vvfl->num_elements = count;
- list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
- if (f->state == IAVF_VLAN_DISABLE) {
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_REMOVE) {
vvfl->vlan_id[i] = f->vlan.vid;
- f->state = IAVF_VLAN_INACTIVE;
- i++;
- if (i == count)
- break;
- } else if (f->state == IAVF_VLAN_REMOVE) {
- vvfl->vlan_id[i] = f->vlan.vid;
- list_del(&f->list);
- kfree(f);
- adapter->num_vlan_filters--;
+ f->state = IAVF_VLAN_REMOVING;
i++;
if (i == count)
break;
@@ -1006,9 +988,8 @@ void iavf_del_vlans(struct iavf_adapter *adapter)
vvfl_v2->vport_id = adapter->vsi_res->vsi_id;
vvfl_v2->num_elements = count;
- list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
- if (f->state == IAVF_VLAN_DISABLE ||
- f->state == IAVF_VLAN_REMOVE) {
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_REMOVE) {
struct virtchnl_vlan_supported_caps *filtering_support =
&adapter->vlan_v2_caps.filtering.filtering_support;
struct virtchnl_vlan *vlan;
@@ -1022,13 +1003,7 @@ void iavf_del_vlans(struct iavf_adapter *adapter)
vlan->tci = f->vlan.vid;
vlan->tpid = f->vlan.tpid;
- if (f->state == IAVF_VLAN_DISABLE) {
- f->state = IAVF_VLAN_INACTIVE;
- } else {
- list_del(&f->list);
- kfree(f);
- adapter->num_vlan_filters--;
- }
+ f->state = IAVF_VLAN_REMOVING;
i++;
if (i == count)
break;
@@ -2391,10 +2366,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
wake_up(&adapter->vc_waitqueue);
break;
- case VIRTCHNL_OP_DEL_VLAN:
- dev_err(&adapter->pdev->dev, "Failed to delete VLAN filter, error %s\n",
- iavf_stat_str(&adapter->hw, v_retval));
- break;
case VIRTCHNL_OP_DEL_ETH_ADDR:
dev_err(&adapter->pdev->dev, "Failed to delete MAC filter, error %s\n",
iavf_stat_str(&adapter->hw, v_retval));
@@ -2905,17 +2876,42 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
spin_unlock_bh(&adapter->adv_rss_lock);
}
break;
+ case VIRTCHNL_OP_ADD_VLAN:
case VIRTCHNL_OP_ADD_VLAN_V2: {
struct iavf_vlan_filter *f;
+ if (v_retval)
+ break;
+
spin_lock_bh(&adapter->mac_vlan_list_lock);
list_for_each_entry(f, &adapter->vlan_filter_list, list) {
- if (f->state == IAVF_VLAN_IS_NEW)
+ if (f->state == IAVF_VLAN_ADDING)
f->state = IAVF_VLAN_ACTIVE;
}
spin_unlock_bh(&adapter->mac_vlan_list_lock);
}
break;
+ case VIRTCHNL_OP_DEL_VLAN:
+ case VIRTCHNL_OP_DEL_VLAN_V2: {
+ struct iavf_vlan_filter *f, *ftmp;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list,
+ list) {
+ if (f->state == IAVF_VLAN_REMOVING) {
+ if (v_retval) {
+ /* PF rejected DEL, keep filter */
+ f->state = IAVF_VLAN_ACTIVE;
+ } else {
+ list_del(&f->list);
+ kfree(f);
+ adapter->num_vlan_filters--;
+ }
+ }
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ }
+ break;
case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
/* PF enabled vlan strip on this VF.
* Update netdev->features if needed to be in sync with ethtool.
diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c
index 6144cee..641d6e2 100644
--- a/drivers/net/ethernet/intel/ice/devlink/devlink.c
+++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c
@@ -1245,6 +1245,8 @@ static int ice_devlink_reinit_up(struct ice_pf *pf)
return err;
}
+ ice_init_dev_hw(pf);
+
/* load MSI-X values */
ice_set_min_max_msix(pf);
diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c
index ce11fea..b617a6b 100644
--- a/drivers/net/ethernet/intel/ice/ice_common.c
+++ b/drivers/net/ethernet/intel/ice/ice_common.c
@@ -1126,8 +1126,6 @@ int ice_init_hw(struct ice_hw *hw)
if (status)
goto err_unroll_fltr_mgmt_struct;
- ice_init_dev_hw(hw->back);
-
mutex_init(&hw->tnl_lock);
ice_init_chk_recipe_reuse_support(hw);
diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
index 16aa255..0bc6dd3 100644
--- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c
@@ -537,14 +537,14 @@ void ice_dcb_rebuild(struct ice_pf *pf)
struct ice_dcbx_cfg *err_cfg;
int ret;
+ mutex_lock(&pf->tc_mutex);
+
ret = ice_query_port_ets(pf->hw.port_info, &buf, sizeof(buf), NULL);
if (ret) {
dev_err(dev, "Query Port ETS failed\n");
goto dcb_error;
}
- mutex_lock(&pf->tc_mutex);
-
if (!pf->hw.port_info->qos_cfg.is_sw_lldp)
ice_cfg_etsrec_defaults(pf->hw.port_info);
diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c
index 62f75701d..892bc7c 100644
--- a/drivers/net/ethernet/intel/ice/ice_dpll.c
+++ b/drivers/net/ethernet/intel/ice/ice_dpll.c
@@ -1155,6 +1155,32 @@ ice_dpll_input_state_get(const struct dpll_pin *pin, void *pin_priv,
}
/**
+ * ice_dpll_sw_pin_notify_peer - notify the paired SW pin after a state change
+ * @d: pointer to dplls struct
+ * @changed: the SW pin that was explicitly changed (already notified by dpll core)
+ *
+ * SMA and U.FL pins share physical signal paths in pairs (SMA1/U.FL1 and
+ * SMA2/U.FL2). When one pin's routing changes via the PCA9575 GPIO
+ * expander, the paired pin's state may also change. Send a change
+ * notification for the peer pin so userspace consumers monitoring the
+ * peer via dpll netlink learn about the update.
+ *
+ * Context: Called from dpll_pin_ops callbacks after pf->dplls.lock is
+ * released. Uses __dpll_pin_change_ntf() because dpll_lock is
+ * still held by the dpll netlink layer.
+ */
+static void ice_dpll_sw_pin_notify_peer(struct ice_dplls *d,
+ struct ice_dpll_pin *changed)
+{
+ struct ice_dpll_pin *peer;
+
+ peer = (changed >= d->sma && changed < d->sma + ICE_DPLL_PIN_SW_NUM) ?
+ &d->ufl[changed->idx] : &d->sma[changed->idx];
+ if (peer->pin)
+ __dpll_pin_change_ntf(peer->pin);
+}
+
+/**
* ice_dpll_sma_direction_set - set direction of SMA pin
* @p: pointer to a pin
* @direction: requested direction of the pin
@@ -1171,6 +1197,8 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p,
enum dpll_pin_direction direction,
struct netlink_ext_ack *extack)
{
+ struct ice_dplls *d = &p->pf->dplls;
+ struct ice_dpll_pin *peer;
u8 data;
int ret;
@@ -1189,8 +1217,9 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p,
case ICE_DPLL_PIN_SW_2_IDX:
if (direction == DPLL_PIN_DIRECTION_INPUT) {
data &= ~ICE_SMA2_DIR_EN;
+ data |= ICE_SMA2_UFL2_RX_DIS;
} else {
- data &= ~ICE_SMA2_TX_EN;
+ data &= ~(ICE_SMA2_TX_EN | ICE_SMA2_UFL2_RX_DIS);
data |= ICE_SMA2_DIR_EN;
}
break;
@@ -1202,6 +1231,34 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p,
ret = ice_dpll_pin_state_update(p->pf, p,
ICE_DPLL_PIN_TYPE_SOFTWARE,
extack);
+ if (ret)
+ return ret;
+
+ /* When a direction change activates the paired U.FL pin, enable
+ * its backing CGU pin so the pin reports as connected. Without
+ * this the U.FL routing is correct but the CGU pin stays disabled
+ * and userspace sees the pin as disconnected. Do not disable the
+ * backing pin when U.FL becomes inactive because the SMA pin may
+ * still be using it.
+ */
+ peer = &d->ufl[p->idx];
+ if (peer->active) {
+ struct ice_dpll_pin *target;
+ enum ice_dpll_pin_type type;
+
+ if (peer->output) {
+ target = peer->output;
+ type = ICE_DPLL_PIN_TYPE_OUTPUT;
+ } else {
+ target = peer->input;
+ type = ICE_DPLL_PIN_TYPE_INPUT;
+ }
+ ret = ice_dpll_pin_enable(&p->pf->hw, target,
+ d->eec.dpll_idx, type, extack);
+ if (!ret)
+ ret = ice_dpll_pin_state_update(p->pf, target,
+ type, extack);
+ }
return ret;
}
@@ -1253,6 +1310,14 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv,
data &= ~ICE_SMA1_MASK;
enable = true;
} else if (state == DPLL_PIN_STATE_DISCONNECTED) {
+ /* Skip if U.FL1 is not active, setting TX_EN
+ * while DIR_EN is set would also deactivate
+ * the paired SMA1 output.
+ */
+ if (data & (ICE_SMA1_DIR_EN | ICE_SMA1_TX_EN)) {
+ ret = 0;
+ goto unlock;
+ }
data |= ICE_SMA1_TX_EN;
enable = false;
} else {
@@ -1267,6 +1332,15 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv,
data &= ~ICE_SMA2_UFL2_RX_DIS;
enable = true;
} else if (state == DPLL_PIN_STATE_DISCONNECTED) {
+ /* Skip if U.FL2 is not active, setting
+ * UFL2_RX_DIS could also disable the paired
+ * SMA2 input.
+ */
+ if (!(data & ICE_SMA2_DIR_EN) ||
+ (data & ICE_SMA2_UFL2_RX_DIS)) {
+ ret = 0;
+ goto unlock;
+ }
data |= ICE_SMA2_UFL2_RX_DIS;
enable = false;
} else {
@@ -1296,6 +1370,8 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv,
unlock:
mutex_unlock(&pf->dplls.lock);
+ if (!ret)
+ ice_dpll_sw_pin_notify_peer(&pf->dplls, p);
return ret;
}
@@ -1414,6 +1490,8 @@ ice_dpll_sma_pin_state_set(const struct dpll_pin *pin, void *pin_priv,
unlock:
mutex_unlock(&pf->dplls.lock);
+ if (!ret)
+ ice_dpll_sw_pin_notify_peer(&pf->dplls, sma);
return ret;
}
@@ -1609,6 +1687,8 @@ ice_dpll_pin_sma_direction_set(const struct dpll_pin *pin, void *pin_priv,
mutex_lock(&pf->dplls.lock);
ret = ice_dpll_sma_direction_set(p, direction, extack);
mutex_unlock(&pf->dplls.lock);
+ if (!ret)
+ ice_dpll_sw_pin_notify_peer(&pf->dplls, p);
return ret;
}
@@ -1915,7 +1995,10 @@ ice_dpll_phase_offset_get(const struct dpll_pin *pin, void *pin_priv,
d->active_input == p->input->pin))
*phase_offset = d->phase_offset * ICE_DPLL_PHASE_OFFSET_FACTOR;
else if (d->phase_offset_monitor_period)
- *phase_offset = p->phase_offset * ICE_DPLL_PHASE_OFFSET_FACTOR;
+ *phase_offset = (p->input &&
+ p->direction == DPLL_PIN_DIRECTION_INPUT ?
+ p->input->phase_offset :
+ p->phase_offset) * ICE_DPLL_PHASE_OFFSET_FACTOR;
else
*phase_offset = 0;
mutex_unlock(&pf->dplls.lock);
@@ -2440,6 +2523,8 @@ ice_dpll_rclk_state_on_pin_set(const struct dpll_pin *pin, void *pin_priv,
if (hw_idx < 0)
goto unlock;
hw_idx -= pf->dplls.base_rclk_idx;
+ if (hw_idx >= ICE_DPLL_RCLK_NUM_MAX)
+ goto unlock;
if ((enable && p->state[hw_idx] == DPLL_PIN_STATE_CONNECTED) ||
(!enable && p->state[hw_idx] == DPLL_PIN_STATE_DISCONNECTED)) {
@@ -2503,6 +2588,9 @@ ice_dpll_rclk_state_on_pin_get(const struct dpll_pin *pin, void *pin_priv,
hw_idx = ice_dpll_pin_get_parent_idx(p, parent_pin);
if (hw_idx < 0)
goto unlock;
+ hw_idx -= pf->dplls.base_rclk_idx;
+ if (hw_idx >= ICE_DPLL_RCLK_NUM_MAX)
+ goto unlock;
ret = ice_dpll_pin_state_update(pf, p, ICE_DPLL_PIN_TYPE_RCLK_INPUT,
extack);
@@ -2610,6 +2698,27 @@ static u64 ice_generate_clock_id(struct ice_pf *pf)
}
/**
+ * ice_dpll_pin_ntf - notify pin change including any SW pin wrappers
+ * @dplls: pointer to dplls struct
+ * @pin: the dpll_pin that changed
+ *
+ * Send a change notification for @pin and for any registered SMA/U.FL pin
+ * whose backing CGU input matches @pin.
+ */
+static void ice_dpll_pin_ntf(struct ice_dplls *dplls, struct dpll_pin *pin)
+{
+ dpll_pin_change_ntf(pin);
+ for (int i = 0; i < ICE_DPLL_PIN_SW_NUM; i++) {
+ if (dplls->sma[i].pin && dplls->sma[i].input &&
+ dplls->sma[i].input->pin == pin)
+ dpll_pin_change_ntf(dplls->sma[i].pin);
+ if (dplls->ufl[i].pin && dplls->ufl[i].input &&
+ dplls->ufl[i].input->pin == pin)
+ dpll_pin_change_ntf(dplls->ufl[i].pin);
+ }
+}
+
+/**
* ice_dpll_notify_changes - notify dpll subsystem about changes
* @d: pointer do dpll
*
@@ -2617,6 +2726,7 @@ static u64 ice_generate_clock_id(struct ice_pf *pf)
*/
static void ice_dpll_notify_changes(struct ice_dpll *d)
{
+ struct ice_dplls *dplls = &d->pf->dplls;
bool pin_notified = false;
if (d->prev_dpll_state != d->dpll_state) {
@@ -2625,17 +2735,17 @@ static void ice_dpll_notify_changes(struct ice_dpll *d)
}
if (d->prev_input != d->active_input) {
if (d->prev_input)
- dpll_pin_change_ntf(d->prev_input);
+ ice_dpll_pin_ntf(dplls, d->prev_input);
d->prev_input = d->active_input;
if (d->active_input) {
- dpll_pin_change_ntf(d->active_input);
+ ice_dpll_pin_ntf(dplls, d->active_input);
pin_notified = true;
}
}
if (d->prev_phase_offset != d->phase_offset) {
d->prev_phase_offset = d->phase_offset;
if (!pin_notified && d->active_input)
- dpll_pin_change_ntf(d->active_input);
+ ice_dpll_pin_ntf(dplls, d->active_input);
}
}
@@ -2664,6 +2774,7 @@ static bool ice_dpll_is_pps_phase_monitor(struct ice_pf *pf)
/**
* ice_dpll_pins_notify_mask - notify dpll subsystem about bulk pin changes
+ * @dplls: pointer to dplls struct
* @pins: array of ice_dpll_pin pointers registered within dpll subsystem
* @pin_num: number of pins
* @phase_offset_ntf_mask: bitmask of pin indexes to notify
@@ -2673,15 +2784,14 @@ static bool ice_dpll_is_pps_phase_monitor(struct ice_pf *pf)
*
* Context: Must be called while pf->dplls.lock is released.
*/
-static void ice_dpll_pins_notify_mask(struct ice_dpll_pin *pins,
+static void ice_dpll_pins_notify_mask(struct ice_dplls *dplls,
+ struct ice_dpll_pin *pins,
u8 pin_num,
u32 phase_offset_ntf_mask)
{
- int i = 0;
-
- for (i = 0; i < pin_num; i++)
- if (phase_offset_ntf_mask & (1 << i))
- dpll_pin_change_ntf(pins[i].pin);
+ for (int i = 0; i < pin_num; i++)
+ if (phase_offset_ntf_mask & BIT(i))
+ ice_dpll_pin_ntf(dplls, pins[i].pin);
}
/**
@@ -2857,7 +2967,7 @@ static void ice_dpll_periodic_work(struct kthread_work *work)
ice_dpll_notify_changes(de);
ice_dpll_notify_changes(dp);
if (phase_offset_ntf)
- ice_dpll_pins_notify_mask(d->inputs, d->num_inputs,
+ ice_dpll_pins_notify_mask(d, d->inputs, d->num_inputs,
phase_offset_ntf);
resched:
@@ -4014,6 +4124,7 @@ static int ice_dpll_init_info_sw_pins(struct ice_pf *pf)
struct ice_dpll_pin *pin;
u32 phase_adj_max, caps;
int i, ret;
+ u8 data;
if (pf->hw.device_id == ICE_DEV_ID_E810C_QSFP)
input_idx_offset = ICE_E810_RCLK_PINS_NUM;
@@ -4073,6 +4184,22 @@ static int ice_dpll_init_info_sw_pins(struct ice_pf *pf)
}
ice_dpll_phase_range_set(&pin->prop.phase_range, phase_adj_max);
}
+
+ /* Initialize the SMA control register to a known-good default state.
+ * Without this write the PCA9575 GPIO expander retains its power-on
+ * default (all outputs high) which makes all SW pins appear inactive.
+ * Set SMA1 and SMA2 as active inputs, disable U.FL1 output and
+ * U.FL2 input.
+ */
+ ret = ice_read_sma_ctrl(&pf->hw, &data);
+ if (ret)
+ return ret;
+ data &= ~ICE_ALL_SMA_MASK;
+ data |= ICE_SMA1_TX_EN | ICE_SMA2_TX_EN | ICE_SMA2_UFL2_RX_DIS;
+ ret = ice_write_sma_ctrl(&pf->hw, data);
+ if (ret)
+ return ret;
+
ret = ice_dpll_pin_state_update(pf, pin, ICE_DPLL_PIN_TYPE_SOFTWARE,
NULL);
if (ret)
diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.h b/drivers/net/ethernet/intel/ice/ice_dpll.h
index ae42cde..8678575 100644
--- a/drivers/net/ethernet/intel/ice/ice_dpll.h
+++ b/drivers/net/ethernet/intel/ice/ice_dpll.h
@@ -8,6 +8,22 @@
#define ICE_DPLL_RCLK_NUM_MAX 4
+#define ICE_CGU_R10 0x28
+#define ICE_CGU_R10_SYNCE_CLKO_SEL GENMASK(8, 5)
+#define ICE_CGU_R10_SYNCE_CLKODIV_M1 GENMASK(13, 9)
+#define ICE_CGU_R10_SYNCE_CLKODIV_LOAD BIT(14)
+#define ICE_CGU_R10_SYNCE_DCK_RST BIT(15)
+#define ICE_CGU_R10_SYNCE_ETHCLKO_SEL GENMASK(18, 16)
+#define ICE_CGU_R10_SYNCE_ETHDIV_M1 GENMASK(23, 19)
+#define ICE_CGU_R10_SYNCE_ETHDIV_LOAD BIT(24)
+#define ICE_CGU_R10_SYNCE_DCK2_RST BIT(25)
+#define ICE_CGU_R10_SYNCE_S_REF_CLK GENMASK(31, 27)
+
+#define ICE_CGU_R11 0x2C
+#define ICE_CGU_R11_SYNCE_S_BYP_CLK GENMASK(6, 1)
+
+#define ICE_CGU_BYPASS_MUX_OFFSET_E825C 3
+
/**
* enum ice_dpll_pin_sw - enumerate ice software pin indices:
* @ICE_DPLL_PIN_SW_1_IDX: index of first SW pin
@@ -157,19 +173,3 @@ static inline void ice_dpll_deinit(struct ice_pf *pf) { }
#endif
#endif
-
-#define ICE_CGU_R10 0x28
-#define ICE_CGU_R10_SYNCE_CLKO_SEL GENMASK(8, 5)
-#define ICE_CGU_R10_SYNCE_CLKODIV_M1 GENMASK(13, 9)
-#define ICE_CGU_R10_SYNCE_CLKODIV_LOAD BIT(14)
-#define ICE_CGU_R10_SYNCE_DCK_RST BIT(15)
-#define ICE_CGU_R10_SYNCE_ETHCLKO_SEL GENMASK(18, 16)
-#define ICE_CGU_R10_SYNCE_ETHDIV_M1 GENMASK(23, 19)
-#define ICE_CGU_R10_SYNCE_ETHDIV_LOAD BIT(24)
-#define ICE_CGU_R10_SYNCE_DCK2_RST BIT(25)
-#define ICE_CGU_R10_SYNCE_S_REF_CLK GENMASK(31, 27)
-
-#define ICE_CGU_R11 0x2C
-#define ICE_CGU_R11_SYNCE_S_BYP_CLK GENMASK(6, 1)
-
-#define ICE_CGU_BYPASS_MUX_OFFSET_E825C 3
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 5f92377..e2fbe11 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -3682,7 +3682,7 @@ int ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
ICE_MCAST_VLAN_PROMISC_BITS,
vid);
- if (ret)
+ if (ret && ret != -EEXIST)
goto finish;
}
@@ -4104,6 +4104,12 @@ int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked)
}
ice_pf_dcb_recfg(pf, locked);
ice_vsi_open(vsi);
+ /* Rx rings are reallocated during VSI rebuild and lose their ptp_rx
+ * flag. Restore timestamp mode so newly allocated rings are set up
+ * for hardware Rx timestamping.
+ */
+ if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
+ ice_ptp_restore_timestamp_mode(pf);
goto done;
rebuild_err:
@@ -5245,6 +5251,8 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
return err;
}
+ ice_init_dev_hw(pf);
+
adapter = ice_adapter_get(pdev);
if (IS_ERR(adapter)) {
err = PTR_ERR(adapter);
@@ -8044,7 +8052,7 @@ int ice_set_rss_hfunc(struct ice_vsi *vsi, u8 hfunc)
ctx->info.q_opt_rss |=
FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_HASH_M, hfunc);
ctx->info.q_opt_tc = vsi->info.q_opt_tc;
- ctx->info.q_opt_flags = vsi->info.q_opt_rss;
+ ctx->info.q_opt_flags = vsi->info.q_opt_flags;
err = ice_update_vsi(hw, vsi->idx, ctx, NULL);
if (err) {
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c
index 36df742..f9e4ec6 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp.c
@@ -2065,11 +2065,13 @@ static const struct ice_crosststamp_cfg ice_crosststamp_cfg_e830 = {
/**
* struct ice_crosststamp_ctx - Device cross timestamp context
* @snapshot: snapshot of system clocks for historic interpolation
+ * @snapshot_clock_id: System clock ID for @snapshot
* @pf: pointer to the PF private structure
* @cfg: pointer to hardware configuration for cross timestamp
*/
struct ice_crosststamp_ctx {
struct system_time_snapshot snapshot;
+ clockid_t snapshot_clock_id;
struct ice_pf *pf;
const struct ice_crosststamp_cfg *cfg;
};
@@ -2115,7 +2117,7 @@ static int ice_capture_crosststamp(ktime_t *device,
}
/* Snapshot system time for historic interpolation */
- ktime_get_snapshot(&ctx->snapshot);
+ ktime_get_snapshot_id(ctx->snapshot_clock_id, &ctx->snapshot);
/* Program cmd to master timer */
ice_ptp_src_cmd(hw, ICE_PTP_READ_TIME);
@@ -2176,6 +2178,7 @@ static int ice_ptp_getcrosststamp(struct ptp_clock_info *info,
{
struct ice_pf *pf = ptp_info_to_pf(info);
struct ice_crosststamp_ctx ctx = {
+ .snapshot_clock_id = cts->clock_id,
.pf = pf,
};
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
index 24fb7a3..2c18e16 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
@@ -2141,16 +2141,23 @@ int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port)
}
incval = (u64)hi << 32 | lo;
+ if (!ice_ptp_lock(hw)) {
+ dev_err(ice_hw_to_dev(hw), "Failed to acquire PTP semaphore\n");
+ return -EBUSY;
+ }
+
err = ice_write_40b_ptp_reg_eth56g(hw, port, PHY_REG_TIMETUS_L, incval);
if (err)
- return err;
+ goto err_ptp_unlock;
err = ice_ptp_one_port_cmd(hw, port, ICE_PTP_INIT_INCVAL);
if (err)
- return err;
+ goto err_ptp_unlock;
ice_ptp_exec_tmr_cmd(hw);
+ ice_ptp_unlock(hw);
+
err = ice_sync_phy_timer_eth56g(hw, port);
if (err)
return err;
@@ -2166,6 +2173,10 @@ int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port)
ice_debug(hw, ICE_DBG_PTP, "Enabled clock on PHY port %u\n", port);
return 0;
+
+err_ptp_unlock:
+ ice_ptp_unlock(hw);
+ return err;
}
/**
@@ -4503,18 +4514,17 @@ static int
ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
{
struct ice_e810_params *params = &hw->ptp.phy.e810;
- unsigned long flags;
u32 val;
int err;
- spin_lock_irqsave(¶ms->atqbal_wq.lock, flags);
+ spin_lock_irq(¶ms->atqbal_wq.lock);
/* Wait for any pending in-progress low latency interrupt */
err = wait_event_interruptible_locked_irq(params->atqbal_wq,
!(params->atqbal_flags &
ATQBAL_FLAGS_INTR_IN_PROGRESS));
if (err) {
- spin_unlock_irqrestore(¶ms->atqbal_wq.lock, flags);
+ spin_unlock_irq(¶ms->atqbal_wq.lock);
return err;
}
@@ -4529,7 +4539,7 @@ ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
REG_LL_PROXY_H);
if (err) {
ice_debug(hw, ICE_DBG_PTP, "Failed to read PTP timestamp using low latency read\n");
- spin_unlock_irqrestore(¶ms->atqbal_wq.lock, flags);
+ spin_unlock_irq(¶ms->atqbal_wq.lock);
return err;
}
@@ -4539,7 +4549,7 @@ ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
/* Read the low 32 bit value and set the TS valid bit */
*lo = rd32(hw, REG_LL_PROXY_L) | TS_VALID;
- spin_unlock_irqrestore(¶ms->atqbal_wq.lock, flags);
+ spin_unlock_irq(¶ms->atqbal_wq.lock);
return 0;
}
@@ -5254,9 +5264,13 @@ static void ice_ptp_init_phy_e830(struct ice_ptp_hw *ptp)
*/
bool ice_ptp_lock(struct ice_hw *hw)
{
+ struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
u32 hw_lock;
int i;
+ if (!ice_is_primary(hw))
+ hw = ice_get_primary_hw(pf);
+
#define MAX_TRIES 15
for (i = 0; i < MAX_TRIES; i++) {
@@ -5283,6 +5297,11 @@ bool ice_ptp_lock(struct ice_hw *hw)
*/
void ice_ptp_unlock(struct ice_hw *hw)
{
+ struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
+
+ if (!ice_is_primary(hw))
+ hw = ice_get_primary_hw(pf);
+
wr32(hw, PFTSYN_SEM + (PFTSYN_SEM_BYTES * hw->pf_id), 0);
}
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
index 772f6b0..b1f4670 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
@@ -804,7 +804,12 @@ void ice_reset_all_vfs(struct ice_pf *pf)
ice_vf_ctrl_invalidate_vsi(vf);
ice_vf_pre_vsi_rebuild(vf);
- ice_vf_rebuild_vsi(vf);
+ if (ice_vf_rebuild_vsi(vf)) {
+ dev_err(dev, "VF %u VSI rebuild failed, leaving VF disabled\n",
+ vf->vf_id);
+ mutex_unlock(&vf->cfg_lock);
+ continue;
+ }
ice_vf_post_vsi_rebuild(vf);
ice_eswitch_attach_vf(pf, vf);
diff --git a/drivers/net/ethernet/intel/ice/virt/queues.c b/drivers/net/ethernet/intel/ice/virt/queues.c
index f73d5a3..31be2f7 100644
--- a/drivers/net/ethernet/intel/ice/virt/queues.c
+++ b/drivers/net/ethernet/intel/ice/virt/queues.c
@@ -840,7 +840,7 @@ int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg)
if (qpi->rxq.databuffer_size != 0 &&
(qpi->rxq.databuffer_size > ((16 * 1024) - 128) ||
- qpi->rxq.databuffer_size < 1024))
+ qpi->rxq.databuffer_size < 128))
goto error_param;
ring->rx_buf_len = qpi->rxq.databuffer_size;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_idc.c b/drivers/net/ethernet/intel/idpf/idpf_idc.c
index 7e4f4ac..b7d6b08 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_idc.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_idc.c
@@ -90,7 +90,10 @@ static int idpf_plug_vport_aux_dev(struct iidc_rdma_core_dev_info *cdev_info,
return 0;
err_aux_dev_add:
+ ida_free(&idpf_idc_ida, adev->id);
+ vdev_info->adev = NULL;
auxiliary_device_uninit(adev);
+ return ret;
err_aux_dev_init:
ida_free(&idpf_idc_ida, adev->id);
err_ida_alloc:
@@ -228,7 +231,10 @@ static int idpf_plug_core_aux_dev(struct iidc_rdma_core_dev_info *cdev_info)
return 0;
err_aux_dev_add:
+ ida_free(&idpf_idc_ida, adev->id);
+ cdev_info->adev = NULL;
auxiliary_device_uninit(adev);
+ return ret;
err_aux_dev_init:
ida_free(&idpf_idc_ida, adev->id);
err_ida_alloc:
diff --git a/drivers/net/ethernet/intel/idpf/idpf_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_ptp.c
index eec91c4..4a51d27 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_ptp.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_ptp.c
@@ -952,6 +952,8 @@ int idpf_ptp_init(struct idpf_adapter *adapter)
goto free_ptp;
}
+ spin_lock_init(&adapter->ptp->read_dev_clk_lock);
+
err = idpf_ptp_create_clock(adapter);
if (err)
goto free_ptp;
@@ -977,8 +979,6 @@ int idpf_ptp_init(struct idpf_adapter *adapter)
goto remove_clock;
}
- spin_lock_init(&adapter->ptp->read_dev_clk_lock);
-
pci_dbg(adapter->pdev, "PTP init successful\n");
return 0;
diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
index 1723681..46d625b 100644
--- a/drivers/net/ethernet/intel/igc/igc.h
+++ b/drivers/net/ethernet/intel/igc/igc.h
@@ -326,6 +326,7 @@ struct igc_adapter {
struct timespec64 prev_ptp_time; /* Pre-reset PTP clock */
ktime_t ptp_reset_start; /* Reset time in clock mono */
struct system_time_snapshot snapshot;
+ clockid_t snapshot_clock_id;
struct mutex ptm_lock; /* Only allow one PTM transaction at a time */
char fw_version[32];
diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c
index 3d6b226..b40aba9 100644
--- a/drivers/net/ethernet/intel/igc/igc_ptp.c
+++ b/drivers/net/ethernet/intel/igc/igc_ptp.c
@@ -1049,7 +1049,7 @@ static int igc_phc_get_syncdevicetime(ktime_t *device,
*/
do {
/* Get a snapshot of system clocks to use as historic value. */
- ktime_get_snapshot(&adapter->snapshot);
+ ktime_get_snapshot_id(adapter->snapshot_clock_id, &adapter->snapshot);
igc_ptm_trigger(hw);
@@ -1103,6 +1103,8 @@ static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp,
/* This blocks until any in progress PTM transactions complete */
mutex_lock(&adapter->ptm_lock);
+ adapter->snapshot_clock_id = cts->clock_id;
+
ret = get_device_system_crosststamp(igc_phc_get_syncdevicetime,
adapter, &adapter->snapshot, cts);
mutex_unlock(&adapter->ptm_lock);
diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c
index 8a11014..52de2bc 100644
--- a/drivers/net/ethernet/intel/igc/igc_tsn.c
+++ b/drivers/net/ethernet/intel/igc/igc_tsn.c
@@ -34,6 +34,7 @@ static int igc_fpe_init_smd_frame(struct igc_ring *ring,
return -ENOMEM;
}
+ buffer->type = IGC_TX_BUFFER_TYPE_SKB;
buffer->skb = skb;
buffer->protocol = 0;
buffer->bytecount = skb->len;
@@ -109,10 +110,16 @@ static int igc_fpe_xmit_smd_frame(struct igc_adapter *adapter,
__netif_tx_lock(nq, cpu);
err = igc_fpe_init_tx_descriptor(ring, skb, type);
+ if (err)
+ goto err_free_skb_any;
+
igc_flush_tx_descriptors(ring);
-
__netif_tx_unlock(nq);
+ return 0;
+err_free_skb_any:
+ __netif_tx_unlock(nq);
+ dev_kfree_skb_any(skb);
return err;
}
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 42f89a1..4ba3be9 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -1221,6 +1221,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
ether_addr_equal(rx_ring->netdev->dev_addr,
eth_hdr(skb)->h_source)) {
dev_kfree_skb_irq(skb);
+ skb = NULL;
continue;
}
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
index 4f33a81..2e94d51 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
@@ -1294,13 +1294,18 @@ static inline void link_status_user_format(u64 lstat,
struct cgx_link_user_info *linfo,
struct cgx *cgx, u8 lmac_id)
{
+ unsigned int speed;
+
linfo->link_up = FIELD_GET(RESP_LINKSTAT_UP, lstat);
linfo->full_duplex = FIELD_GET(RESP_LINKSTAT_FDUPLEX, lstat);
- linfo->speed = cgx_speed_mbps[FIELD_GET(RESP_LINKSTAT_SPEED, lstat)];
linfo->an = FIELD_GET(RESP_LINKSTAT_AN, lstat);
linfo->fec = FIELD_GET(RESP_LINKSTAT_FEC, lstat);
linfo->lmac_type_id = FIELD_GET(RESP_LINKSTAT_LMAC_TYPE, lstat);
+ speed = FIELD_GET(RESP_LINKSTAT_SPEED, lstat);
+ linfo->speed = speed < ARRAY_SIZE(cgx_speed_mbps) ?
+ cgx_speed_mbps[speed] : 0;
+
if (linfo->lmac_type_id >= LMAC_MODE_MAX) {
dev_err(&cgx->pdev->dev, "Unknown lmac_type_id %d reported by firmware on cgx port%d:%d",
linfo->lmac_type_id, cgx->cgx_id, lmac_id);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
index 3debf2f..6f13296 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c
@@ -249,34 +249,21 @@ DEFINE_SHOW_ATTRIBUTE(npc_defrag);
int npc_cn20k_debugfs_init(struct rvu *rvu)
{
struct npc_priv_t *npc_priv = npc_priv_get();
- struct dentry *npc_dentry;
- npc_dentry = debugfs_create_file("mcam_layout", 0444, rvu->rvu_dbg.npc,
- npc_priv, &npc_mcam_layout_fops);
+ debugfs_create_file("mcam_layout", 0444, rvu->rvu_dbg.npc,
+ npc_priv, &npc_mcam_layout_fops);
- if (!npc_dentry)
- return -EFAULT;
+ debugfs_create_file("mcam_default", 0444, rvu->rvu_dbg.npc,
+ rvu, &npc_mcam_default_fops);
- npc_dentry = debugfs_create_file("mcam_default", 0444, rvu->rvu_dbg.npc,
- rvu, &npc_mcam_default_fops);
+ debugfs_create_file("vidx2idx", 0444, rvu->rvu_dbg.npc,
+ npc_priv, &npc_vidx2idx_map_fops);
- if (!npc_dentry)
- return -EFAULT;
+ debugfs_create_file("idx2vidx", 0444, rvu->rvu_dbg.npc,
+ npc_priv, &npc_idx2vidx_map_fops);
- npc_dentry = debugfs_create_file("vidx2idx", 0444, rvu->rvu_dbg.npc,
- npc_priv, &npc_vidx2idx_map_fops);
- if (!npc_dentry)
- return -EFAULT;
-
- npc_dentry = debugfs_create_file("idx2vidx", 0444, rvu->rvu_dbg.npc,
- npc_priv, &npc_idx2vidx_map_fops);
- if (!npc_dentry)
- return -EFAULT;
-
- npc_dentry = debugfs_create_file("defrag", 0444, rvu->rvu_dbg.npc,
- npc_priv, &npc_defrag_fops);
- if (!npc_dentry)
- return -EFAULT;
+ debugfs_create_file("defrag", 0444, rvu->rvu_dbg.npc,
+ npc_priv, &npc_defrag_fops);
return 0;
}
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
index 7291fdb..6b3f453 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
@@ -798,7 +798,7 @@ void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr,
iounmap(mkex_prfl_addr);
}
-void
+int
npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
int index, bool enable)
{
@@ -808,7 +808,12 @@ npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
u64 cfg, hw_prio;
u8 kw_type;
- npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+ if (index < 0 || index >= mcam->total_entries)
+ return -EINVAL;
+
+ if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
+ return -EINVAL;
+
if (kw_type == NPC_MCAM_KEY_X2) {
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
@@ -819,7 +824,7 @@ npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
cfg);
- return;
+ return 0;
}
/* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks
@@ -836,10 +841,12 @@ npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
cfg);
}
+
+ return 0;
}
-void
-npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int bank, int index)
+static void
+npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index)
{
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
@@ -873,6 +880,33 @@ npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int bank, int index)
NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0);
}
+int
+npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx)
+{
+ struct npc_mcam *mcam = &rvu->hw->mcam;
+ int bank = npc_get_bank(mcam, mcam_idx);
+ u8 kw_type;
+ int index;
+
+ if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type))
+ return -EINVAL;
+
+ index = mcam_idx & (mcam->banksize - 1);
+
+ if (kw_type == NPC_MCAM_KEY_X2) {
+ npc_clear_x2_entry(rvu, blkaddr, bank, index);
+ return 0;
+ }
+
+ /* For NPC_MCAM_KEY_X4 keys, both the banks
+ * need to be programmed with the same value.
+ */
+ for (bank = 0; bank < mcam->banks_per_entry; bank++)
+ npc_clear_x2_entry(rvu, blkaddr, bank, index);
+
+ return 0;
+}
+
static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx,
u64 *cam0, u64 *cam1)
{
@@ -1014,48 +1048,27 @@ static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam,
kw, req_kw_type);
}
-static void
-npc_cn20k_set_mcam_bank_cfg(struct rvu *rvu, int blkaddr, int mcam_idx,
- int bank, u8 kw_type, bool enable, u8 hw_prio)
-{
- struct npc_mcam *mcam = &rvu->hw->mcam;
- u64 bank_cfg;
-
- bank_cfg = (u64)hw_prio << 24;
- if (enable)
- bank_cfg |= 0x1;
-
- if (kw_type == NPC_MCAM_KEY_X2) {
- rvu_write64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
- bank_cfg);
- return;
- }
-
- /* For NPC_MCAM_KEY_X4 keys, both the banks
- * need to be programmed with the same value.
- */
- for (bank = 0; bank < mcam->banks_per_entry; bank++) {
- rvu_write64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
- bank_cfg);
- }
-}
-
-void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
- u8 intf, struct cn20k_mcam_entry *entry,
- bool enable, u8 hw_prio, u8 req_kw_type)
+int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
+ u8 intf, struct cn20k_mcam_entry *entry,
+ bool enable, u8 hw_prio, u8 req_kw_type)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int mcam_idx = index % mcam->banksize;
int bank = index / mcam->banksize;
+ u64 bank_cfg = (u64)hw_prio << 24;
int kw = 0;
u8 kw_type;
- /* Disable before mcam entry update */
- npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false);
+ if (index < 0 || index >= mcam->total_entries)
+ return -EINVAL;
- npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+ if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
+ return -EINVAL;
+
+ /* Disable before mcam entry update */
+ if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false))
+ return -EINVAL;
+
/* CAM1 takes the comparison value and
* CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'.
* CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0
@@ -1064,7 +1077,7 @@ void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
*/
if (kw_type == NPC_MCAM_KEY_X2) {
/* Clear mcam entry to avoid writes being suppressed by NPC */
- npc_cn20k_clear_mcam_entry(rvu, blkaddr, bank, mcam_idx);
+ npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx);
npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
mcam_idx, intf, entry,
bank, kw_type, kw, req_kw_type);
@@ -1085,44 +1098,55 @@ void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 1),
entry->vtag_action);
- goto set_cfg;
+
+ /* Set HW priority */
+ rvu_write64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+ bank_cfg);
+
+ } else {
+ /* Clear mcam entry to avoid writes being suppressed by NPC */
+ npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx);
+ npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx);
+
+ npc_cn20k_config_kw_x4(rvu, mcam, blkaddr,
+ mcam_idx, intf, entry,
+ kw_type, req_kw_type);
+ for (bank = 0; bank < mcam->banks_per_entry; bank++) {
+ /* Set 'action' */
+ rvu_write64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+ bank, 0),
+ entry->action);
+
+ /* Set TAG 'action' */
+ rvu_write64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+ bank, 1),
+ entry->vtag_action);
+
+ /* Set 'action2' for inline receive */
+ rvu_write64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
+ bank, 2),
+ entry->action2);
+
+ /* Set HW priority */
+ rvu_write64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
+ bank_cfg);
+ }
}
- /* Clear mcam entry to avoid writes being suppressed by NPC */
- npc_cn20k_clear_mcam_entry(rvu, blkaddr, 0, mcam_idx);
- npc_cn20k_clear_mcam_entry(rvu, blkaddr, 1, mcam_idx);
-
- npc_cn20k_config_kw_x4(rvu, mcam, blkaddr,
- mcam_idx, intf, entry,
- kw_type, req_kw_type);
- for (bank = 0; bank < mcam->banks_per_entry; bank++) {
- /* Set 'action' */
- rvu_write64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
- bank, 0),
- entry->action);
-
- /* Set TAG 'action' */
- rvu_write64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
- bank, 1),
- entry->vtag_action);
-
- /* Set 'action2' for inline receive */
- rvu_write64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
- bank, 2),
- entry->action2);
- }
-
-set_cfg:
/* TODO: */
/* PF installing VF rule */
- npc_cn20k_set_mcam_bank_cfg(rvu, blkaddr, mcam_idx, bank,
- kw_type, enable, hw_prio);
+ if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable))
+ return -EINVAL;
+
+ return 0;
}
-void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
+int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u64 cfg, sreg, dreg, soff, doff;
@@ -1130,12 +1154,20 @@ void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
int bank, i, sb, db;
int dbank, sbank;
+ if (src >= mcam->total_entries || dest >= mcam->total_entries)
+ return -EINVAL;
+
dbank = npc_get_bank(mcam, dest);
sbank = npc_get_bank(mcam, src);
- npc_mcam_idx_2_key_type(rvu, src, &src_kwtype);
- npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype);
+
+ if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype))
+ return -EINVAL;
+
+ if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype))
+ return -EINVAL;
+
if (src_kwtype != dest_kwtype)
- return;
+ return -EINVAL;
src &= (mcam->banksize - 1);
dest &= (mcam->banksize - 1);
@@ -1170,6 +1202,8 @@ void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
if (src_kwtype == NPC_MCAM_KEY_X2)
break;
}
+
+ return 0;
}
static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx,
@@ -1179,21 +1213,37 @@ static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx,
entry->kw_mask[idx] = cam1 ^ cam0;
}
-void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
- struct cn20k_mcam_entry *entry,
- u8 *intf, u8 *ena, u8 *hw_prio)
+int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
+ struct cn20k_mcam_entry *entry,
+ u8 *intf, u8 *ena, u8 *hw_prio)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u64 cam0, cam1, bank_cfg, cfg;
int kw = 0, bank;
u8 kw_type;
- npc_mcam_idx_2_key_type(rvu, index, &kw_type);
+ if (index >= mcam->total_entries)
+ return -EINVAL;
+
+ if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
+ return -EINVAL;
bank = npc_get_bank(mcam, index);
index &= (mcam->banksize - 1);
cfg = rvu_read64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0));
+ entry->action = cfg;
+
+ cfg = rvu_read64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1));
+ entry->vtag_action = cfg;
+
+ cfg = rvu_read64(rvu, blkaddr,
+ NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2));
+ entry->action2 = cfg;
+
+ cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index,
bank, 1)) & 3;
*intf = cfg;
@@ -1242,7 +1292,7 @@ void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
- goto read_action;
+ return 0;
}
for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) {
@@ -1287,17 +1337,7 @@ void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
}
-read_action:
- /* 'action' is set to same value for both bank '0' and '1'.
- * Hence, reading bank '0' should be enough.
- */
- cfg = rvu_read64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 0));
- entry->action = cfg;
-
- cfg = rvu_read64(rvu, blkaddr,
- NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 1));
- entry->vtag_action = cfg;
+ return 0;
}
int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu,
@@ -1335,11 +1375,10 @@ int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu,
if (is_pffunc_af(req->hdr.pcifunc))
nix_intf = req->intf;
- npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf,
- &req->entry_data, req->enable_entry,
- req->hw_prio, req->req_kw_type);
+ rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf,
+ &req->entry_data, req->enable_entry,
+ req->hw_prio, req->req_kw_type);
- rc = 0;
exit:
mutex_unlock(&mcam->lock);
return rc;
@@ -1361,11 +1400,13 @@ int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu,
mutex_lock(&mcam->lock);
rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
- if (!rc)
- npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry,
- &rsp->entry_data, &rsp->intf,
- &rsp->enable, &rsp->hw_prio);
+ if (rc)
+ goto fail;
+ rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry,
+ &rsp->entry_data, &rsp->intf,
+ &rsp->enable, &rsp->hw_prio);
+fail:
mutex_unlock(&mcam->lock);
return rc;
}
@@ -1375,11 +1416,13 @@ int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu,
struct npc_mcam_alloc_and_write_entry_rsp *rsp)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
+ struct npc_mcam_free_entry_req free_req = { 0 };
struct npc_mcam_alloc_entry_req entry_req;
struct npc_mcam_alloc_entry_rsp entry_rsp;
struct npc_mcam *mcam = &rvu->hw->mcam;
u16 entry = NPC_MCAM_ENTRY_INVALID;
- int blkaddr, rc;
+ struct msg_rsp free_rsp;
+ int blkaddr, rc, err;
u8 nix_intf;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
@@ -1415,12 +1458,23 @@ int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu,
else
nix_intf = pfvf->nix_rx_intf;
- npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf,
- &req->entry_data, req->enable_entry,
- req->hw_prio, req->req_kw_type);
+ rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf,
+ &req->entry_data, req->enable_entry,
+ req->hw_prio, req->req_kw_type);
mutex_unlock(&mcam->lock);
+ if (rc) {
+ free_req.hdr.pcifunc = req->hdr.pcifunc;
+ free_req.entry = entry_rsp.entry;
+ err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
+ if (err)
+ dev_err(rvu->dev,
+ "%s: Error to free mcam idx %u\n",
+ __func__, entry_rsp.entry);
+ return rc;
+ }
+
rsp->entry = entry_rsp.entry;
return 0;
}
@@ -1480,9 +1534,9 @@ int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu,
read_entry:
/* Read the mcam entry */
- npc_cn20k_read_mcam_entry(rvu, blkaddr, index,
- &rsp->entry, &intf,
- &enable, &hw_prio);
+ rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index,
+ &rsp->entry, &intf,
+ &enable, &hw_prio);
mutex_unlock(&mcam->lock);
out:
return rc;
@@ -2305,6 +2359,7 @@ static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
__npc_subbank_mark_free(rvu, sb);
err1:
kfree(save);
+ *alloc_cnt = 0;
return rc;
}
@@ -3482,7 +3537,7 @@ static int npc_defrag_alloc_free_slots(struct rvu *rvu,
{
int alloc_cnt1, alloc_cnt2;
struct npc_subbank *sb;
- int rc, sb_off, i;
+ int rc, sb_off, i, err;
bool deleted;
sb = &npc_priv.sb[f->idx];
@@ -3496,6 +3551,7 @@ static int npc_defrag_alloc_free_slots(struct rvu *rvu,
NPC_MCAM_LOWER_PRIO,
false, cnt, save, cnt, true,
&alloc_cnt1);
+
if (alloc_cnt1 < cnt) {
rc = __npc_subbank_alloc(rvu, sb,
NPC_MCAM_KEY_X2, sb->b1b,
@@ -3511,15 +3567,17 @@ static int npc_defrag_alloc_free_slots(struct rvu *rvu,
dev_err(rvu->dev,
"%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n",
__func__, cnt, alloc_cnt1, alloc_cnt2);
+ rc = -ENOSPC;
goto fail_free_alloc;
}
+
return 0;
fail_free_alloc:
for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) {
- rc = npc_mcam_idx_2_subbank_idx(rvu, save[i],
- &sb, &sb_off);
- if (rc) {
+ err = npc_mcam_idx_2_subbank_idx(rvu, save[i],
+ &sb, &sb_off);
+ if (err) {
dev_err(rvu->dev,
"%s: Error to find subbank for mcam idx=%u\n",
__func__, save[i]);
@@ -3565,9 +3623,10 @@ int npc_defrag_move_vdx_to_free(struct rvu *rvu,
struct npc_defrag_node *v,
int cnt, u16 *save)
{
+ u16 new_midx, old_midx, vidx, target_pf;
struct npc_mcam *mcam = &rvu->hw->mcam;
+ struct rvu_npc_mcam_rule *rule, *tmp;
int i, vidx_cnt, rc, sb_off;
- u16 new_midx, old_midx, vidx;
struct npc_subbank *sb;
bool deleted;
u16 pcifunc;
@@ -3607,9 +3666,30 @@ int npc_defrag_move_vdx_to_free(struct rvu *rvu,
NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx,
bank));
- npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false);
- npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx);
- npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true);
+ /* If bug happened during copy/enable mcam, then there is a bug in allocation
+ * algorithm itself. There is no point in rewinding and returning, as it
+ * will face further issue. Return error after printing error
+ */
+ if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) {
+ dev_err(rvu->dev,
+ "%s: Error happened while disabling old_mid=%u\n",
+ __func__, old_midx);
+ return -EFAULT;
+ }
+
+ if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) {
+ dev_err(rvu->dev,
+ "%s: Error happened while copying old_midx=%u new_midx=%u\n",
+ __func__, old_midx, new_midx);
+ return -EFAULT;
+ }
+
+ if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) {
+ dev_err(rvu->dev,
+ "%s: Error happened while enabling new_mid=%u\n",
+ __func__, new_midx);
+ return -EFAULT;
+ }
midx = new_midx % mcam->banksize;
bank = new_midx / mcam->banksize;
@@ -3665,8 +3745,21 @@ int npc_defrag_move_vdx_to_free(struct rvu *rvu,
mcam->entry2pfvf_map[new_midx] = pcifunc;
/* Counter is not preserved */
mcam->entry2cntr_map[new_midx] = new_midx;
+ target_pf = mcam->entry2target_pffunc[old_midx];
+ mcam->entry2target_pffunc[new_midx] = target_pf;
+ mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP;
+
npc_mcam_set_bit(mcam, new_midx);
+ /* Note: list order is not functionally required for mcam_rules */
+ list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
+ if (rule->entry != old_midx)
+ continue;
+
+ rule->entry = new_midx;
+ break;
+ }
+
/* Mark as invalid */
v->vidx[vidx_cnt - i - 1] = -1;
save[cnt - i - 1] = -1;
@@ -3935,6 +4028,13 @@ int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
void *val;
int i, j;
+ for (i = 0; i < ARRAY_SIZE(ptr); i++) {
+ if (!ptr[i])
+ continue;
+
+ *ptr[i] = USHRT_MAX;
+ }
+
if (!npc_priv.init_done)
return 0;
@@ -3950,7 +4050,6 @@ int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
pcifunc);
- *ptr[0] = USHRT_MAX;
return -ESRCH;
}
@@ -3970,7 +4069,6 @@ int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
pcifunc);
- *ptr[3] = USHRT_MAX;
return -ESRCH;
}
@@ -3990,7 +4088,6 @@ int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
__func__,
npc_dft_rule_name[i], pcifunc);
- *ptr[j] = USHRT_MAX;
continue;
}
@@ -4085,7 +4182,7 @@ int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req,
return 0;
}
-static bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
+bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
{
return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) ||
is_lbk_vf(rvu, pcifunc);
@@ -4093,11 +4190,11 @@ static bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc)
{
- struct npc_mcam_free_entry_req free_req = { 0 };
+ struct npc_mcam *mcam = &rvu->hw->mcam;
+ u16 ptr[4] = {[0 ... 3] = USHRT_MAX};
+ struct rvu_npc_mcam_rule *rule, *tmp;
unsigned long index;
- struct msg_rsp rsp;
- u16 ptr[4];
- int rc, i;
+ int blkaddr, rc, i;
void *map;
if (!npc_priv.init_done)
@@ -4155,14 +4252,43 @@ void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc)
}
free_rules:
+ blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
+ if (blkaddr < 0)
+ return;
+ for (int i = 0; i < 4; i++) {
+ if (ptr[i] == USHRT_MAX)
+ continue;
- free_req.hdr.pcifunc = pcifunc;
- free_req.all = 1;
- rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp);
- if (rc)
- dev_err(rvu->dev,
- "%s: Error deleting default entries (pcifunc=%#x\n",
- __func__, pcifunc);
+ mutex_lock(&mcam->lock);
+ npc_mcam_clear_bit(mcam, ptr[i]);
+ mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP;
+ npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false);
+ mcam->entry2target_pffunc[ptr[i]] = 0x0;
+ mutex_unlock(&mcam->lock);
+
+ rc = npc_cn20k_idx_free(rvu, &ptr[i], 1);
+ if (rc) {
+ /* Non recoverable error. Let us WARN and return. Keep system alive to
+ * enable debugging
+ */
+ WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n",
+ __func__, pcifunc, ptr[i]);
+ return;
+ }
+ }
+
+ mutex_lock(&mcam->lock);
+ list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
+ for (int i = 0; i < 4; i++) {
+ if (ptr[i] != rule->entry)
+ continue;
+
+ list_del(&rule->list);
+ kfree(rule);
+ break;
+ }
+ }
+ mutex_unlock(&mcam->lock);
}
int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
index 815d0b2..3d5eb95 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h
@@ -320,21 +320,21 @@ void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc);
int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
u16 *mcast, u16 *promisc, u16 *ucast);
-void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
- u8 intf, struct cn20k_mcam_entry *entry,
- bool enable, u8 hw_prio, u8 req_kw_type);
-void npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
- int index, bool enable);
-void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr,
- u16 src, u16 dest);
-void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
- struct cn20k_mcam_entry *entry, u8 *intf,
- u8 *ena, u8 *hw_prio);
-void npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr,
- int bank, int index);
+int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
+ u8 intf, struct cn20k_mcam_entry *entry,
+ bool enable, u8 hw_prio, u8 req_kw_type);
+int npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
+ int index, bool enable);
+int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr,
+ u16 src, u16 dest);
+int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
+ struct cn20k_mcam_entry *entry, u8 *intf,
+ u8 *ena, u8 *hw_prio);
+int npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int index);
int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type);
u16 npc_cn20k_vidx2idx(u16 index);
u16 npc_cn20k_idx2vidx(u16 idx);
int npc_cn20k_defrag(struct rvu *rvu);
+bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc);
#endif /* NPC_CN20K_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
index e40b790..3cf1315 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
@@ -436,7 +436,7 @@ struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
return &rvu->pf[rvu_get_pf(rvu->pdev, pcifunc)];
}
-static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
+bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
{
int pf, vf, nvfs;
u64 cfg;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index a466181..65397da 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -917,6 +917,7 @@ u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blkaddr);
struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc);
void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf);
bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr);
+bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc);
bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype);
int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot);
int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf);
@@ -1144,6 +1145,7 @@ int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf,
int slot);
int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc);
int rvu_cpt_init(struct rvu *rvu);
+u32 rvu_get_cpt_chan_mask(struct rvu *rvu);
#define NDC_AF_BANK_MASK GENMASK_ULL(7, 0)
#define NDC_AF_BANK_LINE_MASK GENMASK_ULL(31, 16)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index ef5b081..f977734 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -3577,6 +3577,9 @@ static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc,
mcam_index = npc_get_nixlf_mcam_index(mcam,
pcifunc & ~RVU_PFVF_FUNC_MASK,
nixlf, type);
+ if (mcam_index < 0)
+ return -EINVAL;
+
err = nix_update_mce_list(rvu, pcifunc, mce_list,
mce_idx, mcam_index, add);
return err;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
index c2ca5ed..d301a3f 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
@@ -163,14 +163,35 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
if (rc)
return -EFAULT;
+ if (is_lbk_vf(rvu, pcifunc)) {
+ if (promisc == USHRT_MAX)
+ return -EINVAL;
+ return promisc;
+ }
+
+ if (is_cgx_vf(rvu, pcifunc)) {
+ if (ucast == USHRT_MAX)
+ return -EINVAL;
+
+ return ucast;
+ }
+
switch (type) {
case NIXLF_BCAST_ENTRY:
+ if (bcast == USHRT_MAX)
+ return -EINVAL;
return bcast;
case NIXLF_ALLMULTI_ENTRY:
+ if (mcast == USHRT_MAX)
+ return -EINVAL;
return mcast;
case NIXLF_PROMISC_ENTRY:
+ if (promisc == USHRT_MAX)
+ return -EINVAL;
return promisc;
case NIXLF_UCAST_ENTRY:
+ if (ucast == USHRT_MAX)
+ return -EINVAL;
return ucast;
default:
return -EINVAL;
@@ -238,10 +259,10 @@ void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
int actbank = bank;
if (is_cn20k(rvu->pdev)) {
- if (index < 0 || index >= mcam->banksize * mcam->banks)
- return;
-
- return npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable);
+ if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable))
+ dev_err(rvu->dev, "Error to %s mcam %u entry\n",
+ enable ? "enable" : "disable", index);
+ return;
}
index &= (mcam->banksize - 1);
@@ -258,6 +279,13 @@ static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
int bank = npc_get_bank(mcam, index);
int actbank = bank;
+ if (is_cn20k(rvu->pdev)) {
+ if (npc_cn20k_clear_mcam_entry(rvu, blkaddr, index))
+ dev_err(rvu->dev, "%s Failed to clear mcam %u\n",
+ __func__, index);
+ return;
+ }
+
index &= (mcam->banksize - 1);
for (; bank < (actbank + mcam->banks_per_entry); bank++) {
rvu_write64(rvu, blkaddr,
@@ -424,6 +452,15 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam,
index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf,
NIXLF_UCAST_ENTRY);
+
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: failed to get ucast entry pcifunc:0x%x\n",
+ __func__, pf_func);
+ /* Action 0 is drop */
+ return 0;
+ }
+
bank = npc_get_bank(mcam, index);
index &= (mcam->banksize - 1);
@@ -589,8 +626,8 @@ void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1;
}
-static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
- int blkaddr, u16 src, u16 dest)
+static int npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
+ int blkaddr, u16 src, u16 dest)
{
int dbank = npc_get_bank(mcam, dest);
int sbank = npc_get_bank(mcam, src);
@@ -630,6 +667,7 @@ static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
NPC_AF_MCAMEX_BANKX_CFG(src, sbank));
rvu_write64(rvu, blkaddr,
NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg);
+ return 0;
}
u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
@@ -663,6 +701,19 @@ void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
return rvu_write64(rvu, blkaddr, reg, cfg);
}
+u32 rvu_get_cpt_chan_mask(struct rvu *rvu)
+{
+ /* For cn10k the upper two bits of the channel number are
+ * cpt channel number. with masking out these bits in the
+ * mcam entry, same entry used for NIX will allow packets
+ * received from cpt for parsing.
+ */
+ if (!is_rvu_otx2(rvu))
+ return NIX_CHAN_CPT_X2P_MASK;
+ else
+ return 0xFFFu;
+}
+
void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
int nixlf, u64 chan, u8 *mac_addr)
{
@@ -689,6 +740,12 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_UCAST_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
/* Don't change the action if entry is already enabled
* Otherwise RSS action may get overwritten.
@@ -706,7 +763,7 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
eth_broadcast_addr((u8 *)&req.mask.dmac);
req.features = BIT_ULL(NPC_DMAC);
req.channel = chan;
- req.chan_mask = 0xFFFU;
+ req.chan_mask = rvu_get_cpt_chan_mask(rvu);
req.intf = pfvf->nix_rx_intf;
req.op = action.op;
req.hdr.pcifunc = 0; /* AF is requester */
@@ -744,16 +801,38 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_PROMISC_ENTRY);
+ /* In cn20k, default indexes are installed only for CGX mapped
+ * and lbk interfaces
+ */
if (is_cgx_vf(rvu, pcifunc))
index = npc_get_nixlf_mcam_index(mcam,
pcifunc & ~RVU_PFVF_FUNC_MASK,
nixlf, NIXLF_PROMISC_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get promisc entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
+
/* If the corresponding PF's ucast action is RSS,
* use the same action for promisc also
+ * Please note that for lbk(s) "index" and "ucast_idx"
+ * will be same.
*/
- ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
- nixlf, NIXLF_UCAST_ENTRY);
+ if (is_lbk_vf(rvu, pcifunc))
+ ucast_idx = index;
+ else
+ ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
+ nixlf, NIXLF_UCAST_ENTRY);
+ if (ucast_idx < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast/promisc entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
+
if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))
*(u64 *)&action = npc_get_mcam_action(rvu, mcam,
blkaddr, ucast_idx);
@@ -779,11 +858,7 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
* mcam entry, same entry used for NIX will allow packets
* received from cpt for parsing.
*/
- if (!is_rvu_otx2(rvu)) {
- req.chan_mask = NIX_CHAN_CPT_X2P_MASK;
- } else {
- req.chan_mask = 0xFFFU;
- }
+ req.chan_mask = rvu_get_cpt_chan_mask(rvu);
if (chan_cnt > 1) {
if (!is_power_of_2(chan_cnt)) {
@@ -827,6 +902,14 @@ void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc,
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_PROMISC_ENTRY);
+
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get promisc entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
+
npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
}
@@ -867,6 +950,12 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc,
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_BCAST_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get bcast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
if (!hw->cap.nix_rx_multicast) {
/* Early silicon doesn't support pkt replication,
@@ -910,7 +999,7 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
u16 vf_func;
/* Only CGX PF/VF can add allmulticast entry */
- if (is_lbk_vf(rvu, pcifunc) && is_sdp_vf(rvu, pcifunc))
+ if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc))
return;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
@@ -931,12 +1020,25 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_ALLMULTI_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get mcast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
/* If the corresponding PF's ucast action is RSS,
* use the same action for multicast entry also
*/
ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_UCAST_ENTRY);
+ if (ucast_idx < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
+
if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))
*(u64 *)&action = npc_get_mcam_action(rvu, mcam,
blkaddr, ucast_idx);
@@ -960,16 +1062,7 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
ether_addr_copy(req.mask.dmac, mac_addr);
req.features = BIT_ULL(NPC_DMAC);
- /* For cn10k the upper two bits of the channel number are
- * cpt channel number. with masking out these bits in the
- * mcam entry, same entry used for NIX will allow packets
- * received from cpt for parsing.
- */
- if (!is_rvu_otx2(rvu))
- req.chan_mask = NIX_CHAN_CPT_X2P_MASK;
- else
- req.chan_mask = 0xFFFU;
-
+ req.chan_mask = rvu_get_cpt_chan_mask(rvu);
req.channel = chan;
req.intf = pfvf->nix_rx_intf;
req.entry = index;
@@ -1001,6 +1094,13 @@ void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,
NIXLF_ALLMULTI_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get mcast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
+
npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
}
@@ -1113,8 +1213,12 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
index = mcam_index;
}
- if (index >= mcam->total_entries)
+ if (index < 0 || index >= mcam->total_entries) {
+ dev_err(rvu->dev,
+ "%s: Invalid mcam index, pcifunc=%#x\n",
+ __func__, pcifunc);
return;
+ }
bank = npc_get_bank(mcam, index);
index &= (mcam->banksize - 1);
@@ -1158,16 +1262,18 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
/* If PF's promiscuous entry is enabled,
* Set RSS action for that entry as well
*/
- npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index,
- blkaddr, alg_idx);
+ if (index >= 0)
+ npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index,
+ blkaddr, alg_idx);
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_ALLMULTI_ENTRY);
/* If PF's allmulti entry is enabled,
* Set RSS action for that entry as well
*/
- npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index,
- blkaddr, alg_idx);
+ if (index >= 0)
+ npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index,
+ blkaddr, alg_idx);
}
}
@@ -1180,12 +1286,22 @@ void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc,
int index, blkaddr, mce_idx;
struct rvu_pfvf *pfvf;
+ /* multicast pkt replication is not enabled for AF's VFs & SDP links */
+ if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(rvu, pcifunc))
+ return;
+
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return;
index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK,
nixlf, type);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get entry for pcifunc=%#x, type=%u\n",
+ __func__, pcifunc, type);
+ return;
+ }
/* disable MCAM entry when packet replication is not supported by hw */
if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) {
@@ -1214,6 +1330,10 @@ static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc,
struct npc_mcam *mcam = &rvu->hw->mcam;
int index, blkaddr;
+ /* only CGX or LBK interfaces have default entries */
+ if (is_cn20k(rvu->pdev) && !npc_is_cgx_or_lbk(rvu, pcifunc))
+ return;
+
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return;
@@ -1223,6 +1343,12 @@ static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc,
pfvf->nix_rx_intf)) {
index = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_UCAST_ENTRY);
+ if (index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
}
@@ -2066,8 +2192,8 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr)
goto free_entry_cntr_map;
/* Alloc memory for saving target device of mcam rule */
- mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries,
- sizeof(u16), GFP_KERNEL);
+ mcam->entry2target_pffunc = kcalloc(mcam->total_entries,
+ sizeof(u16), GFP_KERNEL);
if (!mcam->entry2target_pffunc)
goto free_cntr_refcnt;
@@ -2504,33 +2630,58 @@ void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index)
static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,
int blkaddr, u16 pcifunc)
{
+ u16 dft_idxs[NPC_DFT_RULE_MAX_ID] = {[0 ... NPC_DFT_RULE_MAX_ID - 1] = USHRT_MAX};
+ bool cn20k_dft_rl;
u16 index, cntr;
int rc;
+ npc_cn20k_dft_rules_idx_get(rvu, pcifunc,
+ &dft_idxs[NPC_DFT_RULE_BCAST_ID],
+ &dft_idxs[NPC_DFT_RULE_MCAST_ID],
+ &dft_idxs[NPC_DFT_RULE_PROMISC_ID],
+ &dft_idxs[NPC_DFT_RULE_UCAST_ID]);
+
/* Scan all MCAM entries and free the ones mapped to 'pcifunc' */
for (index = 0; index < mcam->bmap_entries; index++) {
- if (mcam->entry2pfvf_map[index] == pcifunc) {
+ if (mcam->entry2pfvf_map[index] != pcifunc)
+ continue;
+
+ cn20k_dft_rl = false;
+
+ if (is_cn20k(rvu->pdev)) {
+ if (dft_idxs[NPC_DFT_RULE_BCAST_ID] == index ||
+ dft_idxs[NPC_DFT_RULE_MCAST_ID] == index ||
+ dft_idxs[NPC_DFT_RULE_PROMISC_ID] == index ||
+ dft_idxs[NPC_DFT_RULE_UCAST_ID] == index) {
+ cn20k_dft_rl = true;
+ }
+ }
+
+ /* Disable the entry */
+ npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false);
+
+ if (!cn20k_dft_rl) {
mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP;
/* Free the entry in bitmap */
npc_mcam_clear_bit(mcam, index);
- /* Disable the entry */
- npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false);
-
- /* Update entry2counter mapping */
- cntr = mcam->entry2cntr_map[index];
- if (cntr != NPC_MCAM_INVALID_MAP)
- npc_unmap_mcam_entry_and_cntr(rvu, mcam,
- blkaddr, index,
- cntr);
mcam->entry2target_pffunc[index] = 0x0;
- if (is_cn20k(rvu->pdev)) {
- rc = npc_cn20k_idx_free(rvu, &index, 1);
- if (rc)
- dev_err(rvu->dev,
- "Failed to free mcam idx=%u pcifunc=%#x\n",
- index, pcifunc);
- }
}
+
+ /* Update entry2counter mapping */
+ cntr = mcam->entry2cntr_map[index];
+ if (cntr != NPC_MCAM_INVALID_MAP)
+ npc_unmap_mcam_entry_and_cntr(rvu, mcam,
+ blkaddr, index,
+ cntr);
+
+ if (!is_cn20k(rvu->pdev) || cn20k_dft_rl)
+ continue;
+
+ rc = npc_cn20k_idx_free(rvu, &index, 1);
+ if (rc)
+ dev_err(rvu->dev,
+ "Failed to free mcam idx=%u pcifunc=%#x\n",
+ index, pcifunc);
}
}
@@ -3266,7 +3417,10 @@ int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu,
npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false);
/* Copy rule from old entry to new entry */
- npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry);
+ if (npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry)) {
+ rc = NPC_MCAM_INVALID_REQ;
+ break;
+ }
/* Copy counter mapping, if any */
cntr = mcam->entry2cntr_map[old_entry];
@@ -3284,7 +3438,8 @@ int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu,
/* If shift has failed then report the failed index */
if (index != req->shift_count) {
- rc = NPC_MCAM_PERM_DENIED;
+ if (!rc)
+ rc = NPC_MCAM_PERM_DENIED;
rsp->failed_entry_idx = index;
}
@@ -3851,6 +4006,12 @@ int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu,
/* Read the default ucast entry if there is no pkt steering rule */
index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,
NIXLF_UCAST_ENTRY);
+ if (index < 0) {
+ mutex_unlock(&mcam->lock);
+ rc = NIX_AF_ERR_AF_LF_INVALID;
+ goto out;
+ }
+
read_entry:
/* Read the mcam entry */
npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf,
@@ -3924,6 +4085,12 @@ void rvu_npc_clear_ucast_entry(struct rvu *rvu, int pcifunc, int nixlf)
ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
nixlf, NIXLF_UCAST_ENTRY);
+ if (ucast_idx < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast entry for pcifunc=%#x\n",
+ __func__, pcifunc);
+ return;
+ }
npc_enable_mcam_entry(rvu, mcam, blkaddr, ucast_idx, false);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
index b45798d..34f1e06 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
@@ -1444,7 +1444,7 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
struct msg_rsp write_rsp;
struct mcam_entry *entry;
bool new = false;
- u16 entry_index;
+ int entry_index;
int err;
installed_features = req->features;
@@ -1477,6 +1477,14 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
if (req->default_rule) {
entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf,
NIXLF_UCAST_ENTRY);
+
+ if (entry_index < 0) {
+ dev_err(rvu->dev,
+ "%s: Error to get ucast entry for target=%#x\n",
+ __func__, target);
+ return -EINVAL;
+ }
+
enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index);
}
@@ -1812,7 +1820,7 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
/* ignore chan_mask in case pf func is not AF, revisit later */
if (!is_pffunc_af(req->hdr.pcifunc))
- req->chan_mask = 0xFFF;
+ req->chan_mask = rvu_get_cpt_chan_mask(rvu);
err = npc_check_unsupported_flows(rvu, req->features, req->intf);
if (err) {
@@ -1980,13 +1988,15 @@ static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
- if (is_cn20k(rvu->pdev))
- npc_cn20k_read_mcam_entry(rvu, npcblkaddr, rule->entry,
- cn20k_entry, &intf,
- &enable, &hw_prio);
- else
+ if (is_cn20k(rvu->pdev)) {
+ if (npc_cn20k_read_mcam_entry(rvu, npcblkaddr, rule->entry,
+ cn20k_entry, &intf,
+ &enable, &hw_prio))
+ return -EINVAL;
+ } else {
npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
entry, &intf, &enable);
+ }
npc_update_entry(rvu, NPC_DMAC, &mdata,
ether_addr_to_u64(pfvf->mac_addr), 0,
@@ -2038,8 +2048,12 @@ void npc_mcam_enable_flows(struct rvu *rvu, u16 target)
continue;
}
- if (rule->vfvlan_cfg)
- npc_update_dmac_value(rvu, blkaddr, rule, pfvf);
+ if (rule->vfvlan_cfg) {
+ if (npc_update_dmac_value(rvu, blkaddr, rule, pfvf))
+ dev_err(rvu->dev,
+ "Update dmac failed for %u, target=%#x\n",
+ rule->entry, target);
+ }
if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) {
if (!def_ucast_rule)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c
index 901f6fd..a2781e0 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c
@@ -97,6 +97,14 @@ int rvu_mbox_handler_rep_event_notify(struct rvu *rvu, struct rep_event *req,
{
struct rep_evtq_ent *qentry;
+ /* The mailbox dispatcher normalises only the header pcifunc; the
+ * nested struct rep_event::pcifunc body field is sender-controlled
+ * and is later used by rvu_rep_up_notify() to index rvu->pf[] /
+ * rvu->hwvf[]. Reject out-of-range body selectors before queueing.
+ */
+ if (!is_pf_func_valid(rvu, req->pcifunc))
+ return -EINVAL;
+
qentry = kmalloc_obj(*qentry, GFP_ATOMIC);
if (!qentry)
return -ENOMEM;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
index a5a8f45..dbf1731 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
@@ -619,11 +619,13 @@ static int cn20k_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
err = otx2_sync_mbox_msg(&pfvf->mbox);
if (err) {
qmem_free(pfvf->dev, pool->stack);
+ pool->stack = NULL;
return err;
}
aq = otx2_mbox_alloc_msg_npa_cn20k_aq_enq(&pfvf->mbox);
if (!aq) {
qmem_free(pfvf->dev, pool->stack);
+ pool->stack = NULL;
return -ENOMEM;
}
}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 971fcab..3d25313 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1482,11 +1482,13 @@ int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
err = otx2_sync_mbox_msg(&pfvf->mbox);
if (err) {
qmem_free(pfvf->dev, pool->stack);
+ pool->stack = NULL;
return err;
}
aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
if (!aq) {
qmem_free(pfvf->dev, pool->stack);
+ pool->stack = NULL;
return -ENOMEM;
}
}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index ee62347..f9fbf0c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -3473,7 +3473,7 @@ static void otx2_ndc_sync(struct otx2_nic *pf)
req->nix_lf_rx_sync = 1;
req->npa_lf_sync = 1;
- if (!otx2_sync_mbox_msg(mbox))
+ if (otx2_sync_mbox_msg(mbox))
dev_err(pf->dev, "NDC sync operation failed\n");
mutex_unlock(&mbox->lock);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/rep.c b/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
index 94f155f..0f5d564 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
@@ -609,7 +609,7 @@ static int rvu_rep_rsrc_init(struct otx2_nic *priv)
err = otx2_init_hw_resources(priv);
if (err)
- goto err_free_rsrc;
+ goto err_free_mem;
/* Set maximum frame size allowed in HW */
err = otx2_hw_set_mtu(priv, priv->hw.max_mtu);
@@ -621,6 +621,7 @@ static int rvu_rep_rsrc_init(struct otx2_nic *priv)
err_free_rsrc:
otx2_free_hw_resources(priv);
+err_free_mem:
otx2_free_queue_mem(qset);
return err;
}
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index 8d225bc..7d77116 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -4491,7 +4491,7 @@ static int mtk_free_dev(struct mtk_eth *eth)
for (i = 0; i < ARRAY_SIZE(eth->dsa_meta); i++) {
if (!eth->dsa_meta[i])
break;
- metadata_dst_free(eth->dsa_meta[i]);
+ dst_release(ð->dsa_meta[i]->dst);
}
return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx4/srq.c b/drivers/net/ethernet/mellanox/mlx4/srq.c
index dd890f5..8711689 100644
--- a/drivers/net/ethernet/mellanox/mlx4/srq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/srq.c
@@ -44,13 +44,14 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type)
{
struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
struct mlx4_srq *srq;
+ unsigned long flags;
- rcu_read_lock();
+ spin_lock_irqsave(&srq_table->lock, flags);
srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1));
- rcu_read_unlock();
- if (srq)
- refcount_inc(&srq->refcount);
- else {
+ if (!srq || !refcount_inc_not_zero(&srq->refcount))
+ srq = NULL;
+ spin_unlock_irqrestore(&srq_table->lock, flags);
+ if (!srq) {
mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn);
return;
}
@@ -203,8 +204,8 @@ int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd,
if (err)
goto err_radix;
- refcount_set(&srq->refcount, 1);
init_completion(&srq->free);
+ refcount_set_release(&srq->refcount, 1);
return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index c89417c..e289597 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -1002,12 +1002,13 @@ static void cmd_work_handler(struct work_struct *work)
ent->callback(-EBUSY, ent->context);
mlx5_free_cmd_msg(dev, ent->out);
free_msg(dev, ent->in);
+ complete(&ent->slotted);
cmd_ent_put(ent);
} else {
ent->ret = -EBUSY;
complete(&ent->done);
+ complete(&ent->slotted);
}
- complete(&ent->slotted);
return;
}
alloc_ret = cmd_alloc_index(cmd, ent);
@@ -1017,13 +1018,14 @@ static void cmd_work_handler(struct work_struct *work)
ent->callback(-EAGAIN, ent->context);
mlx5_free_cmd_msg(dev, ent->out);
free_msg(dev, ent->in);
+ complete(&ent->slotted);
cmd_ent_put(ent);
} else {
ent->ret = -EAGAIN;
complete(&ent->done);
+ complete(&ent->slotted);
}
up(&cmd->vars.sem);
- complete(&ent->slotted);
return;
}
} else {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
index afdeb1b..8409ae7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
@@ -160,13 +160,13 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx)
* channels are being closed for other reason and this work is not
* relevant anymore.
*/
- while (!netdev_trylock(sq->netdev)) {
+ while (!netdev_trylock(priv->netdev)) {
if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state))
return 0;
msleep(20);
}
- err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats);
+ err = mlx5e_health_channel_eq_recover(priv->netdev, eq, sq->cq.ch_stats);
if (!err) {
to_ctx->status = 0; /* this sq recovered */
goto out;
@@ -186,7 +186,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx)
"mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n",
err);
out:
- netdev_unlock(sq->netdev);
+ netdev_unlock(priv->netdev);
return err;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 190b8b6..d3bab19 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -708,7 +708,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
page = xdpi.page.page;
- /* No need to check PageNetpp() as we
+ /* No need to check page_pool_page_is_pp() as we
* know this is a page_pool page.
*/
page_pool_recycle_direct(pp_page_to_nmdesc(page)->pp,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index a52e12c..db260e3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -792,8 +792,10 @@ static int mlx5e_xfrm_add_state(struct net_device *dev,
sa_entry->dev = dev;
sa_entry->ipsec = ipsec;
/* Check if this SA is originated from acquire flow temporary SA */
- if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
- goto out;
+ if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) {
+ x->xso.offload_handle = (unsigned long)sa_entry;
+ return 0;
+ }
err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
if (err)
@@ -870,7 +872,6 @@ static int mlx5e_xfrm_add_state(struct net_device *dev,
xa_unlock_bh(&ipsec->sadb);
}
-out:
x->xso.offload_handle = (unsigned long)sa_entry;
if (allow_tunnel_mode)
mlx5_eswitch_unblock_encap(priv->mdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c
index 6a50b6d..d9adb99 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c
@@ -1070,29 +1070,37 @@ static struct psp_dev_ops mlx5_psp_ops = {
void mlx5e_psp_unregister(struct mlx5e_priv *priv)
{
- if (!priv->psp || !priv->psp->psp)
+ struct mlx5e_psp *psp = priv->psp;
+
+ if (!psp || !psp->psp)
return;
- psp_dev_unregister(priv->psp->psp);
+ psp_dev_unregister(psp->psp);
+ psp->psp = NULL;
}
void mlx5e_psp_register(struct mlx5e_priv *priv)
{
+ struct mlx5e_psp *psp = priv->psp;
+ struct psp_dev *psd;
+
/* FW Caps missing */
if (!priv->psp)
return;
- priv->psp->caps.assoc_drv_spc = sizeof(u32);
- priv->psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128;
+ psp->caps.assoc_drv_spc = sizeof(u32);
+ psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128;
if (MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_encrypt) &&
MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_decrypt))
- priv->psp->caps.versions |= 1 << PSP_VERSION_HDR0_AES_GCM_256;
+ psp->caps.versions |= 1 << PSP_VERSION_HDR0_AES_GCM_256;
- priv->psp->psp = psp_dev_create(priv->netdev, &mlx5_psp_ops,
- &priv->psp->caps, NULL);
- if (IS_ERR(priv->psp->psp))
+ psd = psp_dev_create(priv->netdev, &mlx5_psp_ops, &psp->caps, NULL);
+ if (IS_ERR(psd)) {
mlx5_core_err(priv->mdev, "PSP failed to register due to %pe\n",
- priv->psp->psp);
+ psd);
+ return;
+ }
+ psp->psp = psd;
}
int mlx5e_psp_init(struct mlx5e_priv *priv)
@@ -1131,22 +1139,18 @@ int mlx5e_psp_init(struct mlx5e_priv *priv)
if (!psp)
return -ENOMEM;
- priv->psp = psp;
fs = mlx5e_accel_psp_fs_init(priv);
if (IS_ERR(fs)) {
err = PTR_ERR(fs);
- goto out_err;
+ kfree(psp);
+ return err;
}
psp->fs = fs;
+ priv->psp = psp;
mlx5_core_dbg(priv->mdev, "PSP attached to netdevice\n");
return 0;
-
-out_err:
- priv->psp = NULL;
- kfree(psp);
- return err;
}
void mlx5e_psp_cleanup(struct mlx5e_priv *priv)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index bb61e21..99a0034 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -1574,8 +1574,11 @@ static int mlx5e_create_rxfh_context(struct net_device *dev,
rxfh->indir, rxfh->key,
hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc,
rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric);
- if (err)
+ if (err) {
+ WARN_ON(mlx5e_rx_res_rss_destroy(priv->rx_res,
+ rxfh->rss_context));
goto unlock;
+ }
mlx5e_rx_res_rss_get_rxfh(priv->rx_res, rxfh->rss_context,
ethtool_rxfh_context_indir(ctx),
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 5a46870..8f2b3ab 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -6023,7 +6023,6 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
if (take_rtnl)
rtnl_lock();
- mlx5e_psp_register(priv);
/* update XDP supported features */
mlx5e_set_xdp_feature(priv);
@@ -6036,7 +6035,6 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
{
mlx5e_health_destroy_reporters(priv);
- mlx5e_psp_unregister(priv);
mlx5e_ktls_cleanup(priv);
mlx5e_psp_cleanup(priv);
mlx5e_fs_cleanup(priv->fs);
@@ -6160,6 +6158,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
mlx5e_fs_init_l2_addr(priv->fs, netdev);
mlx5e_ipsec_init(priv);
+ mlx5e_psp_register(priv);
err = mlx5e_macsec_init(priv);
if (err)
@@ -6230,6 +6229,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv)
mlx5_lag_remove_netdev(mdev, priv->netdev);
mlx5_vxlan_reset_to_default(mdev->vxlan);
mlx5e_macsec_cleanup(priv);
+ mlx5e_psp_unregister(priv);
mlx5e_ipsec_cleanup(priv);
}
@@ -6774,9 +6774,11 @@ static int mlx5e_resume(struct auxiliary_device *adev)
return err;
actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
- if (actual_adev)
- return _mlx5e_resume(actual_adev);
- return 0;
+ if (actual_adev) {
+ err = _mlx5e_resume(actual_adev);
+ mlx5_sd_put_adev(actual_adev, adev);
+ }
+ return err;
}
static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg)
@@ -6815,6 +6817,8 @@ static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
err = _mlx5e_suspend(actual_adev, false);
mlx5_sd_cleanup(mdev);
+ if (actual_adev)
+ mlx5_sd_put_adev(actual_adev, adev);
return err;
}
@@ -6912,9 +6916,19 @@ static int mlx5e_probe(struct auxiliary_device *adev,
return err;
actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
- if (actual_adev)
- return _mlx5e_probe(actual_adev);
+ if (actual_adev) {
+ err = _mlx5e_probe(actual_adev);
+ if (err)
+ goto sd_cleanup;
+ mlx5_sd_put_adev(actual_adev, adev);
+ }
return 0;
+
+sd_cleanup:
+ mlx5_sd_cleanup(mdev);
+ if (actual_adev)
+ mlx5_sd_put_adev(actual_adev, adev);
+ return err;
}
static void _mlx5e_remove(struct auxiliary_device *adev)
@@ -6966,6 +6980,8 @@ static void mlx5e_remove(struct auxiliary_device *adev)
_mlx5e_remove(actual_adev);
mlx5_sd_cleanup(mdev);
+ if (actual_adev)
+ mlx5_sd_put_adev(actual_adev, adev);
}
static const struct auxiliary_device_id mlx5e_id_table[] = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index b31f689..e90c6c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -252,7 +252,7 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
mlx5e_cq_arm(&c->xdpsq->cq);
if (unlikely(aff_change && busy_xsk)) {
- mlx5e_trigger_irq(&c->icosq);
+ mlx5e_trigger_napi_async_icosq(c);
ch_stats->force_irq++;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
index 3cfe743..ab50d2c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
@@ -142,7 +142,8 @@ static int mlx5_esw_ipsec_modify_flow_dests(struct mlx5_eswitch *esw,
attr = flow->attr;
esw_attr = attr->esw_attr;
- if (esw_attr->out_count - esw_attr->split_count > 1)
+ if (!esw_attr->out_count ||
+ esw_attr->out_count - esw_attr->split_count > 1)
return 0;
err = mlx5_eswitch_restore_ipsec_rule(esw, flow->rule[0], esw_attr,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 123c967..7c8311f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -908,6 +908,24 @@ static void esw_vport_cleanup(struct mlx5_eswitch *esw, struct mlx5_vport *vport
esw_vport_cleanup_acl(esw, vport);
}
+static void mlx5_esw_vport_set_max_tx_speed(struct mlx5_eswitch *esw,
+ struct mlx5_vport *vport)
+{
+ int ret;
+
+ if (!MLX5_CAP_ESW(esw->dev, esw_vport_state_max_tx_speed))
+ return;
+
+ ret = mlx5_modify_vport_max_tx_speed(esw->dev,
+ MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
+ vport->vport, true,
+ vport->agg_max_tx_speed);
+ if (ret)
+ mlx5_core_dbg(esw->dev,
+ "Failed to set vport %d speed %d, err=%d\n",
+ vport->vport, vport->agg_max_tx_speed, ret);
+}
+
int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
enum mlx5_eswitch_vport_event enabled_events)
{
@@ -948,6 +966,9 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
esw->enabled_vports++;
esw_debug(esw->dev, "Enabled VPORT(%d)\n", vport_num);
+
+ if (vport->agg_max_tx_speed)
+ mlx5_esw_vport_set_max_tx_speed(esw, vport);
done:
mutex_unlock(&esw->state_lock);
return ret;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 5128f50..e9cf7c5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -247,6 +247,7 @@ struct mlx5_vport {
enum mlx5_eswitch_vport_event enabled_events;
int index;
struct mlx5_devlink_port *dl_port;
+ u32 agg_max_tx_speed;
};
struct mlx5_esw_indir_table;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 449e4bd..f8e70ac 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1274,6 +1274,11 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev,
if (vport->vport == MLX5_VPORT_UPLINK)
continue;
+ vport->agg_max_tx_speed = speed;
+
+ if (!vport->enabled)
+ continue;
+
ret = mlx5_modify_vport_max_tx_speed(mdev, op_mod,
vport->vport, true, speed);
if (ret)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index d785f1b..5df7861 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -340,7 +340,7 @@ static int mlx5_ptp_getcrosststamp(struct ptp_clock_info *ptp,
goto unlock;
}
- ktime_get_snapshot(&history_begin);
+ ktime_get_snapshot_id(cts->clock_id, &history_begin);
err = get_device_system_crosststamp(mlx5_mtctr_syncdevicetime, mdev,
&history_begin, cts);
@@ -366,7 +366,7 @@ static int mlx5_ptp_getcrosscycles(struct ptp_clock_info *ptp,
goto unlock;
}
- ktime_get_snapshot(&history_begin);
+ ktime_get_snapshot_id(cts->clock_id, &history_begin);
err = get_device_system_crosststamp(mlx5_mtctr_syncdevicecyclestime,
mdev, &history_begin, cts);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
index 762c783..6e19916 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
@@ -18,6 +18,7 @@ struct mlx5_sd {
u8 host_buses;
struct mlx5_devcom_comp_dev *devcom;
struct dentry *dfs;
+ u8 state;
bool primary;
union {
struct { /* primary */
@@ -31,6 +32,11 @@ struct mlx5_sd {
};
};
+enum mlx5_sd_state {
+ MLX5_SD_STATE_DOWN = 0,
+ MLX5_SD_STATE_UP,
+};
+
static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
{
struct mlx5_sd *sd = mlx5_get_sd(dev);
@@ -270,9 +276,6 @@ static void sd_unregister(struct mlx5_core_dev *dev)
{
struct mlx5_sd *sd = mlx5_get_sd(dev);
- mlx5_devcom_comp_lock(sd->devcom);
- mlx5_devcom_comp_set_ready(sd->devcom, false);
- mlx5_devcom_comp_unlock(sd->devcom);
mlx5_devcom_unregister_component(sd->devcom);
}
@@ -426,6 +429,7 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
struct mlx5_core_dev *primary, *pos, *to;
struct mlx5_sd *sd = mlx5_get_sd(dev);
u8 alias_key[ACCESS_KEY_LEN];
+ struct mlx5_sd *primary_sd;
int err, i;
err = sd_init(dev);
@@ -440,10 +444,17 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
if (err)
goto err_sd_cleanup;
+ mlx5_devcom_comp_lock(sd->devcom);
if (!mlx5_devcom_comp_is_ready(sd->devcom))
- return 0;
+ goto out;
primary = mlx5_sd_get_primary(dev);
+ if (!primary)
+ goto out;
+
+ primary_sd = mlx5_get_sd(primary);
+ if (primary_sd->state != MLX5_SD_STATE_DOWN)
+ goto out;
for (i = 0; i < ACCESS_KEY_LEN; i++)
alias_key[i] = get_random_u8();
@@ -452,9 +463,13 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
if (err)
goto err_sd_unregister;
- sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary));
- debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id);
- debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops);
+ primary_sd->dfs =
+ debugfs_create_dir("multi-pf",
+ mlx5_debugfs_get_dev_root(primary));
+ debugfs_create_x32("group_id", 0400, primary_sd->dfs,
+ &primary_sd->group_id);
+ debugfs_create_file("primary", 0400, primary_sd->dfs, primary,
+ &dev_fops);
mlx5_sd_for_each_secondary(i, primary, pos) {
char name[32];
@@ -464,7 +479,8 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
goto err_unset_secondaries;
snprintf(name, sizeof(name), "secondary_%d", i - 1);
- debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops);
+ debugfs_create_file(name, 0400, primary_sd->dfs, pos,
+ &dev_fops);
}
@@ -472,6 +488,9 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
sd->group_id, mlx5_devcom_comp_get_size(sd->devcom));
sd_print_group(primary);
+ primary_sd->state = MLX5_SD_STATE_UP;
+out:
+ mlx5_devcom_comp_unlock(sd->devcom);
return 0;
err_unset_secondaries:
@@ -479,8 +498,18 @@ int mlx5_sd_init(struct mlx5_core_dev *dev)
mlx5_sd_for_each_secondary_to(i, primary, to, pos)
sd_cmd_unset_secondary(pos);
sd_cmd_unset_primary(primary);
- debugfs_remove_recursive(sd->dfs);
+ debugfs_remove_recursive(primary_sd->dfs);
+ primary_sd->dfs = NULL;
err_sd_unregister:
+ mlx5_sd_for_each_secondary(i, primary, pos) {
+ struct mlx5_sd *peer_sd = mlx5_get_sd(pos);
+
+ primary_sd->secondaries[i - 1] = NULL;
+ peer_sd->primary_dev = NULL;
+ }
+ primary_sd->primary = false;
+ mlx5_devcom_comp_set_ready(sd->devcom, false);
+ mlx5_devcom_comp_unlock(sd->devcom);
sd_unregister(dev);
err_sd_cleanup:
sd_cleanup(dev);
@@ -491,42 +520,97 @@ void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
{
struct mlx5_sd *sd = mlx5_get_sd(dev);
struct mlx5_core_dev *primary, *pos;
+ struct mlx5_sd *primary_sd;
int i;
if (!sd)
return;
+ mlx5_devcom_comp_lock(sd->devcom);
if (!mlx5_devcom_comp_is_ready(sd->devcom))
- goto out;
+ goto out_unlock;
primary = mlx5_sd_get_primary(dev);
+ if (!primary)
+ goto out_ready_false;
+
+ primary_sd = mlx5_get_sd(primary);
+ if (primary_sd->state != MLX5_SD_STATE_UP)
+ goto out_clear_peers;
+
mlx5_sd_for_each_secondary(i, primary, pos)
sd_cmd_unset_secondary(pos);
sd_cmd_unset_primary(primary);
- debugfs_remove_recursive(sd->dfs);
+ debugfs_remove_recursive(primary_sd->dfs);
+ primary_sd->dfs = NULL;
sd_info(primary, "group id %#x, uncombined\n", sd->group_id);
-out:
+ primary_sd->state = MLX5_SD_STATE_DOWN;
+out_clear_peers:
+ mlx5_sd_for_each_secondary(i, primary, pos) {
+ struct mlx5_sd *peer_sd = mlx5_get_sd(pos);
+
+ primary_sd->secondaries[i - 1] = NULL;
+ peer_sd->primary_dev = NULL;
+ }
+ primary_sd->primary = false;
+out_ready_false:
+ mlx5_devcom_comp_set_ready(sd->devcom, false);
+out_unlock:
+ mlx5_devcom_comp_unlock(sd->devcom);
sd_unregister(dev);
sd_cleanup(dev);
}
+/* Lock order:
+ * primary: actual_adev_lock -> SD devcom comp lock
+ * secondary: SD devcom comp lock -> (drop) -> actual_adev_lock
+ * The two locks are never held together, so no ABBA.
+ */
struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
struct auxiliary_device *adev,
int idx)
{
struct mlx5_sd *sd = mlx5_get_sd(dev);
struct mlx5_core_dev *primary;
+ struct mlx5_adev *primary_adev;
if (!sd)
return adev;
- if (!mlx5_devcom_comp_is_ready(sd->devcom))
+ mlx5_devcom_comp_lock(sd->devcom);
+ if (!mlx5_devcom_comp_is_ready(sd->devcom)) {
+ mlx5_devcom_comp_unlock(sd->devcom);
return NULL;
+ }
primary = mlx5_sd_get_primary(dev);
- if (dev == primary)
+ if (!primary || dev == primary) {
+ mlx5_devcom_comp_unlock(sd->devcom);
return adev;
+ }
- return &primary->priv.adev[idx]->adev;
+ primary_adev = primary->priv.adev[idx];
+ get_device(&primary_adev->adev.dev);
+ mlx5_devcom_comp_unlock(sd->devcom);
+
+ device_lock(&primary_adev->adev.dev);
+ /* Primary may have completed remove between dropping devcom and
+ * acquiring device_lock; recheck.
+ */
+ if (!mlx5_devcom_comp_is_ready(sd->devcom)) {
+ device_unlock(&primary_adev->adev.dev);
+ put_device(&primary_adev->adev.dev);
+ return NULL;
+ }
+ return &primary_adev->adev;
+}
+
+void mlx5_sd_put_adev(struct auxiliary_device *actual_adev,
+ struct auxiliary_device *adev)
+{
+ if (actual_adev != adev) {
+ device_unlock(&actual_adev->dev);
+ put_device(&actual_adev->dev);
+ }
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h
index 137efaf..9bfd5b9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h
@@ -15,6 +15,8 @@ struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int c
struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
struct auxiliary_device *adev,
int idx);
+void mlx5_sd_put_adev(struct auxiliary_device *actual_adev,
+ struct auxiliary_device *adev);
int mlx5_sd_init(struct mlx5_core_dev *dev);
void mlx5_sd_cleanup(struct mlx5_core_dev *dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/fs_hws.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/fs_hws.c
index aca7785..5a172c5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/fs_hws.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/fs_hws.c
@@ -1320,8 +1320,10 @@ mlx5_cmd_hws_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
break;
case MLX5_REFORMAT_TYPE_REMOVE_HDR:
hws_action = mlx5_fs_get_action_remove_header_vlan(fs_ctx, params);
- if (!hws_action)
+ if (!hws_action) {
mlx5_core_err(dev, "Only vlan remove header supported\n");
+ return -EOPNOTSUPP;
+ }
break;
default:
mlx5_core_err(ns->dev, "Packet-reformat not supported(%d)\n",
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
index c406a3b..4dea2bb 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
@@ -826,7 +826,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
netif_tx_stop_all_queues(netdev);
if (fbnic_phylink_create(netdev)) {
- fbnic_netdev_free(fbd);
+ free_netdev(netdev);
+ fbd->netdev = NULL;
return NULL;
}
diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c
index f333241..ffac228 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.c
+++ b/drivers/net/ethernet/microchip/lan743x_main.c
@@ -1219,6 +1219,36 @@ static void lan743x_mac_set_address(struct lan743x_adapter *adapter,
"MAC address set to %pM\n", addr);
}
+static void lan743x_mac_rx_enable_fse(struct lan743x_adapter *adapter)
+{
+ u32 mac_rx;
+ bool rxen;
+
+ mac_rx = lan743x_csr_read(adapter, MAC_RX);
+ if (mac_rx & MAC_RX_FSE_)
+ return;
+
+ rxen = mac_rx & MAC_RX_RXEN_;
+ if (rxen) {
+ mac_rx &= ~MAC_RX_RXEN_;
+ lan743x_csr_write(adapter, MAC_RX, mac_rx);
+ lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
+ 1, 1000, 20000, 100);
+ }
+
+ /* Per AN2948, hardware prevents modification of the FSE bit while the
+ * MAC receiver is enabled (RXEN bit set). Use separate register write
+ * to assert the FSE bit before enabling the RXEN bit in MAC_RX
+ */
+ mac_rx |= MAC_RX_FSE_;
+ lan743x_csr_write(adapter, MAC_RX, mac_rx);
+
+ if (rxen) {
+ mac_rx |= MAC_RX_RXEN_;
+ lan743x_csr_write(adapter, MAC_RX, mac_rx);
+ }
+}
+
static int lan743x_mac_init(struct lan743x_adapter *adapter)
{
bool mac_address_valid = true;
@@ -1258,6 +1288,8 @@ static int lan743x_mac_init(struct lan743x_adapter *adapter)
lan743x_mac_set_address(adapter, adapter->mac_address);
eth_hw_addr_set(netdev, adapter->mac_address);
+ lan743x_mac_rx_enable_fse(adapter);
+
return 0;
}
diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h
index 160d94a..1573c8f 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.h
+++ b/drivers/net/ethernet/microchip/lan743x_main.h
@@ -182,6 +182,7 @@
#define MAC_RX (0x104)
#define MAC_RX_MAX_SIZE_SHIFT_ (16)
#define MAC_RX_MAX_SIZE_MASK_ (0x3FFF0000)
+#define MAC_RX_FSE_ BIT(2)
#define MAC_RX_RXD_ BIT(1)
#define MAC_RX_RXEN_ BIT(0)
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
index 47752d3..1179a6e 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
@@ -749,11 +749,10 @@ static void lan966x_cleanup_ports(struct lan966x *lan966x)
for (p = 0; p < lan966x->num_phys_ports; p++) {
port = lan966x->ports[p];
- if (!port)
+ if (!port || !port->dev)
continue;
- if (port->dev)
- unregister_netdev(port->dev);
+ unregister_netdev(port->dev);
lan966x_xdp_port_deinit(port);
if (lan966x->fdma && lan966x->fdma_ndev == port->dev)
@@ -873,6 +872,9 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p,
err = register_netdev(dev);
if (err) {
dev_err(lan966x->dev, "register_netdev failed\n");
+ phylink_destroy(phylink);
+ port->phylink = NULL;
+ port->dev = NULL;
return err;
}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
index 6a745bb..eb57b86 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
@@ -31,11 +31,11 @@ enum spx5_target_chiptype {
SPX5_TARGET_CT_7552 = 0x7552, /* SparX-5-128 Enterprise */
SPX5_TARGET_CT_7556 = 0x7556, /* SparX-5-160 Enterprise */
SPX5_TARGET_CT_7558 = 0x7558, /* SparX-5-200 Enterprise */
- SPX5_TARGET_CT_7546TSN = 0x47546, /* SparX-5-64i Industrial */
- SPX5_TARGET_CT_7549TSN = 0x47549, /* SparX-5-90i Industrial */
- SPX5_TARGET_CT_7552TSN = 0x47552, /* SparX-5-128i Industrial */
- SPX5_TARGET_CT_7556TSN = 0x47556, /* SparX-5-160i Industrial */
- SPX5_TARGET_CT_7558TSN = 0x47558, /* SparX-5-200i Industrial */
+ SPX5_TARGET_CT_7546TSN = 0x0546, /* SparX-5-64i Industrial */
+ SPX5_TARGET_CT_7549TSN = 0x0549, /* SparX-5-90i Industrial */
+ SPX5_TARGET_CT_7552TSN = 0x0552, /* SparX-5-128i Industrial */
+ SPX5_TARGET_CT_7556TSN = 0x0556, /* SparX-5-160i Industrial */
+ SPX5_TARGET_CT_7558TSN = 0x0558, /* SparX-5-200i Industrial */
SPX5_TARGET_CT_LAN9694 = 0x9694, /* lan969x-40 */
SPX5_TARGET_CT_LAN9691VAO = 0x9691, /* lan969x-40-VAO */
SPX5_TARGET_CT_LAN9694TSN = 0x9695, /* lan969x-40-TSN */
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_port.c b/drivers/net/ethernet/microchip/sparx5/sparx5_port.c
index 04bc8ff..62c4989 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_port.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_port.c
@@ -1128,7 +1128,8 @@ int sparx5_port_init(struct sparx5 *sparx5,
DEV2G5_PCS1G_SD_CFG(port->portno));
if (conf->portmode == PHY_INTERFACE_MODE_QSGMII ||
- conf->portmode == PHY_INTERFACE_MODE_SGMII) {
+ conf->portmode == PHY_INTERFACE_MODE_SGMII ||
+ conf->portmode == PHY_INTERFACE_MODE_1000BASEX) {
err = sparx5_serdes_set(sparx5, port, conf);
if (err)
return err;
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 098fbda..d8e8168 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -43,8 +43,9 @@ static u64 mana_gd_r64(struct gdma_context *g, u64 offset)
static int mana_gd_init_pf_regs(struct pci_dev *pdev)
{
struct gdma_context *gc = pci_get_drvdata(pdev);
- void __iomem *sriov_base_va;
+ u64 remaining_barsize;
u64 sriov_base_off;
+ u64 sriov_shm_off;
gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;
@@ -73,10 +74,28 @@ static int mana_gd_init_pf_regs(struct pci_dev *pdev)
gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off;
sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);
+ if (sriov_base_off >= gc->bar0_size ||
+ gc->bar0_size - sriov_base_off <
+ GDMA_PF_REG_SHM_OFF + sizeof(u64) ||
+ !IS_ALIGNED(sriov_base_off, sizeof(u64))) {
+ dev_err(gc->dev,
+ "SRIOV base offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n",
+ sriov_base_off, (u64)gc->bar0_size);
+ return -EPROTO;
+ }
- sriov_base_va = gc->bar0_va + sriov_base_off;
- gc->shm_base = sriov_base_va +
- mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);
+ remaining_barsize = gc->bar0_size - sriov_base_off;
+ sriov_shm_off = mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);
+ if (sriov_shm_off >= remaining_barsize ||
+ remaining_barsize - sriov_shm_off < SMC_APERTURE_SIZE ||
+ !IS_ALIGNED(sriov_shm_off, sizeof(u32))) {
+ dev_err(gc->dev,
+ "SRIOV SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n",
+ sriov_shm_off, (u64)gc->bar0_size);
+ return -EPROTO;
+ }
+
+ gc->shm_base = gc->bar0_va + sriov_base_off + sriov_shm_off;
return 0;
}
@@ -84,6 +103,7 @@ static int mana_gd_init_pf_regs(struct pci_dev *pdev)
static int mana_gd_init_vf_regs(struct pci_dev *pdev)
{
struct gdma_context *gc = pci_get_drvdata(pdev);
+ u64 shm_off;
gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;
@@ -111,7 +131,17 @@ static int mana_gd_init_vf_regs(struct pci_dev *pdev)
gc->db_page_base = gc->bar0_va + gc->db_page_off;
gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off;
- gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);
+ shm_off = mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);
+ if (shm_off >= gc->bar0_size ||
+ gc->bar0_size - shm_off < SMC_APERTURE_SIZE ||
+ !IS_ALIGNED(shm_off, sizeof(u32))) {
+ dev_err(gc->dev,
+ "SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n",
+ shm_off, (u64)gc->bar0_size);
+ return -EPROTO;
+ }
+
+ gc->shm_base = gc->bar0_va + shm_off;
return 0;
}
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index dbbde0f..e3c24d5 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -77,21 +77,19 @@ static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
}
static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
- struct hwc_work_request *rx_req)
+ struct hwc_work_request *rx_req, u16 msg_id)
{
const struct gdma_resp_hdr *resp_msg = rx_req->buf_va;
struct hwc_caller_ctx *ctx;
int err;
- if (!test_bit(resp_msg->response.hwc_msg_id,
- hwc->inflight_msg_res.map)) {
- dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
- resp_msg->response.hwc_msg_id);
+ if (!test_bit(msg_id, hwc->inflight_msg_res.map)) {
+ dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", msg_id);
mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
return;
}
- ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;
+ ctx = hwc->caller_ctx + msg_id;
err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
if (err)
goto out;
@@ -251,6 +249,7 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
struct gdma_sge *sge;
u64 rq_base_addr;
u64 rx_req_idx;
+ u16 msg_id;
u8 *wqe;
if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id))
@@ -266,16 +265,26 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;
- rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
- resp = (struct gdma_resp_hdr *)rx_req->buf_va;
-
- if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {
- dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n",
- resp->response.hwc_msg_id);
+ if (rx_req_idx >= hwc_rxq->msg_buf->num_reqs) {
+ dev_err(hwc->dev, "HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\n",
+ rx_req_idx, hwc_rxq->msg_buf->num_reqs);
return;
}
- mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req);
+ rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
+ resp = (struct gdma_resp_hdr *)rx_req->buf_va;
+
+ /* Read msg_id once from DMA buffer to prevent TOCTOU:
+ * DMA memory is shared/unencrypted in CVMs - host can
+ * modify it between reads.
+ */
+ msg_id = READ_ONCE(resp->response.hwc_msg_id);
+ if (msg_id >= hwc->num_inflight_msg) {
+ dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", msg_id);
+ return;
+ }
+
+ mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req, msg_id);
/* Can no longer use 'resp', because the buffer is posted to the HW
* in mana_hwc_handle_resp() above.
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index a654b36..c9b1df1 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -1727,6 +1727,9 @@ static void mana_fence_rqs(struct mana_port_context *apc)
struct mana_rxq *rxq;
int err;
+ if (!apc->rxqs)
+ return;
+
for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
rxq = apc->rxqs[rxq_idx];
err = mana_fence_rq(apc, rxq);
@@ -2520,9 +2523,12 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
napi_disable_locked(napi);
netif_napi_del_locked(napi);
}
- xdp_rxq_info_unreg(&rxq->xdp_rxq);
- mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
+ if (xdp_rxq_info_is_reg(&rxq->xdp_rxq))
+ xdp_rxq_info_unreg(&rxq->xdp_rxq);
+
+ if (rxq->rxobj != INVALID_MANA_HANDLE)
+ mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
mana_deinit_cq(apc, &rxq->rx_cq);
@@ -2796,9 +2802,6 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
mana_destroy_rxq(apc, rxq, false);
- if (cq)
- mana_deinit_cq(apc, cq);
-
return NULL;
}
@@ -2858,13 +2861,16 @@ static void mana_destroy_vport(struct mana_port_context *apc)
struct mana_rxq *rxq;
u32 rxq_idx;
- for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
- rxq = apc->rxqs[rxq_idx];
- if (!rxq)
- continue;
+ if (apc->rxqs) {
- mana_destroy_rxq(apc, rxq, true);
- apc->rxqs[rxq_idx] = NULL;
+ for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
+ rxq = apc->rxqs[rxq_idx];
+ if (!rxq)
+ continue;
+
+ mana_destroy_rxq(apc, rxq, true);
+ apc->rxqs[rxq_idx] = NULL;
+ }
}
mana_destroy_txq(apc);
@@ -3269,7 +3275,8 @@ static int mana_dealloc_queues(struct net_device *ndev)
if (apc->port_is_up)
return -EINVAL;
- mana_chn_setxdp(apc, NULL);
+ if (apc->rxqs)
+ mana_chn_setxdp(apc, NULL);
if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode)
mana_pf_deregister_filter(apc);
@@ -3287,33 +3294,38 @@ static int mana_dealloc_queues(struct net_device *ndev)
* number of queues.
*/
- for (i = 0; i < apc->num_queues; i++) {
- txq = &apc->tx_qp[i].txq;
- tsleep = 1000;
- while (atomic_read(&txq->pending_sends) > 0 &&
- time_before(jiffies, timeout)) {
- usleep_range(tsleep, tsleep + 1000);
- tsleep <<= 1;
- }
- if (atomic_read(&txq->pending_sends)) {
- err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
- if (err) {
- netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
- err, atomic_read(&txq->pending_sends),
- txq->gdma_txq_id);
+ if (apc->tx_qp) {
+ for (i = 0; i < apc->num_queues; i++) {
+ txq = &apc->tx_qp[i].txq;
+ tsleep = 1000;
+ while (atomic_read(&txq->pending_sends) > 0 &&
+ time_before(jiffies, timeout)) {
+ usleep_range(tsleep, tsleep + 1000);
+ tsleep <<= 1;
}
- break;
+ if (atomic_read(&txq->pending_sends)) {
+ err =
+ pcie_flr(to_pci_dev(gd->gdma_context->dev));
+ if (err) {
+ netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
+ err,
+ atomic_read(&txq->pending_sends),
+ txq->gdma_txq_id);
+ }
+ break;
+ }
+ }
+
+ for (i = 0; i < apc->num_queues; i++) {
+ txq = &apc->tx_qp[i].txq;
+ while ((skb = skb_dequeue(&txq->pending_skbs))) {
+ mana_unmap_skb(skb, apc);
+ dev_kfree_skb_any(skb);
+ }
+ atomic_set(&txq->pending_sends, 0);
}
}
- for (i = 0; i < apc->num_queues; i++) {
- txq = &apc->tx_qp[i].txq;
- while ((skb = skb_dequeue(&txq->pending_skbs))) {
- mana_unmap_skb(skb, apc);
- dev_kfree_skb_any(skb);
- }
- atomic_set(&txq->pending_sends, 0);
- }
/* We're 100% sure the queues can no longer be woken up, because
* we're sure now mana_poll_tx_cq() can't be running.
*/
@@ -3338,6 +3350,12 @@ int mana_detach(struct net_device *ndev, bool from_close)
ASSERT_RTNL();
+ /* If already detached (indicates detach succeeded but attach failed
+ * previously). Now skip mana detach and just retry mana_attach.
+ */
+ if (!from_close && !netif_device_present(ndev))
+ return 0;
+
apc->port_st_save = apc->port_is_up;
apc->port_is_up = false;
diff --git a/drivers/net/ethernet/microsoft/mana/shm_channel.c b/drivers/net/ethernet/microsoft/mana/shm_channel.c
index 0f1679e..d21b5db 100644
--- a/drivers/net/ethernet/microsoft/mana/shm_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/shm_channel.c
@@ -61,11 +61,6 @@ union smc_proto_hdr {
};
}; /* HW DATA */
-#define SMC_APERTURE_BITS 256
-#define SMC_BASIC_UNIT (sizeof(u32))
-#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
-#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
-
static int mana_smc_poll_register(void __iomem *base, bool reset)
{
void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
index 9861daa..b70262e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
@@ -1036,11 +1036,13 @@ static void qed_cid_map_free(struct qed_hwfn *p_hwfn)
for (type = 0; type < MAX_CONN_TYPES; type++) {
bitmap_free(p_mngr->acquired[type].cid_map);
+ p_mngr->acquired[type].cid_map = NULL;
p_mngr->acquired[type].max_count = 0;
p_mngr->acquired[type].start_cid = 0;
for (vf = 0; vf < MAX_NUM_VFS; vf++) {
bitmap_free(p_mngr->acquired_vf[type][vf].cid_map);
+ p_mngr->acquired_vf[type][vf].cid_map = NULL;
p_mngr->acquired_vf[type][vf].max_count = 0;
p_mngr->acquired_vf[type][vf].start_cid = 0;
}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index 42c6dcf..dd75c47 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -5103,6 +5103,13 @@ static int qed_init_wfq_param(struct qed_hwfn *p_hwfn,
return -EINVAL;
}
+ /* All vports are already or become configured, nothing to distribute */
+ if (non_requested_count == 0) {
+ p_hwfn->qm_info.wfq_data[vport_id].min_speed = req_rate;
+ p_hwfn->qm_info.wfq_data[vport_id].configured = true;
+ return 0;
+ }
+
total_left_rate = min_pf_rate - total_req_min_rate;
left_rate_per_vp = total_left_rate / non_requested_count;
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index 269c044..78d4df5 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -213,8 +213,8 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head)
ep = rmnet_get_endpoint(real_port, mux_id);
if (ep) {
hlist_del_init_rcu(&ep->hlnode);
- rmnet_vnd_dellink(mux_id, real_port, ep);
- kfree(ep);
+ real_port->nr_rmnet_devs--;
+ kfree_rcu(ep, rcu);
}
netdev_upper_dev_unlink(real_dev, dev);
@@ -238,9 +238,9 @@ static void rmnet_force_unassociate_device(struct net_device *real_dev)
hash_for_each_safe(port->muxed_ep, bkt_ep, tmp_ep, ep, hlnode) {
unregister_netdevice_queue(ep->egress_dev, &list);
netdev_upper_dev_unlink(real_dev, ep->egress_dev);
- rmnet_vnd_dellink(ep->mux_id, port, ep);
hlist_del_init_rcu(&ep->hlnode);
- kfree(ep);
+ port->nr_rmnet_devs--;
+ kfree_rcu(ep, rcu);
}
rmnet_unregister_real_device(real_dev);
unregister_netdevice_many(&list);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index ed112d5..f50fae1 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -18,6 +18,7 @@ struct rmnet_endpoint {
u8 mux_id;
struct net_device *egress_dev;
struct hlist_node hlnode;
+ struct rcu_head rcu;
};
struct rmnet_egress_agg_params {
diff --git a/drivers/net/ethernet/realtek/rtase/rtase_main.c b/drivers/net/ethernet/realtek/rtase/rtase_main.c
index ef13109..55105d3 100644
--- a/drivers/net/ethernet/realtek/rtase/rtase_main.c
+++ b/drivers/net/ethernet/realtek/rtase/rtase_main.c
@@ -239,6 +239,8 @@ static void rtase_tx_clear(struct rtase_private *tp)
rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC);
ring->cur_idx = 0;
ring->dirty_idx = 0;
+
+ netdev_tx_reset_subqueue(tp->dev, i);
}
}
@@ -1563,8 +1565,9 @@ static void rtase_dump_tally_counter(const struct rtase_private *tp)
rtase_w32(tp, RTASE_DTCCR0, cmd);
rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP);
- err = read_poll_timeout(rtase_r32, val, !(val & RTASE_COUNTER_DUMP),
- 10, 250, false, tp, RTASE_DTCCR0);
+ err = read_poll_timeout_atomic(rtase_r32, val,
+ !(val & RTASE_COUNTER_DUMP),
+ 10, 250, false, tp, RTASE_DTCCR0);
if (err == -ETIMEDOUT)
netdev_err(tp->dev, "error occurred in dump tally counter\n");
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 1dbfadb..5f88733 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -1108,9 +1108,12 @@ static int ravb_stop_dma(struct net_device *ndev)
/* Request for transmission suspension */
ravb_modify(ndev, CCC, CCC_DTSR, CCC_DTSR);
- error = ravb_wait(ndev, CSR, CSR_DTS, CSR_DTS);
- if (error)
- netdev_err(ndev, "failed to stop AXI BUS\n");
+ /* Access to URAM will not be suspended if WoL is enabled. */
+ if (!priv->wol_enabled) {
+ error = ravb_wait(ndev, CSR, CSR_DTS, CSR_DTS);
+ if (error)
+ netdev_err(ndev, "failed to stop AXI BUS\n");
+ }
/* Stop AVB-DMAC process */
return ravb_set_opmode(ndev, CCC_OPC_CONFIG);
diff --git a/drivers/net/ethernet/renesas/rtsn.c b/drivers/net/ethernet/renesas/rtsn.c
index 03a2669..ee8381b 100644
--- a/drivers/net/ethernet/renesas/rtsn.c
+++ b/drivers/net/ethernet/renesas/rtsn.c
@@ -797,11 +797,11 @@ static int rtsn_mdio_alloc(struct rtsn_private *priv)
/* Enter config mode before registering the MDIO bus */
ret = rtsn_reset(priv);
if (ret)
- goto out_free_bus;
+ goto out_put_node;
ret = rtsn_change_mode(priv, OCR_OPC_CONFIG);
if (ret)
- goto out_free_bus;
+ goto out_put_node;
rtsn_modify(priv, MPIC, MPIC_PSMCS_MASK | MPIC_PSMHT_MASK,
MPIC_PSMCS_DEFAULT | MPIC_PSMHT_DEFAULT);
@@ -824,6 +824,8 @@ static int rtsn_mdio_alloc(struct rtsn_private *priv)
return 0;
+out_put_node:
+ of_node_put(mdio_node);
out_free_bus:
mdiobus_free(mii);
return ret;
diff --git a/drivers/net/ethernet/sfc/efx_devlink.c b/drivers/net/ethernet/sfc/efx_devlink.c
index d842c60..e5c6f81 100644
--- a/drivers/net/ethernet/sfc/efx_devlink.c
+++ b/drivers/net/ethernet/sfc/efx_devlink.c
@@ -531,7 +531,7 @@ static int efx_devlink_info_running_versions(struct efx_nic *efx,
if (rc || outlength < MC_CMD_GET_VERSION_OUT_LEN) {
netif_err(efx, drv, efx->net_dev,
"mcdi MC_CMD_GET_VERSION failed\n");
- return rc;
+ return rc ?: -EIO;
}
/* Handle previous output */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
index bcb8e00..4ac979d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
@@ -28,13 +28,16 @@
/*
* TX/RX Clock Delay Bit Masks:
- * - TX Delay: bits [14:8] — TX_CLK delay (unit: 0.1ns per bit)
- * - RX Delay: bits [30:24] — RX_CLK delay (unit: 0.1ns per bit)
+ * - TX Delay: bits [14:8] — TX_CLK delay (unit: 0.02ns per bit)
+ * - RX Delay: bits [30:24] — RX_CLK delay (unit: 0.02ns per bit)
*/
#define EIC7700_ETH_TX_ADJ_DELAY GENMASK(14, 8)
#define EIC7700_ETH_RX_ADJ_DELAY GENMASK(30, 24)
-#define EIC7700_MAX_DELAY_UNIT 0x7F
+#define EIC7700_MAX_DELAY_STEPS 0x7F
+#define EIC7700_DELAY_STEP_PS 20
+#define EIC7700_MAX_DELAY_PS \
+ (EIC7700_MAX_DELAY_STEPS * EIC7700_DELAY_STEP_PS)
static const char * const eic7700_clk_names[] = {
"tx", "axi", "cfg",
@@ -42,6 +45,15 @@ static const char * const eic7700_clk_names[] = {
struct eic7700_qos_priv {
struct plat_stmmacenet_data *plat_dat;
+ struct regmap *eic7700_hsp_regmap;
+ u32 eth_axi_lp_ctrl_offset;
+ u32 eth_phy_ctrl_offset;
+ u32 eth_clk_offset;
+ u32 eth_txd_offset;
+ u32 eth_rxd_offset;
+ u32 eth_clk_dly_param;
+ bool has_txd_offset;
+ bool has_rxd_offset;
};
static int eic7700_clks_config(void *priv, bool enabled)
@@ -61,8 +73,34 @@ static int eic7700_clks_config(void *priv, bool enabled)
static int eic7700_dwmac_init(struct device *dev, void *priv)
{
struct eic7700_qos_priv *dwc = priv;
+ int ret;
- return eic7700_clks_config(dwc, true);
+ ret = eic7700_clks_config(dwc, true);
+ if (ret)
+ return ret;
+
+ ret = regmap_set_bits(dwc->eic7700_hsp_regmap,
+ dwc->eth_phy_ctrl_offset,
+ EIC7700_ETH_TX_CLK_SEL |
+ EIC7700_ETH_PHY_INTF_SELI);
+ if (ret) {
+ eic7700_clks_config(dwc, false);
+ return ret;
+ }
+
+ regmap_write(dwc->eic7700_hsp_regmap, dwc->eth_axi_lp_ctrl_offset,
+ EIC7700_ETH_CSYSREQ_VAL);
+
+ if (dwc->has_txd_offset)
+ regmap_write(dwc->eic7700_hsp_regmap, dwc->eth_txd_offset, 0);
+
+ if (dwc->has_rxd_offset)
+ regmap_write(dwc->eic7700_hsp_regmap, dwc->eth_rxd_offset, 0);
+
+ regmap_write(dwc->eic7700_hsp_regmap, dwc->eth_clk_offset,
+ dwc->eth_clk_dly_param);
+
+ return 0;
}
static void eic7700_dwmac_exit(struct device *dev, void *priv)
@@ -93,13 +131,7 @@ static int eic7700_dwmac_probe(struct platform_device *pdev)
struct plat_stmmacenet_data *plat_dat;
struct stmmac_resources stmmac_res;
struct eic7700_qos_priv *dwc_priv;
- struct regmap *eic7700_hsp_regmap;
- u32 eth_axi_lp_ctrl_offset;
- u32 eth_phy_ctrl_offset;
- u32 eth_phy_ctrl_regset;
- u32 eth_rxd_dly_offset;
- u32 eth_dly_param = 0;
- u32 delay_ps;
+ u32 delay_ps, val;
int i, ret;
ret = stmmac_get_platform_resources(pdev, &stmmac_res);
@@ -119,10 +151,20 @@ static int eic7700_dwmac_probe(struct platform_device *pdev)
/* Read rx-internal-delay-ps and update rx_clk delay */
if (!of_property_read_u32(pdev->dev.of_node,
"rx-internal-delay-ps", &delay_ps)) {
- u32 val = min(delay_ps / 100, EIC7700_MAX_DELAY_UNIT);
+ if (delay_ps % EIC7700_DELAY_STEP_PS)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "rx delay must be multiple of %dps\n",
+ EIC7700_DELAY_STEP_PS);
- eth_dly_param &= ~EIC7700_ETH_RX_ADJ_DELAY;
- eth_dly_param |= FIELD_PREP(EIC7700_ETH_RX_ADJ_DELAY, val);
+ if (delay_ps > EIC7700_MAX_DELAY_PS)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "rx delay out of range\n");
+
+ val = delay_ps / EIC7700_DELAY_STEP_PS;
+
+ dwc_priv->eth_clk_dly_param &= ~EIC7700_ETH_RX_ADJ_DELAY;
+ dwc_priv->eth_clk_dly_param |=
+ FIELD_PREP(EIC7700_ETH_RX_ADJ_DELAY, val);
} else {
return dev_err_probe(&pdev->dev, -EINVAL,
"missing required property rx-internal-delay-ps\n");
@@ -131,55 +173,65 @@ static int eic7700_dwmac_probe(struct platform_device *pdev)
/* Read tx-internal-delay-ps and update tx_clk delay */
if (!of_property_read_u32(pdev->dev.of_node,
"tx-internal-delay-ps", &delay_ps)) {
- u32 val = min(delay_ps / 100, EIC7700_MAX_DELAY_UNIT);
+ if (delay_ps % EIC7700_DELAY_STEP_PS)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "tx delay must be multiple of %dps\n",
+ EIC7700_DELAY_STEP_PS);
- eth_dly_param &= ~EIC7700_ETH_TX_ADJ_DELAY;
- eth_dly_param |= FIELD_PREP(EIC7700_ETH_TX_ADJ_DELAY, val);
+ if (delay_ps > EIC7700_MAX_DELAY_PS)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "tx delay out of range\n");
+
+ val = delay_ps / EIC7700_DELAY_STEP_PS;
+
+ dwc_priv->eth_clk_dly_param &= ~EIC7700_ETH_TX_ADJ_DELAY;
+ dwc_priv->eth_clk_dly_param |=
+ FIELD_PREP(EIC7700_ETH_TX_ADJ_DELAY, val);
} else {
return dev_err_probe(&pdev->dev, -EINVAL,
"missing required property tx-internal-delay-ps\n");
}
- eic7700_hsp_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
- "eswin,hsp-sp-csr");
- if (IS_ERR(eic7700_hsp_regmap))
+ dwc_priv->eic7700_hsp_regmap =
+ syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+ "eswin,hsp-sp-csr");
+ if (IS_ERR(dwc_priv->eic7700_hsp_regmap))
return dev_err_probe(&pdev->dev,
- PTR_ERR(eic7700_hsp_regmap),
+ PTR_ERR(dwc_priv->eic7700_hsp_regmap),
"Failed to get hsp-sp-csr regmap\n");
ret = of_property_read_u32_index(pdev->dev.of_node,
"eswin,hsp-sp-csr",
- 1, ð_phy_ctrl_offset);
+ 1, &dwc_priv->eth_phy_ctrl_offset);
if (ret)
return dev_err_probe(&pdev->dev, ret,
"can't get eth_phy_ctrl_offset\n");
- regmap_read(eic7700_hsp_regmap, eth_phy_ctrl_offset,
- ð_phy_ctrl_regset);
- eth_phy_ctrl_regset |=
- (EIC7700_ETH_TX_CLK_SEL | EIC7700_ETH_PHY_INTF_SELI);
- regmap_write(eic7700_hsp_regmap, eth_phy_ctrl_offset,
- eth_phy_ctrl_regset);
-
ret = of_property_read_u32_index(pdev->dev.of_node,
"eswin,hsp-sp-csr",
- 2, ð_axi_lp_ctrl_offset);
+ 2, &dwc_priv->eth_axi_lp_ctrl_offset);
if (ret)
return dev_err_probe(&pdev->dev, ret,
"can't get eth_axi_lp_ctrl_offset\n");
- regmap_write(eic7700_hsp_regmap, eth_axi_lp_ctrl_offset,
- EIC7700_ETH_CSYSREQ_VAL);
+ ret = of_property_read_u32_index(pdev->dev.of_node,
+ "eswin,hsp-sp-csr",
+ 3, &dwc_priv->eth_clk_offset);
+ if (ret)
+ return dev_err_probe(&pdev->dev, ret,
+ "can't get eth_clk_offset\n");
ret = of_property_read_u32_index(pdev->dev.of_node,
"eswin,hsp-sp-csr",
- 3, ð_rxd_dly_offset);
- if (ret)
- return dev_err_probe(&pdev->dev, ret,
- "can't get eth_rxd_dly_offset\n");
+ 4, &dwc_priv->eth_txd_offset);
+ if (!ret)
+ dwc_priv->has_txd_offset = true;
- regmap_write(eic7700_hsp_regmap, eth_rxd_dly_offset,
- eth_dly_param);
+ ret = of_property_read_u32_index(pdev->dev.of_node,
+ "eswin,hsp-sp-csr",
+ 5, &dwc_priv->eth_rxd_offset);
+ if (!ret)
+ dwc_priv->has_rxd_offset = true;
plat_dat->num_clks = ARRAY_SIZE(eic7700_clk_names);
plat_dat->clks = devm_kcalloc(&pdev->dev,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
index 7898b50..b8d467b 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
@@ -6,6 +6,7 @@
#include <linux/pci.h>
#include <linux/dmi.h>
#include <linux/platform_data/x86/intel_pmc_ipc.h>
+#include <asm/cpuid/api.h>
#include "dwmac-intel.h"
#include "dwmac4.h"
#include "stmmac.h"
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c
index e2240b6..2ab6eca 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c
@@ -100,6 +100,8 @@ static int nvt_gmac_probe(struct platform_device *pdev)
if (!priv)
return dev_err_probe(dev, -ENOMEM, "Failed to allocate private data\n");
+ priv->dev = dev;
+
priv->regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node, "nuvoton,sys",
1, &priv->macid);
if (IS_ERR(priv->regmap))
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index ca68248..3591755 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -5549,9 +5549,12 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
break;
/* Prefetch the next RX descriptor */
- rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
- priv->dma_conf.dma_rx_size);
- next_entry = rx_q->cur_rx;
+ next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
+ priv->dma_conf.dma_rx_size);
+ if (unlikely(next_entry == rx_q->dirty_rx))
+ break;
+
+ rx_q->cur_rx = next_entry;
np = stmmac_get_rx_desc(priv, rx_q, next_entry);
@@ -5686,7 +5689,6 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
dma_dir = page_pool_get_dma_dir(rx_q->page_pool);
bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
- limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);
if (netif_msg_rx_status(priv)) {
void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0);
@@ -5733,9 +5735,12 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
if (unlikely(status & dma_own))
break;
- rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
- priv->dma_conf.dma_rx_size);
- next_entry = rx_q->cur_rx;
+ next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx,
+ priv->dma_conf.dma_rx_size);
+ if (unlikely(next_entry == rx_q->dirty_rx))
+ break;
+
+ rx_q->cur_rx = next_entry;
np = stmmac_get_rx_desc(priv, rx_q, next_entry);
diff --git a/drivers/net/ethernet/ti/icssm/icssm_prueth.c b/drivers/net/ethernet/ti/icssm/icssm_prueth.c
index 53bbd92..b7e9424 100644
--- a/drivers/net/ethernet/ti/icssm/icssm_prueth.c
+++ b/drivers/net/ethernet/ti/icssm/icssm_prueth.c
@@ -1825,6 +1825,7 @@ static int icssm_prueth_probe(struct platform_device *pdev)
dev_err(dev, "%pOF error reading port_id %d\n",
eth_node, ret);
of_node_put(eth_node);
+ of_node_put(eth_ports_node);
return ret;
}
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
index d3772d0..2451f6b 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
@@ -2480,8 +2480,11 @@ int wx_sw_init(struct wx *wx)
wx->oem_svid = pdev->subsystem_vendor;
wx->oem_ssid = pdev->subsystem_device;
wx->bus.device = PCI_SLOT(pdev->devfn);
- wx->bus.func = FIELD_GET(WX_CFG_PORT_ST_LANID,
- rd32(wx, WX_CFG_PORT_ST));
+ if (pdev->is_virtfn)
+ wx->bus.func = PCI_FUNC(pdev->devfn);
+ else
+ wx->bus.func = FIELD_GET(WX_CFG_PORT_ST_LANID,
+ rd32(wx, WX_CFG_PORT_ST));
if (wx->oem_svid == PCI_VENDOR_ID_WANGXUN ||
pdev->is_virtfn) {
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c
index 29cdbed..94ff8f5 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c
@@ -99,8 +99,8 @@ int wx_request_msix_irqs_vf(struct wx *wx)
}
}
- err = request_threaded_irq(wx->msix_entry->vector, wx_msix_misc_vf,
- NULL, IRQF_ONESHOT, netdev->name, wx);
+ err = request_irq(wx->msix_entry->vector, wx_msix_misc_vf,
+ 0, netdev->name, wx);
if (err) {
wx_err(wx, "request_irq for msix_other failed: %d\n", err);
goto free_queue_irqs;
diff --git a/drivers/net/fddi/defza.c b/drivers/net/fddi/defza.c
index 064fa48..9bfecc8 100644
--- a/drivers/net/fddi/defza.c
+++ b/drivers/net/fddi/defza.c
@@ -984,7 +984,7 @@ static irqreturn_t fza_interrupt(int irq, void *dev_id)
case FZA_STATE_UNINITIALIZED:
netif_carrier_off(dev);
- timer_delete_sync(&fp->reset_timer);
+ timer_delete_sync_try(&fp->reset_timer);
fp->ring_cmd_index = 0;
fp->ring_uns_index = 0;
fp->ring_rmc_tx_index = 0;
@@ -1018,7 +1018,9 @@ static irqreturn_t fza_interrupt(int irq, void *dev_id)
fp->queue_active = 0;
netif_stop_queue(dev);
pr_debug("%s: queue stopped\n", fp->name);
- timer_delete_sync(&fp->reset_timer);
+
+ spin_lock(&fp->lock);
+ timer_delete(&fp->reset_timer);
pr_warn("%s: halted, reason: %x\n", fp->name,
FZA_STATUS_GET_HALT(status));
fza_regs_dump(fp);
@@ -1027,6 +1029,8 @@ static irqreturn_t fza_interrupt(int irq, void *dev_id)
fp->timer_state = 0;
fp->reset_timer.expires = jiffies + 45 * HZ;
add_timer(&fp->reset_timer);
+ spin_unlock(&fp->lock);
+
break;
default:
@@ -1046,7 +1050,9 @@ static irqreturn_t fza_interrupt(int irq, void *dev_id)
static void fza_reset_timer(struct timer_list *t)
{
struct fza_private *fp = timer_container_of(fp, t, reset_timer);
+ unsigned long flags;
+ spin_lock_irqsave(&fp->lock, flags);
if (!fp->timer_state) {
pr_err("%s: RESET timed out!\n", fp->name);
pr_info("%s: trying harder...\n", fp->name);
@@ -1069,6 +1075,7 @@ static void fza_reset_timer(struct timer_list *t)
fp->reset_timer.expires = jiffies + 45 * HZ;
}
add_timer(&fp->reset_timer);
+ spin_unlock_irqrestore(&fp->lock, flags);
}
static int fza_set_mac_address(struct net_device *dev, void *addr)
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index c656336..715180c 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -632,7 +632,7 @@ static int geneve_post_decap_hint(const struct sock *sk, struct sk_buff *skb,
uh = udp_hdr(skb);
uh->len = htons(skb->len - gro_hint->nested_tp_offset);
if (uh->check) {
- len = skb->len - gro_hint->nested_nh_offset;
+ len = skb->len - gro_hint->nested_tp_offset;
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
if (gro_hint->nested_is_v6)
uh->check = ~udp_v6_check(len, &ipv6h->saddr,
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 5407d2e..43aa1bf 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -211,12 +211,12 @@ static void ifb_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
switch (stringset) {
case ETH_SS_STATS:
- for (i = 0; i < dev->real_num_rx_queues; i++)
+ for (i = 0; i < dev->num_tx_queues; i++)
for (j = 0; j < IFB_Q_STATS_LEN; j++)
ethtool_sprintf(&p, "rx_queue_%u_%.18s",
i, ifb_q_stats_desc[j].desc);
- for (i = 0; i < dev->real_num_tx_queues; i++)
+ for (i = 0; i < dev->num_tx_queues; i++)
for (j = 0; j < IFB_Q_STATS_LEN; j++)
ethtool_sprintf(&p, "tx_queue_%u_%.18s",
i, ifb_q_stats_desc[j].desc);
@@ -229,8 +229,7 @@ static int ifb_get_sset_count(struct net_device *dev, int sset)
{
switch (sset) {
case ETH_SS_STATS:
- return IFB_Q_STATS_LEN * (dev->real_num_rx_queues +
- dev->real_num_tx_queues);
+ return IFB_Q_STATS_LEN * dev->num_tx_queues * 2;
default:
return -EOPNOTSUPP;
}
@@ -262,12 +261,12 @@ static void ifb_get_ethtool_stats(struct net_device *dev,
struct ifb_q_private *txp;
int i;
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (i = 0; i < dev->num_tx_queues; i++) {
txp = dp->tx_private + i;
ifb_fill_stats_data(&data, &txp->rx_stats);
}
- for (i = 0; i < dev->real_num_tx_queues; i++) {
+ for (i = 0; i < dev->num_tx_queues; i++) {
txp = dp->tx_private + i;
ifb_fill_stats_data(&data, &txp->tx_stats);
}
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 6147ee8..fb00912 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -26,6 +26,8 @@
#include <uapi/linux/if_macsec.h>
+static struct workqueue_struct *macsec_wq;
+
/* SecTAG length = macsec_eth_header without the optional SCI */
#define MACSEC_TAG_LEN 6
@@ -174,9 +176,10 @@ static void macsec_rxsc_put(struct macsec_rx_sc *sc)
call_rcu(&sc->rcu_head, free_rx_sc_rcu);
}
-static void free_rxsa(struct rcu_head *head)
+static void free_rxsa_work(struct work_struct *work)
{
- struct macsec_rx_sa *sa = container_of(head, struct macsec_rx_sa, rcu);
+ struct macsec_rx_sa *sa =
+ container_of(to_rcu_work(work), struct macsec_rx_sa, destroy_work);
crypto_free_aead(sa->key.tfm);
free_percpu(sa->stats);
@@ -186,7 +189,7 @@ static void free_rxsa(struct rcu_head *head)
static void macsec_rxsa_put(struct macsec_rx_sa *sa)
{
if (refcount_dec_and_test(&sa->refcnt))
- call_rcu(&sa->rcu, free_rxsa);
+ queue_rcu_work(macsec_wq, &sa->destroy_work);
}
static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
@@ -202,9 +205,10 @@ static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
return sa;
}
-static void free_txsa(struct rcu_head *head)
+static void free_txsa_work(struct work_struct *work)
{
- struct macsec_tx_sa *sa = container_of(head, struct macsec_tx_sa, rcu);
+ struct macsec_tx_sa *sa =
+ container_of(to_rcu_work(work), struct macsec_tx_sa, destroy_work);
crypto_free_aead(sa->key.tfm);
free_percpu(sa->stats);
@@ -214,7 +218,7 @@ static void free_txsa(struct rcu_head *head)
static void macsec_txsa_put(struct macsec_tx_sa *sa)
{
if (refcount_dec_and_test(&sa->refcnt))
- call_rcu(&sa->rcu, free_txsa);
+ queue_rcu_work(macsec_wq, &sa->destroy_work);
}
static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
@@ -804,7 +808,8 @@ static bool macsec_post_decrypt(struct sk_buff *skb, struct macsec_secy *secy, u
if (pn + 1 > rx_sa->next_pn_halves.lower) {
rx_sa->next_pn_halves.lower = pn + 1;
} else if (secy->xpn &&
- !pn_same_half(pn, rx_sa->next_pn_halves.lower)) {
+ (pn + 1 == 0 ||
+ !pn_same_half(pn, rx_sa->next_pn_halves.lower))) {
rx_sa->next_pn_halves.upper++;
rx_sa->next_pn_halves.lower = pn + 1;
}
@@ -1407,6 +1412,7 @@ static int init_rx_sa(struct macsec_rx_sa *rx_sa, char *sak, int key_len,
rx_sa->next_pn = 1;
refcount_set(&rx_sa->refcnt, 1);
spin_lock_init(&rx_sa->lock);
+ INIT_RCU_WORK(&rx_sa->destroy_work, free_rxsa_work);
return 0;
}
@@ -1506,6 +1512,7 @@ static int init_tx_sa(struct macsec_tx_sa *tx_sa, char *sak, int key_len,
tx_sa->active = false;
refcount_set(&tx_sa->refcnt, 1);
spin_lock_init(&tx_sa->lock);
+ INIT_RCU_WORK(&tx_sa->destroy_work, free_txsa_work);
return 0;
}
@@ -4505,25 +4512,35 @@ static int __init macsec_init(void)
{
int err;
+ macsec_wq = alloc_workqueue("macsec", WQ_UNBOUND, 0);
+ if (!macsec_wq)
+ return -ENOMEM;
+
pr_info("MACsec IEEE 802.1AE\n");
err = register_netdevice_notifier(&macsec_notifier);
if (err)
- return err;
+ goto err_destroy_wq;
err = rtnl_link_register(&macsec_link_ops);
if (err)
- goto notifier;
+ goto err_notifier;
err = genl_register_family(&macsec_fam);
if (err)
- goto rtnl;
+ goto err_rtnl;
return 0;
-rtnl:
+err_rtnl:
rtnl_link_unregister(&macsec_link_ops);
-notifier:
+err_notifier:
unregister_netdevice_notifier(&macsec_notifier);
+err_destroy_wq:
+ /* Precautionary, mirrors macsec_exit() to stay safe if work
+ * ever becomes queueable before this point in the future.
+ */
+ rcu_barrier();
+ destroy_workqueue(macsec_wq);
return err;
}
@@ -4533,6 +4550,7 @@ static void __exit macsec_exit(void)
rtnl_link_unregister(&macsec_link_ops);
unregister_netdevice_notifier(&macsec_notifier);
rcu_barrier();
+ destroy_workqueue(macsec_wq);
}
module_init(macsec_init);
diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c
index 15fe4d1..ee29137 100644
--- a/drivers/net/mctp/mctp-i2c.c
+++ b/drivers/net/mctp/mctp-i2c.c
@@ -496,8 +496,6 @@ static void mctp_i2c_xmit(struct mctp_i2c_dev *midev, struct sk_buff *skb)
u8 *pecp;
int rc;
- fs = mctp_i2c_get_tx_flow_state(midev, skb);
-
hdr = (void *)skb_mac_header(skb);
/* Sanity check that packet contents matches skb length,
* and can't exceed MCTP_I2C_BUFSZ
@@ -509,6 +507,8 @@ static void mctp_i2c_xmit(struct mctp_i2c_dev *midev, struct sk_buff *skb)
return;
}
+ fs = mctp_i2c_get_tx_flow_state(midev, skb);
+
if (skb_tailroom(skb) >= 1) {
/* Linear case with space, we can just append the PEC */
skb_put(skb, 1);
diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c
index d0361aa..3f7d3103 100644
--- a/drivers/net/net_failover.c
+++ b/drivers/net/net_failover.c
@@ -502,7 +502,7 @@ static int net_failover_slave_register(struct net_device *slave_dev,
/* Align MTU of slave with failover dev */
orig_mtu = slave_dev->mtu;
- err = dev_set_mtu(slave_dev, failover_dev->mtu);
+ err = netif_set_mtu(slave_dev, failover_dev->mtu);
if (err) {
netdev_err(failover_dev, "unable to change mtu of %s to %u register failed\n",
slave_dev->name, failover_dev->mtu);
@@ -512,11 +512,11 @@ static int net_failover_slave_register(struct net_device *slave_dev,
dev_hold(slave_dev);
if (netif_running(failover_dev)) {
- err = dev_open(slave_dev, NULL);
+ err = netif_open(slave_dev, NULL);
if (err && (err != -EBUSY)) {
netdev_err(failover_dev, "Opening slave %s failed err:%d\n",
slave_dev->name, err);
- goto err_dev_open;
+ goto err_netif_open;
}
}
@@ -562,10 +562,10 @@ static int net_failover_slave_register(struct net_device *slave_dev,
err_vlan_add:
dev_uc_unsync(slave_dev, failover_dev);
dev_mc_unsync(slave_dev, failover_dev);
- dev_close(slave_dev);
-err_dev_open:
+ netif_close(slave_dev);
+err_netif_open:
dev_put(slave_dev);
- dev_set_mtu(slave_dev, orig_mtu);
+ netif_set_mtu(slave_dev, orig_mtu);
done:
return err;
}
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index 205384d..57dd682 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -752,7 +752,7 @@ static ssize_t enabled_store(struct config_item *item,
unregister_netcons_consoles();
}
- ret = strnlen(buf, count);
+ ret = count;
/* Deferred cleanup */
netconsole_process_cleanups();
out_unlock:
@@ -781,7 +781,7 @@ static ssize_t release_store(struct config_item *item, const char *buf,
nt->release = release;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -807,7 +807,7 @@ static ssize_t extended_store(struct config_item *item, const char *buf,
goto out_unlock;
nt->extended = extended;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -817,6 +817,13 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf,
size_t count)
{
struct netconsole_target *nt = to_target(item);
+ size_t len = count;
+
+ /* Account for a trailing newline appended by tools like echo */
+ if (len && buf[len - 1] == '\n')
+ len--;
+ if (len >= IFNAMSIZ)
+ return -ENAMETOOLONG;
dynamic_netconsole_mutex_lock();
if (nt->state == STATE_ENABLED) {
@@ -830,7 +837,7 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf,
trim_newline(nt->np.dev_name, IFNAMSIZ);
dynamic_netconsole_mutex_unlock();
- return strnlen(buf, count);
+ return count;
}
static ssize_t local_port_store(struct config_item *item, const char *buf,
@@ -849,7 +856,7 @@ static ssize_t local_port_store(struct config_item *item, const char *buf,
ret = kstrtou16(buf, 10, &nt->np.local_port);
if (ret < 0)
goto out_unlock;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -871,7 +878,7 @@ static ssize_t remote_port_store(struct config_item *item,
ret = kstrtou16(buf, 10, &nt->np.remote_port);
if (ret < 0)
goto out_unlock;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -896,7 +903,7 @@ static ssize_t local_ip_store(struct config_item *item, const char *buf,
goto out_unlock;
nt->np.ipv6 = !!ipv6;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -921,7 +928,7 @@ static ssize_t remote_ip_store(struct config_item *item, const char *buf,
goto out_unlock;
nt->np.ipv6 = !!ipv6;
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -957,7 +964,7 @@ static ssize_t remote_mac_store(struct config_item *item, const char *buf,
goto out_unlock;
memcpy(nt->np.remote_mac, remote_mac, ETH_ALEN);
- ret = strnlen(buf, count);
+ ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
return ret;
@@ -1072,26 +1079,30 @@ static ssize_t userdatum_value_store(struct config_item *item, const char *buf,
size_t count)
{
struct userdatum *udm = to_userdatum(item);
+ char old_value[MAX_EXTRADATA_VALUE_LEN];
struct netconsole_target *nt;
struct userdata *ud;
ssize_t ret;
- if (count > MAX_EXTRADATA_VALUE_LEN)
+ if (count >= MAX_EXTRADATA_VALUE_LEN)
return -EMSGSIZE;
mutex_lock(&netconsole_subsys.su_mutex);
dynamic_netconsole_mutex_lock();
-
- ret = strscpy(udm->value, buf, sizeof(udm->value));
- if (ret < 0)
- goto out_unlock;
+ /* Snapshot for rollback if update_userdata() fails below */
+ strscpy(old_value, udm->value, sizeof(old_value));
+ /* count is bounded above, so strscpy() cannot truncate here */
+ strscpy(udm->value, buf, sizeof(udm->value));
trim_newline(udm->value, sizeof(udm->value));
ud = to_userdata(item->ci_parent);
nt = userdata_to_target(ud);
ret = update_userdata(nt);
- if (ret < 0)
+ if (ret < 0) {
+ /* Restore the previous value so it matches the live payload */
+ strscpy(udm->value, old_value, sizeof(udm->value));
goto out_unlock;
+ }
ret = count;
out_unlock:
dynamic_netconsole_mutex_unlock();
@@ -1133,7 +1144,7 @@ static ssize_t sysdata_msgid_enabled_store(struct config_item *item,
disable_sysdata_feature(nt, SYSDATA_MSGID);
unlock_ok:
- ret = strnlen(buf, count);
+ ret = count;
dynamic_netconsole_mutex_unlock();
mutex_unlock(&netconsole_subsys.su_mutex);
return ret;
@@ -1162,7 +1173,7 @@ static ssize_t sysdata_release_enabled_store(struct config_item *item,
disable_sysdata_feature(nt, SYSDATA_RELEASE);
unlock_ok:
- ret = strnlen(buf, count);
+ ret = count;
dynamic_netconsole_mutex_unlock();
mutex_unlock(&netconsole_subsys.su_mutex);
return ret;
@@ -1191,7 +1202,7 @@ static ssize_t sysdata_taskname_enabled_store(struct config_item *item,
disable_sysdata_feature(nt, SYSDATA_TASKNAME);
unlock_ok:
- ret = strnlen(buf, count);
+ ret = count;
dynamic_netconsole_mutex_unlock();
mutex_unlock(&netconsole_subsys.su_mutex);
return ret;
@@ -1225,7 +1236,7 @@ static ssize_t sysdata_cpu_nr_enabled_store(struct config_item *item,
disable_sysdata_feature(nt, SYSDATA_CPU_NR);
unlock_ok:
- ret = strnlen(buf, count);
+ ret = count;
dynamic_netconsole_mutex_unlock();
mutex_unlock(&netconsole_subsys.su_mutex);
return ret;
diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c
index 1e06e78..f00fc2f 100644
--- a/drivers/net/netdevsim/dev.c
+++ b/drivers/net/netdevsim/dev.c
@@ -829,7 +829,7 @@ static struct sk_buff *nsim_dev_trap_skb_build(void)
skb->protocol = htons(ETH_P_IP);
skb_set_network_header(skb, skb->len);
- iph = skb_put(skb, sizeof(struct iphdr));
+ iph = skb_put_zero(skb, sizeof(struct iphdr));
iph->protocol = IPPROTO_UDP;
iph->saddr = in_aton("192.0.2.1");
iph->daddr = in_aton("198.51.100.1");
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
index a05af19..a750768 100644
--- a/drivers/net/netdevsim/netdev.c
+++ b/drivers/net/netdevsim/netdev.c
@@ -1182,7 +1182,8 @@ void nsim_destroy(struct netdevsim *ns)
unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
&ns->nn);
- nsim_psp_uninit(ns);
+ if (nsim_dev_port_is_pf(ns->nsim_dev_port))
+ nsim_psp_uninit(ns);
rtnl_lock();
peer = rtnl_dereference(ns->peer);
diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h
index 7e129dd..d909c41 100644
--- a/drivers/net/netdevsim/netdevsim.h
+++ b/drivers/net/netdevsim/netdevsim.h
@@ -120,7 +120,9 @@ struct netdevsim {
u64_stats_t tx_packets;
u64_stats_t tx_bytes;
struct u64_stats_sync syncp;
- struct psp_dev *dev;
+ struct psp_dev __rcu *dev;
+ struct dentry *rereg;
+ struct mutex rereg_lock;
u32 spi;
u32 assoc_cnt;
} psp;
diff --git a/drivers/net/netdevsim/psp.c b/drivers/net/netdevsim/psp.c
index 0b4d717..6936ecb 100644
--- a/drivers/net/netdevsim/psp.c
+++ b/drivers/net/netdevsim/psp.c
@@ -19,6 +19,7 @@ nsim_do_psp(struct sk_buff *skb, struct netdevsim *ns,
struct netdevsim *peer_ns, struct skb_ext **psp_ext)
{
enum skb_drop_reason rc = 0;
+ struct psp_dev *peer_psd;
struct psp_assoc *pas;
struct net *net;
void **ptr;
@@ -48,7 +49,8 @@ nsim_do_psp(struct sk_buff *skb, struct netdevsim *ns,
}
/* Now pretend we just received this frame */
- if (peer_ns->psp.dev->config.versions & (1 << pas->version)) {
+ peer_psd = rcu_dereference(peer_ns->psp.dev);
+ if (peer_psd && peer_psd->config.versions & (1 << pas->version)) {
bool strip_icv = false;
u8 generation;
@@ -61,8 +63,7 @@ nsim_do_psp(struct sk_buff *skb, struct netdevsim *ns,
skb_ext_reset(skb);
skb->mac_len = ETH_HLEN;
- if (psp_dev_rcv(skb, peer_ns->psp.dev->id, generation,
- strip_icv)) {
+ if (psp_dev_rcv(skb, peer_psd->id, generation, strip_icv)) {
rc = SKB_DROP_REASON_PSP_OUTPUT;
goto out_unlock;
}
@@ -209,11 +210,26 @@ static struct psp_dev_caps nsim_psp_caps = {
.assoc_drv_spc = sizeof(void *),
};
+static void __nsim_psp_uninit(struct netdevsim *ns, bool teardown)
+{
+ struct psp_dev *psd;
+
+ psd = rcu_dereference_protected(ns->psp.dev,
+ teardown ||
+ lockdep_is_held(&ns->psp.rereg_lock));
+ if (psd) {
+ rcu_assign_pointer(ns->psp.dev, NULL);
+ synchronize_rcu();
+ psp_dev_unregister(psd);
+ }
+ WARN_ON(ns->psp.assoc_cnt);
+}
+
void nsim_psp_uninit(struct netdevsim *ns)
{
- if (!IS_ERR(ns->psp.dev))
- psp_dev_unregister(ns->psp.dev);
- WARN_ON(ns->psp.assoc_cnt);
+ debugfs_remove(ns->psp.rereg);
+ mutex_destroy(&ns->psp.rereg_lock);
+ __nsim_psp_uninit(ns, true);
}
static ssize_t
@@ -221,14 +237,23 @@ nsim_psp_rereg_write(struct file *file, const char __user *data, size_t count,
loff_t *ppos)
{
struct netdevsim *ns = file->private_data;
- int err;
+ struct psp_dev *psd;
+ ssize_t ret;
- nsim_psp_uninit(ns);
+ mutex_lock(&ns->psp.rereg_lock);
+ __nsim_psp_uninit(ns, false);
- ns->psp.dev = psp_dev_create(ns->netdev, &nsim_psp_ops,
- &nsim_psp_caps, ns);
- err = PTR_ERR_OR_ZERO(ns->psp.dev);
- return err ?: count;
+ psd = psp_dev_create(ns->netdev, &nsim_psp_ops, &nsim_psp_caps, ns);
+ if (IS_ERR(psd)) {
+ ret = PTR_ERR(psd);
+ goto out;
+ }
+
+ rcu_assign_pointer(ns->psp.dev, psd);
+ ret = count;
+out:
+ mutex_unlock(&ns->psp.rereg_lock);
+ return ret;
}
static const struct file_operations nsim_psp_rereg_fops = {
@@ -241,14 +266,16 @@ static const struct file_operations nsim_psp_rereg_fops = {
int nsim_psp_init(struct netdevsim *ns)
{
struct dentry *ddir = ns->nsim_dev_port->ddir;
- int err;
+ struct psp_dev *psd;
- ns->psp.dev = psp_dev_create(ns->netdev, &nsim_psp_ops,
- &nsim_psp_caps, ns);
- err = PTR_ERR_OR_ZERO(ns->psp.dev);
- if (err)
- return err;
+ psd = psp_dev_create(ns->netdev, &nsim_psp_ops, &nsim_psp_caps, ns);
+ if (IS_ERR(psd))
+ return PTR_ERR(psd);
- debugfs_create_file("psp_rereg", 0200, ddir, ns, &nsim_psp_rereg_fops);
+ rcu_assign_pointer(ns->psp.dev, psd);
+
+ mutex_init(&ns->psp.rereg_lock);
+ ns->psp.rereg = debugfs_create_file("psp_rereg", 0200, ddir, ns,
+ &nsim_psp_rereg_fops);
return 0;
}
diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c
index db43a1f..a6b777a 100644
--- a/drivers/net/ovpn/io.c
+++ b/drivers/net/ovpn/io.c
@@ -85,17 +85,24 @@ static void ovpn_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb)
skb_scrub_packet(skb, true);
/* network header reset in ovpn_decrypt_post() */
+ skb_reset_mac_header(skb);
skb_reset_transport_header(skb);
skb_reset_inner_headers(skb);
/* cause packet to be "received" by the interface */
pkt_len = skb->len;
+ /* we may get here in process context in case of TCP connections,
+ * therefore we have to disable BHs to ensure gro_cells_receive()
+ * and dev_dstats_rx_add() do not get corrupted or enter deadlock
+ */
+ local_bh_disable();
ret = gro_cells_receive(&peer->ovpn->gro_cells, skb);
if (likely(ret == NET_RX_SUCCESS)) {
/* update RX stats with the size of decrypted packet */
ovpn_peer_stats_increment_rx(&peer->vpn_stats, pkt_len);
dev_dstats_rx_add(peer->ovpn->dev, pkt_len);
}
+ local_bh_enable();
}
void ovpn_decrypt_post(void *data, int ret)
@@ -194,7 +201,7 @@ void ovpn_decrypt_post(void *data, int ret)
skb = NULL;
drop:
if (unlikely(skb))
- dev_dstats_rx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
kfree_skb(skb);
drop_nocount:
if (likely(peer))
@@ -218,7 +225,7 @@ void ovpn_recv(struct ovpn_peer *peer, struct sk_buff *skb)
net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n",
netdev_name(peer->ovpn->dev), peer->id,
key_id);
- dev_dstats_rx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
kfree_skb(skb);
ovpn_peer_put(peer);
return;
@@ -294,7 +301,7 @@ void ovpn_encrypt_post(void *data, int ret)
rcu_read_unlock();
err:
if (unlikely(skb))
- dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
if (likely(peer))
ovpn_peer_put(peer);
if (likely(ks))
@@ -336,7 +343,7 @@ static void ovpn_send(struct ovpn_priv *ovpn, struct sk_buff *skb,
*/
skb_list_walk_safe(skb, curr, next) {
if (unlikely(!ovpn_encrypt_one(peer, curr))) {
- dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev);
kfree_skb(curr);
}
}
@@ -407,7 +414,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
if (unlikely(!curr)) {
net_err_ratelimited("%s: skb_share_check failed for payload packet\n",
netdev_name(dev));
- dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev);
continue;
}
@@ -433,7 +440,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
drop:
ovpn_peer_put(peer);
drop_no_peer:
- dev_dstats_tx_dropped(ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(ovpn->dev);
skb_tx_error(skb);
kfree_skb_list(skb);
return NETDEV_TX_OK;
diff --git a/drivers/net/ovpn/main.c b/drivers/net/ovpn/main.c
index 2e0420f..9993c1d 100644
--- a/drivers/net/ovpn/main.c
+++ b/drivers/net/ovpn/main.c
@@ -92,6 +92,8 @@ static void ovpn_net_uninit(struct net_device *dev)
{
struct ovpn_priv *ovpn = netdev_priv(dev);
+ disable_delayed_work_sync(&ovpn->keepalive_work);
+ ovpn_peers_free(ovpn, NULL, OVPN_DEL_PEER_REASON_TEARDOWN);
gro_cells_destroy(&ovpn->gro_cells);
}
@@ -208,15 +210,6 @@ static int ovpn_newlink(struct net_device *dev,
return register_netdevice(dev);
}
-static void ovpn_dellink(struct net_device *dev, struct list_head *head)
-{
- struct ovpn_priv *ovpn = netdev_priv(dev);
-
- cancel_delayed_work_sync(&ovpn->keepalive_work);
- ovpn_peers_free(ovpn, NULL, OVPN_DEL_PEER_REASON_TEARDOWN);
- unregister_netdevice_queue(dev, head);
-}
-
static int ovpn_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
struct ovpn_priv *ovpn = netdev_priv(dev);
@@ -235,7 +228,6 @@ static struct rtnl_link_ops ovpn_link_ops = {
.policy = ovpn_policy,
.maxtype = IFLA_OVPN_MAX,
.newlink = ovpn_newlink,
- .dellink = ovpn_dellink,
.fill_info = ovpn_fill_info,
};
diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c
index 291e2e5..4c66c1e 100644
--- a/drivers/net/ovpn/netlink.c
+++ b/drivers/net/ovpn/netlink.c
@@ -462,10 +462,12 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
sock_release:
ovpn_socket_release(peer);
peer_release:
- /* release right away because peer was not yet hashed, thus it is not
- * used in any context
+ /* For UDP, the peer is unreachable until added to the hashtables, so
+ * dropping the initial reference is enough. For TCP, the peer may be
+ * concurrently reachable via sk_user_data->peer until
+ * ovpn_socket_release() detaches; rely on the refcount.
*/
- ovpn_peer_release(peer);
+ ovpn_peer_put(peer);
return ret;
}
diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c
index c02dfab..a09d612 100644
--- a/drivers/net/ovpn/peer.c
+++ b/drivers/net/ovpn/peer.c
@@ -354,7 +354,7 @@ static void ovpn_peer_release_rcu(struct rcu_head *head)
* ovpn_peer_release - release peer private members
* @peer: the peer to release
*/
-void ovpn_peer_release(struct ovpn_peer *peer)
+static void ovpn_peer_release(struct ovpn_peer *peer)
{
ovpn_crypto_state_release(&peer->crypto);
spin_lock_bh(&peer->lock);
@@ -1034,14 +1034,29 @@ static int ovpn_peer_add_p2p(struct ovpn_priv *ovpn, struct ovpn_peer *peer)
*/
int ovpn_peer_add(struct ovpn_priv *ovpn, struct ovpn_peer *peer)
{
+ int ret = -ENODEV;
+
+ /* Prevent adding new peers while destroying the ovpn interface.
+ * Failing to do so would end up holding the device reference
+ * endlessly hostage of the new peer object with no chance of
+ * release..
+ */
+ netdev_lock(ovpn->dev);
+ if (ovpn->dev->reg_state != NETREG_REGISTERED)
+ goto out;
+
switch (ovpn->mode) {
case OVPN_MODE_MP:
- return ovpn_peer_add_mp(ovpn, peer);
+ ret = ovpn_peer_add_mp(ovpn, peer);
+ break;
case OVPN_MODE_P2P:
- return ovpn_peer_add_p2p(ovpn, peer);
+ ret = ovpn_peer_add_p2p(ovpn, peer);
+ break;
}
+out:
+ netdev_unlock(ovpn->dev);
- return -EOPNOTSUPP;
+ return ret;
}
/**
diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h
index 3284015..86c8cff 100644
--- a/drivers/net/ovpn/peer.h
+++ b/drivers/net/ovpn/peer.h
@@ -127,7 +127,6 @@ static inline bool ovpn_peer_hold(struct ovpn_peer *peer)
return kref_get_unless_zero(&peer->refcount);
}
-void ovpn_peer_release(struct ovpn_peer *peer);
void ovpn_peer_release_kref(struct kref *kref);
/**
diff --git a/drivers/net/ovpn/stats.h b/drivers/net/ovpn/stats.h
index 53433d8..3a45b97 100644
--- a/drivers/net/ovpn/stats.h
+++ b/drivers/net/ovpn/stats.h
@@ -11,6 +11,8 @@
#ifndef _NET_OVPN_OVPNSTATS_H_
#define _NET_OVPN_OVPNSTATS_H_
+#include <linux/netdevice.h>
+
/* one stat */
struct ovpn_peer_stat {
atomic64_t bytes;
@@ -44,4 +46,18 @@ static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats,
ovpn_peer_stats_increment(&stats->tx, n);
}
+static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev)
+{
+ local_bh_disable();
+ dev_dstats_tx_dropped(dev);
+ local_bh_enable();
+}
+
+static inline void ovpn_dev_dstats_rx_dropped(struct net_device *dev)
+{
+ local_bh_disable();
+ dev_dstats_rx_dropped(dev);
+ local_bh_enable();
+}
+
#endif /* _NET_OVPN_OVPNSTATS_H_ */
diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c
index 65054cc..433bd07 100644
--- a/drivers/net/ovpn/tcp.c
+++ b/drivers/net/ovpn/tcp.c
@@ -152,7 +152,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb)
if (WARN_ON(!ovpn_peer_hold(peer)))
goto err_nopeer;
schedule_work(&peer->tcp.defer_del_work);
- dev_dstats_rx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
err_nopeer:
kfree_skb(skb);
}
@@ -298,9 +298,9 @@ static void ovpn_tcp_send_sock(struct ovpn_peer *peer, struct sock *sk)
} while (peer->tcp.out_msg.len > 0);
if (!peer->tcp.out_msg.len) {
- preempt_disable();
+ local_bh_disable();
dev_dstats_tx_add(peer->ovpn->dev, skb->len);
- preempt_enable();
+ local_bh_enable();
}
kfree_skb(peer->tcp.out_msg.skb);
@@ -331,7 +331,7 @@ static void ovpn_tcp_send_sock_skb(struct ovpn_peer *peer, struct sock *sk,
ovpn_tcp_send_sock(peer, sk);
if (peer->tcp.out_msg.skb) {
- dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
kfree_skb(skb);
return;
}
@@ -353,7 +353,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
if (sock_owned_by_user(sk)) {
if (skb_queue_len(&peer->tcp.out_queue) >=
READ_ONCE(net_hotdata.max_backlog)) {
- dev_dstats_tx_dropped(peer->ovpn->dev);
+ ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
kfree_skb(skb);
goto unlock;
}
@@ -581,14 +581,19 @@ static void ovpn_tcp_close(struct sock *sk, long timeout)
rcu_read_lock();
sock = rcu_dereference_sk_user_data(sk);
- if (!sock || !sock->peer || !ovpn_peer_hold(sock->peer)) {
+ if (!sock) {
rcu_read_unlock();
return;
}
+
peer = sock->peer;
+ if (!peer || !ovpn_peer_hold(peer)) {
+ rcu_read_unlock();
+ return;
+ }
rcu_read_unlock();
- ovpn_peer_del(sock->peer, OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT);
+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT);
peer->tcp.sk_cb.prot->close(sk, timeout);
ovpn_peer_put(peer);
}
diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c
index 059e896..8811aa9 100644
--- a/drivers/net/ovpn/udp.c
+++ b/drivers/net/ovpn/udp.c
@@ -125,7 +125,7 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
return 0;
drop:
- dev_dstats_rx_dropped(ovpn->dev);
+ ovpn_dev_dstats_rx_dropped(ovpn->dev);
drop_noovpn:
kfree_skb(skb);
return 0;
diff --git a/drivers/net/pcs/pcs-mtk-lynxi.c b/drivers/net/pcs/pcs-mtk-lynxi.c
index c12f808..a753bd8 100644
--- a/drivers/net/pcs/pcs-mtk-lynxi.c
+++ b/drivers/net/pcs/pcs-mtk-lynxi.c
@@ -129,6 +129,9 @@ static int mtk_pcs_config_polarity(struct mtk_pcs_lynxi *mpcs,
unsigned int val = 0;
int ret;
+ if (!fwnode)
+ return 0;
+
if (fwnode_property_read_bool(fwnode, "mediatek,pnswap"))
default_pol = PHY_POL_INVERT;
diff --git a/drivers/net/phy/air_en8811h.c b/drivers/net/phy/air_en8811h.c
index 29ae73e..a86129c 100644
--- a/drivers/net/phy/air_en8811h.c
+++ b/drivers/net/phy/air_en8811h.c
@@ -17,6 +17,7 @@
#include <linux/phy.h>
#include <linux/phy/phy-common-props.h>
#include <linux/firmware.h>
+#include <linux/bitfield.h>
#include <linux/property.h>
#include <linux/wordpart.h>
#include <linux/unaligned.h>
@@ -170,9 +171,23 @@
#define AN8811HB_CLK_DRV_CKO_LDPWD BIT(13)
#define AN8811HB_CLK_DRV_CKO_LPPWD BIT(14)
+#define AN8811HB_MCU_SW_RST 0x5cf9f8
+#define AN8811HB_MCU_SW_RST_HOLD BIT(16)
+#define AN8811HB_MCU_SW_RST_RUN (BIT(16) | BIT(0))
+#define AN8811HB_MCU_SW_START 0x5cf9fc
+#define AN8811HB_MCU_SW_START_EN BIT(16)
+
+/* MII register constants for PBUS access (PHY addr + 8) */
+#define AIR_PBUS_ADDR_HIGH 0x1c
+#define AIR_PBUS_DATA_HIGH 0x10
+#define AIR_PBUS_REG_ADDR_HIGH_MASK GENMASK(15, 6)
+#define AIR_PBUS_REG_ADDR_LOW_MASK GENMASK(5, 2)
+
/* Led definitions */
#define EN8811H_LED_COUNT 3
+#define EN8811H_PBUS_ADDR_OFFS 8
+
/* Default LED setup:
* GPIO5 <-> LED0 On: Link detected, blink Rx/Tx
* GPIO4 <-> LED1 On: Link detected at 2500 or 1000 Mbps
@@ -201,6 +216,7 @@ struct en8811h_priv {
struct clk_hw hw;
struct phy_device *phydev;
unsigned int cko_is_enabled;
+ struct mdio_device *pbusdev;
};
enum {
@@ -254,6 +270,31 @@ static int air_phy_write_page(struct phy_device *phydev, int page)
return __phy_write(phydev, AIR_EXT_PAGE_ACCESS, page);
}
+static int __air_pbus_reg_write(struct mdio_device *mdiodev,
+ u32 pbus_reg, u32 pbus_data)
+{
+ int ret;
+
+ ret = __mdiobus_write(mdiodev->bus, mdiodev->addr, AIR_EXT_PAGE_ACCESS,
+ upper_16_bits(pbus_reg));
+ if (ret < 0)
+ return ret;
+
+ ret = __mdiobus_write(mdiodev->bus, mdiodev->addr, AIR_PBUS_ADDR_HIGH,
+ FIELD_GET(AIR_PBUS_REG_ADDR_HIGH_MASK, pbus_reg));
+ if (ret < 0)
+ return ret;
+
+ ret = __mdiobus_write(mdiodev->bus, mdiodev->addr,
+ FIELD_GET(AIR_PBUS_REG_ADDR_LOW_MASK, pbus_reg),
+ lower_16_bits(pbus_data));
+ if (ret < 0)
+ return ret;
+
+ return __mdiobus_write(mdiodev->bus, mdiodev->addr, AIR_PBUS_DATA_HIGH,
+ upper_16_bits(pbus_data));
+}
+
static int __air_buckpbus_reg_write(struct phy_device *phydev,
u32 pbus_address, u32 pbus_data)
{
@@ -570,10 +611,67 @@ static int an8811hb_load_file(struct phy_device *phydev, const char *name,
return ret;
}
+static int an8811hb_mcu_assert(struct phy_device *phydev)
+{
+ struct en8811h_priv *priv = phydev->priv;
+ int ret;
+
+ phy_lock_mdio_bus(phydev);
+
+ ret = __air_pbus_reg_write(priv->pbusdev, AN8811HB_MCU_SW_RST,
+ AN8811HB_MCU_SW_RST_HOLD);
+ if (ret < 0)
+ goto unlock;
+
+ ret = __air_pbus_reg_write(priv->pbusdev, AN8811HB_MCU_SW_START, 0);
+ if (ret < 0)
+ goto unlock;
+
+ msleep(50);
+ phydev_dbg(phydev, "MCU asserted\n");
+
+unlock:
+ phy_unlock_mdio_bus(phydev);
+ return ret;
+}
+
+static int an8811hb_mcu_deassert(struct phy_device *phydev)
+{
+ struct en8811h_priv *priv = phydev->priv;
+ int ret;
+
+ phy_lock_mdio_bus(phydev);
+
+ ret = __air_pbus_reg_write(priv->pbusdev, AN8811HB_MCU_SW_START,
+ AN8811HB_MCU_SW_START_EN);
+ if (ret < 0)
+ goto unlock;
+
+ ret = __air_pbus_reg_write(priv->pbusdev, AN8811HB_MCU_SW_RST,
+ AN8811HB_MCU_SW_RST_RUN);
+ if (ret < 0)
+ goto unlock;
+
+ msleep(50);
+ phydev_dbg(phydev, "MCU deasserted\n");
+
+unlock:
+ phy_unlock_mdio_bus(phydev);
+ return ret;
+}
+
static int an8811hb_load_firmware(struct phy_device *phydev)
{
int ret;
+ ret = an8811hb_mcu_assert(phydev);
+ if (ret < 0)
+ return ret;
+
+ ret = an8811hb_mcu_deassert(phydev);
+ if (ret < 0)
+ return ret;
+
ret = air_buckpbus_reg_write(phydev, EN8811H_FW_CTRL_1,
EN8811H_FW_CTRL_1_START);
if (ret < 0)
@@ -662,6 +760,16 @@ static int en8811h_restart_mcu(struct phy_device *phydev)
{
int ret;
+ if (phy_id_compare_model(phydev->phy_id, AN8811HB_PHY_ID)) {
+ ret = an8811hb_mcu_assert(phydev);
+ if (ret < 0)
+ return ret;
+
+ ret = an8811hb_mcu_deassert(phydev);
+ if (ret < 0)
+ return ret;
+ }
+
ret = air_buckpbus_reg_write(phydev, EN8811H_FW_CTRL_1,
EN8811H_FW_CTRL_1_START);
if (ret < 0)
@@ -1166,6 +1274,7 @@ static int en8811h_leds_setup(struct phy_device *phydev)
static int an8811hb_probe(struct phy_device *phydev)
{
+ struct mdio_device *mdiodev;
struct en8811h_priv *priv;
int ret;
@@ -1175,10 +1284,28 @@ static int an8811hb_probe(struct phy_device *phydev)
return -ENOMEM;
phydev->priv = priv;
+ /*
+ * The AN8811HB PHY address is restricted to 8-15 (decimal),
+ * depending on the board hardware strapping.
+ * This means the PBUS address is only in the range 16-21 (decimal),
+ * so we do not need to handle the case
+ * where the PBUS address exceeds 31 (decimal).
+ */
+ mdiodev = mdio_device_create(phydev->mdio.bus,
+ phydev->mdio.addr + EN8811H_PBUS_ADDR_OFFS);
+ if (IS_ERR(mdiodev))
+ return PTR_ERR(mdiodev);
+
+ ret = mdio_device_register(mdiodev);
+ if (ret)
+ goto err_dev_free;
+
+ priv->pbusdev = mdiodev;
+
ret = an8811hb_load_firmware(phydev);
if (ret < 0) {
phydev_err(phydev, "Load firmware failed: %d\n", ret);
- return ret;
+ goto err_dev_create;
}
en8811h_print_fw_version(phydev);
@@ -1191,22 +1318,29 @@ static int an8811hb_probe(struct phy_device *phydev)
ret = en8811h_leds_setup(phydev);
if (ret < 0)
- return ret;
+ goto err_dev_create;
priv->phydev = phydev;
/* Co-Clock Output */
ret = an8811hb_clk_provider_setup(&phydev->mdio.dev, &priv->hw);
if (ret)
- return ret;
+ goto err_dev_create;
/* Configure led gpio pins as output */
ret = air_buckpbus_reg_modify(phydev, AN8811HB_GPIO_OUTPUT,
AN8811HB_GPIO_OUTPUT_345,
AN8811HB_GPIO_OUTPUT_345);
if (ret < 0)
- return ret;
+ goto err_dev_create;
return 0;
+
+err_dev_create:
+ mdio_device_remove(mdiodev);
+
+err_dev_free:
+ mdio_device_free(mdiodev);
+ return ret;
}
static int en8811h_probe(struct phy_device *phydev)
@@ -1561,6 +1695,16 @@ static int en8811h_suspend(struct phy_device *phydev)
return genphy_suspend(phydev);
}
+static void an8811hb_remove(struct phy_device *phydev)
+{
+ struct en8811h_priv *priv = phydev->priv;
+
+ if (priv->pbusdev) {
+ mdio_device_remove(priv->pbusdev);
+ mdio_device_free(priv->pbusdev);
+ }
+}
+
static struct phy_driver en8811h_driver[] = {
{
PHY_ID_MATCH_MODEL(EN8811H_PHY_ID),
@@ -1587,6 +1731,7 @@ static struct phy_driver en8811h_driver[] = {
PHY_ID_MATCH_MODEL(AN8811HB_PHY_ID),
.name = "Airoha AN8811HB",
.probe = an8811hb_probe,
+ .remove = an8811hb_remove,
.get_features = en8811h_get_features,
.config_init = an8811hb_config_init,
.get_rate_matching = en8811h_get_rate_matching,
diff --git a/drivers/net/phy/bcm-phy-lib.c b/drivers/net/phy/bcm-phy-lib.c
index 5198d66..b64bead 100644
--- a/drivers/net/phy/bcm-phy-lib.c
+++ b/drivers/net/phy/bcm-phy-lib.c
@@ -563,6 +563,15 @@ void bcm_phy_get_stats(struct phy_device *phydev, u64 *shadow,
}
EXPORT_SYMBOL_GPL(bcm_phy_get_stats);
+void bcm_phy_update_stats_shadow(struct phy_device *phydev, u64 *shadow)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(bcm_phy_hw_stats); i++)
+ bcm_phy_get_stat(phydev, shadow, i);
+}
+EXPORT_SYMBOL_GPL(bcm_phy_update_stats_shadow);
+
void bcm_phy_r_rc_cal_reset(struct phy_device *phydev)
{
/* Reset R_CAL/RC_CAL Engine */
diff --git a/drivers/net/phy/bcm-phy-lib.h b/drivers/net/phy/bcm-phy-lib.h
index bceddbc..bba94ce 100644
--- a/drivers/net/phy/bcm-phy-lib.h
+++ b/drivers/net/phy/bcm-phy-lib.h
@@ -85,6 +85,7 @@ int bcm_phy_get_sset_count(struct phy_device *phydev);
void bcm_phy_get_strings(struct phy_device *phydev, u8 *data);
void bcm_phy_get_stats(struct phy_device *phydev, u64 *shadow,
struct ethtool_stats *stats, u64 *data);
+void bcm_phy_update_stats_shadow(struct phy_device *phydev, u64 *shadow);
void bcm_phy_r_rc_cal_reset(struct phy_device *phydev);
int bcm_phy_28nm_a0b0_afe_config_init(struct phy_device *phydev);
int bcm_phy_enable_jumbo(struct phy_device *phydev);
diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c
index 00e8fa1..71a163f 100644
--- a/drivers/net/phy/bcm7xxx.c
+++ b/drivers/net/phy/bcm7xxx.c
@@ -807,6 +807,17 @@ static void bcm7xxx_28nm_get_phy_stats(struct phy_device *phydev,
bcm_phy_get_stats(phydev, priv->stats, stats, data);
}
+static int bcm7xxx_28nm_suspend(struct phy_device *phydev)
+{
+ struct bcm7xxx_phy_priv *priv = phydev->priv;
+
+ mutex_lock(&phydev->lock);
+ bcm_phy_update_stats_shadow(phydev, priv->stats);
+ mutex_unlock(&phydev->lock);
+
+ return genphy_suspend(phydev);
+}
+
static int bcm7xxx_28nm_probe(struct phy_device *phydev)
{
struct bcm7xxx_phy_priv *priv;
@@ -849,6 +860,7 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
.flags = PHY_IS_INTERNAL, \
.config_init = bcm7xxx_28nm_config_init, \
.resume = bcm7xxx_28nm_resume, \
+ .suspend = bcm7xxx_28nm_suspend, \
.get_tunable = bcm7xxx_28nm_get_tunable, \
.set_tunable = bcm7xxx_28nm_set_tunable, \
.get_sset_count = bcm_phy_get_sset_count, \
@@ -866,6 +878,7 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
.flags = PHY_IS_INTERNAL, \
.config_init = bcm7xxx_28nm_ephy_config_init, \
.resume = bcm7xxx_28nm_ephy_resume, \
+ .suspend = bcm7xxx_28nm_suspend, \
.get_sset_count = bcm_phy_get_sset_count, \
.get_strings = bcm_phy_get_strings, \
.get_stats = bcm7xxx_28nm_get_phy_stats, \
@@ -902,6 +915,7 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
.config_aneg = genphy_config_aneg, \
.read_status = genphy_read_status, \
.resume = bcm7xxx_16nm_ephy_resume, \
+ .suspend = bcm7xxx_28nm_suspend, \
}
static struct phy_driver bcm7xxx_driver[] = {
diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
index bf0c6a0..d1a4edb 100644
--- a/drivers/net/phy/broadcom.c
+++ b/drivers/net/phy/broadcom.c
@@ -592,8 +592,13 @@ static int bcm54xx_set_wakeup_irq(struct phy_device *phydev, bool state)
static int bcm54xx_suspend(struct phy_device *phydev)
{
+ struct bcm54xx_phy_priv *priv = phydev->priv;
int ret = 0;
+ mutex_lock(&phydev->lock);
+ bcm_phy_update_stats_shadow(phydev, priv->stats);
+ mutex_unlock(&phydev->lock);
+
bcm54xx_ptp_stop(phydev);
/* Acknowledge any Wake-on-LAN interrupt prior to suspend */
diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c
index 1f381d7..96a7d25 100644
--- a/drivers/net/phy/dp83869.c
+++ b/drivers/net/phy/dp83869.c
@@ -31,6 +31,7 @@
#define DP83869_RGMIICTL 0x0032
#define DP83869_STRAP_STS1 0x006e
#define DP83869_RGMIIDCTL 0x0086
+#define DP83869_ANA_PLL_PROG_PI 0x00c6
#define DP83869_RXFCFG 0x0134
#define DP83869_RXFPMD1 0x0136
#define DP83869_RXFPMD2 0x0137
@@ -826,12 +827,22 @@ static int dp83869_config_init(struct phy_device *phydev)
dp83869_config_port_mirroring(phydev);
/* Clock output selection if muxing property is set */
- if (dp83869->clk_output_sel != DP83869_CLK_O_SEL_REF_CLK)
+ if (dp83869->clk_output_sel != DP83869_CLK_O_SEL_REF_CLK) {
+ /*
+ * Table 7-121 in datasheet says we have to set register 0xc6
+ * to value 0x10 before CLK_O_SEL can be modified.
+ */
+ ret = phy_write_mmd(phydev, DP83869_DEVADDR,
+ DP83869_ANA_PLL_PROG_PI, 0x10);
+ if (ret)
+ return ret;
+
ret = phy_modify_mmd(phydev,
DP83869_DEVADDR, DP83869_IO_MUX_CFG,
DP83869_IO_MUX_CFG_CLK_O_SEL_MASK,
dp83869->clk_output_sel <<
DP83869_IO_MUX_CFG_CLK_O_SEL_SHIFT);
+ }
if (phy_interface_is_rgmii(phydev)) {
ret = phy_write_mmd(phydev, DP83869_DEVADDR, DP83869_RGMIIDCTL,
diff --git a/drivers/net/phy/dp83tc811.c b/drivers/net/phy/dp83tc811.c
index e480c2a..252fb12 100644
--- a/drivers/net/phy/dp83tc811.c
+++ b/drivers/net/phy/dp83tc811.c
@@ -393,6 +393,7 @@ static struct phy_driver dp83811_driver[] = {
.config_init = dp83811_config_init,
.config_aneg = dp83811_config_aneg,
.soft_reset = dp83811_phy_reset,
+ .get_features = genphy_c45_pma_read_ext_abilities,
.get_wol = dp83811_get_wol,
.set_wol = dp83811_set_wol,
.config_intr = dp83811_config_intr,
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 2aa1dedd..e211a52 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -4548,6 +4548,13 @@ static int lan8814_config_init(struct phy_device *phydev)
struct kszphy_priv *lan8814 = phydev->priv;
int ret;
+ if (phy_package_init_once(phydev))
+ /* Reset the PHY */
+ lanphy_modify_page_reg(phydev, LAN8814_PAGE_COMMON_REGS,
+ LAN8814_QSGMII_SOFT_RESET,
+ LAN8814_QSGMII_SOFT_RESET_BIT,
+ LAN8814_QSGMII_SOFT_RESET_BIT);
+
/* Based on the interface type select how the advertise ability is
* encoded, to set as SGMII or as USGMII.
*/
@@ -4655,13 +4662,7 @@ static int lan8814_probe(struct phy_device *phydev)
priv->is_ptp_available = err == LAN8814_REV_LAN8814 ||
err == LAN8814_REV_LAN8818;
- if (phy_package_init_once(phydev)) {
- /* Reset the PHY */
- lanphy_modify_page_reg(phydev, LAN8814_PAGE_COMMON_REGS,
- LAN8814_QSGMII_SOFT_RESET,
- LAN8814_QSGMII_SOFT_RESET_BIT,
- LAN8814_QSGMII_SOFT_RESET_BIT);
-
+ if (phy_package_probe_once(phydev)) {
err = lan8814_release_coma_mode(phydev);
if (err)
return err;
diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c
index d48aa723..1269517 100644
--- a/drivers/net/phy/phy-c45.c
+++ b/drivers/net/phy/phy-c45.c
@@ -940,6 +940,14 @@ EXPORT_SYMBOL_GPL(genphy_c45_read_eee_abilities);
*/
int genphy_c45_an_config_eee_aneg(struct phy_device *phydev)
{
+ /* Writing MMD AN advertisements while autoneg is disabled has no
+ * effect on link-partner negotiation, but on some PHYs (e.g. the
+ * Broadcom BCM54213PE) the write itself disturbs the receive
+ * datapath. Skip it.
+ */
+ if (phydev->autoneg == AUTONEG_DISABLE)
+ return 0;
+
if (!phydev->eee_cfg.eee_enabled) {
__ETHTOOL_DECLARE_LINK_MODE_MASK(adv) = {};
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index c2cdf1a..3370eb8 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -2877,7 +2877,8 @@ EXPORT_SYMBOL(phy_advertise_supported);
*/
void phy_advertise_eee_all(struct phy_device *phydev)
{
- linkmode_copy(phydev->advertising_eee, phydev->supported_eee);
+ linkmode_andnot(phydev->advertising_eee, phydev->supported_eee,
+ phydev->eee_disabled_modes);
}
EXPORT_SYMBOL_GPL(phy_advertise_eee_all);
@@ -2903,7 +2904,8 @@ EXPORT_SYMBOL_GPL(phy_advertise_eee_all);
*/
void phy_support_eee(struct phy_device *phydev)
{
- linkmode_copy(phydev->advertising_eee, phydev->supported_eee);
+ linkmode_andnot(phydev->advertising_eee, phydev->supported_eee,
+ phydev->eee_disabled_modes);
phydev->eee_cfg.tx_lpi_enabled = true;
phydev->eee_cfg.eee_enabled = true;
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index bd970f7..b94b9c4 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -822,6 +822,7 @@ static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c)
return -EINVAL;
}
+ sfp->i2c_block_size = sfp->i2c_max_block_size;
return 0;
}
diff --git a/drivers/net/pse-pd/pse_core.c b/drivers/net/pse-pd/pse_core.c
index 87aa4f4..69dbdbd 100644
--- a/drivers/net/pse-pd/pse_core.c
+++ b/drivers/net/pse-pd/pse_core.c
@@ -210,7 +210,7 @@ static int of_load_pse_pis(struct pse_controller_dev *pcdev)
ret = of_load_pse_pi_pairsets(node, &pi, ret);
if (ret)
goto out;
- } else if (ret != ENOENT) {
+ } else if (ret != -ENOENT) {
dev_err(pcdev->dev,
"error: wrong number of pairsets. Should be 1 or 2, got %d (%pOF)\n",
ret, node);
diff --git a/drivers/net/tap.c b/drivers/net/tap.c
index b824073..fae1159 100644
--- a/drivers/net/tap.c
+++ b/drivers/net/tap.c
@@ -919,11 +919,11 @@ static long tap_ioctl(struct file *file, unsigned int cmd,
struct tap_queue *q = file->private_data;
struct tap_dev *tap;
void __user *argp = (void __user *)arg;
+ struct sockaddr_storage ss = {};
struct ifreq __user *ifr = argp;
unsigned int __user *up = argp;
unsigned short u;
int __user *sp = argp;
- struct sockaddr_storage ss;
int s;
int ret;
@@ -1052,6 +1052,7 @@ static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp)
int err, depth;
if (unlikely(xdp->data_end - xdp->data < ETH_HLEN)) {
+ put_page(virt_to_head_page(xdp->data));
err = -EINVAL;
goto err;
}
@@ -1061,6 +1062,7 @@ static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp)
skb = build_skb(xdp->data_hard_start, buflen);
if (!skb) {
+ put_page(virt_to_head_page(xdp->data));
err = -ENOMEM;
goto err;
}
diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c
index 0c87f99..f51388d 100644
--- a/drivers/net/team/team_core.c
+++ b/drivers/net/team/team_core.c
@@ -534,21 +534,23 @@ static void team_adjust_ops(struct team *team)
if (!team->tx_en_port_count || !team_is_mode_set(team) ||
!team->mode->ops->transmit)
- team->ops.transmit = team_dummy_transmit;
+ WRITE_ONCE(team->ops.transmit, team_dummy_transmit);
else
- team->ops.transmit = team->mode->ops->transmit;
+ WRITE_ONCE(team->ops.transmit, team->mode->ops->transmit);
if (!team->rx_en_port_count || !team_is_mode_set(team) ||
!team->mode->ops->receive)
- team->ops.receive = team_dummy_receive;
+ WRITE_ONCE(team->ops.receive, team_dummy_receive);
else
- team->ops.receive = team->mode->ops->receive;
+ WRITE_ONCE(team->ops.receive, team->mode->ops->receive);
}
/*
- * We can benefit from the fact that it's ensured no port is present
- * at the time of mode change. Therefore no packets are in fly so there's no
- * need to set mode operations in any special way.
+ * team_change_mode() ensures no ports are present during mode change,
+ * but lockless readers can still reach team_xmit(). Avoid touching
+ * transmit/receive -- they are already set to dummies by
+ * team_adjust_ops() since no ports are enabled. synchronize_net()
+ * drains in-flight readers before destroying old mode state.
*/
static int __team_change_mode(struct team *team,
const struct team_mode *new_mode)
@@ -557,9 +559,21 @@ static int __team_change_mode(struct team *team,
if (team_is_mode_set(team)) {
void (*exit_op)(struct team *team) = team->ops.exit;
- /* Clear ops area so no callback is called any longer */
- memset(&team->ops, 0, sizeof(struct team_mode_ops));
- team_adjust_ops(team);
+ /* Clear cold-path ops used only under RTNL. transmit and
+ * receive are already dummies (no ports) so leave them
+ * alone -- overwriting them is the source of the race.
+ */
+ team->ops.init = NULL;
+ team->ops.exit = NULL;
+ team->ops.port_enter = NULL;
+ team->ops.port_leave = NULL;
+ team->ops.port_change_dev_addr = NULL;
+ team->ops.port_tx_disabled = NULL;
+
+ /* Wait for in-flight readers before tearing down mode
+ * state they may reference.
+ */
+ synchronize_net();
if (exit_op)
exit_op(team);
@@ -582,7 +596,12 @@ static int __team_change_mode(struct team *team,
}
team->mode = new_mode;
- memcpy(&team->ops, new_mode->ops, sizeof(struct team_mode_ops));
+ team->ops.init = new_mode->ops->init;
+ team->ops.exit = new_mode->ops->exit;
+ team->ops.port_enter = new_mode->ops->port_enter;
+ team->ops.port_leave = new_mode->ops->port_leave;
+ team->ops.port_change_dev_addr = new_mode->ops->port_change_dev_addr;
+ team->ops.port_tx_disabled = new_mode->ops->port_tx_disabled;
team_adjust_ops(team);
return 0;
@@ -743,7 +762,7 @@ static rx_handler_result_t team_handle_frame(struct sk_buff **pskb)
/* allow exact match delivery for disabled ports */
res = RX_HANDLER_EXACT;
} else {
- res = team->ops.receive(team, port, skb);
+ res = READ_ONCE(team->ops.receive)(team, port, skb);
}
if (res == RX_HANDLER_ANOTHER) {
struct team_pcpu_stats *pcpu_stats;
@@ -1845,7 +1864,7 @@ static netdev_tx_t team_xmit(struct sk_buff *skb, struct net_device *dev)
tx_success = team_queue_override_transmit(team, skb);
if (!tx_success)
- tx_success = team->ops.transmit(team, skb);
+ tx_success = READ_ONCE(team->ops.transmit)(team, skb);
if (tx_success) {
struct team_pcpu_stats *pcpu_stats;
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index b183189..9e7744e 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -2394,8 +2394,10 @@ static int tun_xdp_one(struct tun_struct *tun,
bool skb_xdp = false;
struct page *page;
- if (unlikely(datasize < ETH_HLEN))
+ if (unlikely(datasize < ETH_HLEN)) {
+ put_page(virt_to_head_page(xdp->data));
return -EINVAL;
+ }
xdp_prog = rcu_dereference(tun->xdp_prog);
if (xdp_prog) {
@@ -2437,6 +2439,7 @@ static int tun_xdp_one(struct tun_struct *tun,
build:
skb = build_skb(xdp->data_hard_start, buflen);
if (!skb) {
+ put_page(virt_to_head_page(xdp->data));
ret = -ENOMEM;
goto out;
}
diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c
index df0bcfed..293ef80 100644
--- a/drivers/net/usb/asix_devices.c
+++ b/drivers/net/usb/asix_devices.c
@@ -756,6 +756,7 @@ static void ax88772_mac_link_down(struct phylink_config *config,
struct usbnet *dev = netdev_priv(to_net_dev(config->dev));
asix_write_medium_mode(dev, 0, 0);
+ usbnet_link_change(dev, false, false);
}
static void ax88772_mac_link_up(struct phylink_config *config,
@@ -786,6 +787,7 @@ static void ax88772_mac_link_up(struct phylink_config *config,
m |= AX_MEDIUM_RFC;
asix_write_medium_mode(dev, m, 0);
+ usbnet_link_change(dev, true, false);
}
static const struct phylink_mac_ops ax88772_phylink_mac_ops = {
diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c
index bb99297..0223a17 100644
--- a/drivers/net/usb/cdc_ncm.c
+++ b/drivers/net/usb/cdc_ncm.c
@@ -2012,6 +2012,14 @@ static const struct usb_device_id cdc_devs[] = {
.driver_info = (unsigned long)&apple_private_interface_info,
},
+ /* Mac */
+ { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x1905, 0),
+ .driver_info = (unsigned long)&apple_private_interface_info,
+ },
+ { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x1905, 2),
+ .driver_info = (unsigned long)&apple_private_interface_info,
+ },
+
/* Ericsson MBM devices like F5521gw */
{ .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
| USB_DEVICE_ID_MATCH_VENDOR,
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 7337bf1..1ace1d2 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -10138,6 +10138,7 @@ static const struct usb_device_id rtl8152_table[] = {
{ USB_DEVICE(VENDOR_ID_DELL, 0xb097) },
{ USB_DEVICE(VENDOR_ID_ASUS, 0x1976) },
{ USB_DEVICE(VENDOR_ID_TRENDNET, 0xe02b) },
+ { USB_DEVICE(VENDOR_ID_TRENDNET, 0xe02c) },
{}
};
diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c
index 4cda064..c880c95 100644
--- a/drivers/net/usb/rtl8150.c
+++ b/drivers/net/usb/rtl8150.c
@@ -683,6 +683,7 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb,
struct net_device *netdev)
{
rtl8150_t *dev = netdev_priv(netdev);
+ unsigned int skb_len;
int count, res;
/* pad the frame and ensure terminating USB packet, datasheet 9.2.3 */
@@ -694,6 +695,8 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb,
return NETDEV_TX_OK;
}
+ skb_len = skb->len;
+
netif_stop_queue(netdev);
dev->tx_skb = skb;
usb_fill_bulk_urb(dev->tx_urb, dev->udev, usb_sndbulkpipe(dev->udev, 2),
@@ -707,9 +710,16 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb,
netdev->stats.tx_errors++;
netif_start_queue(netdev);
}
+ /*
+ * The URB was not submitted, so write_bulk_callback() will
+ * never run to free dev->tx_skb. Drop the skb here and
+ * clear tx_skb to avoid leaving a stale pointer.
+ */
+ dev->tx_skb = NULL;
+ dev_kfree_skb_any(skb);
} else {
netdev->stats.tx_packets++;
- netdev->stats.tx_bytes += skb->len;
+ netdev->stats.tx_bytes += skb_len;
netif_trans_update(netdev);
}
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index e35df71..0cfb19b 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -972,7 +972,8 @@ static int veth_poll(struct napi_struct *napi, int budget)
/* NAPI functions as RCU section */
peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held());
- peer_txq = peer_dev ? netdev_get_tx_queue(peer_dev, queue_idx) : NULL;
+ peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ?
+ netdev_get_tx_queue(peer_dev, queue_idx) : NULL;
xdp_set_return_frame_no_direct();
done = veth_xdp_rcv(rq, budget, &bq, &stats);
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 2cf2dbd..4620991 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -1034,6 +1034,7 @@ static int do_vrf_add_slave(struct net_device *dev, struct net_device *port_dev,
err:
port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE;
+ synchronize_net();
return ret;
}
@@ -1053,10 +1054,16 @@ static int vrf_add_slave(struct net_device *dev, struct net_device *port_dev,
}
/* inverse of do_vrf_add_slave */
-static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev)
+static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev,
+ bool needs_sync)
{
netdev_upper_dev_unlink(port_dev, dev);
port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE;
+ /* Make sure that concurrent RCU readers that identified the device
+ * as a VRF port see a VRF master or no master at all.
+ */
+ if (needs_sync)
+ synchronize_net();
cycle_netdev(port_dev, NULL);
@@ -1065,7 +1072,7 @@ static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev)
static int vrf_del_slave(struct net_device *dev, struct net_device *port_dev)
{
- return do_vrf_del_slave(dev, port_dev);
+ return do_vrf_del_slave(dev, port_dev, true);
}
static void vrf_dev_uninit(struct net_device *dev)
@@ -1619,7 +1626,7 @@ static void vrf_dellink(struct net_device *dev, struct list_head *head)
struct list_head *iter;
netdev_for_each_lower_dev(dev, port_dev, iter)
- vrf_del_slave(dev, port_dev);
+ do_vrf_del_slave(dev, port_dev, false);
vrf_map_unregister_dev(dev);
@@ -1751,7 +1758,7 @@ static int vrf_device_event(struct notifier_block *unused,
goto out;
vrf_dev = netdev_master_upper_dev_get(dev);
- vrf_del_slave(vrf_dev, dev);
+ do_vrf_del_slave(vrf_dev, dev, false);
}
out:
return NOTIFY_DONE;
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index e887984..b5b1253 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -2531,7 +2531,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
goto out_unlock;
}
- tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
+ tos = ip_tunnel_ecn_encap(tos, ip_hdr(skb), skb);
ttl = ttl ? : ip4_dst_hoplimit(&rt->dst);
err = vxlan_build_skb(skb, ndst, sizeof(struct iphdr),
vni, md, flags, udp_sum);
@@ -2605,7 +2605,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
goto out_unlock;
}
- tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
+ tos = ip_tunnel_ecn_encap(tos, ip_hdr(skb), skb);
ttl = ttl ? : ip6_dst_hoplimit(ndst);
skb_scrub_packet(skb, xnet);
err = vxlan_build_skb(skb, ndst, sizeof(struct ipv6hdr),
diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c
index 2042369..3e76f4e 100644
--- a/drivers/net/vxlan/vxlan_vnifilter.c
+++ b/drivers/net/vxlan/vxlan_vnifilter.c
@@ -661,7 +661,7 @@ static int vxlan_vni_update(struct vxlan_dev *vxlan,
if (ret)
return ret;
- if (changed)
+ if (*changed)
vxlan_vnifilter_notify(vxlan, vninode, RTM_NEWTUNNEL);
return 0;
@@ -759,8 +759,7 @@ static int vxlan_vni_add(struct vxlan_dev *vxlan,
err = vxlan_vni_update_group(vxlan, vninode, group, true, &changed,
extack);
- if (changed)
- vxlan_vnifilter_notify(vxlan, vninode, RTM_NEWTUNNEL);
+ vxlan_vnifilter_notify(vxlan, vninode, RTM_NEWTUNNEL);
return err;
}
diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c
index 3bd57527..809f21f 100644
--- a/drivers/net/wan/fsl_ucc_hdlc.c
+++ b/drivers/net/wan/fsl_ucc_hdlc.c
@@ -740,6 +740,8 @@ static int uhdlc_open(struct net_device *dev)
static void uhdlc_memclean(struct ucc_hdlc_private *priv)
{
+ int i;
+
qe_muram_free(ioread16be(&priv->ucc_pram->riptr));
qe_muram_free(ioread16be(&priv->ucc_pram->tiptr));
@@ -770,14 +772,14 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv)
kfree(priv->rx_skbuff);
priv->rx_skbuff = NULL;
+ for (i = 0; i < TX_BD_RING_LEN; i++) {
+ dev_kfree_skb(priv->tx_skbuff[i]);
+ priv->tx_skbuff[i] = NULL;
+ }
+
kfree(priv->tx_skbuff);
priv->tx_skbuff = NULL;
- if (priv->uf_regs) {
- iounmap(priv->uf_regs);
- priv->uf_regs = NULL;
- }
-
if (priv->uccf) {
ucc_fast_free(priv->uccf);
priv->uccf = NULL;
@@ -1255,12 +1257,12 @@ static void ucc_hdlc_remove(struct platform_device *pdev)
uhdlc_memclean(priv);
- if (priv->utdm->si_regs) {
+ if (priv->utdm && priv->utdm->si_regs) {
iounmap(priv->utdm->si_regs);
priv->utdm->si_regs = NULL;
}
- if (priv->utdm->siram) {
+ if (priv->utdm && priv->utdm->siram) {
iounmap(priv->utdm->siram);
priv->utdm->siram = NULL;
}
diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c
index 26e09c3..67d0147 100644
--- a/drivers/net/wireguard/send.c
+++ b/drivers/net/wireguard/send.c
@@ -177,16 +177,6 @@ static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
trailer_len = padding_len + noise_encrypted_len(0);
plaintext_len = skb->len + padding_len;
- /* Expand data section to have room for padding and auth tag. */
- num_frags = skb_cow_data(skb, trailer_len, &trailer);
- if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
- return false;
-
- /* Set the padding to zeros, and make sure it and the auth tag are part
- * of the skb.
- */
- memset(skb_tail_pointer(trailer), 0, padding_len);
-
/* Expand head section to have room for our header and the network
* stack's headers.
*/
@@ -198,6 +188,16 @@ static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
skb_checksum_help(skb)))
return false;
+ /* Expand data section to have room for padding and auth tag. */
+ num_frags = skb_cow_data(skb, trailer_len, &trailer);
+ if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
+ return false;
+
+ /* Set the padding to zeros, and make sure it and the auth tag are part
+ * of the skb.
+ */
+ memset(skb_tail_pointer(trailer), 0, padding_len);
+
/* Only after checksumming can we safely add on the padding at the end
* and the header.
*/
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index 876aed7..efb9f02 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -46,6 +46,7 @@
depends on ARCH_QCOM || COMPILE_TEST
depends on QCOM_SMEM
depends on QCOM_RPROC_COMMON || QCOM_RPROC_COMMON=n
+ select POWER_SEQUENCING
select QCOM_SCM
select QCOM_QMI_HELPERS
help
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index 0bdb38e..e57588c19 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -3,7 +3,6 @@
* Copyright (c) 2005-2011 Atheros Communications Inc.
* Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
*/
@@ -1947,15 +1946,15 @@ int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id)
ret = -ESHUTDOWN;
ath10k_dbg(ar, ATH10K_DBG_WMI,
"drop wmi command %d, hardware is wedged\n", cmd_id);
+ } else {
+ /* try to send pending beacons first. they take priority */
+ ath10k_wmi_tx_beacons_nowait(ar);
+
+ ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id);
+
+ if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
+ ret = -ESHUTDOWN;
}
- /* try to send pending beacons first. they take priority */
- ath10k_wmi_tx_beacons_nowait(ar);
-
- ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id);
-
- if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
- ret = -ESHUTDOWN;
-
(ret != -EAGAIN);
}), 3 * HZ);
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index fe79109..2a413e3 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -1761,6 +1761,7 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
int buf_first_hdr_len, buf_first_len;
struct hal_rx_desc *ldesc;
int space_extra, rem_len, buf_len;
+ bool is_continuation;
u32 hal_rx_desc_sz = ar->ab->hw_params.hal_desc_sz;
/* As the msdu is spread across multiple rx buffers,
@@ -1810,7 +1811,8 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
rem_len = msdu_len - buf_first_len;
while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) {
rxcb = ATH11K_SKB_RXCB(skb);
- if (rxcb->is_continuation)
+ is_continuation = rxcb->is_continuation;
+ if (is_continuation)
buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz;
else
buf_len = rem_len;
@@ -1828,7 +1830,7 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
dev_kfree_skb_any(skb);
rem_len -= buf_len;
- if (!rxcb->is_continuation)
+ if (!is_continuation)
break;
}
@@ -2214,8 +2216,7 @@ ath11k_dp_rx_h_find_peer(struct ath11k_base *ab, struct sk_buff *msdu)
lockdep_assert_held(&ab->base_lock);
- if (rxcb->peer_id)
- peer = ath11k_peer_find_by_id(ab, rxcb->peer_id);
+ peer = ath11k_peer_find_by_id(ab, rxcb->peer_id);
if (peer)
return peer;
diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c
index e821e5a..98bd9e3 100644
--- a/drivers/net/wireless/ath/ath11k/hal.c
+++ b/drivers/net/wireless/ath/ath11k/hal.c
@@ -1387,14 +1387,22 @@ EXPORT_SYMBOL(ath11k_hal_srng_deinit);
void ath11k_hal_srng_clear(struct ath11k_base *ab)
{
- /* No need to memset rdp and wrp memory since each individual
- * segment would get cleared in ath11k_hal_srng_src_hw_init()
- * and ath11k_hal_srng_dst_hw_init().
+ /*
+ * Preserve the shared pointer buffers, but clear the previous
+ * firmware instance's hp/tp state before handing them back to FW.
+ * LMAC rings reuse this shared memory without going through the
+ * normal SRNG hw-init path that zeros non-LMAC ring pointers.
*/
memset(ab->hal.srng_list, 0,
sizeof(ab->hal.srng_list));
memset(ab->hal.shadow_reg_addr, 0,
sizeof(ab->hal.shadow_reg_addr));
+ if (ab->hal.rdp.vaddr)
+ memset(ab->hal.rdp.vaddr, 0,
+ sizeof(*ab->hal.rdp.vaddr) * HAL_SRNG_RING_ID_MAX);
+ if (ab->hal.wrp.vaddr)
+ memset(ab->hal.wrp.vaddr, 0,
+ sizeof(*ab->hal.wrp.vaddr) * HAL_SRNG_NUM_LMAC_RINGS);
ab->hal.avail_blk_resource = 0;
ab->hal.current_blk_index = 0;
ab->hal.num_shadow_reg_configured = 0;
diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.c b/drivers/net/wireless/ath/ath11k/hal_rx.c
index 753bd93..51e0840 100644
--- a/drivers/net/wireless/ath/ath11k/hal_rx.c
+++ b/drivers/net/wireless/ath/ath11k/hal_rx.c
@@ -1467,11 +1467,8 @@ ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
case HAL_RX_MPDU_START: {
struct hal_rx_mpdu_info *mpdu_info =
(struct hal_rx_mpdu_info *)tlv_data;
- u16 peer_id;
- peer_id = ath11k_hal_rx_mpduinfo_get_peerid(ab, mpdu_info);
- if (peer_id)
- ppdu_info->peer_id = peer_id;
+ ppdu_info->peer_id = ath11k_hal_rx_mpduinfo_get_peerid(ab, mpdu_info);
break;
}
case HAL_RXPCU_PPDU_END_INFO: {
diff --git a/drivers/net/wireless/ath/ath11k/testmode.c b/drivers/net/wireless/ath/ath11k/testmode.c
index a9751ea..c72eed3 100644
--- a/drivers/net/wireless/ath/ath11k/testmode.c
+++ b/drivers/net/wireless/ath/ath11k/testmode.c
@@ -457,6 +457,7 @@ static int ath11k_tm_cmd_wmi_ftm(struct ath11k *ar, struct nlattr *tb[])
ret = ath11k_wmi_cmd_send(wmi, skb, cmd_id);
if (ret) {
ath11k_warn(ar->ab, "failed to send wmi ftm command: %d\n", ret);
+ dev_kfree_skb(skb);
goto out;
}
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index 40747fb..dca6e01 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -9299,7 +9299,7 @@ int ath11k_wmi_hw_data_filter_cmd(struct ath11k *ar, u32 vdev_id,
{
struct wmi_hw_data_filter_cmd *cmd;
struct sk_buff *skb;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9324,7 +9324,13 @@ int ath11k_wmi_hw_data_filter_cmd(struct ath11k *ar, u32 vdev_id,
"hw data filter enable %d filter_bitmap 0x%x\n",
enable, filter_bitmap);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_HW_DATA_FILTER_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
@@ -9332,6 +9338,7 @@ int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
struct wmi_wow_host_wakeup_ind *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9345,14 +9352,20 @@ int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow host wakeup ind\n");
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_wow_enable(struct ath11k *ar)
{
struct wmi_wow_enable_cmd *cmd;
struct sk_buff *skb;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9367,7 +9380,13 @@ int ath11k_wmi_wow_enable(struct ath11k *ar)
cmd->pause_iface_config = WOW_IFACE_PAUSE_ENABLED;
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow enable\n");
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
@@ -9376,7 +9395,7 @@ int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
struct sk_buff *skb;
struct wmi_scan_prob_req_oui_cmd *cmd;
u32 prob_req_oui;
- int len;
+ int ret, len;
prob_req_oui = (((u32)mac_addr[0]) << 16) |
(((u32)mac_addr[1]) << 8) | mac_addr[2];
@@ -9395,7 +9414,13 @@ int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "scan prob req oui %d\n",
prob_req_oui);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SCAN_PROB_REQ_OUI_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SCAN_PROB_REQ_OUI_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_SCAN_PROB_REQ_OUI_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
@@ -9405,6 +9430,7 @@ int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
struct wmi_wow_add_del_event_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9422,7 +9448,13 @@ int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow add wakeup event %s enable %d vdev_id %d\n",
wow_wakeup_event(event), enable, vdev_id);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
@@ -9435,6 +9467,7 @@ int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
struct sk_buff *skb;
u8 *ptr;
size_t len;
+ int ret;
len = sizeof(*cmd) +
sizeof(*tlv) + /* array struct */
@@ -9527,7 +9560,13 @@ int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow add pattern vdev_id %d pattern_id %d pattern_offset %d\n",
vdev_id, pattern_id, pattern_offset);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_WOW_ADD_WAKE_PATTERN_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
@@ -9535,6 +9574,7 @@ int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
struct wmi_wow_del_pattern_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9553,7 +9593,13 @@ int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow del pattern vdev_id %d pattern_id %d\n",
vdev_id, pattern_id);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_WOW_DEL_WAKE_PATTERN_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
static struct sk_buff *
@@ -9697,6 +9743,7 @@ int ath11k_wmi_wow_config_pno(struct ath11k *ar, u32 vdev_id,
struct wmi_pno_scan_req *pno_scan)
{
struct sk_buff *skb;
+ int ret;
if (pno_scan->enable)
skb = ath11k_wmi_op_gen_config_pno_start(ar, vdev_id, pno_scan);
@@ -9706,7 +9753,13 @@ int ath11k_wmi_wow_config_pno(struct ath11k *ar, u32 vdev_id,
if (IS_ERR_OR_NULL(skb))
return -ENOMEM;
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
static void ath11k_wmi_fill_ns_offload(struct ath11k *ar,
@@ -9824,6 +9877,7 @@ int ath11k_wmi_arp_ns_offload(struct ath11k *ar,
u8 *buf_ptr;
size_t len;
u8 ns_cnt, ns_ext_tuples = 0;
+ int ret;
offload = &arvif->arp_ns_offload;
ns_cnt = offload->ipv6_count;
@@ -9862,7 +9916,13 @@ int ath11k_wmi_arp_ns_offload(struct ath11k *ar,
if (ns_ext_tuples)
ath11k_wmi_fill_ns_offload(ar, offload, &buf_ptr, enable, 1);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_SET_ARP_NS_OFFLOAD_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
@@ -9870,7 +9930,7 @@ int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
{
struct wmi_gtk_rekey_offload_cmd *cmd;
struct ath11k_rekey_data *rekey_data = &arvif->rekey_data;
- int len;
+ int ret, len;
struct sk_buff *skb;
__le64 replay_ctr;
@@ -9904,14 +9964,20 @@ int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "offload gtk rekey vdev: %d %d\n",
arvif->vdev_id, enable);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID offload\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_gtk_rekey_getinfo(struct ath11k *ar,
struct ath11k_vif *arvif)
{
struct wmi_gtk_rekey_offload_cmd *cmd;
- int len;
+ int ret, len;
struct sk_buff *skb;
len = sizeof(*cmd);
@@ -9928,7 +9994,13 @@ int ath11k_wmi_gtk_rekey_getinfo(struct ath11k *ar,
ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "get gtk rekey vdev_id: %d\n",
arvif->vdev_id);
- return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID getinfo\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_val)
@@ -9938,6 +10010,7 @@ int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_va
struct sk_buff *skb;
u8 *buf_ptr;
u32 len, sar_len_aligned, rsvd_len_aligned;
+ int ret;
sar_len_aligned = roundup(BIOS_SAR_TABLE_LEN, sizeof(u32));
rsvd_len_aligned = roundup(BIOS_SAR_RSVD1_LEN, sizeof(u32));
@@ -9968,7 +10041,13 @@ int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_va
tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) |
FIELD_PREP(WMI_TLV_LEN, rsvd_len_aligned);
- return ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID);
+ ret = ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
@@ -9979,6 +10058,7 @@ int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
struct sk_buff *skb;
u8 *buf_ptr;
u32 len, rsvd_len_aligned;
+ int ret;
rsvd_len_aligned = roundup(BIOS_SAR_RSVD2_LEN, sizeof(u32));
len = sizeof(*cmd) + TLV_HDR_SIZE + rsvd_len_aligned;
@@ -9998,7 +10078,13 @@ int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) |
FIELD_PREP(WMI_TLV_LEN, rsvd_len_aligned);
- return ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID);
+ ret = ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath11k_wmi_sta_keepalive(struct ath11k *ar,
@@ -10009,6 +10095,7 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar,
struct wmi_sta_keepalive_arp_resp *arp;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd) + sizeof(*arp);
skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, len);
@@ -10040,7 +10127,13 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar,
"sta keepalive vdev %d enabled %d method %d interval %d\n",
arg->vdev_id, arg->enabled, arg->method, arg->interval);
- return ath11k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+ ret = ath11k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+ if (ret) {
+ ath11k_warn(ar->ab, "failed to send WMI_STA_KEEPALIVE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
bool ath11k_wmi_supports_6ghz_cc_ext(struct ath11k *ar)
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index 2519e24..980a12fb 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -1838,10 +1838,22 @@ static struct ath12k_hw_group *ath12k_core_hw_group_alloc(struct ath12k_base *ab
return ag;
}
+static void ath12k_core_free_wsi_info(struct ath12k_hw_group *ag)
+{
+ int i;
+
+ for (i = 0; i < ag->num_devices; i++) {
+ of_node_put(ag->wsi_node[i]);
+ ag->wsi_node[i] = NULL;
+ }
+ ag->num_devices = 0;
+}
+
static void ath12k_core_hw_group_free(struct ath12k_hw_group *ag)
{
mutex_lock(&ath12k_hw_group_mutex);
+ ath12k_core_free_wsi_info(ag);
list_del(&ag->list);
kfree(ag);
@@ -1867,52 +1879,59 @@ static struct ath12k_hw_group *ath12k_core_hw_group_find_by_dt(struct ath12k_bas
static int ath12k_core_get_wsi_info(struct ath12k_hw_group *ag,
struct ath12k_base *ab)
{
- struct device_node *wsi_dev = ab->dev->of_node, *next_wsi_dev;
- struct device_node *tx_endpoint, *next_rx_endpoint;
- int device_count = 0;
+ struct device_node *next_wsi_dev;
+ int device_count = 0, ret = 0;
+ struct device_node *wsi_dev;
- next_wsi_dev = wsi_dev;
-
- if (!next_wsi_dev)
+ wsi_dev = of_node_get(ab->dev->of_node);
+ if (!wsi_dev)
return -ENODEV;
do {
- ag->wsi_node[device_count] = next_wsi_dev;
+ if (device_count >= ATH12K_MAX_DEVICES) {
+ ath12k_warn(ab, "device count in DT %d is more than limit %d\n",
+ device_count, ATH12K_MAX_DEVICES);
+ ret = -EINVAL;
+ break;
+ }
- tx_endpoint = of_graph_get_endpoint_by_regs(next_wsi_dev, 0, -1);
+ ag->wsi_node[device_count++] = of_node_get(wsi_dev);
+
+ struct device_node *tx_endpoint __free(device_node) =
+ of_graph_get_endpoint_by_regs(wsi_dev, 0, -1);
if (!tx_endpoint) {
- of_node_put(next_wsi_dev);
- return -ENODEV;
+ ret = -ENODEV;
+ break;
}
- next_rx_endpoint = of_graph_get_remote_endpoint(tx_endpoint);
+ struct device_node *next_rx_endpoint __free(device_node) =
+ of_graph_get_remote_endpoint(tx_endpoint);
if (!next_rx_endpoint) {
- of_node_put(next_wsi_dev);
- of_node_put(tx_endpoint);
- return -ENODEV;
+ ret = -ENODEV;
+ break;
}
- of_node_put(tx_endpoint);
- of_node_put(next_wsi_dev);
-
next_wsi_dev = of_graph_get_port_parent(next_rx_endpoint);
if (!next_wsi_dev) {
- of_node_put(next_rx_endpoint);
- return -ENODEV;
+ ret = -ENODEV;
+ break;
}
- of_node_put(next_rx_endpoint);
+ of_node_put(wsi_dev);
+ wsi_dev = next_wsi_dev;
+ } while (ab->dev->of_node != wsi_dev);
- device_count++;
- if (device_count > ATH12K_MAX_DEVICES) {
- ath12k_warn(ab, "device count in DT %d is more than limit %d\n",
- device_count, ATH12K_MAX_DEVICES);
- of_node_put(next_wsi_dev);
- return -EINVAL;
+ if (ret) {
+ while (--device_count >= 0) {
+ of_node_put(ag->wsi_node[device_count]);
+ ag->wsi_node[device_count] = NULL;
}
- } while (wsi_dev != next_wsi_dev);
- of_node_put(next_wsi_dev);
+ of_node_put(wsi_dev);
+ return ret;
+ }
+
+ of_node_put(wsi_dev);
ag->num_devices = device_count;
return 0;
@@ -1983,9 +2002,9 @@ static struct ath12k_hw_group *ath12k_core_hw_group_assign(struct ath12k_base *a
ath12k_core_get_wsi_index(ag, ab)) {
ath12k_dbg(ab, ATH12K_DBG_BOOT,
"unable to get wsi info from dt, grouping single device");
+ ath12k_core_free_wsi_info(ag);
ag->id = ATH12K_INVALID_GROUP_ID;
ag->num_devices = 1;
- memset(ag->wsi_node, 0, sizeof(ag->wsi_node));
wsi->index = 0;
}
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index 250459f..b108ccd 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -565,6 +565,9 @@ static int ath12k_dp_prepare_reo_update_elem(struct ath12k_dp *dp,
lockdep_assert_held(&dp->dp_lock);
+ if (!peer->primary_link)
+ return 0;
+
elem = kzalloc_obj(*elem, GFP_ATOMIC);
if (!elem)
return -ENOMEM;
@@ -1337,7 +1340,7 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
bool is_mcbc = rxcb->is_mcbc;
bool is_eapol = rxcb->is_eapol;
- peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rx_info->peer_id);
+ peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rxcb->peer_id);
pubsta = peer ? peer->sta : NULL;
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index fbdfe64..2cff948 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -788,7 +788,7 @@ struct ath12k_link_vif *ath12k_mac_get_arvif(struct ath12k *ar, u32 vdev_id)
/* To use the arvif returned, caller must have held rcu read lock.
*/
- WARN_ON(!rcu_read_lock_any_held());
+ lockdep_assert_in_rcu_read_lock();
arvif_iter.vdev_id = vdev_id;
arvif_iter.ar = ar;
@@ -3446,7 +3446,9 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar,
arg->peer_eht_mcs_count++;
fallthrough;
default:
- if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+ if ((vif->type == NL80211_IFTYPE_AP ||
+ vif->type == NL80211_IFTYPE_MESH_POINT) &&
+ !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
bw_20 = &eht_cap->eht_mcs_nss_supp.only_20mhz;
@@ -3475,7 +3477,9 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar,
arg->punct_bitmap = ~arvif->punct_bitmap;
arg->eht_disable_mcs15 = link_conf->eht_disable_mcs15;
- if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+ if ((vif->type == NL80211_IFTYPE_AP ||
+ vif->type == NL80211_IFTYPE_MESH_POINT) &&
+ !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
if (bw_20->rx_tx_mcs13_max_nss)
max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs13_max_nss,
diff --git a/drivers/net/wireless/ath/ath12k/p2p.c b/drivers/net/wireless/ath/ath12k/p2p.c
index 5958974..19ebcd1 100644
--- a/drivers/net/wireless/ath/ath12k/p2p.c
+++ b/drivers/net/wireless/ath/ath12k/p2p.c
@@ -123,7 +123,7 @@ static void ath12k_p2p_noa_update_vdev_iter(void *data, u8 *mac,
struct ath12k_p2p_noa_arg *arg = data;
struct ath12k_link_vif *arvif;
- WARN_ON(!rcu_read_lock_any_held());
+ lockdep_assert_in_rcu_read_lock();
arvif = &ahvif->deflink;
if (!arvif->is_created || arvif->ar != arg->ar || arvif->vdev_id != arg->vdev_id)
return;
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index 65a05a95..b5e904a 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -9778,7 +9778,7 @@ static void
ath12k_wmi_rssi_dbm_conversion_params_info_event(struct ath12k_base *ab,
struct sk_buff *skb)
{
- struct ath12k_wmi_rssi_dbm_conv_info_arg rssi_info;
+ struct ath12k_wmi_rssi_dbm_conv_info_arg rssi_info = {};
struct ath12k *ar;
s32 noise_floor;
u32 pdev_id;
@@ -10251,7 +10251,7 @@ int ath12k_wmi_hw_data_filter_cmd(struct ath12k *ar, struct wmi_hw_data_filter_a
{
struct wmi_hw_data_filter_cmd *cmd;
struct sk_buff *skb;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10275,7 +10275,13 @@ int ath12k_wmi_hw_data_filter_cmd(struct ath12k *ar, struct wmi_hw_data_filter_a
"wmi hw data filter enable %d filter_bitmap 0x%x\n",
arg->enable, arg->hw_filter_bitmap);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_HW_DATA_FILTER_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_wow_host_wakeup_ind(struct ath12k *ar)
@@ -10283,6 +10289,7 @@ int ath12k_wmi_wow_host_wakeup_ind(struct ath12k *ar)
struct wmi_wow_host_wakeup_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10295,14 +10302,20 @@ int ath12k_wmi_wow_host_wakeup_ind(struct ath12k *ar)
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "wmi tlv wow host wakeup ind\n");
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_wow_enable(struct ath12k *ar)
{
struct wmi_wow_enable_cmd *cmd;
struct sk_buff *skb;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10317,7 +10330,13 @@ int ath12k_wmi_wow_enable(struct ath12k *ar)
cmd->pause_iface_config = cpu_to_le32(WOW_IFACE_PAUSE_ENABLED);
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "wmi tlv wow enable\n");
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_wow_add_wakeup_event(struct ath12k *ar, u32 vdev_id,
@@ -10327,6 +10346,7 @@ int ath12k_wmi_wow_add_wakeup_event(struct ath12k *ar, u32 vdev_id,
struct wmi_wow_add_del_event_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10343,7 +10363,13 @@ int ath12k_wmi_wow_add_wakeup_event(struct ath12k *ar, u32 vdev_id,
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "wmi tlv wow add wakeup event %s enable %d vdev_id %d\n",
wow_wakeup_event(event), enable, vdev_id);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_wow_add_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id,
@@ -10356,6 +10382,7 @@ int ath12k_wmi_wow_add_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id,
struct sk_buff *skb;
void *ptr;
size_t len;
+ int ret;
len = sizeof(*cmd) +
sizeof(*tlv) + /* array struct */
@@ -10435,7 +10462,13 @@ int ath12k_wmi_wow_add_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id,
ath12k_dbg_dump(ar->ab, ATH12K_DBG_WMI, NULL, "wow bitmask: ",
bitmap->bitmaskbuf, pattern_len);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_WOW_ADD_WAKE_PATTERN_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_wow_del_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id)
@@ -10443,6 +10476,7 @@ int ath12k_wmi_wow_del_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id)
struct wmi_wow_del_pattern_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10459,7 +10493,13 @@ int ath12k_wmi_wow_del_pattern(struct ath12k *ar, u32 vdev_id, u32 pattern_id)
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "wmi tlv wow del pattern vdev_id %d pattern_id %d\n",
vdev_id, pattern_id);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_WOW_DEL_WAKE_PATTERN_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
static struct sk_buff *
@@ -10595,6 +10635,7 @@ int ath12k_wmi_wow_config_pno(struct ath12k *ar, u32 vdev_id,
struct wmi_pno_scan_req_arg *pno_scan)
{
struct sk_buff *skb;
+ int ret;
if (pno_scan->enable)
skb = ath12k_wmi_op_gen_config_pno_start(ar, vdev_id, pno_scan);
@@ -10604,7 +10645,13 @@ int ath12k_wmi_wow_config_pno(struct ath12k *ar, u32 vdev_id,
if (IS_ERR_OR_NULL(skb))
return -ENOMEM;
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
static void ath12k_wmi_fill_ns_offload(struct ath12k *ar,
@@ -10717,6 +10764,7 @@ int ath12k_wmi_arp_ns_offload(struct ath12k *ar,
void *buf_ptr;
size_t len;
u8 ns_cnt, ns_ext_tuples = 0;
+ int ret;
ns_cnt = offload->ipv6_count;
@@ -10752,7 +10800,13 @@ int ath12k_wmi_arp_ns_offload(struct ath12k *ar,
if (ns_ext_tuples)
ath12k_wmi_fill_ns_offload(ar, offload, &buf_ptr, enable, 1);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_SET_ARP_NS_OFFLOAD_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_gtk_rekey_offload(struct ath12k *ar,
@@ -10762,7 +10816,7 @@ int ath12k_wmi_gtk_rekey_offload(struct ath12k *ar,
struct wmi_gtk_rekey_offload_cmd *cmd;
struct sk_buff *skb;
__le64 replay_ctr;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10789,7 +10843,13 @@ int ath12k_wmi_gtk_rekey_offload(struct ath12k *ar,
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "offload gtk rekey vdev: %d %d\n",
arvif->vdev_id, enable);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID offload\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_gtk_rekey_getinfo(struct ath12k *ar,
@@ -10797,7 +10857,7 @@ int ath12k_wmi_gtk_rekey_getinfo(struct ath12k *ar,
{
struct wmi_gtk_rekey_offload_cmd *cmd;
struct sk_buff *skb;
- int len;
+ int ret, len;
len = sizeof(*cmd);
skb = ath12k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -10811,7 +10871,13 @@ int ath12k_wmi_gtk_rekey_getinfo(struct ath12k *ar,
ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "get gtk rekey vdev_id: %d\n",
arvif->vdev_id);
- return ath12k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ ret = ath12k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID getinfo\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_sta_keepalive(struct ath12k *ar,
@@ -10822,6 +10888,7 @@ int ath12k_wmi_sta_keepalive(struct ath12k *ar,
struct wmi_sta_keepalive_cmd *cmd;
struct sk_buff *skb;
size_t len;
+ int ret;
len = sizeof(*cmd) + sizeof(*arp);
skb = ath12k_wmi_alloc_skb(wmi->wmi_ab, len);
@@ -10849,7 +10916,13 @@ int ath12k_wmi_sta_keepalive(struct ath12k *ar,
"wmi sta keepalive vdev %d enabled %d method %d interval %d\n",
arg->vdev_id, arg->enabled, arg->method, arg->interval);
- return ath12k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+ ret = ath12k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+ if (ret) {
+ ath12k_warn(ar->ab, "failed to send WMI_STA_KEEPALIVE_CMDID\n");
+ dev_kfree_skb(skb);
+ }
+
+ return ret;
}
int ath12k_wmi_mlo_setup(struct ath12k *ar, struct wmi_mlo_setup_arg *mlo_params)
diff --git a/drivers/net/wireless/ath/ath5k/base.c b/drivers/net/wireless/ath/ath5k/base.c
index 05c9c07..6ca31d4 100644
--- a/drivers/net/wireless/ath/ath5k/base.c
+++ b/drivers/net/wireless/ath/ath5k/base.c
@@ -1738,7 +1738,8 @@ ath5k_tx_frame_completed(struct ath5k_hw *ah, struct sk_buff *skb,
}
info->status.rates[ts->ts_final_idx].count = ts->ts_final_retry;
- info->status.rates[ts->ts_final_idx + 1].idx = -1;
+ if (ts->ts_final_idx + 1 < IEEE80211_TX_MAX_RATES)
+ info->status.rates[ts->ts_final_idx + 1].idx = -1;
if (unlikely(ts->ts_status)) {
ah->stats.ack_fail++;
diff --git a/drivers/net/wireless/broadcom/b43/xmit.c b/drivers/net/wireless/broadcom/b43/xmit.c
index 7651b1b..f0b0825 100644
--- a/drivers/net/wireless/broadcom/b43/xmit.c
+++ b/drivers/net/wireless/broadcom/b43/xmit.c
@@ -702,7 +702,8 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr)
* key index, but the ucode passed it slightly different.
*/
keyidx = b43_kidx_to_raw(dev, keyidx);
- B43_WARN_ON(keyidx >= ARRAY_SIZE(dev->key));
+ if (B43_WARN_ON(keyidx >= ARRAY_SIZE(dev->key)))
+ goto drop;
if (dev->key[keyidx].algorithm != B43_SEC_ALGO_NONE) {
wlhdr_len = ieee80211_hdrlen(fctl);
diff --git a/drivers/net/wireless/broadcom/b43legacy/xmit.c b/drivers/net/wireless/broadcom/b43legacy/xmit.c
index efd63f4..ee199d4e 100644
--- a/drivers/net/wireless/broadcom/b43legacy/xmit.c
+++ b/drivers/net/wireless/broadcom/b43legacy/xmit.c
@@ -476,7 +476,8 @@ void b43legacy_rx(struct b43legacy_wldev *dev,
* key index, but the ucode passed it slightly different.
*/
keyidx = b43legacy_kidx_to_raw(dev, keyidx);
- B43legacy_WARN_ON(keyidx >= dev->max_nr_keys);
+ if (B43legacy_WARN_ON(keyidx >= dev->max_nr_keys))
+ goto drop;
if (dev->key[keyidx].algorithm != B43legacy_SEC_ALGO_NONE) {
/* Remove PROTECTED flag to mark it as decrypted. */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 30f6fcb..8fb5957 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -2476,8 +2476,9 @@ static void brcmf_sdio_bus_stop(struct device *dev)
brcmf_dbg(TRACE, "Enter\n");
if (bus->watchdog_tsk) {
+ get_task_struct(bus->watchdog_tsk);
send_sig(SIGTERM, bus->watchdog_tsk, 1);
- kthread_stop(bus->watchdog_tsk);
+ kthread_stop_put(bus->watchdog_tsk);
bus->watchdog_tsk = NULL;
}
@@ -4567,8 +4568,9 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus)
if (bus) {
/* Stop watchdog task */
if (bus->watchdog_tsk) {
+ get_task_struct(bus->watchdog_tsk);
send_sig(SIGTERM, bus->watchdog_tsk, 1);
- kthread_stop(bus->watchdog_tsk);
+ kthread_stop_put(bus->watchdog_tsk);
bus->watchdog_tsk = NULL;
}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ap.c b/drivers/net/wireless/intel/iwlwifi/mld/ap.c
index 5c59acc..6598d933 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ap.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ap.c
@@ -9,7 +9,6 @@
#include "ap.h"
#include "hcmd.h"
#include "tx.h"
-#include "power.h"
#include "key.h"
#include "phy.h"
#include "iwl-utils.h"
@@ -273,9 +272,6 @@ int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw,
struct ieee80211_chanctx_conf *ctx;
int ret;
- if (vif->type == NL80211_IFTYPE_AP)
- iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link);
-
ret = iwl_mld_update_beacon_template(mld, vif, link);
if (ret)
return ret;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/constants.h b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
index e2a5eec..890abca 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/constants.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
@@ -1,11 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
/*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
*/
#ifndef __iwl_mld_constants_h__
#define __iwl_mld_constants_h__
-#define IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD 4
+#define IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD 6
#define IWL_MLD_MISSED_BEACONS_THRESHOLD 8
#define IWL_MLD_MISSED_BEACONS_THRESHOLD_LONG 19
#define IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS 5
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
index ef98efc..3a595a1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
@@ -1930,12 +1930,12 @@ int iwl_mld_wowlan_suspend(struct iwl_mld *mld, struct cfg80211_wowlan *wowlan)
if (WARN_ON(!wowlan))
return 1;
- IWL_DEBUG_WOWLAN(mld, "Starting the wowlan suspend flow\n");
-
bss_vif = iwl_mld_get_bss_vif(mld);
- if (WARN_ON(!bss_vif))
+ if (!bss_vif)
return 1;
+ IWL_DEBUG_WOWLAN(mld, "Starting the wowlan suspend flow\n");
+
if (!bss_vif->cfg.assoc) {
int ret;
/* If we're not associated, this must be netdetect */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/link.c b/drivers/net/wireless/intel/iwlwifi/mld/link.c
index b66e84d..be2cdf4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/link.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/link.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
*/
#include "constants.h"
@@ -504,7 +504,6 @@ void iwl_mld_remove_link(struct iwl_mld *mld,
struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_conf->vif);
struct iwl_mld_link *link = iwl_mld_link_from_mac80211(bss_conf);
bool is_deflink = link == &mld_vif->deflink;
- u8 fw_id = link->fw_id;
if (WARN_ON(!link || link->active))
return;
@@ -512,15 +511,15 @@ void iwl_mld_remove_link(struct iwl_mld *mld,
iwl_mld_rm_link_from_fw(mld, bss_conf);
/* Continue cleanup on failure */
- if (!is_deflink)
- kfree_rcu(link, rcu_head);
-
RCU_INIT_POINTER(mld_vif->link[bss_conf->link_id], NULL);
- if (WARN_ON(fw_id >= mld->fw->ucode_capa.num_links))
+ if (WARN_ON(link->fw_id >= mld->fw->ucode_capa.num_links))
return;
- RCU_INIT_POINTER(mld->fw_id_to_bss_conf[fw_id], NULL);
+ RCU_INIT_POINTER(mld->fw_id_to_bss_conf[link->fw_id], NULL);
+
+ if (!is_deflink)
+ kfree_rcu(link, rcu_head);
}
void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index da6fd74..3c8dadd 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -1150,6 +1150,13 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw,
if (iwl_mld_can_activate_link(mld, vif, link)) {
iwl_mld_tlc_update_phy(mld, vif, link);
+ /* FW requires AP_TX_POWER_CONSTRAINTS_CMD before link
+ * activation for AP and after link activation for STA,
+ * for an unknown reason.
+ */
+ if (vif->type == NL80211_IFTYPE_AP)
+ iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link);
+
ret = iwl_mld_activate_link(mld, link);
if (ret)
goto err;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c
index 49b0d9f..266fe16 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c
@@ -366,7 +366,7 @@ iwl_mld_send_ap_tx_power_constraint_cmd(struct iwl_mld *mld,
lockdep_assert_wiphy(mld->wiphy);
- if (!mld_link->active)
+ if (!mld_link->active && vif->type != NL80211_IFTYPE_AP)
return;
if (link->chanreq.oper.chan->band != NL80211_BAND_6GHZ)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
index c65f4b5..f829156 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
@@ -250,7 +250,8 @@ iwl_mld_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
/* System (wall) time */
ktime_t sys_time;
- memset(xtstamp, 0, sizeof(struct system_device_crosststamp));
+ if (xtstamp->clock_id != CLOCK_REALTIME)
+ return -ENOTSUPP;
ret = iwl_mld_get_crosstimestamp_fw(mld, &gp2, &sys_time);
if (ret) {
@@ -270,7 +271,7 @@ iwl_mld_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
/* System monotonic raw time is not used */
xtstamp->device = ns_to_ktime(gp2_ns);
- xtstamp->sys_realtime = sys_time;
+ xtstamp->sys_systime = sys_time;
return ret;
}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
index 546d09a..0bcb1ae6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
@@ -834,7 +834,7 @@ static int iwl_mld_tx_tso_segment(struct iwl_mld *mld, struct sk_buff *skb,
return -EINVAL;
max_tid_amsdu_len = sta->cur->max_tid_amsdu_len[tid];
- if (!max_tid_amsdu_len)
+ if (!max_tid_amsdu_len || max_tid_amsdu_len == 1)
return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
/* Sub frame header + SNAP + IP header + TCP header + MSS */
@@ -846,6 +846,9 @@ static int iwl_mld_tx_tso_segment(struct iwl_mld *mld, struct sk_buff *skb,
*/
num_subframes = (max_tid_amsdu_len + pad) / (subf_len + pad);
+ if (WARN_ON_ONCE(!num_subframes))
+ return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
+
if (sta->max_amsdu_subframes &&
num_subframes > sta->max_amsdu_subframes)
num_subframes = sta->max_amsdu_subframes;
@@ -971,6 +974,16 @@ void iwl_mld_tx_from_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
u8 zero_addr[ETH_ALEN] = {};
/*
+ * Don't transmit during firmware restart. The firmware is dead,
+ * so iwl_trans_tx() would return -EIO for each frame. Avoid the
+ * overhead of dequeuing from mac80211 only to immediately free
+ * the skbs, and the potential memory pressure from rapid skb
+ * allocation churn during high-throughput restart scenarios.
+ */
+ if (unlikely(mld->fw_status.in_hw_restart))
+ return;
+
+ /*
* No need for threads to be pending here, they can leave the first
* taker all the work.
*
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index f05df3a..6e507d6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
* Copyright (C) 2013-2015 Intel Mobile Communications GmbH
* Copyright (C) 2016-2017 Intel Deutschland GmbH
*/
@@ -459,9 +459,14 @@ static void iwl_mvm_phy_filter_init(struct iwl_mvm *mvm,
static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
{
+ struct iwl_mcc_allowed_ap_type_cmd_v1 *cmd __free(kfree) = NULL;
int cmd_id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
MCC_ALLOWED_AP_TYPE_CMD);
- struct iwl_mcc_allowed_ap_type_cmd_v1 cmd = {};
+ struct iwl_host_cmd hcmd = {
+ .id = cmd_id,
+ .len[0] = sizeof(*cmd),
+ .dataflags[0] = IWL_HCMD_DFL_NOCOPY,
+ };
u8 cmd_ver;
int ret;
@@ -485,14 +490,25 @@ static void iwl_mvm_uats_init(struct iwl_mvm *mvm)
if (!mvm->fwrt.ap_type_cmd_valid)
return;
- BUILD_BUG_ON(sizeof(mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map) !=
- sizeof(cmd.mcc_to_ap_type_map));
+ /* Since we free the command immediately after iwl_mvm_send_cmd, we
+ * must send this command in SYNC mode.
+ */
+ lockdep_assert_held(&mvm->mutex);
- memcpy(cmd.mcc_to_ap_type_map,
+ cmd = kzalloc_obj(*cmd);
+ if (!cmd)
+ return;
+
+ BUILD_BUG_ON(sizeof(mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map) !=
+ sizeof(cmd->mcc_to_ap_type_map));
+
+ memcpy(cmd->mcc_to_ap_type_map,
mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map,
sizeof(mvm->fwrt.ap_type_cmd.mcc_to_ap_type_map));
- ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
+ hcmd.data[0] = cmd;
+
+ ret = iwl_mvm_send_cmd(mvm, &hcmd);
if (ret < 0)
IWL_ERR(mvm, "failed to send MCC_ALLOWED_AP_TYPE_CMD (%d)\n",
ret);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index c523c5e..8ffa72a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
* Copyright (C) 2013-2014 Intel Mobile Communications GmbH
* Copyright (C) 2015-2017 Intel Deutschland GmbH
*/
@@ -927,13 +927,18 @@ u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
u16 iwl_mvm_mac_ctxt_get_beacon_flags(const struct iwl_fw *fw, u8 rate_idx)
{
- u16 flags = iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
bool is_new_rate = iwl_fw_lookup_cmd_ver(fw, BEACON_TEMPLATE_CMD, 0) > 10;
+ u16 flags = 0;
if (rate_idx <= IWL_LAST_CCK_RATE)
flags |= is_new_rate ? IWL_MAC_BEACON_CCK
: IWL_MAC_BEACON_CCK_V1;
+ if (iwl_fw_lookup_cmd_ver(fw, TX_CMD, 0) > 8)
+ flags |= iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
+ else
+ flags |= iwl_fw_rate_idx_to_plcp(rate_idx);
+
return flags;
}
@@ -962,6 +967,7 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
{
struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
struct ieee80211_tx_info *info;
+ u32 rate_n_flags = 0;
u8 rate;
u32 tx_flags;
@@ -981,18 +987,21 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
IWL_UCODE_TLV_CAPA_BEACON_ANT_SELECTION)) {
iwl_mvm_toggle_tx_ant(mvm, &mvm->mgmt_last_antenna_idx);
- tx_params->rate_n_flags =
- cpu_to_le32(BIT(mvm->mgmt_last_antenna_idx) <<
- RATE_MCS_ANT_POS);
+ rate_n_flags |= BIT(mvm->mgmt_last_antenna_idx) <<
+ RATE_MCS_ANT_POS;
}
rate = iwl_mvm_mac_ctxt_get_beacon_rate(mvm, info, vif);
- tx_params->rate_n_flags |=
- cpu_to_le32(iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate));
- if (rate == IWL_FIRST_CCK_RATE)
- tx_params->rate_n_flags |= cpu_to_le32(RATE_MCS_CCK_MSK_V1);
+ if (rate < IWL_FIRST_OFDM_RATE)
+ rate_n_flags |= RATE_MCS_MOD_TYPE_CCK;
+ else
+ rate_n_flags |= RATE_MCS_MOD_TYPE_LEGACY_OFDM;
+ rate_n_flags |= iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate);
+
+ tx_params->rate_n_flags = iwl_mvm_v3_rate_to_fw(rate_n_flags,
+ mvm->fw_rates_ver);
}
int iwl_mvm_mac_ctxt_send_beacon_cmd(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index ae17747..384bed9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -1416,6 +1416,12 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
fw_has_capa(&mvm->fw->ucode_capa,
IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE);
+ /* Those firmware versions claim to support the fw_reset_handshake
+ * but they are buggy.
+ */
+ if (IWL_UCODE_MAJOR(mvm->fw->ucode_ver) <= 77)
+ trans->conf.fw_reset_handshake = false;
+
trans->conf.queue_alloc_cmd_ver =
iwl_fw_lookup_cmd_ver(mvm->fw,
WIDE_ID(DATA_PATH_GROUP,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
index f7b6201..bcd6f7c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
@@ -160,13 +160,14 @@ iwl_mvm_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
/* System (wall) time */
ktime_t sys_time;
- memset(xtstamp, 0, sizeof(struct system_device_crosststamp));
-
if (!mvm->ptp_data.ptp_clock) {
IWL_ERR(mvm, "No PHC clock registered\n");
return -ENODEV;
}
+ if (xtstamp->clock_id != CLOCK_REALTIME)
+ return -ENOTSUPP;
+
mutex_lock(&mvm->mutex);
if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_SYNCED_TIME)) {
ret = iwl_mvm_get_crosstimestamp_fw(mvm, &gp2, &sys_time);
@@ -184,7 +185,7 @@ iwl_mvm_phc_get_crosstimestamp(struct ptp_clock_info *ptp,
/* System monotonic raw time is not used */
xtstamp->device = (ktime_t)gp2_ns;
- xtstamp->sys_realtime = sys_time;
+ xtstamp->sys_systime = sys_time;
out:
mutex_unlock(&mvm->mutex);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 4a33a03..f052537 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
* Copyright (C) 2013-2014 Intel Mobile Communications GmbH
* Copyright (C) 2015-2017 Intel Deutschland GmbH
*/
@@ -159,15 +159,9 @@ int iwl_mvm_legacy_rate_to_mac80211_idx(u32 rate_n_flags,
u8 iwl_mvm_mac80211_idx_to_hwrate(const struct iwl_fw *fw, int rate_idx)
{
- if (iwl_fw_lookup_cmd_ver(fw, TX_CMD, 0) > 8)
- /* In the new rate legacy rates are indexed:
- * 0 - 3 for CCK and 0 - 7 for OFDM.
- */
- return (rate_idx >= IWL_FIRST_OFDM_RATE ?
- rate_idx - IWL_FIRST_OFDM_RATE :
- rate_idx);
-
- return iwl_fw_rate_idx_to_plcp(rate_idx);
+ return rate_idx >= IWL_FIRST_OFDM_RATE ?
+ rate_idx - IWL_FIRST_OFDM_RATE :
+ rate_idx;
}
u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac)
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index dc99e7a..eb3c5a6 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -1225,33 +1225,41 @@ static int _iwl_pci_resume(struct device *device, bool restore)
if (!trans->op_mode)
return 0;
- /*
- * Scratch value was altered, this means the device was powered off, we
- * need to reset it completely.
- * Note: MAC (bits 0:7) will be cleared upon suspend even with wowlan,
- * but not bits [15:8]. So if we have bits set in lower word, assume
- * the device is alive.
- * Alternatively, if the scratch value is 0xFFFFFFFF, then we no longer
- * have access to the device and consider it powered off.
- * For older devices, just try silently to grab the NIC.
- */
- if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
- u32 scratch = iwl_read32(trans, CSR_FUNC_SCRATCH);
-
- if (!(scratch & CSR_FUNC_SCRATCH_POWER_OFF_MASK) ||
- scratch == ~0U)
- device_was_powered_off = true;
- } else {
+ if (test_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
/*
- * bh are re-enabled by iwl_trans_pcie_release_nic_access,
- * so re-enable them if _iwl_trans_pcie_grab_nic_access fails.
+ * Scratch value was altered, this means the device was powered
+ * off, we need to reset it completely.
+ * Note: MAC (bits 0:7) will be cleared upon suspend even with
+ * wowlan, but not bits [15:8]. So if we have bits set in lower
+ * word, assume the device is alive.
+ * Alternatively, if the scratch value is 0xFFFFFFFF, then we
+ * no longer have access to the device and consider it powered
+ * off.
+ * For older devices, just try silently to grab the NIC.
*/
- local_bh_disable();
- if (_iwl_trans_pcie_grab_nic_access(trans, true)) {
- iwl_trans_pcie_release_nic_access(trans);
+ if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
+ u32 scratch = iwl_read32(trans, CSR_FUNC_SCRATCH);
+
+ if (!(scratch & CSR_FUNC_SCRATCH_POWER_OFF_MASK) ||
+ scratch == ~0U) {
+ IWL_DEBUG_WOWLAN(trans,
+ "Scratch 0x%08x indicates device was powered off\n",
+ scratch);
+ device_was_powered_off = true;
+ }
} else {
- device_was_powered_off = true;
- local_bh_enable();
+ /*
+ * bh are re-enabled by iwl_trans_pcie_release_nic_access,
+ * so re-enable them if _iwl_trans_pcie_grab_nic_access
+ * fails.
+ */
+ local_bh_disable();
+ if (_iwl_trans_pcie_grab_nic_access(trans, true)) {
+ iwl_trans_pcie_release_nic_access(trans);
+ } else {
+ device_was_powered_off = true;
+ local_bh_enable();
+ }
}
}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
index a50e845..64262bc 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
@@ -398,9 +398,9 @@ void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans)
mutex_unlock(&trans_pcie->mutex);
if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
- trans->step_urm = !!(iwl_read_umac_prph(trans,
- CNVI_PMU_STEP_FLOW) &
- CNVI_PMU_STEP_FLOW_FORCE_URM);
+ trans->step_urm = !!(iwl_read_prph(trans,
+ CNVI_PMU_STEP_FLOW) &
+ CNVI_PMU_STEP_FLOW_FORCE_URM);
}
static bool iwl_pcie_set_ltr(struct iwl_trans *trans)
diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c
index 4fae0e3..5cc0c5c 100644
--- a/drivers/net/wireless/marvell/libertas/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas/if_usb.c
@@ -310,6 +310,7 @@ static void if_usb_disconnect(struct usb_interface *intf)
struct lbs_private *priv = cardp->priv;
cardp->surprise_removed = 1;
+ wake_up(&cardp->fw_wq);
if (priv) {
lbs_stop_card(priv);
@@ -633,9 +634,10 @@ static inline void process_cmdrequest(int recvlength, uint8_t *recvbuff,
unsigned long flags;
u8 i;
- if (recvlength > LBS_CMD_BUFFER_SIZE) {
+ if (recvlength < MESSAGE_HEADER_LEN ||
+ recvlength > LBS_CMD_BUFFER_SIZE) {
lbs_deb_usbd(&cardp->udev->dev,
- "The receive buffer is too large\n");
+ "The receive buffer is invalid: %d\n", recvlength);
kfree_skb(skb);
return;
}
diff --git a/drivers/net/wireless/microchip/wilc1000/wlan.c b/drivers/net/wireless/microchip/wilc1000/wlan.c
index 3fa8592..4b116fe 100644
--- a/drivers/net/wireless/microchip/wilc1000/wlan.c
+++ b/drivers/net/wireless/microchip/wilc1000/wlan.c
@@ -1265,7 +1265,7 @@ int wilc_wlan_firmware_download(struct wilc *wilc, const u8 *buffer,
ret = acquire_bus(wilc, WILC_BUS_ACQUIRE_AND_WAKEUP);
if (ret)
- return ret;
+ goto fail;
wilc->hif_func->hif_read_reg(wilc, WILC_GLB_RESET_0, ®);
reg &= ~BIT(10);
diff --git a/drivers/net/wireless/rsi/rsi_common.h b/drivers/net/wireless/rsi/rsi_common.h
index 591602b..3cdf9de 100644
--- a/drivers/net/wireless/rsi/rsi_common.h
+++ b/drivers/net/wireless/rsi/rsi_common.h
@@ -70,12 +70,11 @@ static inline int rsi_create_kthread(struct rsi_common *common,
return 0;
}
-static inline int rsi_kill_thread(struct rsi_thread *handle)
+static inline void rsi_kill_thread(struct rsi_thread *handle)
{
atomic_inc(&handle->thread_done);
rsi_set_event(&handle->event);
-
- return kthread_stop(handle->task);
+ wait_for_completion(&handle->completion);
}
void rsi_mac80211_detach(struct rsi_hw *hw);
diff --git a/drivers/net/wireless/st/cw1200/pm.c b/drivers/net/wireless/st/cw1200/pm.c
index 84eb15d..120f037 100644
--- a/drivers/net/wireless/st/cw1200/pm.c
+++ b/drivers/net/wireless/st/cw1200/pm.c
@@ -264,14 +264,12 @@ int cw1200_wow_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan)
wiphy_err(priv->hw->wiphy,
"PM request failed: %d. WoW is disabled.\n", ret);
cw1200_wow_resume(hw);
- mutex_unlock(&priv->conf_mutex);
return -EBUSY;
}
/* Force resume if event is coming from the device. */
if (atomic_read(&priv->bh_rx)) {
cw1200_wow_resume(hw);
- mutex_unlock(&priv->conf_mutex);
return -EAGAIN;
}
diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem.c b/drivers/net/wwan/iosm/iosm_ipc_imem.c
index 1b7bc7d6..4405c85 100644
--- a/drivers/net/wwan/iosm/iosm_ipc_imem.c
+++ b/drivers/net/wwan/iosm/iosm_ipc_imem.c
@@ -1425,6 +1425,8 @@ struct iosm_imem *ipc_imem_init(struct iosm_pcie *pcie, unsigned int device_id,
protocol_init_fail:
cancel_work_sync(&ipc_imem->run_state_worker);
ipc_task_deinit(ipc_imem->ipc_task);
+ if (ipc_imem->ipc_protocol)
+ ipc_protocol_deinit(ipc_imem->ipc_protocol);
ipc_task_init_fail:
kfree(ipc_imem->ipc_task);
ipc_task_fail:
diff --git a/drivers/net/wwan/t7xx/t7xx_modem_ops.c b/drivers/net/wwan/t7xx/t7xx_modem_ops.c
index 7968e20..adb29d3 100644
--- a/drivers/net/wwan/t7xx/t7xx_modem_ops.c
+++ b/drivers/net/wwan/t7xx/t7xx_modem_ops.c
@@ -457,8 +457,20 @@ static int t7xx_parse_host_rt_data(struct t7xx_fsm_ctl *ctl, struct t7xx_sys_inf
offset = sizeof(struct feature_query);
for (i = 0; i < FEATURE_COUNT && offset < data_length; i++) {
+ size_t remaining = data_length - offset;
+ size_t feat_data_len, feat_total;
+
+ if (remaining < sizeof(*rt_feature))
+ break;
+
rt_feature = data + offset;
- offset += sizeof(*rt_feature) + le32_to_cpu(rt_feature->data_len);
+ feat_data_len = le32_to_cpu(rt_feature->data_len);
+
+ if (feat_data_len > remaining - sizeof(*rt_feature))
+ break;
+
+ feat_total = sizeof(*rt_feature) + feat_data_len;
+ offset += feat_total;
ft_spt_cfg = FIELD_GET(FEATURE_MSK, core->feature_set[i]);
if (ft_spt_cfg != MTK_FEATURE_MUST_BE_SUPPORTED)
@@ -468,8 +480,10 @@ static int t7xx_parse_host_rt_data(struct t7xx_fsm_ctl *ctl, struct t7xx_sys_inf
if (ft_spt_st != MTK_FEATURE_MUST_BE_SUPPORTED)
return -EINVAL;
- if (i == RT_ID_MD_PORT_ENUM || i == RT_ID_AP_PORT_ENUM)
- t7xx_port_enum_msg_handler(ctl->md, rt_feature->data);
+ if (i == RT_ID_MD_PORT_ENUM || i == RT_ID_AP_PORT_ENUM) {
+ t7xx_port_enum_msg_handler(ctl->md, rt_feature->data,
+ feat_data_len);
+ }
}
return 0;
diff --git a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c
index ae632ef..f869e4e 100644
--- a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c
+++ b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c
@@ -117,6 +117,7 @@ static int fsm_ee_message_handler(struct t7xx_port *port, struct t7xx_fsm_ctl *c
* t7xx_port_enum_msg_handler() - Parse the port enumeration message to create/remove nodes.
* @md: Modem context.
* @msg: Message.
+ * @msg_len: Length of @msg in bytes.
*
* Used to control create/remove device node.
*
@@ -124,12 +125,18 @@ static int fsm_ee_message_handler(struct t7xx_port *port, struct t7xx_fsm_ctl *c
* * 0 - Success.
* * -EFAULT - Message check failure.
*/
-int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg)
+int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg, size_t msg_len)
{
struct device *dev = &md->t7xx_dev->pdev->dev;
unsigned int version, port_count, i;
struct port_msg *port_msg = msg;
+ if (msg_len < sizeof(*port_msg)) {
+ dev_err(dev, "Port enum msg too short for header: need %zu, have %zu\n",
+ sizeof(*port_msg), msg_len);
+ return -EINVAL;
+ }
+
version = FIELD_GET(PORT_MSG_VERSION, le32_to_cpu(port_msg->info));
if (version != PORT_ENUM_VER ||
le32_to_cpu(port_msg->head_pattern) != PORT_ENUM_HEAD_PATTERN ||
@@ -141,6 +148,13 @@ int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg)
}
port_count = FIELD_GET(PORT_MSG_PRT_CNT, le32_to_cpu(port_msg->info));
+
+ if (msg_len < struct_size(port_msg, data, port_count)) {
+ dev_err(dev, "Port enum msg too short: need %zu, have %zu\n",
+ struct_size(port_msg, data, port_count), msg_len);
+ return -EINVAL;
+ }
+
for (i = 0; i < port_count; i++) {
u32 port_info = le32_to_cpu(port_msg->data[i]);
unsigned int ch_id;
@@ -191,7 +205,7 @@ static int control_msg_handler(struct t7xx_port *port, struct sk_buff *skb)
case CTL_ID_PORT_ENUM:
skb_pull(skb, sizeof(*ctrl_msg_h));
- ret = t7xx_port_enum_msg_handler(ctl->md, (struct port_msg *)skb->data);
+ ret = t7xx_port_enum_msg_handler(ctl->md, (struct port_msg *)skb->data, skb->len);
if (!ret)
ret = port_ctl_send_msg_to_md(port, CTL_ID_PORT_ENUM, 0);
else
diff --git a/drivers/net/wwan/t7xx/t7xx_port_proxy.h b/drivers/net/wwan/t7xx/t7xx_port_proxy.h
index f0918b3..7c3190b 100644
--- a/drivers/net/wwan/t7xx/t7xx_port_proxy.h
+++ b/drivers/net/wwan/t7xx/t7xx_port_proxy.h
@@ -103,7 +103,7 @@ void t7xx_port_proxy_reset(struct port_proxy *port_prox);
void t7xx_port_proxy_uninit(struct port_proxy *port_prox);
int t7xx_port_proxy_init(struct t7xx_modem *md);
void t7xx_port_proxy_md_status_notify(struct port_proxy *port_prox, unsigned int state);
-int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg);
+int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg, size_t msg_len);
int t7xx_port_proxy_chl_enable_disable(struct port_proxy *port_prox, unsigned int ch_id,
bool en_flag);
void t7xx_port_proxy_set_cfg(struct t7xx_modem *md, enum port_cfg_id cfg_id);
diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c
index b3d3443..a6c0817 100644
--- a/drivers/nfc/nxp-nci/i2c.c
+++ b/drivers/nfc/nxp-nci/i2c.c
@@ -16,6 +16,7 @@
#include <linux/delay.h>
#include <linux/i2c.h>
#include <linux/interrupt.h>
+#include <linux/irq.h>
#include <linux/module.h>
#include <linux/nfc.h>
#include <linux/gpio/consumer.h>
@@ -267,6 +268,7 @@ static int nxp_nci_i2c_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct nxp_nci_i2c_phy *phy;
+ unsigned long irqflags;
int r;
if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
@@ -303,9 +305,26 @@ static int nxp_nci_i2c_probe(struct i2c_client *client)
if (r < 0)
return r;
+ /*
+ * ACPI platforms may report incorrect IRQ trigger types
+ * (e.g. level-high), which can lead to interrupt storms.
+ *
+ * Use the historically stable rising-edge trigger for ACPI devices.
+ *
+ * On non-ACPI systems (e.g. Device Tree), prefer the firmware-
+ * provided trigger type, falling back to rising-edge if not set.
+ */
+ if (ACPI_COMPANION(dev)) {
+ irqflags = IRQF_TRIGGER_RISING;
+ } else {
+ irqflags = irq_get_trigger_type(client->irq);
+ if (!irqflags)
+ irqflags = IRQF_TRIGGER_RISING;
+ }
+
r = request_threaded_irq(client->irq, NULL,
nxp_nci_i2c_irq_thread_fn,
- IRQF_ONESHOT,
+ irqflags | IRQF_ONESHOT,
NXP_NCI_I2C_DRIVER_NAME, phy);
if (r < 0)
nfc_err(&client->dev, "Unable to register IRQ handler\n");
diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c
index d17c701c7..08c27bb 100644
--- a/drivers/nfc/trf7970a.c
+++ b/drivers/nfc/trf7970a.c
@@ -317,6 +317,7 @@
#define TRF7970A_RSSI_OSC_STATUS_RSSI_MASK (BIT(2) | BIT(1) | BIT(0))
#define TRF7970A_RSSI_OSC_STATUS_RSSI_X_MASK (BIT(5) | BIT(4) | BIT(3))
#define TRF7970A_RSSI_OSC_STATUS_RSSI_OSC_OK BIT(6)
+#define TRF7970A_RSSI_OSC_STATUS_RSSI_NOISE_LEVEL 1
#define TRF7970A_SPECIAL_FCN_REG1_COL_7_6 BIT(0)
#define TRF7970A_SPECIAL_FCN_REG1_14_ANTICOLL BIT(1)
@@ -1300,7 +1301,7 @@ static int trf7970a_is_rf_field(struct trf7970a *trf, bool *is_rf_field)
if (ret)
return ret;
- if (rssi & TRF7970A_RSSI_OSC_STATUS_RSSI_MASK)
+ if ((rssi & TRF7970A_RSSI_OSC_STATUS_RSSI_MASK) > TRF7970A_RSSI_OSC_STATUS_RSSI_NOISE_LEVEL)
*is_rf_field = true;
else
*is_rf_field = false;
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index b199eea..18b6455 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -632,8 +632,11 @@ u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,
u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region);
void nvdimm_bus_lock(struct device *dev);
void nvdimm_bus_unlock(struct device *dev);
-DEFINE_GUARD(nvdimm_bus, struct device *,
- if (_T) nvdimm_bus_lock(_T), if (_T) nvdimm_bus_unlock(_T));
+DEFINE_CLASS(nvdimm_bus, struct device *,
+ if (_T) nvdimm_bus_unlock(_T),
+ ({ if (_T) nvdimm_bus_lock(_T); _T; }),
+ struct device *_T);
+DEFINE_CLASS_IS_GUARD(nvdimm_bus);
bool is_nvdimm_bus_locked(struct device *dev);
void nvdimm_check_and_set_ro(struct gendisk *disk);
diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c
index 2d325fb..77f1d22 100644
--- a/drivers/nvme/common/auth.c
+++ b/drivers/nvme/common/auth.c
@@ -351,18 +351,29 @@ struct nvme_dhchap_key *nvme_auth_transform_key(
}
EXPORT_SYMBOL_GPL(nvme_auth_transform_key);
+/**
+ * nvme_auth_augmented_challenge() - Compute the augmented DH-HMAC-CHAP challenge
+ * @hmac_id: Hash algorithm identifier
+ * @skey: Session key
+ * @skey_len: Length of @skey
+ * @challenge: Challenge value
+ * @aug: Output buffer for the augmented challenge
+ * @hlen: Hash output length (length of @challenge and @aug)
+ *
+ * NVMe base specification 8.3.5.5.4: The augmented challenge is computed
+ * applying the HMAC function using the hash function H() selected by the
+ * HashID parameter ... with the hash of the ephemeral DH key ... as HMAC key
+ * to the challenge C (i.e., Ca = HMAC(H(g^xy mod p), C)).
+ *
+ * As the session key skey is already H(g^xy mod p) per section 8.3.5.5.9, use
+ * it directly as the HMAC key without additional hashing.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
int nvme_auth_augmented_challenge(u8 hmac_id, const u8 *skey, size_t skey_len,
const u8 *challenge, u8 *aug, size_t hlen)
{
- u8 hashed_key[NVME_AUTH_MAX_DIGEST_SIZE];
- int ret;
-
- ret = nvme_auth_hash(hmac_id, skey, skey_len, hashed_key);
- if (ret)
- return ret;
- ret = nvme_auth_hmac(hmac_id, hashed_key, hlen, challenge, hlen, aug);
- memzero_explicit(hashed_key, sizeof(hashed_key));
- return ret;
+ return nvme_auth_hmac(hmac_id, skey, skey_len, challenge, hlen, aug);
}
EXPORT_SYMBOL_GPL(nvme_auth_augmented_challenge);
@@ -403,33 +414,76 @@ int nvme_auth_gen_pubkey(struct crypto_kpp *dh_tfm,
}
EXPORT_SYMBOL_GPL(nvme_auth_gen_pubkey);
-int nvme_auth_gen_shared_secret(struct crypto_kpp *dh_tfm,
- const u8 *ctrl_key, size_t ctrl_key_len,
- u8 *sess_key, size_t sess_key_len)
+/**
+ * nvme_auth_gen_session_key() - Generate an ephemeral session key
+ * @dh_tfm: Diffie-Hellman transform with local private key already set
+ * @public_key: Peer's public key
+ * @public_key_len: Length of @public_key
+ * @sess_key: Output buffer for the session key
+ * @sess_key_len: Size of @sess_key buffer
+ * @hash_id: Hash algorithm identifier
+ *
+ * NVMe base specification 8.3.5.5.9: The session key Ks shall be computed from
+ * the ephemeral DH key (i.e., g^xy mod p) ... by applying the hash function
+ * H() selected by the HashID parameter ... (i.e., Ks = H(g^xy mod p)).
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int nvme_auth_gen_session_key(struct crypto_kpp *dh_tfm,
+ const u8 *public_key, size_t public_key_len,
+ u8 *sess_key, size_t sess_key_len, u8 hash_id)
{
struct kpp_request *req;
struct crypto_wait wait;
struct scatterlist src, dst;
+ u8 *dh_secret;
+ size_t dh_secret_len, hash_len;
int ret;
- req = kpp_request_alloc(dh_tfm, GFP_KERNEL);
- if (!req)
+ hash_len = nvme_auth_hmac_hash_len(hash_id);
+ if (!hash_len) {
+ pr_warn("%s: invalid hash algorithm %d\n", __func__, hash_id);
+ return -EINVAL;
+ }
+
+ if (sess_key_len != hash_len) {
+ pr_warn("%s: sess_key buffer missized (%zu != %zu)\n",
+ __func__, sess_key_len, hash_len);
+ return -EINVAL;
+ }
+
+ dh_secret_len = crypto_kpp_maxsize(dh_tfm);
+ dh_secret = kzalloc(dh_secret_len, GFP_KERNEL);
+ if (!dh_secret)
return -ENOMEM;
+ req = kpp_request_alloc(dh_tfm, GFP_KERNEL);
+ if (!req) {
+ ret = -ENOMEM;
+ goto out_free_secret;
+ }
+
crypto_init_wait(&wait);
- sg_init_one(&src, ctrl_key, ctrl_key_len);
- kpp_request_set_input(req, &src, ctrl_key_len);
- sg_init_one(&dst, sess_key, sess_key_len);
- kpp_request_set_output(req, &dst, sess_key_len);
+ sg_init_one(&src, public_key, public_key_len);
+ kpp_request_set_input(req, &src, public_key_len);
+ sg_init_one(&dst, dh_secret, dh_secret_len);
+ kpp_request_set_output(req, &dst, dh_secret_len);
kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
crypto_req_done, &wait);
ret = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
-
kpp_request_free(req);
+
+ if (ret)
+ goto out_free_secret;
+
+ ret = nvme_auth_hash(hash_id, dh_secret, dh_secret_len, sess_key);
+
+out_free_secret:
+ kfree_sensitive(dh_secret);
return ret;
}
-EXPORT_SYMBOL_GPL(nvme_auth_gen_shared_secret);
+EXPORT_SYMBOL_GPL(nvme_auth_gen_session_key);
int nvme_auth_parse_key(const char *secret, struct nvme_dhchap_key **ret_key)
{
diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c
index ed61b97..c692fc7 100644
--- a/drivers/nvme/host/apple.c
+++ b/drivers/nvme/host/apple.c
@@ -1009,6 +1009,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q)
unsigned int depth = apple_nvme_queue_depth(q);
struct apple_nvme *anv = queue_to_apple_nvme(q);
+ q->sq_tail = 0;
q->cq_head = 0;
q->cq_phase = 1;
if (anv->hw->has_lsq_nvmmu)
@@ -1267,11 +1268,7 @@ static int apple_nvme_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
static void apple_nvme_free_ctrl(struct nvme_ctrl *ctrl)
{
- struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl);
-
- if (anv->ctrl.admin_q)
- blk_put_queue(anv->ctrl.admin_q);
- put_device(anv->dev);
+ put_device(ctrl->dev);
}
static const struct nvme_ctrl_ops nvme_ctrl_ops = {
diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c
index bbedbe1..16de449 100644
--- a/drivers/nvme/host/auth.c
+++ b/drivers/nvme/host/auth.c
@@ -535,11 +535,12 @@ static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
put_unaligned_le16(chap->transaction, buf);
nvme_auth_hmac_update(&hmac, buf, 2);
- memset(buf, 0, 4);
+ *buf = chap->sc_c;
nvme_auth_hmac_update(&hmac, buf, 1);
nvme_auth_hmac_update(&hmac, "Controller", 10);
nvme_auth_hmac_update(&hmac, ctrl->opts->subsysnqn,
strlen(ctrl->opts->subsysnqn));
+ memset(buf, 0, 4);
nvme_auth_hmac_update(&hmac, buf, 1);
nvme_auth_hmac_update(&hmac, ctrl->opts->host->nqn,
strlen(ctrl->opts->host->nqn));
@@ -587,7 +588,7 @@ static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
}
gen_sesskey:
- chap->sess_key_len = chap->host_key_len;
+ chap->sess_key_len = chap->hash_len;
chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
if (!chap->sess_key) {
chap->sess_key_len = 0;
@@ -595,16 +596,17 @@ static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
return -ENOMEM;
}
- ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
- chap->ctrl_key, chap->ctrl_key_len,
- chap->sess_key, chap->sess_key_len);
+ ret = nvme_auth_gen_session_key(chap->dh_tfm,
+ chap->ctrl_key, chap->ctrl_key_len,
+ chap->sess_key, chap->sess_key_len,
+ chap->hash_id);
if (ret) {
dev_dbg(ctrl->device,
- "failed to generate shared secret, error %d\n", ret);
+ "failed to generate session key, error %d\n", ret);
chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
return ret;
}
- dev_dbg(ctrl->device, "shared secret %*ph\n",
+ dev_dbg(ctrl->device, "session key %*ph\n",
(int)chap->sess_key_len, chap->sess_key);
return 0;
}
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 1e33af9..c3032d6 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -454,11 +454,10 @@ void nvme_end_req(struct request *req)
blk_mq_end_request(req, status);
}
-void nvme_complete_rq(struct request *req)
+static void __nvme_complete_rq(struct request *req)
{
struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
- trace_nvme_complete_rq(req);
nvme_cleanup_cmd(req);
/*
@@ -493,6 +492,12 @@ void nvme_complete_rq(struct request *req)
return;
}
}
+
+void nvme_complete_rq(struct request *req)
+{
+ trace_nvme_complete_rq(req);
+ __nvme_complete_rq(req);
+}
EXPORT_SYMBOL_GPL(nvme_complete_rq);
void nvme_complete_batch_req(struct request *req)
@@ -513,7 +518,7 @@ blk_status_t nvme_host_path_error(struct request *req)
{
nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
blk_mq_set_request_complete(req);
- nvme_complete_rq(req);
+ __nvme_complete_rq(req);
return BLK_STS_OK;
}
EXPORT_SYMBOL_GPL(nvme_host_path_error);
@@ -3044,7 +3049,7 @@ static const struct nvme_core_quirk_entry core_quirks[] = {
*
* The device is left in a state where it is also not possible
* to use "nvme set-feature" to disable APST, but booting with
- * nvme_core.default_ps_max_latency=0 works.
+ * nvme_core.default_ps_max_latency_us=0 works.
*/
.vid = 0x1e0f,
.mn = "KCD6XVUL6T40",
@@ -3744,6 +3749,10 @@ int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
ret = nvme_hwmon_init(ctrl);
if (ret == -EINTR)
return ret;
+
+ if (!nvme_ctrl_sgl_supported(ctrl))
+ dev_info(ctrl->device,
+ "passthrough uses implicit buffer lengths\n");
}
clear_bit(NVME_CTRL_DIRTY_CAPABILITY, &ctrl->flags);
@@ -4083,7 +4092,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
mutex_unlock(&ctrl->subsys->lock);
#ifdef CONFIG_NVME_MULTIPATH
- cancel_delayed_work(&head->remove_work);
+ if (cancel_delayed_work(&head->remove_work))
+ module_put(THIS_MODULE);
#endif
return 0;
@@ -5035,8 +5045,8 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)
nvme_mpath_update(ctrl);
}
- nvme_change_uevent(ctrl, "NVME_EVENT=connected");
set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags);
+ nvme_change_uevent(ctrl, "NVME_EVENT=connected");
}
EXPORT_SYMBOL_GPL(nvme_start_ctrl);
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index e1bb470..e4f4528 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -3968,3 +3968,4 @@ module_exit(nvme_fc_exit_module);
MODULE_DESCRIPTION("NVMe host FC transport driver");
MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("nvme-fc");
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 9597a87..08889b2 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -120,21 +120,11 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
struct nvme_ns *ns = q->queuedata;
struct block_device *bdev = ns ? ns->disk->part0 : NULL;
bool supports_metadata = bdev && blk_get_integrity(bdev->bd_disk);
- struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
bool has_metadata = meta_buffer && meta_len;
- struct bio *bio = NULL;
int ret;
- if (!nvme_ctrl_sgl_supported(ctrl))
- dev_warn_once(ctrl->device, "using unchecked data buffer\n");
- if (has_metadata) {
- if (!supports_metadata)
- return -EINVAL;
-
- if (!nvme_ctrl_meta_sgl_supported(ctrl))
- dev_warn_once(ctrl->device,
- "using unchecked metadata buffer\n");
- }
+ if (has_metadata && !supports_metadata)
+ return -EINVAL;
if (iter)
ret = blk_rq_map_user_iov(q, req, NULL, iter, GFP_KERNEL);
@@ -154,8 +144,8 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
return ret;
out_unmap:
- if (bio)
- blk_rq_unmap_user(bio);
+ if (req->bio)
+ blk_rq_unmap_user(req->bio);
return ret;
}
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index ba00f0b72..263161c 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -231,16 +231,12 @@ bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
bool changed = false;
int node;
- if (!head)
- goto out;
-
for_each_node(node) {
if (ns == rcu_access_pointer(head->current_path[node])) {
rcu_assign_pointer(head->current_path[node], NULL);
changed = true;
}
}
-out:
return changed;
}
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index db5fc9b..b5f8462 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -966,7 +966,8 @@ static bool nvme_pci_prp_save_mapping(struct request *req,
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
- if (dma_use_iova(&iod->dma_state) || !dma_need_unmap(dma_dev))
+ if (dma_use_iova(&iod->dma_state) || !dma_need_unmap(dma_dev) ||
+ (iod->flags & IOD_DATA_P2P))
return true;
if (!iod->nr_dma_vecs) {
@@ -996,6 +997,23 @@ static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev,
return nvme_pci_prp_save_mapping(req, dma_dev, iter);
}
+static void nvme_unmap_iter(struct request *req, struct blk_dma_iter *iter,
+ struct dma_iova_state *state)
+{
+ struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+ struct device *dev = nvmeq->dev->dev;
+
+ if (!blk_rq_dma_unmap(req, dev, state, iter->len, iter->p2pdma.map)) {
+ unsigned int attrs = 0;
+
+ if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE)
+ attrs |= DMA_ATTR_MMIO;
+
+ dma_unmap_phys(dev, iter->addr, iter->len, rq_dma_dir(req),
+ attrs);
+ }
+}
+
static blk_status_t nvme_pci_setup_data_prp(struct request *req,
struct blk_dma_iter *iter)
{
@@ -1006,8 +1024,10 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req,
unsigned int prp_len, i;
__le64 *prp_list;
- if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter))
+ if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter)) {
+ nvme_unmap_iter(req, iter, &iod->dma_state);
return iter->status;
+ }
/*
* PRP1 always points to the start of the DMA transfers.
@@ -1112,6 +1132,7 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req,
dev_err_once(nvmeq->dev->dev,
"Incorrectly formed request for payload:%d nents:%d\n",
blk_rq_payload_bytes(req), blk_rq_nr_phys_segments(req));
+ nvme_unmap_data(req);
return BLK_STS_IOERR;
}
@@ -1155,8 +1176,11 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req,
sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
&sgl_dma);
- if (!sg_list)
+ if (!sg_list) {
+ nvme_unmap_iter(req, iter, &iod->dma_state);
return BLK_STS_RESOURCE;
+ }
+
iod->descriptors[iod->nr_descriptors++] = sg_list;
do {
@@ -1313,8 +1337,10 @@ static blk_status_t nvme_pci_setup_meta_iter(struct request *req)
sg_list = dma_pool_alloc(nvmeq->descriptor_pools.small, GFP_ATOMIC,
&sgl_dma);
- if (!sg_list)
+ if (!sg_list) {
+ nvme_unmap_iter(req, &iter, &iod->meta_dma_state);
return BLK_STS_RESOURCE;
+ }
iod->meta_descriptor = sg_list;
iod->meta_dma = sgl_dma;
@@ -2241,6 +2267,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
static const struct blk_mq_ops nvme_mq_admin_ops = {
.queue_rq = nvme_queue_rq,
.complete = nvme_pci_complete_rq,
+ .commit_rqs = nvme_commit_rqs,
.init_hctx = nvme_admin_init_hctx,
.init_request = nvme_pci_init_request,
.timeout = nvme_timeout,
@@ -2532,11 +2559,13 @@ static void nvme_free_host_mem_multi(struct nvme_dev *dev)
static void nvme_free_host_mem(struct nvme_dev *dev)
{
- if (dev->hmb_sgt)
+ if (dev->hmb_sgt) {
dma_free_noncontiguous(dev->dev, dev->host_mem_size,
dev->hmb_sgt, DMA_BIDIRECTIONAL);
- else
+ dev->hmb_sgt = NULL;
+ } else {
nvme_free_host_mem_multi(dev);
+ }
dma_free_coherent(dev->dev, dev->host_mem_descs_size,
dev->host_mem_descs, dev->host_mem_descs_dma);
@@ -4104,6 +4133,8 @@ static const struct pci_device_id nvme_id_table[] = {
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
{ PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
+ { PCI_DEVICE(0x1c5f, 0x0555), /* Memblaze Pblaze5 adapter */
+ .driver_data = NVME_QUIRK_NO_NS_DESC_LIST, },
{ PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */
.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
{ PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 5711113..f77c960 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -2189,6 +2189,13 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
nvme_rdma_reconnect_or_remove(ctrl, ret);
}
+static bool nvme_rdma_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
+{
+ struct nvme_rdma_ctrl *r_ctrl = to_rdma_ctrl(ctrl);
+
+ return ib_dma_pci_p2p_dma_supported(r_ctrl->device->dev);
+}
+
static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
.name = "rdma",
.module = THIS_MODULE,
@@ -2203,6 +2210,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
.get_address = nvmf_get_address,
.stop_ctrl = nvme_rdma_stop_ctrl,
.get_virt_boundary = nvme_get_virt_boundary,
+ .supports_pci_p2pdma = nvme_rdma_supports_pci_p2pdma,
};
/*
@@ -2432,3 +2440,4 @@ module_exit(nvme_rdma_cleanup_module);
MODULE_DESCRIPTION("NVMe host RDMA transport driver");
MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("nvme-rdma");
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 7bf2e97..e597586 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -883,10 +883,26 @@ static ssize_t tls_keyring_show(struct device *dev,
}
static DEVICE_ATTR_RO(tls_keyring);
+static ssize_t tls_mode_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
+ const char *mode;
+
+ if (ctrl->opts->tls)
+ mode = "tls";
+ else
+ mode = "concat";
+
+ return sysfs_emit(buf, "%s\n", mode);
+}
+static DEVICE_ATTR_RO(tls_mode);
+
static struct attribute *nvme_tls_attrs[] = {
&dev_attr_tls_key.attr,
&dev_attr_tls_configured_key.attr,
&dev_attr_tls_keyring.attr,
+ &dev_attr_tls_mode.attr,
NULL,
};
@@ -908,6 +924,9 @@ static umode_t nvme_tls_attrs_are_visible(struct kobject *kobj,
if (a == &dev_attr_tls_keyring.attr &&
!ctrl->opts->keyring)
return 0;
+ if (a == &dev_attr_tls_mode.attr &&
+ !ctrl->opts->tls && !ctrl->opts->concat)
+ return 0;
return a->mode;
}
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 243dab8..68a1d76 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -1438,18 +1438,32 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
{
struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
struct nvme_tcp_queue *queue = &ctrl->queues[qid];
- unsigned int noreclaim_flag;
+ unsigned int noio_flag;
if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return;
page_frag_cache_drain(&queue->pf_cache);
- noreclaim_flag = memalloc_noreclaim_save();
- /* ->sock will be released by fput() */
- fput(queue->sock->file);
+ /**
+ * Prevent memory reclaim from triggering block I/O during socket
+ * teardown. The socket release path fput -> tcp_close ->
+ * tcp_disconnect -> tcp_send_active_reset may allocate memory, and
+ * allowing reclaim to issue I/O could deadlock if we're being called
+ * from block device teardown (e.g., del_gendisk -> elevator cleanup)
+ * which holds locks that the I/O completion path needs.
+ */
+ noio_flag = memalloc_noio_save();
+
+ /**
+ * Release the socket synchronously. During reset in
+ * nvme_reset_ctrl_work(), queue teardown is immediately followed by
+ * re-allocation. fput() defers socket cleanup to delayed_fput_work
+ * in workqueue context, which can race with new queue setup.
+ */
+ __fput_sync(queue->sock->file);
queue->sock = NULL;
- memalloc_noreclaim_restore(noreclaim_flag);
+ memalloc_noio_restore(noio_flag);
kfree(queue->pdu);
mutex_destroy(&queue->send_mutex);
@@ -1688,7 +1702,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
qid, pskid, status);
if (status) {
- queue->tls_err = -status;
+ queue->tls_err = status;
goto out_complete;
}
@@ -1901,8 +1915,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
err_rcv_pdu:
kfree(queue->pdu);
err_sock:
- /* ->sock will be released by fput() */
- fput(queue->sock->file);
+ /* Use sync variant - see nvme_tcp_free_queue() for explanation */
+ __fput_sync(queue->sock->file);
queue->sock = NULL;
err_destroy_mutex:
mutex_destroy(&queue->send_mutex);
@@ -3071,3 +3085,4 @@ module_exit(nvme_tcp_cleanup_module);
MODULE_DESCRIPTION("NVMe host TCP transport driver");
MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("nvme-tcp");
diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index 4904097..69bde27 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -117,6 +117,15 @@
If unsure, say N.
+config NVME_TARGET_AUTH_DEBUG
+ bool "NVMe over Fabrics In-band Authentication debug messages"
+ depends on NVME_TARGET_AUTH
+ help
+ This enables additional debug messages including the generated
+ DH-HMAC-CHAP secrets to help debugging authentication failures.
+
+ If unsure, say N.
+
config NVME_TARGET_PCI_EPF
tristate "NVMe PCI Endpoint Function target support"
depends on NVME_TARGET && PCI_ENDPOINT
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index e4fd1ca..01b799e 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -687,12 +687,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
id->cmic = NVME_CTRL_CMIC_MULTI_PORT | NVME_CTRL_CMIC_MULTI_CTRL |
NVME_CTRL_CMIC_ANA;
- /* Limit MDTS according to transport capability */
- if (ctrl->ops->get_mdts)
- id->mdts = ctrl->ops->get_mdts(ctrl);
- else
- id->mdts = 0;
-
+ /* Limit MDTS according to port config or transport capability */
+ id->mdts = nvmet_ctrl_mdts(req);
id->cntlid = cpu_to_le16(ctrl->cntlid);
id->ver = cpu_to_le32(ctrl->subsys->ver);
diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c
index b34610e..edb9627d 100644
--- a/drivers/nvme/target/auth.c
+++ b/drivers/nvme/target/auth.c
@@ -144,7 +144,6 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
goto out_unlock;
list_for_each_entry(p, &ctrl->subsys->hosts, entry) {
- pr_debug("check %s\n", nvmet_host_name(p->host));
if (strcmp(nvmet_host_name(p->host), ctrl->hostnqn))
continue;
host = p->host;
@@ -189,11 +188,12 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
ctrl->host_key = NULL;
goto out_free_hash;
}
+#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
pr_debug("%s: using hash %s key %*ph\n", __func__,
ctrl->host_key->hash > 0 ?
nvme_auth_hmac_name(ctrl->host_key->hash) : "none",
(int)ctrl->host_key->len, ctrl->host_key->key);
-
+#endif
nvme_auth_free_key(ctrl->ctrl_key);
if (!host->dhchap_ctrl_secret) {
ctrl->ctrl_key = NULL;
@@ -207,11 +207,12 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
ctrl->ctrl_key = NULL;
goto out_free_hash;
}
+#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
pr_debug("%s: using ctrl hash %s key %*ph\n", __func__,
ctrl->ctrl_key->hash > 0 ?
nvme_auth_hmac_name(ctrl->ctrl_key->hash) : "none",
(int)ctrl->ctrl_key->len, ctrl->ctrl_key->key);
-
+#endif
out_free_hash:
if (ret) {
if (ctrl->host_key) {
@@ -229,9 +230,6 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
void nvmet_auth_sq_free(struct nvmet_sq *sq)
{
cancel_delayed_work(&sq->auth_expired_work);
-#ifdef CONFIG_NVME_TARGET_TCP_TLS
- sq->tls_key = NULL;
-#endif
kfree(sq->dhchap_c1);
sq->dhchap_c1 = NULL;
kfree(sq->dhchap_c2);
@@ -320,7 +318,6 @@ int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response,
if (ret)
goto out_free_challenge;
}
-
pr_debug("ctrl %d qid %d host response seq %u transaction %d\n",
ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
req->sq->dhchap_tid);
@@ -402,11 +399,12 @@ int nvmet_auth_ctrl_hash(struct nvmet_req *req, u8 *response,
put_unaligned_le16(req->sq->dhchap_tid, buf);
nvme_auth_hmac_update(&hmac, buf, 2);
- memset(buf, 0, 4);
+ *buf = req->sq->sc_c;
nvme_auth_hmac_update(&hmac, buf, 1);
nvme_auth_hmac_update(&hmac, "Controller", 10);
nvme_auth_hmac_update(&hmac, ctrl->subsys->subsysnqn,
strlen(ctrl->subsys->subsysnqn));
+ memset(buf, 0, 4);
nvme_auth_hmac_update(&hmac, buf, 1);
nvme_auth_hmac_update(&hmac, ctrl->hostnqn, strlen(ctrl->hostnqn));
nvme_auth_hmac_final(&hmac, response);
@@ -436,8 +434,10 @@ int nvmet_auth_ctrl_exponential(struct nvmet_req *req,
ret = -EINVAL;
} else {
memcpy(buf, ctrl->dh_key, buf_size);
+#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
pr_debug("%s: ctrl %d public key %*ph\n", __func__,
ctrl->cntlid, (int)buf_size, buf);
+#endif
}
return ret;
@@ -449,21 +449,23 @@ int nvmet_auth_ctrl_sesskey(struct nvmet_req *req,
struct nvmet_ctrl *ctrl = req->sq->ctrl;
int ret;
- req->sq->dhchap_skey_len = ctrl->dh_keysize;
+ req->sq->dhchap_skey_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
req->sq->dhchap_skey = kzalloc(req->sq->dhchap_skey_len, GFP_KERNEL);
if (!req->sq->dhchap_skey)
return -ENOMEM;
- ret = nvme_auth_gen_shared_secret(ctrl->dh_tfm,
- pkey, pkey_size,
- req->sq->dhchap_skey,
- req->sq->dhchap_skey_len);
+ ret = nvme_auth_gen_session_key(ctrl->dh_tfm,
+ pkey, pkey_size,
+ req->sq->dhchap_skey,
+ req->sq->dhchap_skey_len,
+ ctrl->shash_id);
if (ret)
- pr_debug("failed to compute shared secret, err %d\n", ret);
+ pr_debug("failed to compute session key, err %d\n", ret);
+#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
else
- pr_debug("%s: shared secret %*ph\n", __func__,
+ pr_debug("%s: session key %*ph\n", __func__,
(int)req->sq->dhchap_skey_len,
req->sq->dhchap_skey);
-
+#endif
return ret;
}
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index 463348c..b88f897 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -301,6 +301,31 @@ static ssize_t nvmet_param_max_queue_size_store(struct config_item *item,
CONFIGFS_ATTR(nvmet_, param_max_queue_size);
+static ssize_t nvmet_param_mdts_show(struct config_item *item, char *page)
+{
+ struct nvmet_port *port = to_nvmet_port(item);
+
+ return snprintf(page, PAGE_SIZE, "%d\n", port->mdts);
+}
+
+static ssize_t nvmet_param_mdts_store(struct config_item *item,
+ const char *page, size_t count)
+{
+ struct nvmet_port *port = to_nvmet_port(item);
+ int ret;
+
+ if (nvmet_is_port_enabled(port, __func__))
+ return -EACCES;
+ ret = kstrtoint(page, 0, &port->mdts);
+ if (ret) {
+ pr_err("Invalid value '%s' for mdts\n", page);
+ return -EINVAL;
+ }
+ return count;
+}
+
+CONFIGFS_ATTR(nvmet_, param_mdts);
+
#ifdef CONFIG_BLK_DEV_INTEGRITY
static ssize_t nvmet_param_pi_enable_show(struct config_item *item,
char *page)
@@ -1995,6 +2020,7 @@ static struct configfs_attribute *nvmet_port_attrs[] = {
&nvmet_attr_addr_tsas,
&nvmet_attr_param_inline_data_size,
&nvmet_attr_param_max_queue_size,
+ &nvmet_attr_param_mdts,
#ifdef CONFIG_BLK_DEV_INTEGRITY
&nvmet_attr_param_pi_enable,
#endif
@@ -2053,6 +2079,7 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
INIT_LIST_HEAD(&port->referrals);
port->inline_data_size = -1; /* < 0 == let the transport choose */
port->max_queue_size = -1; /* < 0 == let the transport choose */
+ port->mdts = -1; /* < 0 == let the transport choose */
port->disc_addr.trtype = NVMF_TRTYPE_MAX;
port->disc_addr.portid = cpu_to_le16(portid);
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 45f6861..62dd59b 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -370,6 +370,14 @@ int nvmet_enable_port(struct nvmet_port *port)
NVMET_MIN_QUEUE_SIZE,
NVMET_MAX_QUEUE_SIZE);
+ /*
+ * If the transport didn't set the mdts properly, then clamp it to the
+ * target limits. Also set default values in case the transport didn't
+ * set it at all.
+ */
+ if (port->mdts < 0 || port->mdts > NVMET_MAX_MDTS)
+ port->mdts = 0;
+
port->enabled = true;
port->tr_ops = ops;
return 0;
@@ -1743,7 +1751,7 @@ static void nvmet_ctrl_free(struct kref *ref)
nvmet_stop_keep_alive_timer(ctrl);
- flush_work(&ctrl->async_event_work);
+ cancel_work_sync(&ctrl->async_event_work);
cancel_work_sync(&ctrl->fatal_err_work);
nvmet_destroy_auth(ctrl);
diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c
index b9ab80c..f1e613e 100644
--- a/drivers/nvme/target/fabrics-cmd-auth.c
+++ b/drivers/nvme/target/fabrics-cmd-auth.c
@@ -395,10 +395,9 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
goto complete;
}
/* Final states, clear up variables */
- if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
- nvmet_auth_sq_free(req->sq);
+ nvmet_auth_sq_free(req->sq);
+ if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
nvmet_ctrl_fatal_error(ctrl);
- }
complete:
nvmet_req_complete(req, status);
@@ -574,7 +573,9 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
status = nvmet_copy_to_sgl(req, 0, d, al);
kfree(d);
done:
- if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
+ if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
+ nvmet_auth_sq_free(req->sq);
+ else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
nvmet_auth_sq_free(req->sq);
nvmet_ctrl_fatal_error(ctrl);
}
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 50070cf..3305a88 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -214,6 +214,7 @@ struct nvmet_port {
bool enabled;
int inline_data_size;
int max_queue_size;
+ int mdts;
const struct nvmet_fabrics_ops *tr_ops;
bool pi_enable;
};
@@ -673,6 +674,7 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
#define NVMET_MAX_QUEUE_SIZE 1024
#define NVMET_NR_QUEUES 128
#define NVMET_MAX_CMD(ctrl) (NVME_CAP_MQES(ctrl->cap) + 1)
+#define NVMET_MAX_MDTS 255
/*
* Nice round number that makes a list of nsids fit into a page.
@@ -761,6 +763,17 @@ static inline bool nvmet_is_pci_ctrl(struct nvmet_ctrl *ctrl)
return ctrl->port->disc_addr.trtype == NVMF_TRTYPE_PCI;
}
+/* Limit MDTS according to port config or transport capability */
+static inline u8 nvmet_ctrl_mdts(struct nvmet_req *req)
+{
+ struct nvmet_ctrl *ctrl = req->sq->ctrl;
+ u8 mdts = req->port->mdts;
+
+ if (!ctrl->ops->get_mdts)
+ return mdts;
+ return min_not_zero(ctrl->ops->get_mdts(ctrl), mdts);
+}
+
#ifdef CONFIG_NVME_TARGET_PASSTHRU
void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys);
int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys);
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 4b8b023..20f150d 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -349,9 +349,7 @@ static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd)
cmd->req.sg = NULL;
}
-static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue);
-
-static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
+static int nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
{
struct bio_vec *iov = cmd->iov;
struct scatterlist *sg;
@@ -364,22 +362,19 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
offset = cmd->rbytes_done;
cmd->sg_idx = offset / PAGE_SIZE;
sg_offset = offset % PAGE_SIZE;
- if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) {
- nvmet_tcp_fatal_error(cmd->queue);
- return;
- }
+ if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt)
+ return -EPROTO;
+
sg = &cmd->req.sg[cmd->sg_idx];
sg_remaining = cmd->req.sg_cnt - cmd->sg_idx;
while (length) {
- if (!sg_remaining) {
- nvmet_tcp_fatal_error(cmd->queue);
- return;
- }
- if (!sg->length || sg->length <= sg_offset) {
- nvmet_tcp_fatal_error(cmd->queue);
- return;
- }
+ if (!sg_remaining)
+ return -EPROTO;
+
+ if (!sg->length || sg->length <= sg_offset)
+ return -EPROTO;
+
u32 iov_len = min_t(u32, length, sg->length - sg_offset);
bvec_set_page(iov, sg_page(sg), iov_len,
@@ -394,24 +389,29 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
iov_iter_bvec(&cmd->recv_msg.msg_iter, ITER_DEST, cmd->iov,
nr_pages, cmd->pdu_len);
-}
-
-static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue)
-{
- queue->rcv_state = NVMET_TCP_RECV_ERR;
- if (queue->nvme_sq.ctrl)
- nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
- else
- kernel_sock_shutdown(queue->sock, SHUT_RDWR);
+ return 0;
}
static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status)
{
+ /*
+ * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path.
+ * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has
+ * already been consumed and queue teardown has started.
+ *
+ * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues
+ * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work()
+ * cancels it, the queue must not keep that old receive state.
+ * Otherwise the next nvmet_tcp_io_work() run can reach
+ * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again.
+ *
+ * That is why queue->rcv_state needs to be updated before we return.
+ */
queue->rcv_state = NVMET_TCP_RECV_ERR;
- if (status == -EPIPE || status == -ECONNRESET)
+ if (status == -EPIPE || status == -ECONNRESET || !queue->nvme_sq.ctrl)
kernel_sock_shutdown(queue->sock, SHUT_RDWR);
else
- nvmet_tcp_fatal_error(queue);
+ nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
}
static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
@@ -887,7 +887,6 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) {
pr_err("bad nvme-tcp pdu length (%d)\n",
le32_to_cpu(icreq->hdr.plen));
- nvmet_tcp_fatal_error(queue);
return -EPROTO;
}
@@ -922,16 +921,29 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
iov.iov_len = sizeof(*icresp);
ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
if (ret < 0) {
+ spin_lock_bh(&queue->state_lock);
+ if (queue->state == NVMET_TCP_Q_DISCONNECTING) {
+ spin_unlock_bh(&queue->state_lock);
+ return -ESHUTDOWN;
+ }
queue->state = NVMET_TCP_Q_FAILED;
+ spin_unlock_bh(&queue->state_lock);
return ret; /* queue removal will cleanup */
}
+ spin_lock_bh(&queue->state_lock);
+ if (queue->state == NVMET_TCP_Q_DISCONNECTING) {
+ spin_unlock_bh(&queue->state_lock);
+ /* Tell nvmet_tcp_socket_error() teardown is in progress. */
+ return -ESHUTDOWN;
+ }
queue->state = NVMET_TCP_Q_LIVE;
+ spin_unlock_bh(&queue->state_lock);
nvmet_prepare_receive_pdu(queue);
return 0;
}
-static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
+static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
{
size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
@@ -947,19 +959,22 @@ static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
if (!nvme_is_write(cmd->req.cmd) || !data_len ||
data_len > cmd->req.port->inline_data_size) {
nvmet_prepare_receive_pdu(queue);
- return;
+ return 0;
}
ret = nvmet_tcp_map_data(cmd);
if (unlikely(ret)) {
pr_err("queue %d: failed to map data\n", queue->idx);
- nvmet_tcp_fatal_error(queue);
- return;
+ return -EPROTO;
}
queue->rcv_state = NVMET_TCP_RECV_DATA;
- nvmet_tcp_build_pdu_iovec(cmd);
cmd->flags |= NVMET_TCP_F_INIT_FAILED;
+ ret = nvmet_tcp_build_pdu_iovec(cmd);
+ if (unlikely(ret))
+ pr_err("queue %d: failed to build PDU iovec\n", queue->idx);
+
+ return ret;
}
static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
@@ -1011,7 +1026,10 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
goto err_proto;
}
cmd->pdu_recv = 0;
- nvmet_tcp_build_pdu_iovec(cmd);
+ if (unlikely(nvmet_tcp_build_pdu_iovec(cmd))) {
+ pr_err("queue %d: failed to build PDU iovec\n", queue->idx);
+ goto err_proto;
+ }
queue->cmd = cmd;
queue->rcv_state = NVMET_TCP_RECV_DATA;
@@ -1019,7 +1037,6 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
err_proto:
/* FIXME: use proper transport errors */
- nvmet_tcp_fatal_error(queue);
return -EPROTO;
}
@@ -1034,7 +1051,6 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
if (hdr->type != nvme_tcp_icreq) {
pr_err("unexpected pdu type (%d) before icreq\n",
hdr->type);
- nvmet_tcp_fatal_error(queue);
return -EPROTO;
}
return nvmet_tcp_handle_icreq(queue);
@@ -1043,7 +1059,6 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
if (unlikely(hdr->type == nvme_tcp_icreq)) {
pr_err("queue %d: received icreq pdu in state %d\n",
queue->idx, queue->state);
- nvmet_tcp_fatal_error(queue);
return -EPROTO;
}
@@ -1060,7 +1075,6 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d",
queue->idx, queue->nr_cmds, queue->send_list_len,
nvme_cmd->common.opcode);
- nvmet_tcp_fatal_error(queue);
return -ENOMEM;
}
@@ -1074,17 +1088,16 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
le32_to_cpu(req->cmd->common.dptr.sgl.length),
le16_to_cpu(req->cqe->status));
- nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
- return 0;
+ return nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
}
ret = nvmet_tcp_map_data(queue->cmd);
if (unlikely(ret)) {
pr_err("queue %d: failed to map data\n", queue->idx);
if (nvmet_tcp_has_inline_data(queue->cmd))
- nvmet_tcp_fatal_error(queue);
- else
- nvmet_req_complete(req, ret);
+ return -EPROTO;
+
+ nvmet_req_complete(req, ret);
ret = -EAGAIN;
goto out;
}
@@ -1092,8 +1105,11 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
if (nvmet_tcp_need_data_in(queue->cmd)) {
if (nvmet_tcp_has_inline_data(queue->cmd)) {
queue->rcv_state = NVMET_TCP_RECV_DATA;
- nvmet_tcp_build_pdu_iovec(queue->cmd);
- return 0;
+ ret = nvmet_tcp_build_pdu_iovec(queue->cmd);
+ if (unlikely(ret))
+ pr_err("queue %d: failed to build PDU iovec\n",
+ queue->idx);
+ return ret;
}
/* send back R2T */
nvmet_tcp_queue_response(&queue->cmd->req);
@@ -1204,7 +1220,6 @@ static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {
pr_err("unexpected pdu type %d\n", hdr->type);
- nvmet_tcp_fatal_error(queue);
return -EIO;
}
@@ -1218,16 +1233,12 @@ static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
}
if (queue->hdr_digest &&
- nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) {
- nvmet_tcp_fatal_error(queue); /* fatal */
+ nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen))
return -EPROTO;
- }
if (queue->data_digest &&
- nvmet_tcp_check_ddgst(queue, &queue->pdu)) {
- nvmet_tcp_fatal_error(queue); /* fatal */
+ nvmet_tcp_check_ddgst(queue, &queue->pdu))
return -EPROTO;
- }
return nvmet_tcp_done_recv_pdu(queue);
}
@@ -1310,9 +1321,11 @@ static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
queue->idx, cmd->req.cmd->common.command_id,
queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),
le32_to_cpu(cmd->exp_ddgst));
- nvmet_req_uninit(&cmd->req);
+ if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED)) {
+ cmd->req.cqe->status = NVME_SC_CMD_SEQ_ERROR;
+ nvmet_req_uninit(&cmd->req);
+ }
nvmet_tcp_free_cmd_buffers(cmd);
- nvmet_tcp_fatal_error(queue);
ret = -EPROTO;
goto out;
}
diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c
index aeaf73b..f009219 100644
--- a/drivers/nvme/target/zns.c
+++ b/drivers/nvme/target/zns.c
@@ -69,7 +69,6 @@ bool nvmet_bdev_zns_enable(struct nvmet_ns *ns)
void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req)
{
u8 zasl = req->sq->ctrl->subsys->zasl;
- struct nvmet_ctrl *ctrl = req->sq->ctrl;
struct nvme_id_ctrl_zns *id;
u16 status;
@@ -79,10 +78,7 @@ void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req)
goto out;
}
- if (ctrl->ops->get_mdts)
- id->zasl = min_t(u8, ctrl->ops->get_mdts(ctrl), zasl);
- else
- id->zasl = zasl;
+ id->zasl = min_not_zero(nvmet_ctrl_mdts(req), zasl);
status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
diff --git a/drivers/parisc/lasi.c b/drivers/parisc/lasi.c
index ef6125d..a5b80cd 100644
--- a/drivers/parisc/lasi.c
+++ b/drivers/parisc/lasi.c
@@ -193,8 +193,7 @@ static int __init lasi_init_chip(struct parisc_device *dev)
ret = request_irq(lasi->gsc_irq.irq, gsc_asic_intr, 0, "lasi", lasi);
if (ret < 0) {
- kfree(lasi);
- return ret;
+ goto err_free;
}
/* enable IRQ's for devices below LASI */
@@ -203,8 +202,7 @@ static int __init lasi_init_chip(struct parisc_device *dev)
/* Done init'ing, register this driver */
ret = gsc_common_setup(dev, lasi);
if (ret) {
- kfree(lasi);
- return ret;
+ goto err_irq;
}
gsc_fixup_irqs(dev, lasi, lasi_choose_irq);
@@ -214,6 +212,12 @@ static int __init lasi_init_chip(struct parisc_device *dev)
SYS_OFF_PRIO_DEFAULT, lasi_power_off, lasi);
return ret;
+
+err_irq:
+ free_irq(lasi->gsc_irq.irq, lasi);
+err_free:
+ kfree(lasi);
+ return ret;
}
static struct parisc_device_id lasi_tbl[] __initdata = {
diff --git a/drivers/parisc/led.c b/drivers/parisc/led.c
index b299fcc..016c9d5 100644
--- a/drivers/parisc/led.c
+++ b/drivers/parisc/led.c
@@ -543,10 +543,8 @@ static void __init register_led_regions(void)
static int __init startup_leds(void)
{
- if (platform_device_register(&platform_leds)) {
- pr_info("LED: failed to register LEDs\n");
- platform_device_put(&platform_leds);
- }
+ if (platform_device_register(&platform_leds))
+ printk(KERN_INFO "LED: failed to register LEDs\n");
register_led_regions();
return 0;
}
diff --git a/drivers/parport/share.c b/drivers/parport/share.c
index ba52928..eb0977c 100644
--- a/drivers/parport/share.c
+++ b/drivers/parport/share.c
@@ -214,10 +214,14 @@ static void get_lowlevel_driver(void)
static int port_check(struct device *dev, void *dev_drv)
{
struct parport_driver *drv = dev_drv;
+ struct parport *port;
/* only send ports, do not send other devices connected to bus */
- if (is_parport(dev))
- drv->match_port(to_parport_dev(dev));
+ if (is_parport(dev)) {
+ port = to_parport_dev(dev);
+ if (test_bit(PARPORT_ANNOUNCED, &port->devflags))
+ drv->match_port(port);
+ }
return 0;
}
@@ -532,6 +536,7 @@ void parport_announce_port(struct parport *port)
if (slave)
attach_driver_chain(slave);
}
+ set_bit(PARPORT_ANNOUNCED, &port->devflags);
mutex_unlock(®istration_lock);
}
EXPORT_SYMBOL(parport_announce_port);
@@ -561,6 +566,8 @@ void parport_remove_port(struct parport *port)
mutex_lock(®istration_lock);
+ clear_bit(PARPORT_ANNOUNCED, &port->devflags);
+
/* Spread the word. */
detach_driver_chain(port);
diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
index 714bcab..08a0e70 100644
--- a/drivers/pci/controller/pcie-brcmstb.c
+++ b/drivers/pci/controller/pcie-brcmstb.c
@@ -2072,8 +2072,10 @@ static int brcm_pcie_probe(struct platform_device *pdev)
return PTR_ERR(pcie->clk);
ret = of_pci_get_max_link_speed(np);
- if (pcie_get_link_speed(ret) == PCI_SPEED_UNKNOWN)
+ if (ret < 0 || ret > 3)
pcie->gen = 0;
+ else
+ pcie->gen = ret;
pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index d10ece0..e3f5900 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -179,6 +179,11 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
return NULL;
}
+static void _pci_free_device(struct device *dev)
+{
+ kfree(to_pci_dev(dev));
+}
+
/**
* new_id_store - sysfs frontend to pci_add_dynid()
* @driver: target device driver
@@ -214,11 +219,13 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf,
pdev->subsystem_vendor = subvendor;
pdev->subsystem_device = subdevice;
pdev->class = class;
+ pdev->dev.release = _pci_free_device;
+ device_initialize(&pdev->dev);
if (pci_match_device(pdrv, pdev))
retval = -EEXIST;
- kfree(pdev);
+ put_device(&pdev->dev);
if (retval)
return retval;
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 8f7cfcc..d342666 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -5607,13 +5607,14 @@ static int pci_try_reset_bus(struct pci_bus *bus)
* reset for affected devices
*
* This function will first try to reset the slots on this bus if the method is
- * available. If slot reset fails or is not available, this will fall back to a
+ * available. If slot reset is not available, this will fall back to a
* secondary bus reset.
*/
static int pci_reset_bridge(struct pci_dev *bridge, bool restore)
{
struct pci_bus *bus = bridge->subordinate;
struct pci_slot *slot;
+ int ret = 0;
if (!bus)
return -ENOTTY;
@@ -5627,19 +5628,17 @@ static int pci_reset_bridge(struct pci_dev *bridge, bool restore)
goto bus_reset;
list_for_each_entry(slot, &bus->slots, list) {
- int ret;
-
if (restore)
ret = pci_try_reset_slot(slot);
else
ret = pci_slot_reset(slot, PCI_RESET_DO_RESET);
if (ret)
- goto bus_reset;
+ break;
}
mutex_unlock(&pci_slot_mutex);
- return 0;
+ return ret;
bus_reset:
mutex_unlock(&pci_slot_mutex);
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index fbc05cd..991d3ed 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -102,6 +102,7 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno)
}
pci_write_config_dword(dev, reg, new);
+ dev->saved_config_space[reg / 4] = new;
pci_read_config_dword(dev, reg, &check);
if ((new ^ check) & mask) {
@@ -112,6 +113,7 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno)
if (res->flags & IORESOURCE_MEM_64) {
new = region.start >> 16 >> 16;
pci_write_config_dword(dev, reg + 4, new);
+ dev->saved_config_space[(reg + 4) / 4] = new;
pci_read_config_dword(dev, reg + 4, &check);
if (check != new) {
pci_err(dev, "%s: error updating (high %#010x != %#010x)\n",
diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c
index e9d106f..4156fab 100644
--- a/drivers/phy/apple/atc.c
+++ b/drivers/phy/apple/atc.c
@@ -628,9 +628,6 @@ struct apple_atcphy {
struct reset_controller_dev rcdev;
- struct typec_switch *sw;
- struct typec_mux *mux;
-
struct mutex lock;
};
@@ -2066,15 +2063,25 @@ static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation ori
return 0;
}
+static void atcphy_typec_switch_unregister(void *data)
+{
+ typec_switch_unregister(data);
+}
+
static int atcphy_probe_switch(struct apple_atcphy *atcphy)
{
+ struct typec_switch_dev *sw;
struct typec_switch_desc sw_desc = {
.drvdata = atcphy,
.fwnode = atcphy->dev->fwnode,
.set = atcphy_sw_set,
};
- return PTR_ERR_OR_ZERO(typec_switch_register(atcphy->dev, &sw_desc));
+ sw = typec_switch_register(atcphy->dev, &sw_desc);
+ if (IS_ERR(sw))
+ return PTR_ERR(sw);
+
+ return devm_add_action_or_reset(atcphy->dev, atcphy_typec_switch_unregister, sw);
}
static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state)
@@ -2146,15 +2153,25 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta
return atcphy_configure(atcphy, target_mode);
}
+static void atcphy_typec_mux_unregister(void *data)
+{
+ typec_mux_unregister(data);
+}
+
static int atcphy_probe_mux(struct apple_atcphy *atcphy)
{
+ struct typec_mux_dev *mux;
struct typec_mux_desc mux_desc = {
.drvdata = atcphy,
.fwnode = atcphy->dev->fwnode,
.set = atcphy_mux_set,
};
- return PTR_ERR_OR_ZERO(typec_mux_register(atcphy->dev, &mux_desc));
+ mux = typec_mux_register(atcphy->dev, &mux_desc);
+ if (IS_ERR(mux))
+ return PTR_ERR(mux);
+
+ return devm_add_action_or_reset(atcphy->dev, atcphy_typec_mux_unregister, mux);
}
static int atcphy_load_tunables(struct apple_atcphy *atcphy)
diff --git a/drivers/phy/eswin/phy-eic7700-sata.c b/drivers/phy/eswin/phy-eic7700-sata.c
index c33653d..76774b9 100644
--- a/drivers/phy/eswin/phy-eic7700-sata.c
+++ b/drivers/phy/eswin/phy-eic7700-sata.c
@@ -216,8 +216,8 @@ static int eic7700_sata_phy_probe(struct platform_device *pdev)
return -ENOENT;
regs = devm_ioremap(dev, res->start, resource_size(res));
- if (IS_ERR(regs))
- return PTR_ERR(regs);
+ if (!regs)
+ return -ENOMEM;
sata_phy->regmap = devm_regmap_init_mmio
(dev, regs, &eic7700_sata_phy_regmap_config);
diff --git a/drivers/phy/marvell/phy-mvebu-a3700-utmi.c b/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
index 04f4fb4..f882bc5 100644
--- a/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
+++ b/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
@@ -168,9 +168,8 @@ static int mvebu_a3700_utmi_phy_power_off(struct phy *phy)
u32 reg;
/* Disable PHY pull-up and enable USB2 suspend */
- reg = readl(utmi->regs + USB2_PHY_CTRL(usb32));
- reg &= ~(RB_USB2PHY_PU | RB_USB2PHY_SUSPM(usb32));
- writel(reg, utmi->regs + USB2_PHY_CTRL(usb32));
+ regmap_update_bits(utmi->usb_misc, USB2_PHY_CTRL(usb32),
+ RB_USB2PHY_PU | RB_USB2PHY_SUSPM(usb32), 0);
/* Power down OTG module */
if (usb32) {
diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c
index 7372de0..a3c893f 100644
--- a/drivers/phy/qualcomm/phy-qcom-edp.c
+++ b/drivers/phy/qualcomm/phy-qcom-edp.c
@@ -81,13 +81,15 @@ struct phy_ver_ops {
int (*com_clk_fwd_cfg)(const struct qcom_edp *edp);
int (*com_configure_pll)(const struct qcom_edp *edp);
int (*com_configure_ssc)(const struct qcom_edp *edp);
+ int (*com_ldo_config)(const struct qcom_edp *edp);
};
struct qcom_edp_phy_cfg {
bool is_edp;
const u8 *aux_cfg;
const u8 *vco_div_cfg;
- const struct qcom_edp_swing_pre_emph_cfg *swing_pre_emph_cfg;
+ const struct qcom_edp_swing_pre_emph_cfg *dp_swing_pre_emph_cfg;
+ const struct qcom_edp_swing_pre_emph_cfg *edp_swing_pre_emph_cfg;
const struct phy_ver_ops *ver_ops;
};
@@ -116,17 +118,17 @@ struct qcom_edp {
};
static const u8 dp_swing_hbr_rbr[4][4] = {
- { 0x08, 0x0f, 0x16, 0x1f },
+ { 0x07, 0x0f, 0x16, 0x1f },
{ 0x11, 0x1e, 0x1f, 0xff },
{ 0x16, 0x1f, 0xff, 0xff },
{ 0x1f, 0xff, 0xff, 0xff }
};
static const u8 dp_pre_emp_hbr_rbr[4][4] = {
- { 0x00, 0x0d, 0x14, 0x1a },
+ { 0x00, 0x0e, 0x15, 0x1a },
{ 0x00, 0x0e, 0x15, 0xff },
{ 0x00, 0x0e, 0xff, 0xff },
- { 0x03, 0xff, 0xff, 0xff }
+ { 0x04, 0xff, 0xff, 0xff }
};
static const u8 dp_swing_hbr2_hbr3[4][4] = {
@@ -150,6 +152,47 @@ static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg = {
.pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3,
};
+static const u8 dp_pre_emp_hbr_rbr_v8[4][4] = {
+ { 0x00, 0x0e, 0x15, 0x1a },
+ { 0x00, 0x0e, 0x15, 0xff },
+ { 0x00, 0x0e, 0xff, 0xff },
+ { 0x00, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg_v8 = {
+ .swing_hbr_rbr = &dp_swing_hbr_rbr,
+ .swing_hbr3_hbr2 = &dp_swing_hbr2_hbr3,
+ .pre_emphasis_hbr_rbr = &dp_pre_emp_hbr_rbr_v8,
+ .pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3,
+};
+
+static const u8 dp_swing_hbr2_hbr3_v2[4][4] = {
+ { 0x27, 0x2f, 0x36, 0xff },
+ { 0x31, 0x3e, 0x3f, 0xff },
+ { 0x3a, 0x3f, 0xff, 0xff },
+ { 0xff, 0xff, 0xff, 0xff }
+};
+
+static const u8 dp_pre_emp_hbr2_hbr3_v2[4][4] = {
+ { 0x20, 0x2e, 0x35, 0xff },
+ { 0x20, 0x2e, 0x35, 0xff },
+ { 0x20, 0x2e, 0xff, 0xff },
+ { 0xff, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg_v2 = {
+ /*
+ * NOTE: The HPG does not specify a separate swing_hbr_rbr table.
+ * Reuse the HBR2/HBR3 table for now.
+ *
+ * TODO: Update this once the HPG explicitly defines RBR/HBR swing values.
+ */
+ .swing_hbr_rbr = &dp_swing_hbr2_hbr3_v2,
+ .swing_hbr3_hbr2 = &dp_swing_hbr2_hbr3_v2,
+ .pre_emphasis_hbr_rbr = &dp_pre_emp_hbr2_hbr3_v2,
+ .pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3_v2,
+};
+
static const u8 edp_swing_hbr_rbr[4][4] = {
{ 0x07, 0x0f, 0x16, 0x1f },
{ 0x0d, 0x16, 0x1e, 0xff },
@@ -158,7 +201,7 @@ static const u8 edp_swing_hbr_rbr[4][4] = {
};
static const u8 edp_pre_emp_hbr_rbr[4][4] = {
- { 0x05, 0x12, 0x17, 0x1d },
+ { 0x05, 0x11, 0x17, 0x1d },
{ 0x05, 0x11, 0x18, 0xff },
{ 0x06, 0x11, 0xff, 0xff },
{ 0x00, 0xff, 0xff, 0xff }
@@ -172,10 +215,10 @@ static const u8 edp_swing_hbr2_hbr3[4][4] = {
};
static const u8 edp_pre_emp_hbr2_hbr3[4][4] = {
- { 0x08, 0x11, 0x17, 0x1b },
- { 0x00, 0x0c, 0x13, 0xff },
- { 0x05, 0x10, 0xff, 0xff },
- { 0x00, 0xff, 0xff, 0xff }
+ { 0x0c, 0x15, 0x19, 0x1e },
+ { 0x0b, 0x15, 0x19, 0xff },
+ { 0x0e, 0x14, 0xff, 0xff },
+ { 0x0d, 0xff, 0xff, 0xff }
};
static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg = {
@@ -193,25 +236,46 @@ static const u8 edp_phy_vco_div_cfg_v4[4] = {
0x01, 0x01, 0x02, 0x00,
};
-static const u8 edp_pre_emp_hbr_rbr_v5[4][4] = {
- { 0x05, 0x11, 0x17, 0x1d },
+static const u8 edp_pre_emp_hbr_rbr_v2[4][4] = {
+ { 0x05, 0x12, 0x17, 0x1d },
{ 0x05, 0x11, 0x18, 0xff },
{ 0x06, 0x11, 0xff, 0xff },
{ 0x00, 0xff, 0xff, 0xff }
};
-static const u8 edp_pre_emp_hbr2_hbr3_v5[4][4] = {
+static const u8 edp_pre_emp_hbr2_hbr3_v2[4][4] = {
{ 0x0c, 0x15, 0x19, 0x1e },
- { 0x0b, 0x15, 0x19, 0xff },
+ { 0x08, 0x15, 0x19, 0xff },
{ 0x0e, 0x14, 0xff, 0xff },
{ 0x0d, 0xff, 0xff, 0xff }
};
-static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v5 = {
+static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v2 = {
.swing_hbr_rbr = &edp_swing_hbr_rbr,
.swing_hbr3_hbr2 = &edp_swing_hbr2_hbr3,
- .pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr_v5,
- .pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v5,
+ .pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr_v2,
+ .pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v2,
+};
+
+static const u8 edp_swing_hbr2_hbr3_v3[4][4] = {
+ { 0x06, 0x11, 0x16, 0x1b },
+ { 0x0b, 0x19, 0x1f, 0xff },
+ { 0x18, 0x1f, 0xff, 0xff },
+ { 0x1f, 0xff, 0xff, 0xff }
+};
+
+static const u8 edp_pre_emp_hbr2_hbr3_v3[4][4] = {
+ { 0x0c, 0x15, 0x19, 0x1e },
+ { 0x09, 0x14, 0x19, 0xff },
+ { 0x0f, 0x14, 0xff, 0xff },
+ { 0x0d, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v3 = {
+ .swing_hbr_rbr = &edp_swing_hbr_rbr,
+ .swing_hbr3_hbr2 = &edp_swing_hbr2_hbr3_v3,
+ .pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr,
+ .pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v3,
};
static const u8 edp_phy_aux_cfg_v5[DP_AUX_CFG_SIZE] = {
@@ -262,12 +326,7 @@ static int qcom_edp_phy_init(struct phy *phy)
DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN,
edp->edp + DP_PHY_PD_CTL);
- /*
- * TODO: Re-work the conditions around setting the cfg8 value
- * when more information becomes available about why this is
- * even needed.
- */
- if (edp->cfg->swing_pre_emph_cfg && !edp->is_edp)
+ if (!edp->is_edp)
aux_cfg[8] = 0xb7;
writel(0xfc, edp->edp + DP_PHY_MODE);
@@ -291,19 +350,18 @@ static int qcom_edp_phy_init(struct phy *phy)
static int qcom_edp_set_voltages(struct qcom_edp *edp, const struct phy_configure_opts_dp *dp_opts)
{
- const struct qcom_edp_swing_pre_emph_cfg *cfg = edp->cfg->swing_pre_emph_cfg;
+ const struct qcom_edp_swing_pre_emph_cfg *cfg;
unsigned int v_level = 0;
unsigned int p_level = 0;
- u8 ldo_config;
+ int ret;
u8 swing;
u8 emph;
int i;
- if (!cfg)
- return 0;
-
if (edp->is_edp)
- cfg = &edp_phy_swing_pre_emph_cfg;
+ cfg = edp->cfg->edp_swing_pre_emph_cfg;
+ else
+ cfg = edp->cfg->dp_swing_pre_emph_cfg;
for (i = 0; i < dp_opts->lanes; i++) {
v_level = max(v_level, dp_opts->voltage[i]);
@@ -321,13 +379,13 @@ static int qcom_edp_set_voltages(struct qcom_edp *edp, const struct phy_configur
if (swing == 0xff || emph == 0xff)
return -EINVAL;
- ldo_config = edp->is_edp ? 0x0 : 0x1;
+ ret = edp->cfg->ver_ops->com_ldo_config(edp);
+ if (ret)
+ return ret;
- writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
writel(swing, edp->tx0 + TXn_TX_DRV_LVL);
writel(emph, edp->tx0 + TXn_TX_EMP_POST1_LVL);
- writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG);
writel(swing, edp->tx1 + TXn_TX_DRV_LVL);
writel(emph, edp->tx1 + TXn_TX_EMP_POST1_LVL);
@@ -551,6 +609,52 @@ static int qcom_edp_com_configure_pll_v4(const struct qcom_edp *edp)
return 0;
}
+static int qcom_edp_ldo_config_v3(const struct qcom_edp *edp)
+{
+ const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+ u32 ldo_config;
+
+ if (!edp->is_edp)
+ ldo_config = 0x0;
+ else if (dp_opts->link_rate <= 2700)
+ ldo_config = 0x81;
+ else
+ ldo_config = 0x41;
+
+ writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+ writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+ return 0;
+}
+
+static int qcom_edp_ldo_config_v4(const struct qcom_edp *edp)
+{
+ const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+ u32 ldo_config;
+
+ if (!edp->is_edp)
+ ldo_config = 0x0;
+ else if (dp_opts->link_rate <= 2700)
+ ldo_config = 0xc1;
+ else
+ ldo_config = 0x81;
+
+ writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+ writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+ return 0;
+}
+
+static const struct phy_ver_ops qcom_edp_phy_ops_v3 = {
+ .com_power_on = qcom_edp_phy_power_on_v4,
+ .com_resetsm_cntrl = qcom_edp_phy_com_resetsm_cntrl_v4,
+ .com_bias_en_clkbuflr = qcom_edp_com_bias_en_clkbuflr_v4,
+ .com_clk_fwd_cfg = qcom_edp_com_clk_fwd_cfg_v4,
+ .com_configure_pll = qcom_edp_com_configure_pll_v4,
+ .com_configure_ssc = qcom_edp_com_configure_ssc_v4,
+ .com_ldo_config = qcom_edp_ldo_config_v3,
+};
+
static const struct phy_ver_ops qcom_edp_phy_ops_v4 = {
.com_power_on = qcom_edp_phy_power_on_v4,
.com_resetsm_cntrl = qcom_edp_phy_com_resetsm_cntrl_v4,
@@ -558,26 +662,39 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v4 = {
.com_clk_fwd_cfg = qcom_edp_com_clk_fwd_cfg_v4,
.com_configure_pll = qcom_edp_com_configure_pll_v4,
.com_configure_ssc = qcom_edp_com_configure_ssc_v4,
+ .com_ldo_config = qcom_edp_ldo_config_v4,
};
static const struct qcom_edp_phy_cfg sa8775p_dp_phy_cfg = {
.is_edp = false,
.aux_cfg = edp_phy_aux_cfg_v5,
.vco_div_cfg = edp_phy_vco_div_cfg_v4,
- .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v5,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
.ver_ops = &qcom_edp_phy_ops_v4,
};
static const struct qcom_edp_phy_cfg sc7280_dp_phy_cfg = {
.aux_cfg = edp_phy_aux_cfg_v4,
.vco_div_cfg = edp_phy_vco_div_cfg_v4,
- .ver_ops = &qcom_edp_phy_ops_v4,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v3,
+ .ver_ops = &qcom_edp_phy_ops_v3,
+};
+
+static const struct qcom_edp_phy_cfg sc8180x_dp_phy_cfg = {
+ .aux_cfg = edp_phy_aux_cfg_v4,
+ .vco_div_cfg = edp_phy_vco_div_cfg_v4,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg_v2,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v2,
+ .ver_ops = &qcom_edp_phy_ops_v3,
};
static const struct qcom_edp_phy_cfg sc8280xp_dp_phy_cfg = {
.aux_cfg = edp_phy_aux_cfg_v4,
.vco_div_cfg = edp_phy_vco_div_cfg_v4,
- .swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
.ver_ops = &qcom_edp_phy_ops_v4,
};
@@ -585,7 +702,8 @@ static const struct qcom_edp_phy_cfg sc8280xp_edp_phy_cfg = {
.is_edp = true,
.aux_cfg = edp_phy_aux_cfg_v4,
.vco_div_cfg = edp_phy_vco_div_cfg_v4,
- .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
.ver_ops = &qcom_edp_phy_ops_v4,
};
@@ -754,6 +872,24 @@ static int qcom_edp_com_configure_pll_v6(const struct qcom_edp *edp)
return 0;
}
+static int qcom_edp_ldo_config_v6(const struct qcom_edp *edp)
+{
+ const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+ u32 ldo_config;
+
+ if (!edp->is_edp)
+ ldo_config = 0x0;
+ else if (dp_opts->link_rate <= 2700)
+ ldo_config = 0x51;
+ else
+ ldo_config = 0x91;
+
+ writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+ writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+ return 0;
+}
+
static const struct phy_ver_ops qcom_edp_phy_ops_v6 = {
.com_power_on = qcom_edp_phy_power_on_v6,
.com_resetsm_cntrl = qcom_edp_phy_com_resetsm_cntrl_v6,
@@ -761,12 +897,14 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v6 = {
.com_clk_fwd_cfg = qcom_edp_com_clk_fwd_cfg_v4,
.com_configure_pll = qcom_edp_com_configure_pll_v6,
.com_configure_ssc = qcom_edp_com_configure_ssc_v6,
+ .com_ldo_config = qcom_edp_ldo_config_v6,
};
static struct qcom_edp_phy_cfg x1e80100_phy_cfg = {
.aux_cfg = edp_phy_aux_cfg_v4,
.vco_div_cfg = edp_phy_vco_div_cfg_v4,
- .swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
.ver_ops = &qcom_edp_phy_ops_v6,
};
@@ -940,12 +1078,14 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v8 = {
.com_clk_fwd_cfg = qcom_edp_com_clk_fwd_cfg_v8,
.com_configure_pll = qcom_edp_com_configure_pll_v8,
.com_configure_ssc = qcom_edp_com_configure_ssc_v8,
+ .com_ldo_config = qcom_edp_ldo_config_v6,
};
static struct qcom_edp_phy_cfg glymur_phy_cfg = {
.aux_cfg = edp_phy_aux_cfg_v8,
.vco_div_cfg = edp_phy_vco_div_cfg_v8,
- .swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v5,
+ .dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg_v8,
+ .edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
.ver_ops = &qcom_edp_phy_ops_v8,
};
@@ -954,7 +1094,6 @@ static int qcom_edp_phy_power_on(struct phy *phy)
const struct qcom_edp *edp = phy_get_drvdata(phy);
u32 bias0_en, drvr0_en, bias1_en, drvr1_en;
unsigned long pixel_freq;
- u8 ldo_config = 0x0;
int ret;
u32 val;
u8 cfg1;
@@ -963,11 +1102,10 @@ static int qcom_edp_phy_power_on(struct phy *phy)
if (ret)
return ret;
- if (edp->cfg->swing_pre_emph_cfg && !edp->is_edp)
- ldo_config = 0x1;
+ ret = edp->cfg->ver_ops->com_ldo_config(edp);
+ if (ret)
+ return ret;
- writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
- writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG);
writel(0x00, edp->tx0 + TXn_LANE_MODE_1);
writel(0x00, edp->tx1 + TXn_LANE_MODE_1);
@@ -1347,7 +1485,7 @@ static const struct of_device_id qcom_edp_phy_match_table[] = {
{ .compatible = "qcom,glymur-dp-phy", .data = &glymur_phy_cfg, },
{ .compatible = "qcom,sa8775p-edp-phy", .data = &sa8775p_dp_phy_cfg, },
{ .compatible = "qcom,sc7280-edp-phy", .data = &sc7280_dp_phy_cfg, },
- { .compatible = "qcom,sc8180x-edp-phy", .data = &sc7280_dp_phy_cfg, },
+ { .compatible = "qcom,sc8180x-edp-phy", .data = &sc8180x_dp_phy_cfg, },
{ .compatible = "qcom,sc8280xp-dp-phy", .data = &sc8280xp_dp_phy_cfg, },
{ .compatible = "qcom,sc8280xp-edp-phy", .data = &sc8280xp_edp_phy_cfg, },
{ .compatible = "qcom,x1e80100-dp-phy", .data = &x1e80100_phy_cfg, },
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
index 771bc7c..b87314c 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
@@ -1112,6 +1112,7 @@ static const struct qmp_phy_init_tbl sm8750_ufsphy_pcs[] = {
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43),
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0x40),
+ QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33),
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f),
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68),
QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e),
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
index c342479..dff27d3 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
@@ -794,7 +794,7 @@ static int qmp_v2_configure_dp_swing(struct qmp_usbc *qmp)
p_level = max(p_level, dp_opts->pre[i]);
}
- if (v_level > 4 || p_level > 4) {
+ if (v_level >= 4 || p_level >= 4) {
dev_err(qmp->dev, "Invalid v(%d) | p(%d) level)\n",
v_level, p_level);
return -EINVAL;
diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c
index 5a181cb..8711a3b 100644
--- a/drivers/phy/samsung/phy-exynos5-usbdrd.c
+++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c
@@ -1958,13 +1958,14 @@ const struct exynos5_usbdrd_phy_tuning exynos7870_tunes_utmi_postinit[] = {
PHYPARAM0_TXPREEMPAMPTUNE | PHYPARAM0_TXHSXVTUNE |
PHYPARAM0_TXFSLSTUNE | PHYPARAM0_SQRXTUNE |
PHYPARAM0_OTGTUNE | PHYPARAM0_COMPDISTUNE),
- (FIELD_PREP_CONST(PHYPARAM0_TXVREFTUNE, 14) |
+ (FIELD_PREP_CONST(PHYPARAM0_TXVREFTUNE, 3) |
FIELD_PREP_CONST(PHYPARAM0_TXRISETUNE, 1) |
- FIELD_PREP_CONST(PHYPARAM0_TXRESTUNE, 3) |
+ FIELD_PREP_CONST(PHYPARAM0_TXRESTUNE, 2) |
+ FIELD_PREP_CONST(PHYPARAM0_TXPREEMPPULSETUNE, 0) |
FIELD_PREP_CONST(PHYPARAM0_TXPREEMPAMPTUNE, 0) |
FIELD_PREP_CONST(PHYPARAM0_TXHSXVTUNE, 0) |
FIELD_PREP_CONST(PHYPARAM0_TXFSLSTUNE, 3) |
- FIELD_PREP_CONST(PHYPARAM0_SQRXTUNE, 6) |
+ FIELD_PREP_CONST(PHYPARAM0_SQRXTUNE, 5) |
FIELD_PREP_CONST(PHYPARAM0_OTGTUNE, 2) |
FIELD_PREP_CONST(PHYPARAM0_COMPDISTUNE, 3))),
PHY_TUNING_ENTRY_LAST
diff --git a/drivers/phy/spacemit/phy-k1-usb2.c b/drivers/phy/spacemit/phy-k1-usb2.c
index 9215d0b..e8c1e26 100644
--- a/drivers/phy/spacemit/phy-k1-usb2.c
+++ b/drivers/phy/spacemit/phy-k1-usb2.c
@@ -97,7 +97,6 @@ static int spacemit_usb2phy_init(struct phy *phy)
ret = clk_enable(sphy->clk);
if (ret) {
dev_err(&phy->dev, "failed to enable clock\n");
- clk_disable(sphy->clk);
return ret;
}
diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c
index 1ddf112..60156ae 100644
--- a/drivers/phy/tegra/xusb-tegra186.c
+++ b/drivers/phy/tegra/xusb-tegra186.c
@@ -20,8 +20,8 @@
/* FUSE USB_CALIB registers */
#define HS_CURR_LEVEL_PADX_SHIFT(x) ((x) ? (11 + (x - 1) * 6) : 0)
#define HS_CURR_LEVEL_PAD_MASK 0x3f
-#define HS_TERM_RANGE_ADJ_SHIFT 7
-#define HS_TERM_RANGE_ADJ_MASK 0xf
+#define HS_TERM_RANGE_ADJ_PADX_SHIFT(x) ((x) ? (5 + (x - 1) * 4) : 7)
+#define HS_TERM_RANGE_ADJ_PAD_MASK 0xf
#define HS_SQUELCH_SHIFT 29
#define HS_SQUELCH_MASK 0x7
@@ -253,7 +253,7 @@
struct tegra_xusb_fuse_calibration {
u32 *hs_curr_level;
u32 hs_squelch;
- u32 hs_term_range_adj;
+ u32 *hs_term_range_adj;
u32 rpd_ctrl;
};
@@ -930,7 +930,7 @@ static int tegra186_utmi_phy_power_on(struct phy *phy)
value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
value &= ~TERM_RANGE_ADJ(~0);
- value |= TERM_RANGE_ADJ(priv->calib.hs_term_range_adj);
+ value |= TERM_RANGE_ADJ(priv->calib.hs_term_range_adj[index]);
value &= ~RPD_CTRL(~0);
value |= RPD_CTRL(priv->calib.rpd_ctrl);
padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
@@ -1464,17 +1464,23 @@ static const char * const tegra186_usb3_functions[] = {
static int
tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
{
+ const struct tegra_xusb_padctl_soc *soc = padctl->base.soc;
struct device *dev = padctl->base.dev;
unsigned int i, count;
u32 value, *level;
+ u32 *hs_term_range_adj;
int err;
- count = padctl->base.soc->ports.usb2.count;
+ count = soc->ports.usb2.count;
level = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL);
if (!level)
return -ENOMEM;
+ hs_term_range_adj = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL);
+ if (!hs_term_range_adj)
+ return -ENOMEM;
+
err = tegra_fuse_readl(TEGRA_FUSE_SKU_CALIB_0, &value);
if (err)
return dev_err_probe(dev, err,
@@ -1490,8 +1496,8 @@ tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
padctl->calib.hs_squelch = (value >> HS_SQUELCH_SHIFT) &
HS_SQUELCH_MASK;
- padctl->calib.hs_term_range_adj = (value >> HS_TERM_RANGE_ADJ_SHIFT) &
- HS_TERM_RANGE_ADJ_MASK;
+ hs_term_range_adj[0] = (value >> HS_TERM_RANGE_ADJ_PADX_SHIFT(0)) &
+ HS_TERM_RANGE_ADJ_PAD_MASK;
err = tegra_fuse_readl(TEGRA_FUSE_USB_CALIB_EXT_0, &value);
if (err) {
@@ -1503,6 +1509,17 @@ tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
padctl->calib.rpd_ctrl = (value >> RPD_CTRL_SHIFT) & RPD_CTRL_MASK;
+ for (i = 1; i < count; i++) {
+ if (soc->has_per_pad_term)
+ hs_term_range_adj[i] =
+ (value >> HS_TERM_RANGE_ADJ_PADX_SHIFT(i)) &
+ HS_TERM_RANGE_ADJ_PAD_MASK;
+ else
+ hs_term_range_adj[i] = hs_term_range_adj[0];
+ }
+
+ padctl->calib.hs_term_range_adj = hs_term_range_adj;
+
return 0;
}
@@ -1708,6 +1725,7 @@ const struct tegra_xusb_padctl_soc tegra194_xusb_padctl_soc = {
.num_supplies = ARRAY_SIZE(tegra194_xusb_padctl_supply_names),
.supports_gen2 = true,
.poll_trk_completed = true,
+ .has_per_pad_term = true,
};
EXPORT_SYMBOL_GPL(tegra194_xusb_padctl_soc);
@@ -1732,6 +1750,7 @@ const struct tegra_xusb_padctl_soc tegra234_xusb_padctl_soc = {
.trk_hw_mode = false,
.trk_update_on_idle = true,
.supports_lp_cfg_en = true,
+ .has_per_pad_term = true,
};
EXPORT_SYMBOL_GPL(tegra234_xusb_padctl_soc);
#endif
diff --git a/drivers/phy/tegra/xusb.h b/drivers/phy/tegra/xusb.h
index cd277d0..77609e5 100644
--- a/drivers/phy/tegra/xusb.h
+++ b/drivers/phy/tegra/xusb.h
@@ -435,6 +435,7 @@ struct tegra_xusb_padctl_soc {
bool trk_hw_mode;
bool trk_update_on_idle;
bool supports_lp_cfg_en;
+ bool has_per_pad_term;
};
struct tegra_xusb_padctl {
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
index b36c8a1..b7bd4ef 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
@@ -540,10 +540,34 @@ static int imx1_pinctrl_parse_functions(struct device_node *np,
return 0;
}
+/*
+ * Check if the DT contains pins in the direct child nodes. This indicates the
+ * newer DT format to store pins. This function returns true if the first found
+ * fsl,pins property is in a child of np. Otherwise false is returned.
+ */
+static bool imx1_pinctrl_dt_is_flat_functions(struct device_node *np)
+{
+ struct device_node *function_np;
+ struct device_node *pinctrl_np;
+
+ for_each_child_of_node(np, function_np) {
+ if (of_property_present(function_np, "fsl,pins"))
+ return true;
+
+ for_each_child_of_node(function_np, pinctrl_np) {
+ if (of_property_present(pinctrl_np, "fsl,pins"))
+ return false;
+ }
+ }
+
+ return true;
+}
+
static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
struct imx1_pinctrl *pctl, struct imx1_pinctrl_soc_info *info)
{
struct device_node *np = pdev->dev.of_node;
+ bool flat_funcs;
int ret;
u32 nfuncs = 0;
u32 ngroups = 0;
@@ -552,9 +576,15 @@ static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
if (!np)
return -ENODEV;
- for_each_child_of_node_scoped(np, child) {
- ++nfuncs;
- ngroups += of_get_child_count(child);
+ flat_funcs = imx1_pinctrl_dt_is_flat_functions(np);
+ if (flat_funcs) {
+ nfuncs = 1;
+ ngroups = of_get_child_count(np);
+ } else {
+ for_each_child_of_node_scoped(np, child) {
+ ++nfuncs;
+ ngroups += of_get_child_count(child);
+ }
}
if (!nfuncs) {
@@ -574,10 +604,14 @@ static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
if (!info->functions || !info->groups)
return -ENOMEM;
- for_each_child_of_node_scoped(np, child) {
- ret = imx1_pinctrl_parse_functions(child, info, ifunc++);
- if (ret == -ENOMEM)
- return -ENOMEM;
+ if (flat_funcs) {
+ imx1_pinctrl_parse_functions(np, info, 0);
+ } else {
+ for_each_child_of_node_scoped(np, child) {
+ ret = imx1_pinctrl_parse_functions(child, info, ifunc++);
+ if (ret == -ENOMEM)
+ return -ENOMEM;
+ }
}
return 0;
diff --git a/drivers/pinctrl/mediatek/pinctrl-moore.c b/drivers/pinctrl/mediatek/pinctrl-moore.c
index 70f6083..071ba84 100644
--- a/drivers/pinctrl/mediatek/pinctrl-moore.c
+++ b/drivers/pinctrl/mediatek/pinctrl-moore.c
@@ -520,6 +520,23 @@ static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio,
return pinctrl_gpio_direction_output(chip, gpio);
}
+static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+ struct mtk_pinctrl *hw = gpiochip_get_data(chip);
+ const struct mtk_pin_desc *desc;
+ int ret, dir;
+
+ desc = (const struct mtk_pin_desc *)&hw->soc->pins[offset];
+ if (!desc->name)
+ return -ENOTSUPP;
+
+ ret = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &dir);
+ if (ret)
+ return ret;
+
+ return dir ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
{
struct mtk_pinctrl *hw = gpiochip_get_data(chip);
@@ -566,6 +583,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw)
chip->parent = hw->dev;
chip->request = gpiochip_generic_request;
chip->free = gpiochip_generic_free;
+ chip->get_direction = mtk_gpio_get_direction;
chip->direction_input = pinctrl_gpio_direction_input;
chip->direction_output = mtk_gpio_direction_output;
chip->get = mtk_gpio_get;
diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
index e2293a8..35d2762 100644
--- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
+++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
@@ -292,7 +292,7 @@ static int aml_calc_reg_and_bit(struct pinctrl_gpio_range *range,
static int aml_pinconf_get_pull(struct aml_pinctrl *info, unsigned int pin)
{
struct pinctrl_gpio_range *range =
- pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+ pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
unsigned int reg, bit, val;
int ret, conf;
@@ -326,7 +326,7 @@ static int aml_pinconf_get_drive_strength(struct aml_pinctrl *info,
u16 *drive_strength_ua)
{
struct pinctrl_gpio_range *range =
- pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+ pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
unsigned int reg, bit;
unsigned int val;
@@ -365,7 +365,7 @@ static int aml_pinconf_get_gpio_bit(struct aml_pinctrl *info,
unsigned int reg_type)
{
struct pinctrl_gpio_range *range =
- pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+ pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
unsigned int reg, bit, val;
int ret;
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index e3128b0..64315b0 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -26,6 +26,7 @@
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/pinctrl/pinconf.h>
+#include <linux/dmi.h>
#include <linux/pinctrl/pinconf-generic.h>
#include <linux/pinctrl/pinmux.h>
#include <linux/string_choices.h>
@@ -39,6 +40,39 @@
static struct amd_gpio *pinctrl_dev;
#endif
+static const struct dmi_system_id amd_gpio_quirk_yoga7_14agp11[] = {
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "83TD"),
+ DMI_MATCH(DMI_BOARD_NAME, "LNVNB161216"),
+ },
+ },
+ { }
+};
+
+static void amd_gpio_apply_quirks(struct amd_gpio *gpio_dev)
+{
+ const unsigned int pin = 157; /* WACF2200 GpioInt per ACPI _CRS */
+ unsigned long flags;
+ u32 reg;
+
+ if (!dmi_check_system(amd_gpio_quirk_yoga7_14agp11))
+ return;
+ if (pin >= gpio_dev->gc.ngpio)
+ return;
+
+ raw_spin_lock_irqsave(&gpio_dev->lock, flags);
+ reg = readl(gpio_dev->base + pin * 4);
+ reg |= BIT(INTERRUPT_ENABLE_OFF) | BIT(INTERRUPT_MASK_OFF);
+ writel(reg, gpio_dev->base + pin * 4);
+ raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
+
+ dev_info(&gpio_dev->pdev->dev,
+ "Enabled IRQ for GPIO %u (Yoga 7 14AGP11 touchscreen)\n",
+ pin);
+}
+
static int amd_gpio_get_direction(struct gpio_chip *gc, unsigned offset)
{
unsigned long flags;
@@ -1219,6 +1253,7 @@ static int amd_gpio_probe(struct platform_device *pdev)
/* Disable and mask interrupts */
amd_gpio_irq_init(gpio_dev);
+ amd_gpio_apply_quirks(gpio_dev);
girq = &gpio_dev->gc.irq;
gpio_irq_chip_set_chip(girq, &amd_gpio_irqchip);
diff --git a/drivers/pinctrl/qcom/pinctrl-eliza.c b/drivers/pinctrl/qcom/pinctrl-eliza.c
index c1f756c..dd8c040 100644
--- a/drivers/pinctrl/qcom/pinctrl-eliza.c
+++ b/drivers/pinctrl/qcom/pinctrl-eliza.c
@@ -1340,7 +1340,7 @@ static const struct msm_pingroup eliza_groups[] = {
[51] = PINGROUP(51, _, _, _, _, _, _, _, _, _, _, _),
[52] = PINGROUP(52, qup1_se2, pcie1_clk_req_n, qup1_se2, ddr_bist_complete, qdss_gpio_tracedata, _, vsense_trigger_mirnat, _, _, _, _),
[53] = PINGROUP(53, qup1_se2, qup1_se2, gcc_gp1, ddr_bist_stop, _, qdss_gpio_tracedata, _, _, _, _, _),
- [54] = PINGROUP(54, qup1_se2, qup1_se6, qdss_gpio_tracedata, gnss_adc1, atest_usb, ddr_pxi0, _, _, _, _, _),
+ [54] = PINGROUP(54, qup1_se2, qup1_se6, qdss_gpio_traceclk, gnss_adc1, atest_usb, ddr_pxi0, _, _, _, _, _),
[55] = PINGROUP(55, qup1_se2, dp0_hot, qup1_se6, _, gnss_adc0, atest_usb, ddr_pxi0, _, _, _, _),
[56] = PINGROUP(56, usb0_hs, tsense_pwm1, tsense_pwm2, tsense_pwm3, tsense_pwm4, _, _, _, _, _, _),
[57] = PINGROUP(57, sd_write_protect, _, _, _, _, _, _, _, _, _, _),
@@ -1358,7 +1358,7 @@ static const struct msm_pingroup eliza_groups[] = {
[69] = PINGROUP(69, cam_mclk, audio_ext_mclk0, resout_gpio, prng_rosc1, _, _, _, _, _, _, _),
[70] = PINGROUP(70, cci_i2c_sda, tmess_prng2, _, phase_flag, atest_char, _, _, _, _, _, _),
[71] = PINGROUP(71, cci_i2c_scl, tmess_prng3, _, phase_flag, atest_char, _, _, _, _, _, _),
- [72] = PINGROUP(72, cci_i2c_sda, tmess_prng1, qdss_gpio_tracedata, atest_char, _, _, _, _, _, _, _),
+ [72] = PINGROUP(72, cci_i2c_sda, tmess_prng1, qdss_gpio_tracectl, atest_char, _, _, _, _, _, _, _),
[73] = PINGROUP(73, cci_i2c_scl, tmess_prng0, qdss_cti, atest_char, _, _, _, _, _, _, _),
[74] = PINGROUP(74, cci_i2c_sda, prng_rosc3, qdss_cti, atest_char, _, _, _, _, _, _, _),
[75] = PINGROUP(75, cci_i2c_scl, _, phase_flag, _, _, _, _, _, _, _, _),
@@ -1430,10 +1430,10 @@ static const struct msm_pingroup eliza_groups[] = {
[141] = PINGROUP(141, _, _, _, _, _, _, _, _, _, _, egpio),
[142] = PINGROUP(142, _, _, _, _, _, _, _, _, _, _, egpio),
[143] = PINGROUP(143, _, _, _, _, _, _, _, _, _, _, egpio),
- [144] = PINGROUP(144, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+ [144] = PINGROUP(144, _, qdss_gpio_tracectl, _, _, _, _, _, _, _, _, egpio),
[145] = PINGROUP(145, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _, egpio),
[146] = PINGROUP(146, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
- [147] = PINGROUP(147, ddr_bist_fail, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+ [147] = PINGROUP(147, ddr_bist_fail, _, qdss_gpio_traceclk, _, _, _, _, _, _, _, egpio),
[148] = PINGROUP(148, _, _, _, _, _, _, _, _, _, _, egpio),
[149] = PINGROUP(149, _, _, _, _, _, _, _, _, _, _, egpio),
[150] = PINGROUP(150, _, _, _, _, _, _, _, _, _, _, egpio),
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq4019.c b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
index c5f0dec..05fdd73 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq4019.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
@@ -479,7 +479,7 @@ static const struct pinfunction ipq4019_functions[] = {
QCA_PIN_FUNCTION(blsp_uart0),
QCA_PIN_FUNCTION(blsp_uart1),
QCA_PIN_FUNCTION(chip_rst),
- QCA_PIN_FUNCTION(gpio),
+ QCA_GPIO_PIN_FUNCTION(gpio),
QCA_PIN_FUNCTION(i2s_rx),
QCA_PIN_FUNCTION(i2s_spdif_in),
QCA_PIN_FUNCTION(i2s_spdif_out),
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h
index a4af279..4fbff61 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.h
+++ b/drivers/pinctrl/qcom/pinctrl-msm.h
@@ -39,6 +39,11 @@ struct pinctrl_pin_desc;
fname##_groups, \
ARRAY_SIZE(fname##_groups))
+#define QCA_GPIO_PIN_FUNCTION(fname) \
+ [qca_mux_##fname] = PINCTRL_GPIO_PINFUNCTION(#fname, \
+ fname##_groups, \
+ ARRAY_SIZE(fname##_groups))
+
/**
* struct msm_pingroup - Qualcomm pingroup definition
* @grp: Generic data of the pin group (name and pins)
diff --git a/drivers/pinctrl/qcom/pinctrl-qcs615.c b/drivers/pinctrl/qcom/pinctrl-qcs615.c
index 0ed4332..f066b3a 100644
--- a/drivers/pinctrl/qcom/pinctrl-qcs615.c
+++ b/drivers/pinctrl/qcom/pinctrl-qcs615.c
@@ -1040,11 +1040,11 @@ static const struct msm_pingroup qcs615_groups[] = {
static const struct msm_gpio_wakeirq_map qcs615_pdc_map[] = {
{ 1, 45 }, { 3, 31 }, { 7, 55 }, { 9, 110 }, { 11, 34 },
{ 13, 33 }, { 14, 35 }, { 17, 46 }, { 19, 48 }, { 21, 83 },
- { 22, 36 }, { 26, 38 }, { 35, 37 }, { 39, 125 }, { 41, 47 },
- { 47, 49 }, { 48, 51 }, { 50, 52 }, { 51, 123 }, { 55, 56 },
+ { 22, 36 }, { 26, 38 }, { 35, 37 }, { 39, 118 }, { 41, 47 },
+ { 47, 49 }, { 48, 51 }, { 50, 52 }, { 51, 116 }, { 55, 56 },
{ 56, 57 }, { 57, 58 }, { 60, 60 }, { 71, 54 }, { 80, 73 },
{ 81, 64 }, { 82, 50 }, { 83, 65 }, { 84, 92 }, { 85, 99 },
- { 86, 67 }, { 87, 84 }, { 88, 124 }, { 89, 122 }, { 90, 69 },
+ { 86, 67 }, { 87, 84 }, { 88, 117 }, { 89, 115 }, { 90, 69 },
{ 92, 88 }, { 93, 75 }, { 94, 91 }, { 95, 72 }, { 96, 82 },
{ 97, 74 }, { 98, 95 }, { 99, 94 }, { 100, 100 }, { 101, 40 },
{ 102, 93 }, { 103, 77 }, { 104, 78 }, { 105, 96 }, { 107, 97 },
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8150.c b/drivers/pinctrl/qcom/pinctrl-sm8150.c
index 0767261..1271367 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8150.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8150.c
@@ -1493,18 +1493,18 @@ static const struct msm_gpio_wakeirq_map sm8150_pdc_map[] = {
{ 3, 31 }, { 5, 32 }, { 8, 33 }, { 9, 34 }, { 10, 100 },
{ 12, 104 }, { 24, 37 }, { 26, 38 }, { 27, 41 }, { 28, 42 },
{ 30, 39 }, { 36, 43 }, { 37, 44 }, { 38, 30 }, { 39, 118 },
- { 39, 125 }, { 41, 47 }, { 42, 48 }, { 46, 50 }, { 47, 49 },
- { 48, 51 }, { 49, 53 }, { 50, 52 }, { 51, 116 }, { 51, 123 },
+ { 41, 47 }, { 42, 48 }, { 46, 50 }, { 47, 49 },
+ { 48, 51 }, { 49, 53 }, { 50, 52 }, { 51, 116 },
{ 53, 54 }, { 54, 55 }, { 55, 56 }, { 56, 57 }, { 58, 58 },
{ 60, 60 }, { 61, 61 }, { 68, 62 }, { 70, 63 }, { 76, 71 },
{ 77, 66 }, { 81, 64 }, { 83, 65 }, { 86, 67 }, { 87, 84 },
- { 88, 117 }, { 88, 124 }, { 90, 69 }, { 91, 70 }, { 93, 75 },
+ { 88, 117 }, { 90, 69 }, { 91, 70 }, { 93, 75 },
{ 95, 72 }, { 96, 73 }, { 97, 74 }, { 101, 40 }, { 103, 77 },
{ 104, 78 }, { 108, 79 }, { 112, 80 }, { 113, 81 }, { 114, 82 },
{ 117, 85 }, { 118, 101 }, { 119, 87 }, { 120, 88 }, { 121, 89 },
{ 122, 90 }, { 123, 91 }, { 124, 92 }, { 125, 93 }, { 129, 94 },
{ 132, 105 }, { 133, 83 }, { 134, 36 }, { 136, 97 }, { 142, 103 },
- { 144, 115 }, { 144, 122 }, { 147, 102 }, { 150, 107 },
+ { 144, 115 }, { 147, 102 }, { 150, 107 },
{ 152, 108 }, { 153, 109 }
};
diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
index 561e601..1c6b115 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
@@ -335,7 +335,7 @@ struct rzg2l_pinctrl_reg_cache {
u32 *iolh[2];
u32 *ien[2];
u32 *pupd[2];
- u32 *smt;
+ u32 *smt[2];
u8 sd_ch[2];
u8 eth_poc[2];
u8 oen;
@@ -2737,10 +2737,6 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl)
if (!cache->pfc)
return -ENOMEM;
- cache->smt = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->smt), GFP_KERNEL);
- if (!cache->smt)
- return -ENOMEM;
-
for (u8 i = 0; i < 2; i++) {
u32 n_dedicated_pins = pctrl->data->n_dedicated_pins;
@@ -2759,6 +2755,11 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl)
if (!cache->pupd[i])
return -ENOMEM;
+ cache->smt[i] = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->smt[i]),
+ GFP_KERNEL);
+ if (!cache->smt[i])
+ return -ENOMEM;
+
/* Allocate dedicated cache. */
dedicated_cache->iolh[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins,
sizeof(*dedicated_cache->iolh[i]),
@@ -3049,7 +3050,7 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off),
cache->pupd[0][port]);
if (pincnt >= 4) {
- RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off),
+ RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off) + 4,
cache->pupd[1][port]);
}
}
@@ -3066,8 +3067,14 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
}
}
- if (has_smt)
- RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off), cache->smt[port]);
+ if (has_smt) {
+ RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off),
+ cache->smt[0][port]);
+ if (pincnt >= 4) {
+ RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off) + 4,
+ cache->smt[1][port]);
+ }
+ }
}
}
diff --git a/drivers/platform/chrome/cros_typec_altmode.c b/drivers/platform/chrome/cros_typec_altmode.c
index 557340b..66c546b 100644
--- a/drivers/platform/chrome/cros_typec_altmode.c
+++ b/drivers/platform/chrome/cros_typec_altmode.c
@@ -359,6 +359,7 @@ cros_typec_register_thunderbolt(struct cros_typec_port *port,
}
INIT_WORK(&adata->work, cros_typec_altmode_work);
+ mutex_init(&adata->lock);
adata->alt = alt;
adata->port = port;
adata->ap_mode_entry = true;
diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c
index 0599d5a..f0881ed 100644
--- a/drivers/platform/surface/surface_aggregator_registry.c
+++ b/drivers/platform/surface/surface_aggregator_registry.c
@@ -295,8 +295,6 @@ static const struct software_node *ssam_node_group_sl6[] = {
/* Devices for Surface Laptop 7. */
static const struct software_node *ssam_node_group_sl7[] = {
&ssam_node_root,
- &ssam_node_bat_ac,
- &ssam_node_bat_main,
&ssam_node_tmp_perf_profile_with_fan,
&ssam_node_fan_speed,
&ssam_node_hid_sam_keyboard,
diff --git a/drivers/platform/surface/surfacepro3_button.c b/drivers/platform/surface/surfacepro3_button.c
index 0293bc5..388a3e1 100644
--- a/drivers/platform/surface/surfacepro3_button.c
+++ b/drivers/platform/surface/surfacepro3_button.c
@@ -185,12 +185,15 @@ static bool surface_button_check_MSHW0040(struct device *dev, acpi_handle handle
static int surface_button_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct surface_button *button;
+ struct acpi_device *device;
struct input_dev *input;
- const char *hid = acpi_device_hid(device);
int error;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
if (strncmp(acpi_device_bid(device), SURFACE_BUTTON_OBJ_NAME,
strlen(SURFACE_BUTTON_OBJ_NAME)))
return -ENODEV;
@@ -210,7 +213,8 @@ static int surface_button_probe(struct platform_device *pdev)
}
strscpy(acpi_device_name(device), SURFACE_BUTTON_DEVICE_NAME);
- snprintf(button->phys, sizeof(button->phys), "%s/buttons", hid);
+ snprintf(button->phys, sizeof(button->phys), "%s/buttons",
+ acpi_device_hid(device));
input->name = acpi_device_name(device);
input->phys = button->phys;
diff --git a/drivers/platform/wmi/core.c b/drivers/platform/wmi/core.c
index 7aa40da..5a2ffcb 100644
--- a/drivers/platform/wmi/core.c
+++ b/drivers/platform/wmi/core.c
@@ -411,6 +411,9 @@ int wmidev_invoke_method(struct wmi_device *wdev, u8 instance, u32 method_id,
obj = aout.pointer;
if (!obj) {
+ if (min_size != 0)
+ return -ENOMSG;
+
out->length = 0;
out->data = ZERO_SIZE_PTR;
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 2ffa4ec..7a49560 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -118,6 +118,7 @@
depends on ACPI_WMI
depends on HWMON
depends on INPUT
+ depends on LEDS_CLASS
depends on POWER_SUPPLY
select ACPI_PLATFORM_PROFILE
select INPUT_SPARSEKMAP
diff --git a/drivers/platform/x86/acer-wireless.c b/drivers/platform/x86/acer-wireless.c
index f464b13..fae8e5a 100644
--- a/drivers/platform/x86/acer-wireless.c
+++ b/drivers/platform/x86/acer-wireless.c
@@ -37,9 +37,14 @@ static void acer_wireless_notify(acpi_handle handle, u32 event, void *data)
static int acer_wireless_probe(struct platform_device *pdev)
{
+ struct acpi_device *adev;
struct input_dev *idev;
int ret;
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
+
idev = devm_input_allocate_device(&pdev->dev);
if (!idev)
return -ENOMEM;
@@ -57,8 +62,7 @@ static int acer_wireless_probe(struct platform_device *pdev)
if (ret)
return ret;
- return acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
- ACPI_DEVICE_NOTIFY,
+ return acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
acer_wireless_notify,
&pdev->dev);
}
diff --git a/drivers/platform/x86/adv_swbutton.c b/drivers/platform/x86/adv_swbutton.c
index 6fa60f3..8f7a26e 100644
--- a/drivers/platform/x86/adv_swbutton.c
+++ b/drivers/platform/x86/adv_swbutton.c
@@ -48,10 +48,14 @@ static int adv_swbutton_probe(struct platform_device *device)
{
struct adv_swbutton *button;
struct input_dev *input;
- acpi_handle handle = ACPI_HANDLE(&device->dev);
+ acpi_handle handle;
acpi_status status;
int error;
+ handle = ACPI_HANDLE(&device->dev);
+ if (!handle)
+ return -ENODEV;
+
button = devm_kzalloc(&device->dev, sizeof(*button), GFP_KERNEL);
if (!button)
return -ENOMEM;
diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c
index 5b0987c..495dc1e 100644
--- a/drivers/platform/x86/asus-armoury.c
+++ b/drivers/platform/x86/asus-armoury.c
@@ -370,7 +370,7 @@ static ssize_t mini_led_mode_current_value_show(struct kobject *kobj,
if (err)
return err;
- mode = FIELD_GET(ASUS_MINI_LED_MODE_MASK, 0);
+ mode = FIELD_GET(ASUS_MINI_LED_MODE_MASK, mode);
for (i = 0; i < mini_led_mode_map_size; i++)
if (mode == mini_led_mode_map[i])
@@ -386,6 +386,7 @@ static ssize_t mini_led_mode_current_value_store(struct kobject *kobj,
{
u32 *mini_led_mode_map;
size_t mini_led_mode_map_size;
+ char mapped_value[12];
u32 mode;
int err;
@@ -414,9 +415,16 @@ static ssize_t mini_led_mode_current_value_store(struct kobject *kobj,
return -ENODEV;
}
- return armoury_attr_uint_store(kobj, attr, buf, count,
- 0, mini_led_mode_map[mode],
- NULL, asus_armoury.mini_led_dev_id);
+ /*
+ * armoury_attr_uint_store() parses and sends the value from the
+ * passed buffer; hand it the mapped firmware value so the device
+ * receives the translated mode instead of the raw index.
+ */
+ snprintf(mapped_value, sizeof(mapped_value), "%u", mini_led_mode_map[mode]);
+
+ return armoury_attr_uint_store(kobj, attr, mapped_value, count, 0,
+ mini_led_mode_map[mode], NULL,
+ asus_armoury.mini_led_dev_id);
}
static ssize_t mini_led_mode_possible_values_show(struct kobject *kobj,
diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h
index c30d2b4..692978b 100644
--- a/drivers/platform/x86/asus-armoury.h
+++ b/drivers/platform/x86/asus-armoury.h
@@ -348,6 +348,29 @@ struct power_data {
static const struct dmi_system_id power_limits[] = {
{
.matches = {
+ DMI_MATCH(DMI_BOARD_NAME, "FA401EA"),
+ },
+ .driver_data = &(struct power_data) {
+ .ac_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 15,
+ .ppt_pl1_spl_max = 95,
+ .ppt_pl2_sppt_min = 35,
+ .ppt_pl2_sppt_max = 100,
+ .ppt_pl3_fppt_min = 35,
+ .ppt_pl3_fppt_max = 115,
+ },
+ .dc_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 15,
+ .ppt_pl1_spl_max = 71,
+ .ppt_pl2_sppt_min = 35,
+ .ppt_pl2_sppt_max = 71,
+ .ppt_pl3_fppt_min = 35,
+ .ppt_pl3_fppt_max = 71,
+ },
+ },
+ },
+ {
+ .matches = {
DMI_MATCH(DMI_BOARD_NAME, "FA401UM"),
},
.driver_data = &(struct power_data) {
@@ -888,6 +911,33 @@ static const struct dmi_system_id power_limits[] = {
},
{
.matches = {
+ DMI_MATCH(DMI_BOARD_NAME, "FX607VU"),
+ },
+ .driver_data = &(struct power_data) {
+ .ac_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 28,
+ .ppt_pl1_spl_def = 115,
+ .ppt_pl1_spl_max = 135,
+ .ppt_pl2_sppt_min = 28,
+ .ppt_pl2_sppt_max = 135,
+ .nv_dynamic_boost_min = 5,
+ .nv_dynamic_boost_max = 25,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ },
+ .dc_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 25,
+ .ppt_pl1_spl_max = 45,
+ .ppt_pl2_sppt_min = 35,
+ .ppt_pl2_sppt_max = 60,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ },
+ .requires_fan_curve = true,
+ },
+ },
+ {
+ .matches = {
DMI_MATCH(DMI_BOARD_NAME, "GA401Q"),
},
.driver_data = &(struct power_data) {
@@ -1255,6 +1305,35 @@ static const struct dmi_system_id power_limits[] = {
},
{
.matches = {
+ DMI_MATCH(DMI_BOARD_NAME, "GU605CP"),
+ },
+ .driver_data = &(struct power_data) {
+ .ac_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 45,
+ .ppt_pl1_spl_max = 75,
+ .ppt_pl2_sppt_min = 56,
+ .ppt_pl2_sppt_max = 95,
+ .nv_dynamic_boost_min = 5,
+ .nv_dynamic_boost_max = 15,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ .nv_tgp_min = 55,
+ .nv_tgp_def = 75,
+ .nv_tgp_max = 95,
+ },
+ .dc_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 25,
+ .ppt_pl1_spl_max = 75,
+ .ppt_pl2_sppt_min = 32,
+ .ppt_pl2_sppt_max = 95,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ },
+ .requires_fan_curve = true,
+ },
+ },
+ {
+ .matches = {
DMI_MATCH(DMI_BOARD_NAME, "GU605CR"),
},
.driver_data = &(struct power_data) {
@@ -1761,6 +1840,40 @@ static const struct dmi_system_id power_limits[] = {
},
{
.matches = {
+ DMI_MATCH(DMI_BOARD_NAME, "G614FR"),
+ },
+ .driver_data = &(struct power_data) {
+ .ac_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 30,
+ .ppt_pl1_spl_max = 120,
+ .ppt_pl2_sppt_min = 65,
+ .ppt_pl2_sppt_def = 140,
+ .ppt_pl2_sppt_max = 162,
+ .ppt_pl3_fppt_min = 65,
+ .ppt_pl3_fppt_def = 140,
+ .ppt_pl3_fppt_max = 162,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ .nv_dynamic_boost_min = 5,
+ .nv_dynamic_boost_max = 25,
+ .nv_tgp_min = 65,
+ .nv_tgp_max = 115,
+ },
+ .dc_data = &(struct power_limits) {
+ .ppt_pl1_spl_min = 25,
+ .ppt_pl1_spl_max = 65,
+ .ppt_pl2_sppt_min = 25,
+ .ppt_pl2_sppt_max = 65,
+ .ppt_pl3_fppt_min = 35,
+ .ppt_pl3_fppt_max = 75,
+ .nv_temp_target_min = 75,
+ .nv_temp_target_max = 87,
+ },
+ .requires_fan_curve = true,
+ },
+ },
+ {
+ .matches = {
DMI_MATCH(DMI_BOARD_NAME, "G614J"),
},
.driver_data = &(struct power_data) {
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index dbbb629..140ac8a 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -1826,10 +1826,14 @@ static bool asus_device_present;
static int asus_acpi_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
struct asus_laptop *asus;
int result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
pr_notice("Asus Laptop Support version %s\n",
ASUS_LAPTOP_VERSION);
asus = kzalloc_obj(struct asus_laptop);
diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c
index b4677c5..8005c08 100644
--- a/drivers/platform/x86/asus-nb-wmi.c
+++ b/drivers/platform/x86/asus-nb-wmi.c
@@ -546,6 +546,15 @@ static const struct dmi_system_id asus_quirks[] = {
},
{
.callback = dmi_matched,
+ .ident = "ASUS Zenbook Duo UX8407AA",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook Duo UX8407AA"),
+ },
+ .driver_data = &quirk_asus_zenbook_duo_kbd,
+ },
+ {
+ .callback = dmi_matched,
.ident = "ASUS ROG Z13",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
diff --git a/drivers/platform/x86/dell/dell-rbtn.c b/drivers/platform/x86/dell/dell-rbtn.c
index 34af9f4..180b8c6 100644
--- a/drivers/platform/x86/dell/dell-rbtn.c
+++ b/drivers/platform/x86/dell/dell-rbtn.c
@@ -396,11 +396,15 @@ static void rbtn_cleanup(struct device *dev)
static int rbtn_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct rbtn_data *rbtn_data;
+ struct acpi_device *device;
enum rbtn_type type;
int ret = 0;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
type = rbtn_check(device);
if (type == RBTN_UNKNOWN) {
dev_info(&pdev->dev, "Unknown device type\n");
diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c
index 02a7109..d18a809 100644
--- a/drivers/platform/x86/eeepc-laptop.c
+++ b/drivers/platform/x86/eeepc-laptop.c
@@ -1363,10 +1363,14 @@ static bool eeepc_device_present;
static int eeepc_acpi_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
struct eeepc_laptop *eeepc;
int result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
pr_notice(EEEPC_LAPTOP_NAME "\n");
eeepc = kzalloc_obj(struct eeepc_laptop);
if (!eeepc)
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index 2e265be..54d0b9c 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -530,10 +530,14 @@ static void acpi_fujitsu_bl_notify(acpi_handle handle, u32 event, void *data)
static int acpi_fujitsu_bl_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
struct fujitsu_bl *priv;
int ret;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
return -ENODEV;
@@ -993,10 +997,14 @@ static void acpi_fujitsu_laptop_notify(acpi_handle handle, u32 event, void *data
static int acpi_fujitsu_laptop_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct fujitsu_laptop *priv;
+ struct acpi_device *device;
int ret, i = 0;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
diff --git a/drivers/platform/x86/fujitsu-tablet.c b/drivers/platform/x86/fujitsu-tablet.c
index 8319df2..2f8c1b8 100644
--- a/drivers/platform/x86/fujitsu-tablet.c
+++ b/drivers/platform/x86/fujitsu-tablet.c
@@ -445,10 +445,14 @@ static acpi_status fujitsu_walk_resources(struct acpi_resource *res, void *data)
static int acpi_fujitsu_probe(struct platform_device *pdev)
{
- struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *adev;
acpi_status status;
int error;
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
+
status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS,
fujitsu_walk_resources, NULL);
if (ACPI_FAILURE(status) || !fujitsu.irq || !fujitsu.io_base)
diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c
index d1cc6e7..f63bc00 100644
--- a/drivers/platform/x86/hp/hp-wmi.c
+++ b/drivers/platform/x86/hp/hp-wmi.c
@@ -190,6 +190,10 @@ static const char * const victus_thermal_profile_boards[] = {
/* DMI Board names of Victus 16-r and Victus 16-s laptops */
static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = {
{
+ .matches = { DMI_MATCH(DMI_BOARD_NAME, "8902") },
+ .driver_data = (void *)&omen_v1_legacy_thermal_params,
+ },
+ {
.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A44") },
.driver_data = (void *)&omen_v1_legacy_thermal_params,
},
@@ -206,6 +210,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst
.driver_data = (void *)&victus_s_thermal_params,
},
{
+ .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BC2") },
+ .driver_data = (void *)&omen_v1_thermal_params,
+ },
+ {
.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCA") },
.driver_data = (void *)&omen_v1_thermal_params,
},
@@ -243,7 +251,7 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst
},
{
.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") },
- .driver_data = (void *)&victus_s_thermal_params,
+ .driver_data = (void *)&omen_v1_no_ec_thermal_params,
},
{
.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D87") },
diff --git a/drivers/platform/x86/hp/hp_accel.c b/drivers/platform/x86/hp/hp_accel.c
index 10d5af1..39b73dc 100644
--- a/drivers/platform/x86/hp/hp_accel.c
+++ b/drivers/platform/x86/hp/hp_accel.c
@@ -300,6 +300,9 @@ static int lis3lv02d_probe(struct platform_device *device)
int ret;
lis3_dev.bus_priv = ACPI_COMPANION(&device->dev);
+ if (!lis3_dev.bus_priv)
+ return -ENODEV;
+
lis3_dev.init = lis3lv02d_acpi_init;
lis3_dev.read = lis3lv02d_acpi_read;
lis3_dev.write = lis3lv02d_acpi_write;
diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
index 2ddd8af..0850935 100644
--- a/drivers/platform/x86/intel/hid.c
+++ b/drivers/platform/x86/intel/hid.c
@@ -688,12 +688,16 @@ static bool button_array_present(struct platform_device *device)
static int intel_hid_probe(struct platform_device *device)
{
- acpi_handle handle = ACPI_HANDLE(&device->dev);
unsigned long long mode, dummy;
struct intel_hid_priv *priv;
+ acpi_handle handle;
acpi_status status;
int err;
+ handle = ACPI_HANDLE(&device->dev);
+ if (!handle)
+ return -ENODEV;
+
intel_hid_init_dsm(handle);
if (!intel_hid_evaluate_method(handle, INTEL_HID_DSM_HDMM_FN, &mode)) {
diff --git a/drivers/platform/x86/intel/int1092/intel_sar.c b/drivers/platform/x86/intel/int1092/intel_sar.c
index 8882202..849f7b4 100644
--- a/drivers/platform/x86/intel/int1092/intel_sar.c
+++ b/drivers/platform/x86/intel/int1092/intel_sar.c
@@ -245,15 +245,20 @@ static void sar_get_data(int reg, struct wwan_sar_context *context)
static int sar_probe(struct platform_device *device)
{
struct wwan_sar_context *context;
+ acpi_handle handle;
int reg;
int result;
+ handle = ACPI_HANDLE(&device->dev);
+ if (!handle)
+ return -ENODEV;
+
context = kzalloc_obj(*context);
if (!context)
return -ENOMEM;
context->sar_device = device;
- context->handle = ACPI_HANDLE(&device->dev);
+ context->handle = handle;
dev_set_drvdata(&device->dev, context);
result = guid_parse(SAR_DSM_UUID, &context->guid);
diff --git a/drivers/platform/x86/intel/plr_tpmi.c b/drivers/platform/x86/intel/plr_tpmi.c
index 0572716..8faecc3 100644
--- a/drivers/platform/x86/intel/plr_tpmi.c
+++ b/drivers/platform/x86/intel/plr_tpmi.c
@@ -22,6 +22,7 @@
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/mutex.h>
+#include <linux/notifier.h>
#include <linux/seq_file.h>
#include <linux/sprintf.h>
#include <linux/types.h>
@@ -60,6 +61,8 @@ struct tpmi_plr {
struct tpmi_plr_die *die_info;
int num_dies;
struct auxiliary_device *auxdev;
+ struct notifier_block nb;
+ struct mutex lock; /* Protect access to dbgfs_dir */
};
static const char * const plr_coarse_reasons[] = {
@@ -255,6 +258,30 @@ static ssize_t plr_status_write(struct file *filp, const char __user *ubuf,
}
DEFINE_SHOW_STORE_ATTRIBUTE(plr_status);
+static int intel_plr_notify(struct notifier_block *self, unsigned long action, void *data)
+{
+ struct tpmi_plr *plr = container_of(self, struct tpmi_plr, nb);
+
+ if (action == TPMI_CORE_EXIT) {
+ guard(mutex)(&plr->lock);
+ plr->dbgfs_dir = NULL;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static int intel_plr_register_notifier(struct notifier_block *nb)
+{
+ nb->notifier_call = intel_plr_notify;
+ nb->priority = 0;
+ return tpmi_register_notifier(nb);
+}
+
+static void intel_plr_unregister_notifier(struct notifier_block *nb)
+{
+ tpmi_unregister_notifier(nb);
+}
+
static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
{
struct oobmsm_plat_info *plat_info;
@@ -282,10 +309,18 @@ static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxilia
if (!plr)
return -ENOMEM;
+ err = devm_mutex_init(&auxdev->dev, &plr->lock);
+ if (err)
+ return err;
+
+ intel_plr_register_notifier(&plr->nb);
+
plr->die_info = devm_kcalloc(&auxdev->dev, num_resources, sizeof(*plr->die_info),
GFP_KERNEL);
- if (!plr->die_info)
- return -ENOMEM;
+ if (!plr->die_info) {
+ err = -ENOMEM;
+ goto err_notify;
+ }
plr->num_dies = num_resources;
plr->dbgfs_dir = debugfs_create_dir("plr", dentry);
@@ -326,6 +361,9 @@ static int intel_plr_probe(struct auxiliary_device *auxdev, const struct auxilia
err:
debugfs_remove_recursive(plr->dbgfs_dir);
+err_notify:
+ intel_plr_unregister_notifier(&plr->nb);
+
return err;
}
@@ -333,6 +371,9 @@ static void intel_plr_remove(struct auxiliary_device *auxdev)
{
struct tpmi_plr *plr = auxiliary_get_drvdata(auxdev);
+ intel_plr_unregister_notifier(&plr->nb);
+
+ guard(mutex)(&plr->lock);
debugfs_remove_recursive(plr->dbgfs_dir);
}
diff --git a/drivers/platform/x86/intel/rst.c b/drivers/platform/x86/intel/rst.c
index 4bd1092..bb19f0d 100644
--- a/drivers/platform/x86/intel/rst.c
+++ b/drivers/platform/x86/intel/rst.c
@@ -102,9 +102,13 @@ static struct device_attribute irst_timeout_attr = {
static int irst_probe(struct platform_device *pdev)
{
- struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *acpi;
int error;
+ acpi = ACPI_COMPANION(&pdev->dev);
+ if (!acpi)
+ return -ENODEV;
+
error = device_create_file(&acpi->dev, &irst_timeout_attr);
if (unlikely(error))
return error;
diff --git a/drivers/platform/x86/intel/smartconnect.c b/drivers/platform/x86/intel/smartconnect.c
index 4d866b6..71e91ac 100644
--- a/drivers/platform/x86/intel/smartconnect.c
+++ b/drivers/platform/x86/intel/smartconnect.c
@@ -12,10 +12,14 @@ MODULE_LICENSE("GPL");
static int smartconnect_acpi_probe(struct platform_device *pdev)
{
- acpi_handle handle = ACPI_HANDLE(&pdev->dev);
unsigned long long value;
+ acpi_handle handle;
acpi_status status;
+ handle = ACPI_HANDLE(&pdev->dev);
+ if (!handle)
+ return -ENODEV;
+
status = acpi_evaluate_integer(handle, "GAOS", NULL, &value);
if (ACPI_FAILURE(status))
return -EINVAL;
diff --git a/drivers/platform/x86/intel/vbtn.c b/drivers/platform/x86/intel/vbtn.c
index 9ca87e7..874023c 100644
--- a/drivers/platform/x86/intel/vbtn.c
+++ b/drivers/platform/x86/intel/vbtn.c
@@ -275,12 +275,16 @@ static bool intel_vbtn_has_switches(acpi_handle handle, bool dual_accel)
static int intel_vbtn_probe(struct platform_device *device)
{
- acpi_handle handle = ACPI_HANDLE(&device->dev);
bool dual_accel, has_buttons, has_switches;
struct intel_vbtn_priv *priv;
+ acpi_handle handle;
acpi_status status;
int err;
+ handle = ACPI_HANDLE(&device->dev);
+ if (!handle)
+ return -ENODEV;
+
dual_accel = dual_accel_detect();
has_buttons = acpi_has_method(handle, "VBDL");
has_switches = intel_vbtn_has_switches(handle, dual_accel);
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 7d5dbc1..18e4a89 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -649,29 +649,13 @@ static void intel_vsec_skip_missing_dependencies(struct pci_dev *pdev)
}
}
-static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int intel_vsec_pci_init(struct pci_dev *pdev)
{
- const struct intel_vsec_platform_info *info;
- struct vsec_priv *priv;
- int num_caps, ret;
+ struct vsec_priv *priv = pci_get_drvdata(pdev);
+ const struct intel_vsec_platform_info *info = priv->info;
int run_once = 0;
bool found_any = false;
-
- ret = pcim_enable_device(pdev);
- if (ret)
- return ret;
-
- pci_save_state(pdev);
- info = (const struct intel_vsec_platform_info *)id->driver_data;
- if (!info)
- return -EINVAL;
-
- priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
-
- priv->info = info;
- pci_set_drvdata(pdev, priv);
+ int num_caps;
num_caps = hweight_long(info->caps);
while (num_caps--) {
@@ -692,6 +676,31 @@ static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id
return 0;
}
+static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ const struct intel_vsec_platform_info *info;
+ struct vsec_priv *priv;
+ int ret;
+
+ ret = pcim_enable_device(pdev);
+ if (ret)
+ return ret;
+
+ pci_save_state(pdev);
+ info = (const struct intel_vsec_platform_info *)id->driver_data;
+ if (!info)
+ return -EINVAL;
+
+ priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->info = info;
+ pci_set_drvdata(pdev, priv);
+
+ return intel_vsec_pci_init(pdev);
+}
+
int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info,
struct intel_vsec_device *vsec_dev)
{
@@ -832,7 +841,6 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
{
struct intel_vsec_device *intel_vsec_dev;
pci_ers_result_t status = PCI_ERS_RESULT_DISCONNECT;
- const struct pci_device_id *pci_dev_id;
unsigned long index;
dev_info(&pdev->dev, "Resetting PCI slot\n");
@@ -853,10 +861,8 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
devm_release_action(&pdev->dev, intel_vsec_remove_aux,
&intel_vsec_dev->auxdev);
}
- pci_disable_device(pdev);
pci_restore_state(pdev);
- pci_dev_id = pci_match_id(intel_vsec_pci_ids, pdev);
- intel_vsec_pci_probe(pdev, pci_dev_id);
+ intel_vsec_pci_init(pdev);
out:
return status;
diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c
index 7fc6ff8..16fd7aa 100644
--- a/drivers/platform/x86/intel/vsec_tpmi.c
+++ b/drivers/platform/x86/intel/vsec_tpmi.c
@@ -56,6 +56,7 @@
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/module.h>
+#include <linux/notifier.h>
#include <linux/pci.h>
#include <linux/security.h>
#include <linux/sizes.h>
@@ -188,6 +189,20 @@ struct tpmi_feature_state {
/* Used during auxbus device creation */
static DEFINE_IDA(intel_vsec_tpmi_ida);
+static BLOCKING_NOTIFIER_HEAD(tpmi_notify_list);
+
+int tpmi_register_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&tpmi_notify_list, nb);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_register_notifier, "INTEL_TPMI");
+
+int tpmi_unregister_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&tpmi_notify_list, nb);
+}
+EXPORT_SYMBOL_NS_GPL(tpmi_unregister_notifier, "INTEL_TPMI");
+
struct oobmsm_plat_info *tpmi_get_platform_data(struct auxiliary_device *auxdev)
{
struct intel_vsec_device *vsec_dev = auxdev_to_ivdev(auxdev);
@@ -817,10 +832,6 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev)
auxiliary_set_drvdata(auxdev, tpmi_info);
- ret = tpmi_create_devices(tpmi_info);
- if (ret)
- return ret;
-
/*
* Allow debugfs when security policy allows. Everything this debugfs
* interface provides, can also be done via /dev/mem access. If
@@ -830,6 +841,14 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev)
if (!security_locked_down(LOCKDOWN_DEV_MEM) && capable(CAP_SYS_RAWIO))
tpmi_dbgfs_register(tpmi_info);
+ ret = tpmi_create_devices(tpmi_info);
+ if (ret) {
+ debugfs_remove_recursive(tpmi_info->dbgfs_dir);
+ return ret;
+ }
+
+ blocking_notifier_call_chain(&tpmi_notify_list, TPMI_CORE_INIT, auxdev);
+
return 0;
}
@@ -843,6 +862,8 @@ static void tpmi_remove(struct auxiliary_device *auxdev)
{
struct intel_tpmi_info *tpmi_info = auxiliary_get_drvdata(auxdev);
+ blocking_notifier_call_chain(&tpmi_notify_list, TPMI_CORE_EXIT, auxdev);
+
debugfs_remove_recursive(tpmi_info->dbgfs_dir);
}
diff --git a/drivers/platform/x86/lenovo/Kconfig b/drivers/platform/x86/lenovo/Kconfig
index f885127..09b1b055 100644
--- a/drivers/platform/x86/lenovo/Kconfig
+++ b/drivers/platform/x86/lenovo/Kconfig
@@ -252,7 +252,6 @@
select ACPI_PLATFORM_PROFILE
select LENOVO_WMI_EVENTS
select LENOVO_WMI_HELPERS
- select LENOVO_WMI_TUNING
help
Say Y here if you have a WMI aware Lenovo Legion device and would like to use the
platform-profile firmware interface to manage power usage.
diff --git a/drivers/platform/x86/lenovo/wmi-capdata.c b/drivers/platform/x86/lenovo/wmi-capdata.c
index b73d378..714aa6f 100644
--- a/drivers/platform/x86/lenovo/wmi-capdata.c
+++ b/drivers/platform/x86/lenovo/wmi-capdata.c
@@ -27,7 +27,6 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/acpi.h>
-#include <linux/bitfield.h>
#include <linux/bug.h>
#include <linux/cleanup.h>
#include <linux/component.h>
@@ -48,6 +47,7 @@
#include <linux/wmi.h>
#include "wmi-capdata.h"
+#include "wmi-helpers.h"
#define LENOVO_CAPABILITY_DATA_00_GUID "362A3AFE-3D96-4665-8530-96DAD5BB300E"
#define LENOVO_CAPABILITY_DATA_01_GUID "7A8F5407-CB67-4D6E-B547-39B3BE018154"
@@ -57,9 +57,9 @@
#define LWMI_FEATURE_ID_FAN_TEST 0x05
-#define LWMI_ATTR_ID_FAN_TEST \
- (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, LWMI_DEVICE_ID_FAN) | \
- FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, LWMI_FEATURE_ID_FAN_TEST))
+#define LWMI_ATTR_ID_FAN_TEST \
+ lwmi_attr_id(LWMI_DEVICE_ID_FAN, LWMI_FEATURE_ID_FAN_TEST, \
+ LWMI_GZ_THERMAL_MODE_NONE, LWMI_TYPE_ID_NONE)
enum lwmi_cd_type {
LENOVO_CAPABILITY_DATA_00,
diff --git a/drivers/platform/x86/lenovo/wmi-capdata.h b/drivers/platform/x86/lenovo/wmi-capdata.h
index 8c1df3e..c3e760b 100644
--- a/drivers/platform/x86/lenovo/wmi-capdata.h
+++ b/drivers/platform/x86/lenovo/wmi-capdata.h
@@ -6,6 +6,7 @@
#define _LENOVO_WMI_CAPDATA_H_
#include <linux/bits.h>
+#include <linux/bitfield.h>
#include <linux/types.h>
#define LWMI_SUPP_VALID BIT(0)
@@ -19,6 +20,8 @@
#define LWMI_DEVICE_ID_FAN 0x04
+#define LWMI_TYPE_ID_NONE 0x00
+
struct component_match;
struct device;
struct cd_list;
@@ -57,6 +60,23 @@ struct lwmi_cd_binder {
cd_list_cb_t cd_fan_list_cb;
};
+/**
+ * lwmi_attr_id() - Formats a capability data attribute ID
+ * @dev_id: The u8 corresponding to the device ID.
+ * @feat_id: The u8 corresponding to the feature ID on the device.
+ * @mode_id: The u8 corresponding to the wmi-gamezone mode for set/get.
+ * @type_id: The u8 corresponding to the sub-device.
+ *
+ * Return: encoded capability data attribute ID.
+ */
+static inline u32 lwmi_attr_id(u8 dev_id, u8 feat_id, u8 mode_id, u8 type_id)
+{
+ return (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, dev_id) |
+ FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, feat_id) |
+ FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode_id) |
+ FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, type_id));
+}
+
void lwmi_cd_match_add_all(struct device *master, struct component_match **matchptr);
int lwmi_cd00_get_data(struct cd_list *list, u32 attribute_id, struct capdata00 *output);
int lwmi_cd01_get_data(struct cd_list *list, u32 attribute_id, struct capdata01 *output);
diff --git a/drivers/platform/x86/lenovo/wmi-events.c b/drivers/platform/x86/lenovo/wmi-events.c
index 4a6a2c8..fc25bba 100644
--- a/drivers/platform/x86/lenovo/wmi-events.c
+++ b/drivers/platform/x86/lenovo/wmi-events.c
@@ -17,7 +17,7 @@
#include <linux/wmi.h>
#include "wmi-events.h"
-#include "wmi-gamezone.h"
+#include "wmi-helpers.h"
#define THERMAL_MODE_EVENT_GUID "D320289E-8FEA-41E0-86F9-911D83151B5F"
diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.c b/drivers/platform/x86/lenovo/wmi-gamezone.c
index c7fe7e3..109c0b5 100644
--- a/drivers/platform/x86/lenovo/wmi-gamezone.c
+++ b/drivers/platform/x86/lenovo/wmi-gamezone.c
@@ -21,9 +21,7 @@
#include <linux/wmi.h>
#include "wmi-events.h"
-#include "wmi-gamezone.h"
#include "wmi-helpers.h"
-#include "wmi-other.h"
#define LENOVO_GAMEZONE_GUID "887B54E3-DDDC-4B2C-8B88-68A26A8835D0"
@@ -201,7 +199,7 @@ static int lwmi_gz_profile_set(struct device *dev,
enum platform_profile_option profile)
{
struct lwmi_gz_priv *priv = dev_get_drvdata(dev);
- struct wmi_method_args_32 args;
+ struct wmi_method_args_32 args = {};
enum thermal_mode mode;
int ret;
@@ -383,7 +381,7 @@ static int lwmi_gz_probe(struct wmi_device *wdev, const void *context)
return ret;
priv->mode_nb.notifier_call = lwmi_gz_mode_call;
- return devm_lwmi_om_register_notifier(&wdev->dev, &priv->mode_nb);
+ return devm_lwmi_tm_register_notifier(&wdev->dev, &priv->mode_nb);
}
static const struct wmi_device_id lwmi_gz_id_table[] = {
@@ -405,7 +403,6 @@ module_wmi_driver(lwmi_gz_driver);
MODULE_IMPORT_NS("LENOVO_WMI_EVENTS");
MODULE_IMPORT_NS("LENOVO_WMI_HELPERS");
-MODULE_IMPORT_NS("LENOVO_WMI_OTHER");
MODULE_DEVICE_TABLE(wmi, lwmi_gz_id_table);
MODULE_AUTHOR("Derek J. Clark <derekjohn.clark@gmail.com>");
MODULE_DESCRIPTION("Lenovo GameZone WMI Driver");
diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.h b/drivers/platform/x86/lenovo/wmi-gamezone.h
deleted file mode 100644
index 6b163a5..0000000
--- a/drivers/platform/x86/lenovo/wmi-gamezone.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-
-/* Copyright (C) 2025 Derek J. Clark <derekjohn.clark@gmail.com> */
-
-#ifndef _LENOVO_WMI_GAMEZONE_H_
-#define _LENOVO_WMI_GAMEZONE_H_
-
-enum gamezone_events_type {
- LWMI_GZ_GET_THERMAL_MODE = 1,
-};
-
-enum thermal_mode {
- LWMI_GZ_THERMAL_MODE_QUIET = 0x01,
- LWMI_GZ_THERMAL_MODE_BALANCED = 0x02,
- LWMI_GZ_THERMAL_MODE_PERFORMANCE = 0x03,
- LWMI_GZ_THERMAL_MODE_EXTREME = 0xE0, /* Ver 6+ */
- LWMI_GZ_THERMAL_MODE_CUSTOM = 0xFF,
-};
-
-#endif /* !_LENOVO_WMI_GAMEZONE_H_ */
diff --git a/drivers/platform/x86/lenovo/wmi-helpers.c b/drivers/platform/x86/lenovo/wmi-helpers.c
index 7379def..7a19825 100644
--- a/drivers/platform/x86/lenovo/wmi-helpers.c
+++ b/drivers/platform/x86/lenovo/wmi-helpers.c
@@ -21,11 +21,15 @@
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/module.h>
+#include <linux/notifier.h>
#include <linux/unaligned.h>
#include <linux/wmi.h>
#include "wmi-helpers.h"
+/* Thermal mode notifier chain. */
+static BLOCKING_NOTIFIER_HEAD(tm_chain_head);
+
/**
* lwmi_dev_evaluate_int() - Helper function for calling WMI methods that
* return an integer.
@@ -46,7 +50,6 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id,
unsigned char *buf, size_t size, u32 *retval)
{
struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
- union acpi_object *ret_obj __free(kfree) = NULL;
struct acpi_buffer input = { size, buf };
acpi_status status;
@@ -55,8 +58,9 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id,
if (ACPI_FAILURE(status))
return -EIO;
+ union acpi_object *ret_obj __free(kfree) = output.pointer;
+
if (retval) {
- ret_obj = output.pointer;
if (!ret_obj)
return -ENODATA;
@@ -84,6 +88,103 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id,
};
EXPORT_SYMBOL_NS_GPL(lwmi_dev_evaluate_int, "LENOVO_WMI_HELPERS");
+/**
+ * lwmi_tm_register_notifier() - Add a notifier to the blocking notifier chain
+ * @nb: The notifier_block struct to register
+ *
+ * Call blocking_notifier_chain_register to register the notifier block to the
+ * thermal mode notifier chain.
+ *
+ * Return: 0 on success, %-EEXIST on error.
+ */
+int lwmi_tm_register_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&tm_chain_head, nb);
+}
+EXPORT_SYMBOL_NS_GPL(lwmi_tm_register_notifier, "LENOVO_WMI_HELPERS");
+
+/**
+ * lwmi_tm_unregister_notifier() - Remove a notifier from the blocking notifier
+ * chain.
+ * @nb: The notifier_block struct to register
+ *
+ * Call blocking_notifier_chain_unregister to unregister the notifier block from the
+ * thermal mode notifier chain.
+ *
+ * Return: 0 on success, %-ENOENT on error.
+ */
+int lwmi_tm_unregister_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&tm_chain_head, nb);
+}
+EXPORT_SYMBOL_NS_GPL(lwmi_tm_unregister_notifier, "LENOVO_WMI_HELPERS");
+
+/**
+ * devm_lwmi_tm_unregister_notifier() - Remove a notifier from the blocking
+ * notifier chain.
+ * @data: Void pointer to the notifier_block struct to register.
+ *
+ * Call lwmi_tm_unregister_notifier to unregister the notifier block from the
+ * thermal mode notifier chain.
+ *
+ * Return: 0 on success, %-ENOENT on error.
+ */
+static void devm_lwmi_tm_unregister_notifier(void *data)
+{
+ struct notifier_block *nb = data;
+
+ lwmi_tm_unregister_notifier(nb);
+}
+
+/**
+ * devm_lwmi_tm_register_notifier() - Add a notifier to the blocking notifier
+ * chain.
+ * @dev: The parent device of the notifier_block struct.
+ * @nb: The notifier_block struct to register
+ *
+ * Call lwmi_tm_register_notifier to register the notifier block to the
+ * thermal mode notifier chain. Then add devm_lwmi_tm_unregister_notifier
+ * as a device managed action to automatically unregister the notifier block
+ * upon parent device removal.
+ *
+ * Return: 0 on success, or an error code.
+ */
+int devm_lwmi_tm_register_notifier(struct device *dev,
+ struct notifier_block *nb)
+{
+ int ret;
+
+ ret = lwmi_tm_register_notifier(nb);
+ if (ret < 0)
+ return ret;
+
+ return devm_add_action_or_reset(dev, devm_lwmi_tm_unregister_notifier,
+ nb);
+}
+EXPORT_SYMBOL_NS_GPL(devm_lwmi_tm_register_notifier, "LENOVO_WMI_HELPERS");
+
+/**
+ * lwmi_tm_notifier_call() - Call functions for the notifier call chain.
+ * @mode: Pointer to a thermal mode enum to retrieve the data from.
+ *
+ * Call blocking_notifier_call_chain to retrieve the thermal mode from the
+ * lenovo-wmi-gamezone driver.
+ *
+ * Return: 0 on success, or an error code.
+ */
+int lwmi_tm_notifier_call(enum thermal_mode *mode)
+{
+ int ret;
+
+ ret = blocking_notifier_call_chain(&tm_chain_head,
+ LWMI_GZ_GET_THERMAL_MODE, &mode);
+ if ((ret & ~NOTIFY_STOP_MASK) != NOTIFY_OK)
+ return -EINVAL;
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(lwmi_tm_notifier_call, "LENOVO_WMI_HELPERS");
+
MODULE_AUTHOR("Derek J. Clark <derekjohn.clark@gmail.com>");
MODULE_DESCRIPTION("Lenovo WMI Helpers Driver");
MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/lenovo/wmi-helpers.h b/drivers/platform/x86/lenovo/wmi-helpers.h
index 20fd217..ed7db3e 100644
--- a/drivers/platform/x86/lenovo/wmi-helpers.h
+++ b/drivers/platform/x86/lenovo/wmi-helpers.h
@@ -7,6 +7,8 @@
#include <linux/types.h>
+struct device;
+struct notifier_block;
struct wmi_device;
struct wmi_method_args_32 {
@@ -14,7 +16,26 @@ struct wmi_method_args_32 {
u32 arg1;
};
+enum lwmi_event_type {
+ LWMI_GZ_GET_THERMAL_MODE = 0x01,
+};
+
+enum thermal_mode {
+ LWMI_GZ_THERMAL_MODE_NONE = 0x00,
+ LWMI_GZ_THERMAL_MODE_QUIET = 0x01,
+ LWMI_GZ_THERMAL_MODE_BALANCED = 0x02,
+ LWMI_GZ_THERMAL_MODE_PERFORMANCE = 0x03,
+ LWMI_GZ_THERMAL_MODE_EXTREME = 0xE0, /* Ver 6+ */
+ LWMI_GZ_THERMAL_MODE_CUSTOM = 0xFF,
+};
+
int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id,
unsigned char *buf, size_t size, u32 *retval);
+int lwmi_tm_register_notifier(struct notifier_block *nb);
+int lwmi_tm_unregister_notifier(struct notifier_block *nb);
+int devm_lwmi_tm_register_notifier(struct device *dev,
+ struct notifier_block *nb);
+int lwmi_tm_notifier_call(enum thermal_mode *mode);
+
#endif /* !_LENOVO_WMI_HELPERS_H_ */
diff --git a/drivers/platform/x86/lenovo/wmi-other.c b/drivers/platform/x86/lenovo/wmi-other.c
index 6040f45..d318ba4 100644
--- a/drivers/platform/x86/lenovo/wmi-other.c
+++ b/drivers/platform/x86/lenovo/wmi-other.c
@@ -40,16 +40,13 @@
#include <linux/kobject.h>
#include <linux/limits.h>
#include <linux/module.h>
-#include <linux/notifier.h>
#include <linux/platform_profile.h>
#include <linux/types.h>
#include <linux/wmi.h>
#include "wmi-capdata.h"
#include "wmi-events.h"
-#include "wmi-gamezone.h"
#include "wmi-helpers.h"
-#include "wmi-other.h"
#include "../firmware_attributes_class.h"
#define LENOVO_OTHER_MODE_GUID "DC2A8805-3A8C-41BA-A6F7-092E0089CD3B"
@@ -62,8 +59,6 @@
#define LWMI_FEATURE_ID_FAN_RPM 0x03
-#define LWMI_TYPE_ID_NONE 0x00
-
#define LWMI_FEATURE_VALUE_GET 17
#define LWMI_FEATURE_VALUE_SET 18
@@ -71,17 +66,15 @@
#define LWMI_FAN_NR 4
#define LWMI_FAN_ID(x) ((x) + LWMI_FAN_ID_BASE)
-#define LWMI_ATTR_ID_FAN_RPM(x) \
- (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, LWMI_DEVICE_ID_FAN) | \
- FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, LWMI_FEATURE_ID_FAN_RPM) | \
- FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, LWMI_FAN_ID(x)))
-
#define LWMI_FAN_DIV 100
+#define LWMI_ATTR_ID_FAN_RPM(x) \
+ lwmi_attr_id(LWMI_DEVICE_ID_FAN, LWMI_FEATURE_ID_FAN_RPM, \
+ LWMI_GZ_THERMAL_MODE_NONE, LWMI_FAN_ID(x))
+
#define LWMI_OM_FW_ATTR_BASE_PATH "lenovo-wmi-other"
#define LWMI_OM_HWMON_NAME "lenovo_wmi_other"
-static BLOCKING_NOTIFIER_HEAD(om_chain_head);
static DEFINE_IDA(lwmi_om_ida);
enum attribute_property {
@@ -109,7 +102,6 @@ struct lwmi_om_priv {
struct device *hwmon_dev;
struct device *fw_attr_dev;
struct kset *fw_attr_kset;
- struct notifier_block nb;
struct wmi_device *wdev;
int ida_id;
@@ -166,7 +158,7 @@ MODULE_PARM_DESC(relax_fan_constraint,
*/
static int lwmi_om_fan_get_set(struct lwmi_om_priv *priv, int channel, u32 *val, bool set)
{
- struct wmi_method_args_32 args;
+ struct wmi_method_args_32 args = {};
u32 method_id, retval;
int err;
@@ -349,6 +341,8 @@ static int lwmi_om_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
*/
if (!relax_fan_constraint)
raw = val / LWMI_FAN_DIV * LWMI_FAN_DIV;
+ else
+ raw = val;
err = lwmi_om_fan_get_set(priv, channel, &raw, true);
if (err)
@@ -546,13 +540,26 @@ static void lwmi_om_fan_info_collect_cd_fan(struct device *dev, struct cd_list *
/* ======== fw_attributes (component: lenovo-wmi-capdata 01) ======== */
struct tunable_attr_01 {
- struct capdata01 *capdata;
struct device *dev;
- u32 feature_id;
- u32 device_id;
- u32 type_id;
+ u8 feature_id;
+ u8 device_id;
+ u8 type_id;
+ u8 cd_mode_id; /* mode arg for searching capdata */
+ u8 cv_mode_id; /* mode arg for set/get current_value */
};
+/**
+ * tunable_attr_01_id() - Formats a tunable_attr_01 to a capdata attribute ID
+ * @attr: The tunable_attr_01 to format.
+ * @mode: The u8 corresponding to the wmi-gamezone mode for set/get.
+ *
+ * Return: encoded capability data attribute ID.
+ */
+static u32 tunable_attr_01_id(struct tunable_attr_01 *attr, u8 mode)
+{
+ return lwmi_attr_id(attr->device_id, attr->feature_id, mode, attr->type_id);
+}
+
static struct tunable_attr_01 ppt_pl1_spl = {
.device_id = LWMI_DEVICE_ID_CPU,
.feature_id = LWMI_FEATURE_ID_CPU_SPL,
@@ -576,102 +583,6 @@ struct capdata01_attr_group {
struct tunable_attr_01 *tunable_attr;
};
-/**
- * lwmi_om_register_notifier() - Add a notifier to the blocking notifier chain
- * @nb: The notifier_block struct to register
- *
- * Call blocking_notifier_chain_register to register the notifier block to the
- * lenovo-wmi-other driver notifier chain.
- *
- * Return: 0 on success, %-EEXIST on error.
- */
-int lwmi_om_register_notifier(struct notifier_block *nb)
-{
- return blocking_notifier_chain_register(&om_chain_head, nb);
-}
-EXPORT_SYMBOL_NS_GPL(lwmi_om_register_notifier, "LENOVO_WMI_OTHER");
-
-/**
- * lwmi_om_unregister_notifier() - Remove a notifier from the blocking notifier
- * chain.
- * @nb: The notifier_block struct to register
- *
- * Call blocking_notifier_chain_unregister to unregister the notifier block from the
- * lenovo-wmi-other driver notifier chain.
- *
- * Return: 0 on success, %-ENOENT on error.
- */
-int lwmi_om_unregister_notifier(struct notifier_block *nb)
-{
- return blocking_notifier_chain_unregister(&om_chain_head, nb);
-}
-EXPORT_SYMBOL_NS_GPL(lwmi_om_unregister_notifier, "LENOVO_WMI_OTHER");
-
-/**
- * devm_lwmi_om_unregister_notifier() - Remove a notifier from the blocking
- * notifier chain.
- * @data: Void pointer to the notifier_block struct to register.
- *
- * Call lwmi_om_unregister_notifier to unregister the notifier block from the
- * lenovo-wmi-other driver notifier chain.
- *
- * Return: 0 on success, %-ENOENT on error.
- */
-static void devm_lwmi_om_unregister_notifier(void *data)
-{
- struct notifier_block *nb = data;
-
- lwmi_om_unregister_notifier(nb);
-}
-
-/**
- * devm_lwmi_om_register_notifier() - Add a notifier to the blocking notifier
- * chain.
- * @dev: The parent device of the notifier_block struct.
- * @nb: The notifier_block struct to register
- *
- * Call lwmi_om_register_notifier to register the notifier block to the
- * lenovo-wmi-other driver notifier chain. Then add devm_lwmi_om_unregister_notifier
- * as a device managed action to automatically unregister the notifier block
- * upon parent device removal.
- *
- * Return: 0 on success, or an error code.
- */
-int devm_lwmi_om_register_notifier(struct device *dev,
- struct notifier_block *nb)
-{
- int ret;
-
- ret = lwmi_om_register_notifier(nb);
- if (ret < 0)
- return ret;
-
- return devm_add_action_or_reset(dev, devm_lwmi_om_unregister_notifier,
- nb);
-}
-EXPORT_SYMBOL_NS_GPL(devm_lwmi_om_register_notifier, "LENOVO_WMI_OTHER");
-
-/**
- * lwmi_om_notifier_call() - Call functions for the notifier call chain.
- * @mode: Pointer to a thermal mode enum to retrieve the data from.
- *
- * Call blocking_notifier_call_chain to retrieve the thermal mode from the
- * lenovo-wmi-gamezone driver.
- *
- * Return: 0 on success, or an error code.
- */
-static int lwmi_om_notifier_call(enum thermal_mode *mode)
-{
- int ret;
-
- ret = blocking_notifier_call_chain(&om_chain_head,
- LWMI_GZ_GET_THERMAL_MODE, &mode);
- if ((ret & ~NOTIFY_STOP_MASK) != NOTIFY_OK)
- return -EINVAL;
-
- return 0;
-}
-
/* Attribute Methods */
/**
@@ -716,12 +627,7 @@ static ssize_t attr_capdata01_show(struct kobject *kobj,
u32 attribute_id;
int value, ret;
- attribute_id =
- FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) |
- FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) |
- FIELD_PREP(LWMI_ATTR_MODE_ID_MASK,
- LWMI_GZ_THERMAL_MODE_CUSTOM) |
- FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id);
+ attribute_id = tunable_attr_01_id(tunable_attr, tunable_attr->cd_mode_id);
ret = lwmi_cd01_get_data(priv->cd01_list, attribute_id, &capdata);
if (ret)
@@ -773,27 +679,22 @@ static ssize_t attr_current_value_store(struct kobject *kobj,
struct tunable_attr_01 *tunable_attr)
{
struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev);
- struct wmi_method_args_32 args;
+ struct wmi_method_args_32 args = {};
struct capdata01 capdata;
enum thermal_mode mode;
- u32 attribute_id;
u32 value;
int ret;
- ret = lwmi_om_notifier_call(&mode);
+ ret = lwmi_tm_notifier_call(&mode);
if (ret)
return ret;
if (mode != LWMI_GZ_THERMAL_MODE_CUSTOM)
return -EBUSY;
- attribute_id =
- FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) |
- FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) |
- FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode) |
- FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id);
+ args.arg0 = tunable_attr_01_id(tunable_attr, tunable_attr->cd_mode_id);
- ret = lwmi_cd01_get_data(priv->cd01_list, attribute_id, &capdata);
+ ret = lwmi_cd01_get_data(priv->cd01_list, args.arg0, &capdata);
if (ret)
return ret;
@@ -804,7 +705,7 @@ static ssize_t attr_current_value_store(struct kobject *kobj,
if (value < capdata.min_value || value > capdata.max_value)
return -EINVAL;
- args.arg0 = attribute_id;
+ args.arg0 = tunable_attr_01_id(tunable_attr, tunable_attr->cv_mode_id);
args.arg1 = value;
ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_SET,
@@ -836,23 +737,20 @@ static ssize_t attr_current_value_show(struct kobject *kobj,
struct tunable_attr_01 *tunable_attr)
{
struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev);
- struct wmi_method_args_32 args;
+ struct wmi_method_args_32 args = {};
enum thermal_mode mode;
- u32 attribute_id;
int retval;
int ret;
- ret = lwmi_om_notifier_call(&mode);
+ ret = lwmi_tm_notifier_call(&mode);
if (ret)
return ret;
- attribute_id =
- FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) |
- FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) |
- FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode) |
- FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id);
+ /* If "no-mode" is the supported mode, ensure we never send current mode */
+ if (tunable_attr->cv_mode_id == LWMI_GZ_THERMAL_MODE_NONE)
+ mode = tunable_attr->cv_mode_id;
- args.arg0 = attribute_id;
+ args.arg0 = tunable_attr_01_id(tunable_attr, mode);
ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_GET,
(unsigned char *)&args, sizeof(args),
@@ -863,6 +761,81 @@ static ssize_t attr_current_value_show(struct kobject *kobj,
return sysfs_emit(buf, "%d\n", retval);
}
+/**
+ * lwmi_attr_01_is_supported() - Determine if the given attribute is supported.
+ * @tunable_attr: The attribute to verify.
+ *
+ * For an attribute to be supported it must have a functional get/set method,
+ * as well as associated capability data stored in the capdata01 table.
+ *
+ * First check if the attribute has a corresponding data table under custom mode
+ * (0xff), then under no mode (0x00). If either of those passes, check if the
+ * supported field of the capdata struct is > 0. If it is supported, store the
+ * successful mode in the cd_mode_id field of tunable_attr.
+ *
+ * If the attribute capdata shows it is supported, attempt to determine the mode
+ * for the current value property get/set methods using a similar pattern to the
+ * capdata table check. If the value returned by either mode is 0 or an error,
+ * assume that mode is not supported. Otherwise, store the successful mode in the
+ * cv_mode_id field of tunable_attr.
+ *
+ * If any of the above checks fail then the attribute is not fully supported.
+ *
+ * Return: true if capdata and set/get modes are found, otherwise false.
+ */
+static bool lwmi_attr_01_is_supported(struct tunable_attr_01 *tunable_attr)
+{
+ u8 modes[2] = { LWMI_GZ_THERMAL_MODE_CUSTOM, LWMI_GZ_THERMAL_MODE_NONE };
+ struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev);
+ struct wmi_method_args_32 args = {};
+ bool cd_mode_found = false;
+ bool cv_mode_found = false;
+ struct capdata01 capdata;
+ int retval, ret, i;
+
+ /* Determine tunable_attr->cd_mode_id */
+ for (i = 0; i < ARRAY_SIZE(modes); i++) {
+ args.arg0 = tunable_attr_01_id(tunable_attr, modes[i]);
+
+ ret = lwmi_cd01_get_data(priv->cd01_list, args.arg0, &capdata);
+ if (ret || !capdata.supported)
+ continue;
+
+ tunable_attr->cd_mode_id = modes[i];
+ cd_mode_found = true;
+ break;
+ }
+
+ if (!cd_mode_found)
+ return cd_mode_found;
+
+ dev_dbg(tunable_attr->dev,
+ "cd_mode_id: %#010x\n", args.arg0);
+
+ /* Determine tunable_attr->cv_mode_id, returns 1 if supported */
+ for (i = 0; i < ARRAY_SIZE(modes); i++) {
+ args.arg0 = tunable_attr_01_id(tunable_attr, modes[i]);
+
+ ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_GET,
+ (u8 *)&args, sizeof(args),
+ &retval);
+ if (ret || !retval)
+ continue;
+
+ tunable_attr->cv_mode_id = modes[i];
+ cv_mode_found = true;
+ break;
+ }
+
+ if (!cv_mode_found)
+ return cv_mode_found;
+
+ dev_dbg(tunable_attr->dev, "cv_mode_id: %#010x, attribute support level: %#010x\n",
+ args.arg0, capdata.supported);
+
+ return capdata.supported > 0;
+}
+
/* Lenovo WMI Other Mode Attribute macros */
#define __LWMI_ATTR_RO(_func, _name) \
{ \
@@ -957,17 +930,17 @@ static struct capdata01_attr_group cd01_attr_groups[] = {
/**
* lwmi_om_fw_attr_add() - Register all firmware_attributes_class members
* @priv: The Other Mode driver data.
- *
- * Return: Either 0, or an error code.
*/
-static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv)
+static void lwmi_om_fw_attr_add(struct lwmi_om_priv *priv)
{
unsigned int i;
int err;
- priv->ida_id = ida_alloc(&lwmi_om_ida, GFP_KERNEL);
- if (priv->ida_id < 0)
- return priv->ida_id;
+ err = ida_alloc(&lwmi_om_ida, GFP_KERNEL);
+ if (err < 0)
+ goto err_no_ida;
+
+ priv->ida_id = err;
priv->fw_attr_dev = device_create(&firmware_attributes_class, NULL,
MKDEV(0, 0), NULL, "%s-%u",
@@ -986,14 +959,16 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv)
}
for (i = 0; i < ARRAY_SIZE(cd01_attr_groups) - 1; i++) {
+ cd01_attr_groups[i].tunable_attr->dev = &priv->wdev->dev;
+ if (!lwmi_attr_01_is_supported(cd01_attr_groups[i].tunable_attr))
+ continue;
+
err = sysfs_create_group(&priv->fw_attr_kset->kobj,
cd01_attr_groups[i].attr_group);
if (err)
goto err_remove_groups;
-
- cd01_attr_groups[i].tunable_attr->dev = &priv->wdev->dev;
}
- return 0;
+ return;
err_remove_groups:
while (i--)
@@ -1007,7 +982,12 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv)
err_free_ida:
ida_free(&lwmi_om_ida, priv->ida_id);
- return err;
+
+err_no_ida:
+ priv->ida_id = -EIDRM;
+
+ dev_warn(&priv->wdev->dev,
+ "failed to register firmware-attributes device: %d\n", err);
}
/**
@@ -1016,12 +996,17 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv)
*/
static void lwmi_om_fw_attr_remove(struct lwmi_om_priv *priv)
{
+ if (priv->ida_id < 0)
+ return;
+
for (unsigned int i = 0; i < ARRAY_SIZE(cd01_attr_groups) - 1; i++)
sysfs_remove_group(&priv->fw_attr_kset->kobj,
cd01_attr_groups[i].attr_group);
kset_unregister(priv->fw_attr_kset);
device_unregister(priv->fw_attr_dev);
+ ida_free(&lwmi_om_ida, priv->ida_id);
+ priv->ida_id = -EIDRM;
}
/* ======== Self (master: lenovo-wmi-other) ======== */
@@ -1058,12 +1043,17 @@ static int lwmi_om_master_bind(struct device *dev)
priv->cd00_list = binder.cd00_list;
priv->cd01_list = binder.cd01_list;
- if (!priv->cd00_list || !priv->cd01_list)
+ if (!priv->cd00_list || !priv->cd01_list) {
+ component_unbind_all(dev, NULL);
+
return -ENODEV;
+ }
lwmi_om_fan_info_collect_cd00(priv);
- return lwmi_om_fw_attr_add(priv);
+ lwmi_om_fw_attr_add(priv);
+
+ return 0;
}
/**
@@ -1115,13 +1105,7 @@ static int lwmi_other_probe(struct wmi_device *wdev, const void *context)
static void lwmi_other_remove(struct wmi_device *wdev)
{
- struct lwmi_om_priv *priv = dev_get_drvdata(&wdev->dev);
-
component_master_del(&wdev->dev, &lwmi_om_master_ops);
-
- /* No IDA to free if the driver is never bound to its components. */
- if (priv->ida_id >= 0)
- ida_free(&lwmi_om_ida, priv->ida_id);
}
static const struct wmi_device_id lwmi_other_id_table[] = {
diff --git a/drivers/platform/x86/lenovo/wmi-other.h b/drivers/platform/x86/lenovo/wmi-other.h
deleted file mode 100644
index 8ebf560..0000000
--- a/drivers/platform/x86/lenovo/wmi-other.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-
-/* Copyright (C) 2025 Derek J. Clark <derekjohn.clark@gmail.com> */
-
-#ifndef _LENOVO_WMI_OTHER_H_
-#define _LENOVO_WMI_OTHER_H_
-
-struct device;
-struct notifier_block;
-
-int lwmi_om_register_notifier(struct notifier_block *nb);
-int lwmi_om_unregister_notifier(struct notifier_block *nb);
-int devm_lwmi_om_register_notifier(struct device *dev,
- struct notifier_block *nb);
-
-#endif /* !_LENOVO_WMI_OTHER_H_ */
diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c
index 9681412..a8f2f46 100644
--- a/drivers/platform/x86/lg-laptop.c
+++ b/drivers/platform/x86/lg-laptop.c
@@ -761,12 +761,11 @@ static void lg_laptop_remove_address_space_handler(void *data)
static int acpi_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct platform_device_info pdev_info = {
- .fwnode = acpi_fwnode_handle(device),
.name = PLATFORM_NAME,
.id = PLATFORM_DEVID_NONE,
};
+ struct acpi_device *device;
acpi_status status;
int ret;
const char *product;
@@ -775,6 +774,12 @@ static int acpi_probe(struct platform_device *pdev)
if (pf_device)
return 0;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
+ pdev_info.fwnode = acpi_fwnode_handle(device),
+
status = acpi_install_address_space_handler(device->handle, LG_ADDRESS_SPACE_ID,
&lg_laptop_address_space_handler,
NULL, &pdev->dev);
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index 1337f7c..b83113c 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -981,11 +981,15 @@ static int acpi_pcc_hotkey_resume(struct device *dev)
static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct backlight_properties props;
+ struct acpi_device *device;
struct pcc_acpi *pcc;
int num_sifr, result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
num_sifr = acpi_pcc_get_sqty(device);
/*
diff --git a/drivers/platform/x86/samsung-galaxybook.c b/drivers/platform/x86/samsung-galaxybook.c
index 755cb82..6382af0 100644
--- a/drivers/platform/x86/samsung-galaxybook.c
+++ b/drivers/platform/x86/samsung-galaxybook.c
@@ -53,7 +53,7 @@ struct samsung_galaxybook {
void *i8042_filter_ptr;
struct work_struct block_recording_hotkey_work;
- struct input_dev *camera_lens_cover_switch;
+ struct input_dev *input;
struct acpi_battery_hook battery_hook;
@@ -197,6 +197,9 @@ static const guid_t performance_mode_guid =
#define GB_ACPI_NOTIFY_DEVICE_ON_TABLE 0x6c
#define GB_ACPI_NOTIFY_DEVICE_OFF_TABLE 0x6d
#define GB_ACPI_NOTIFY_HOTKEY_PERFORMANCE_MODE 0x70
+#define GB_ACPI_NOTIFY_HOTKEY_KBD_BACKLIGHT 0x7d
+#define GB_ACPI_NOTIFY_HOTKEY_MICMUTE 0x6e
+#define GB_ACPI_NOTIFY_HOTKEY_CAMERA 0x6f
#define GB_KEY_KBD_BACKLIGHT_KEYDOWN 0x2c
#define GB_KEY_KBD_BACKLIGHT_KEYUP 0xac
@@ -859,13 +862,29 @@ static int block_recording_acpi_set(struct samsung_galaxybook *galaxybook, const
if (err)
return err;
- input_report_switch(galaxybook->camera_lens_cover_switch,
+ input_report_switch(galaxybook->input,
SW_CAMERA_LENS_COVER, value ? 1 : 0);
- input_sync(galaxybook->camera_lens_cover_switch);
+ input_sync(galaxybook->input);
return 0;
}
+static int galaxybook_input_init(struct samsung_galaxybook *galaxybook)
+{
+ galaxybook->input = devm_input_allocate_device(&galaxybook->platform->dev);
+ if (!galaxybook->input)
+ return -ENOMEM;
+
+ galaxybook->input->name = "Samsung Galaxy Book Camera Lens Cover";
+ galaxybook->input->phys = DRIVER_NAME "/input0";
+ galaxybook->input->id.bustype = BUS_HOST;
+
+ input_set_capability(galaxybook->input, EV_KEY, KEY_MICMUTE);
+ input_set_capability(galaxybook->input, EV_SW, SW_CAMERA_LENS_COVER);
+
+ return input_register_device(galaxybook->input);
+}
+
static int galaxybook_block_recording_init(struct samsung_galaxybook *galaxybook)
{
bool value;
@@ -887,24 +906,8 @@ static int galaxybook_block_recording_init(struct samsung_galaxybook *galaxybook
return GB_NOT_SUPPORTED;
}
- galaxybook->camera_lens_cover_switch =
- devm_input_allocate_device(&galaxybook->platform->dev);
- if (!galaxybook->camera_lens_cover_switch)
- return -ENOMEM;
-
- galaxybook->camera_lens_cover_switch->name = "Samsung Galaxy Book Camera Lens Cover";
- galaxybook->camera_lens_cover_switch->phys = DRIVER_NAME "/input0";
- galaxybook->camera_lens_cover_switch->id.bustype = BUS_HOST;
-
- input_set_capability(galaxybook->camera_lens_cover_switch, EV_SW, SW_CAMERA_LENS_COVER);
-
- err = input_register_device(galaxybook->camera_lens_cover_switch);
- if (err)
- return err;
-
- input_report_switch(galaxybook->camera_lens_cover_switch,
- SW_CAMERA_LENS_COVER, value ? 1 : 0);
- input_sync(galaxybook->camera_lens_cover_switch);
+ input_report_switch(galaxybook->input, SW_CAMERA_LENS_COVER, value ? 1 : 0);
+ input_sync(galaxybook->input);
return 0;
}
@@ -1260,6 +1263,25 @@ static void galaxybook_acpi_notify(acpi_handle handle, u32 event, void *data)
if (galaxybook->has_performance_mode)
platform_profile_cycle();
break;
+ case GB_ACPI_NOTIFY_HOTKEY_KBD_BACKLIGHT:
+ if (galaxybook->has_kbd_backlight)
+ schedule_work(&galaxybook->kbd_backlight_hotkey_work);
+ break;
+ case GB_ACPI_NOTIFY_HOTKEY_MICMUTE:
+ input_report_key(galaxybook->input, KEY_MICMUTE, 1);
+ input_sync(galaxybook->input);
+ input_report_key(galaxybook->input, KEY_MICMUTE, 0);
+ input_sync(galaxybook->input);
+ break;
+ case GB_ACPI_NOTIFY_HOTKEY_CAMERA:
+ if (galaxybook->has_block_recording) {
+ schedule_work(&galaxybook->block_recording_hotkey_work);
+ } else {
+ input_report_switch(galaxybook->input, SW_CAMERA_LENS_COVER,
+ !test_bit(SW_CAMERA_LENS_COVER, galaxybook->input->sw));
+ input_sync(galaxybook->input);
+ }
+ break;
default:
dev_warn(&galaxybook->platform->dev,
"unknown ACPI notification event: 0x%x\n", event);
@@ -1392,6 +1414,11 @@ static int galaxybook_probe(struct platform_device *pdev)
return dev_err_probe(&galaxybook->platform->dev, err,
"failed to initialize kbd_backlight\n");
+ err = galaxybook_input_init(galaxybook);
+ if (err)
+ return dev_err_probe(&galaxybook->platform->dev, err,
+ "failed to initialize input device\n");
+
err = galaxybook_fw_attrs_init(galaxybook);
if (err)
return dev_err_probe(&galaxybook->platform->dev, err,
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index b18f00e..6737096 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -3147,11 +3147,15 @@ static void sony_nc_backlight_cleanup(void)
static int sony_nc_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
acpi_status status;
int result = 0;
struct sony_nc_value *item;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
sony_nc_acpi_device = device;
strscpy(acpi_device_class(device), "sony/hotkey");
@@ -4509,11 +4513,15 @@ static void sony_pic_remove(struct platform_device *pdev)
static int sony_pic_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct sony_pic_ioport *io, *tmp_io;
struct sony_pic_irq *irq, *tmp_irq;
+ struct acpi_device *device;
int result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
spic_dev.acpi_dev = device;
strscpy(acpi_device_class(device), "sony/hotkey");
sony_pic_detect_device_type(&spic_dev);
diff --git a/drivers/platform/x86/system76_acpi.c b/drivers/platform/x86/system76_acpi.c
index 693cbb4..dd7b1b0 100644
--- a/drivers/platform/x86/system76_acpi.c
+++ b/drivers/platform/x86/system76_acpi.c
@@ -674,10 +674,14 @@ static void system76_notify(acpi_handle handle, u32 event, void *context)
// Probe a System76 platform device
static int system76_probe(struct platform_device *pdev)
{
- struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *acpi_dev;
struct system76_data *data;
int err;
+ acpi_dev = ACPI_COMPANION(&pdev->dev);
+ if (!acpi_dev)
+ return -ENODEV;
+
data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index 35d899c..7cecb3a 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -3374,7 +3374,7 @@ static const struct dmi_system_id toshiba_dmi_quirks[] __initconst = {
static int toshiba_acpi_probe(struct platform_device *pdev)
{
- struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *acpi_dev;
struct toshiba_acpi_dev *dev;
const char *hci_method;
u32 dummy;
@@ -3383,6 +3383,10 @@ static int toshiba_acpi_probe(struct platform_device *pdev)
if (toshiba_acpi)
return -EBUSY;
+ acpi_dev = ACPI_COMPANION(&pdev->dev);
+ if (!acpi_dev)
+ return -ENODEV;
+
pr_info("Toshiba Laptop ACPI Extras version %s\n",
TOSHIBA_ACPI_VERSION);
diff --git a/drivers/platform/x86/toshiba_bluetooth.c b/drivers/platform/x86/toshiba_bluetooth.c
index e50d4fc..e00abba 100644
--- a/drivers/platform/x86/toshiba_bluetooth.c
+++ b/drivers/platform/x86/toshiba_bluetooth.c
@@ -230,10 +230,14 @@ static int toshiba_bt_resume(struct device *dev)
static int toshiba_bt_rfkill_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
struct toshiba_bluetooth_dev *bt_dev;
+ struct acpi_device *device;
int result;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
result = toshiba_bluetooth_present(device->handle);
if (result)
return result;
diff --git a/drivers/platform/x86/toshiba_haps.c b/drivers/platform/x86/toshiba_haps.c
index 1486252..8d12241 100644
--- a/drivers/platform/x86/toshiba_haps.c
+++ b/drivers/platform/x86/toshiba_haps.c
@@ -182,13 +182,17 @@ static int toshiba_haps_available(acpi_handle handle)
static int toshiba_haps_probe(struct platform_device *pdev)
{
- struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
struct toshiba_haps_dev *haps;
+ struct acpi_device *acpi_dev;
int ret;
if (toshiba_haps)
return -EBUSY;
+ acpi_dev = ACPI_COMPANION(&pdev->dev);
+ if (!acpi_dev)
+ return -ENODEV;
+
if (!toshiba_haps_available(acpi_dev->handle))
return -ENODEV;
diff --git a/drivers/platform/x86/uniwill/uniwill-acpi.c b/drivers/platform/x86/uniwill/uniwill-acpi.c
index 945df50..8cc01be 100644
--- a/drivers/platform/x86/uniwill/uniwill-acpi.c
+++ b/drivers/platform/x86/uniwill/uniwill-acpi.c
@@ -1359,6 +1359,16 @@ static int uniwill_led_init(struct uniwill_data *data)
&init_data);
}
+static unsigned int uniwill_sanitize_battery_threshold(unsigned int value)
+{
+ /* 0 means "charging threshold not active" */
+ if (!value)
+ return 100;
+
+ /* Guard against invalid values */
+ return min(value, 100);
+}
+
static int uniwill_get_property(struct power_supply *psy, const struct power_supply_ext *ext,
void *drvdata, enum power_supply_property psp,
union power_supply_propval *val)
@@ -1405,7 +1415,8 @@ static int uniwill_get_property(struct power_supply *psy, const struct power_sup
if (ret < 0)
return ret;
- val->intval = clamp_val(FIELD_GET(CHARGE_CTRL_MASK, regval), 0, 100);
+ regval = FIELD_GET(CHARGE_CTRL_MASK, regval);
+ val->intval = uniwill_sanitize_battery_threshold(regval);
return 0;
default:
return -EINVAL;
@@ -1420,11 +1431,11 @@ static int uniwill_set_property(struct power_supply *psy, const struct power_sup
switch (psp) {
case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD:
- if (val->intval < 1 || val->intval > 100)
+ if (val->intval < 0 || val->intval > 100)
return -EINVAL;
return regmap_update_bits(data->regmap, EC_ADDR_CHARGE_CTRL, CHARGE_CTRL_MASK,
- val->intval);
+ max(val->intval, 1));
default:
return -EINVAL;
}
@@ -1500,11 +1511,33 @@ static int uniwill_remove_battery(struct power_supply *battery, struct acpi_batt
static int uniwill_battery_init(struct uniwill_data *data)
{
+ unsigned int value, threshold, sanitized;
int ret;
if (!uniwill_device_supports(data, UNIWILL_FEATURE_BATTERY))
return 0;
+ ret = regmap_read(data->regmap, EC_ADDR_CHARGE_CTRL, &value);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * The charge control threshold might be initialized with 0 by
+ * the EC to signal that said threshold is uninitialized. We thus
+ * need to replace this placeholder value with a valid one (100)
+ * to signal that we want to take control of battery charging.
+ * For the sake of completeness we also apply this to other
+ * invalid threshold values.
+ */
+ threshold = FIELD_GET(CHARGE_CTRL_MASK, value);
+ sanitized = uniwill_sanitize_battery_threshold(threshold);
+ if (threshold != sanitized) {
+ FIELD_MODIFY(CHARGE_CTRL_MASK, &value, sanitized);
+ ret = regmap_write(data->regmap, EC_ADDR_CHARGE_CTRL, value);
+ if (ret < 0)
+ return ret;
+ }
+
ret = devm_mutex_init(data->dev, &data->battery_lock);
if (ret < 0)
return ret;
@@ -2456,8 +2489,6 @@ static int __init uniwill_init(void)
if (!force)
return -ENODEV;
- /* Assume that the device supports all features */
- device_descriptor.features = UINT_MAX;
pr_warn("Loading on a potentially unsupported device\n");
} else {
/*
@@ -2475,6 +2506,12 @@ static int __init uniwill_init(void)
device_descriptor = *descriptor;
}
+ if (force) {
+ /* Assume that the device supports all features except the charge limit */
+ device_descriptor.features = UINT_MAX & ~UNIWILL_FEATURE_BATTERY;
+ pr_warn("Enabling potentially unsupported features\n");
+ }
+
ret = platform_driver_register(&uniwill_driver);
if (ret < 0)
return ret;
diff --git a/drivers/platform/x86/wireless-hotkey.c b/drivers/platform/x86/wireless-hotkey.c
index f680d8f..3151844 100644
--- a/drivers/platform/x86/wireless-hotkey.c
+++ b/drivers/platform/x86/wireless-hotkey.c
@@ -89,9 +89,14 @@ static void wl_notify(acpi_handle handle, u32 event, void *data)
static int wl_probe(struct platform_device *pdev)
{
+ struct acpi_device *adev;
struct wl_button *button;
int err;
+ adev = ACPI_COMPANION(&pdev->dev);
+ if (!adev)
+ return -ENODEV;
+
button = kzalloc_obj(struct wl_button);
if (!button)
return -ENOMEM;
@@ -104,8 +109,8 @@ static int wl_probe(struct platform_device *pdev)
kfree(button);
return err;
}
- err = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
- ACPI_DEVICE_NOTIFY, wl_notify, button);
+ err = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
+ wl_notify, button);
if (err) {
pr_err("Failed to install ACPI notify handler\n");
wireless_input_destroy(&pdev->dev);
diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c
index 4d32fc6..71e930e 100644
--- a/drivers/pmdomain/core.c
+++ b/drivers/pmdomain/core.c
@@ -3089,6 +3089,7 @@ static const struct bus_type genpd_bus_type = {
static void genpd_dev_pm_detach(struct device *dev, bool power_off)
{
struct generic_pm_domain *pd;
+ bool is_virt_dev;
unsigned int i;
int ret = 0;
@@ -3098,6 +3099,13 @@ static void genpd_dev_pm_detach(struct device *dev, bool power_off)
dev_dbg(dev, "removing from PM domain %s\n", pd->name);
+ /* Check if the device was created by genpd at attach. */
+ is_virt_dev = dev->bus == &genpd_bus_type;
+
+ /* Disable runtime PM if we enabled it at attach. */
+ if (is_virt_dev)
+ pm_runtime_disable(dev);
+
/* Drop the default performance state */
if (dev_gpd_data(dev)->default_pstate) {
dev_pm_genpd_set_performance_state(dev, 0);
@@ -3123,7 +3131,7 @@ static void genpd_dev_pm_detach(struct device *dev, bool power_off)
genpd_queue_power_off_work(pd);
/* Unregister the device if it was created by genpd. */
- if (dev->bus == &genpd_bus_type)
+ if (is_virt_dev)
device_unregister(dev);
}
diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c
index d2b8d03..e1cfd42 100644
--- a/drivers/pmdomain/mediatek/mtk-pm-domains.c
+++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c
@@ -1015,6 +1015,7 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s
struct device_node *node, *smi_np;
int num_regmaps = 0, i, j;
struct regmap *regmap[3];
+ int ret = 0;
/*
* Legacy code retrieves a maximum of three bus protection handles:
@@ -1065,11 +1066,14 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s
if (node) {
regmap[2] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg-nao");
num_regmaps++;
- of_node_put(node);
- if (IS_ERR(regmap[2]))
- return dev_err_probe(dev, PTR_ERR(regmap[2]),
+ if (IS_ERR(regmap[2])) {
+ ret = dev_err_probe(dev, PTR_ERR(regmap[2]),
"%pOF: failed to get infracfg regmap\n",
node);
+ of_node_put(node);
+ return ret;
+ }
+ of_node_put(node);
} else {
regmap[2] = NULL;
}
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index c49e0e4..1b0239c 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -881,26 +881,22 @@ static bool cm_setup_timer(void)
mutex_unlock(&cm_list_mtx);
if (timer_req && cm_timer) {
- ktime_t now, add;
-
/*
* Set alarm with the polling interval (wakeup_ms)
* The alarm time should be NOW + CM_RTC_SMALL or later.
*/
- if (wakeup_ms == UINT_MAX ||
- wakeup_ms < CM_RTC_SMALL * MSEC_PER_SEC)
+ if (wakeup_ms == UINT_MAX || wakeup_ms < CM_RTC_SMALL * MSEC_PER_SEC)
wakeup_ms = 2 * CM_RTC_SMALL * MSEC_PER_SEC;
pr_info("Charger Manager wakeup timer: %u ms\n", wakeup_ms);
- now = ktime_get_boottime();
- add = ktime_set(wakeup_ms / MSEC_PER_SEC,
- (wakeup_ms % MSEC_PER_SEC) * NSEC_PER_MSEC);
- alarm_start(cm_timer, ktime_add(now, add));
-
cm_suspend_duration_ms = wakeup_ms;
- return true;
+ /*
+ * The timer should always be queued as the timeout is at least
+ * two seconds out. Handle it correctly nevertheless.
+ */
+ return alarm_start_timer(cm_timer, ktime_add_ms(0, wakeup_ms), true);
}
return false;
}
diff --git a/drivers/pps/generators/pps_gen-dummy.c b/drivers/pps/generators/pps_gen-dummy.c
index 547fa7f..a439554 100644
--- a/drivers/pps/generators/pps_gen-dummy.c
+++ b/drivers/pps/generators/pps_gen-dummy.c
@@ -39,11 +39,7 @@ static void pps_gen_ktimer_event(struct timer_list *unused)
static int pps_gen_dummy_get_time(struct pps_gen_device *pps_gen,
struct timespec64 *time)
{
- struct system_time_snapshot snap;
-
- ktime_get_snapshot(&snap);
- *time = ktime_to_timespec64(snap.real);
-
+ ktime_get_real_ts64(time);
return 0;
}
diff --git a/drivers/pps/generators/pps_gen_tio.c b/drivers/pps/generators/pps_gen_tio.c
index de00a85..9483d12 100644
--- a/drivers/pps/generators/pps_gen_tio.c
+++ b/drivers/pps/generators/pps_gen_tio.c
@@ -189,11 +189,7 @@ static int pps_tio_gen_enable(struct pps_gen_device *pps_gen, bool enable)
static int pps_tio_get_time(struct pps_gen_device *pps_gen,
struct timespec64 *time)
{
- struct system_time_snapshot snap;
-
- ktime_get_snapshot(&snap);
- *time = ktime_to_timespec64(snap.real);
-
+ ktime_get_real_ts64(time);
return 0;
}
diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c
index c61cf9e..dc23cd7 100644
--- a/drivers/ptp/ptp_chardev.c
+++ b/drivers/ptp/ptp_chardev.c
@@ -317,8 +317,8 @@ typedef int (*ptp_crosststamp_fn)(struct ptp_clock_info *,
static long ptp_sys_offset_precise(struct ptp_clock *ptp, void __user *arg,
ptp_crosststamp_fn crosststamp_fn)
{
+ struct system_device_crosststamp xtstamp = { .clock_id = CLOCK_REALTIME };
struct ptp_sys_offset_precise precise_offset;
- struct system_device_crosststamp xtstamp;
struct timespec64 ts;
int err;
@@ -333,7 +333,7 @@ static long ptp_sys_offset_precise(struct ptp_clock *ptp, void __user *arg,
ts = ktime_to_timespec64(xtstamp.device);
precise_offset.device.sec = ts.tv_sec;
precise_offset.device.nsec = ts.tv_nsec;
- ts = ktime_to_timespec64(xtstamp.sys_realtime);
+ ts = ktime_to_timespec64(xtstamp.sys_systime);
precise_offset.sys_realtime.sec = ts.tv_sec;
precise_offset.sys_realtime.nsec = ts.tv_nsec;
ts = ktime_to_timespec64(xtstamp.sys_monoraw);
@@ -386,15 +386,19 @@ static long ptp_sys_offset_extended(struct ptp_clock *ptp, void __user *arg,
return err;
/* Filter out disabled or unavailable clocks */
- if (sts.pre_ts.tv_sec < 0 || sts.post_ts.tv_sec < 0)
+ if (!sts.pre_sts.valid || !sts.post_sts.valid)
return -EINVAL;
- extoff->ts[i][0].sec = sts.pre_ts.tv_sec;
- extoff->ts[i][0].nsec = sts.pre_ts.tv_nsec;
extoff->ts[i][1].sec = ts.tv_sec;
extoff->ts[i][1].nsec = ts.tv_nsec;
- extoff->ts[i][2].sec = sts.post_ts.tv_sec;
- extoff->ts[i][2].nsec = sts.post_ts.tv_nsec;
+
+ ts = ktime_to_timespec64(sts.pre_sts.systime);
+ extoff->ts[i][0].sec = ts.tv_sec;
+ extoff->ts[i][0].nsec = ts.tv_nsec;
+
+ ts = ktime_to_timespec64(sts.post_sts.systime);
+ extoff->ts[i][2].sec = ts.tv_sec;
+ extoff->ts[i][2].nsec = ts.tv_nsec;
}
return copy_to_user(arg, extoff, sizeof(*extoff)) ? -EFAULT : 0;
diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
index beacc2f..28b0302 100644
--- a/drivers/ptp/ptp_ocp.c
+++ b/drivers/ptp/ptp_ocp.c
@@ -1491,11 +1491,8 @@ __ptp_ocp_gettime_locked(struct ptp_ocp *bp, struct timespec64 *ts,
}
ptp_read_system_postts(sts);
- if (sts && bp->ts_window_adjust) {
- s64 ns = timespec64_to_ns(&sts->post_ts);
-
- sts->post_ts = ns_to_timespec64(ns - bp->ts_window_adjust);
- }
+ if (sts && bp->ts_window_adjust)
+ sts->post_sts.systime -= bp->ts_window_adjust;
time_ns = ioread32(&bp->reg->time_ns);
time_sec = ioread32(&bp->reg->time_sec);
@@ -4595,8 +4592,8 @@ ptp_ocp_summary_show(struct seq_file *s, void *data)
struct timespec64 sys_ts;
s64 pre_ns, post_ns, ns;
- pre_ns = timespec64_to_ns(&sts.pre_ts);
- post_ns = timespec64_to_ns(&sts.post_ts);
+ pre_ns = ktime_to_ns(sts.pre_sts.systime);
+ post_ns = ktime_to_ns(sts.post_sts.systime);
ns = (pre_ns + post_ns) / 2;
ns += (s64)bp->utc_tai_offset * NSEC_PER_SEC;
sys_ts = ns_to_timespec64(ns);
diff --git a/drivers/ptp/ptp_vclock.c b/drivers/ptp/ptp_vclock.c
index 915a4f6..84cb527 100644
--- a/drivers/ptp/ptp_vclock.c
+++ b/drivers/ptp/ptp_vclock.c
@@ -19,6 +19,8 @@ static DEFINE_SPINLOCK(vclock_hash_lock);
static DEFINE_READ_MOSTLY_HASHTABLE(vclock_hash, 8);
+DEFINE_STATIC_SRCU(vclock_srcu);
+
static void ptp_vclock_hash_add(struct ptp_vclock *vclock)
{
spin_lock(&vclock_hash_lock);
@@ -37,7 +39,7 @@ static void ptp_vclock_hash_del(struct ptp_vclock *vclock)
spin_unlock(&vclock_hash_lock);
- synchronize_rcu();
+ synchronize_srcu(&vclock_srcu);
}
static int ptp_vclock_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
@@ -276,14 +278,16 @@ ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp, int vclock_index)
{
unsigned int hash = vclock_index % HASH_SIZE(vclock_hash);
struct ptp_vclock *vclock;
- u64 ns;
u64 vclock_ns = 0;
+ int srcu_idx;
+ u64 ns;
ns = ktime_to_ns(*hwtstamp);
- rcu_read_lock();
+ srcu_idx = srcu_read_lock(&vclock_srcu);
- hlist_for_each_entry_rcu(vclock, &vclock_hash[hash], vclock_hash_node) {
+ hlist_for_each_entry_srcu(vclock, &vclock_hash[hash], vclock_hash_node,
+ srcu_read_lock_held(&vclock_srcu)) {
if (vclock->clock->index != vclock_index)
continue;
@@ -294,7 +298,7 @@ ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp, int vclock_index)
break;
}
- rcu_read_unlock();
+ srcu_read_unlock(&vclock_srcu, srcu_idx);
return ns_to_ktime(vclock_ns);
}
diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c
index 8b630eb..eebdcd5 100644
--- a/drivers/ptp/ptp_vmclock.c
+++ b/drivers/ptp/ptp_vmclock.c
@@ -101,7 +101,6 @@ static int vmclock_get_crosststamp(struct vmclock_state *st,
struct timespec64 *tspec)
{
ktime_t deadline = ktime_add(ktime_get(), VMCLOCK_MAX_WAIT);
- struct system_time_snapshot systime_snapshot;
uint64_t cycle, delta, seq, frac_sec;
#ifdef CONFIG_X86
@@ -132,17 +131,19 @@ static int vmclock_get_crosststamp(struct vmclock_state *st,
* will be derived from the *same* counter value.
*
* If the system isn't using the same counter, then the value
- * from ktime_get_snapshot() will still be used as pre_ts, and
- * ptp_read_system_postts() is called to populate postts after
- * calling get_cycles().
- *
- * The conversion to timespec64 happens further down, outside
- * the seq_count loop.
+ * from ptp_read_system_prets() will still be used as pre_ts,
+ * and ptp_read_system_postts() is called to populate postts
+ * after calling get_cycles().
*/
if (sts) {
- ktime_get_snapshot(&systime_snapshot);
- if (systime_snapshot.cs_id == st->cs_id) {
- cycle = systime_snapshot.cycles;
+ ptp_read_system_prets(sts);
+ if (sts->pre_sts.cs_id == st->cs_id) {
+ cycle = sts->pre_sts.cycles;
+ sts->post_sts = sts->pre_sts;
+ } else if (sts->pre_sts.hw_csid == st->cs_id &&
+ sts->pre_sts.hw_cycles) {
+ cycle = sts->pre_sts.hw_cycles;
+ sts->post_sts = sts->pre_sts;
} else {
cycle = get_cycles();
ptp_read_system_postts(sts);
@@ -180,12 +181,6 @@ static int vmclock_get_crosststamp(struct vmclock_state *st,
system_counter->cs_id = st->cs_id;
}
- if (sts) {
- sts->pre_ts = ktime_to_timespec64(systime_snapshot.real);
- if (systime_snapshot.cs_id == st->cs_id)
- sts->post_ts = sts->pre_ts;
- }
-
return 0;
}
@@ -272,7 +267,7 @@ static int ptp_vmclock_getcrosststamp(struct ptp_clock_info *ptp,
if (ret == -ENODEV) {
struct system_time_snapshot systime_snapshot;
- ktime_get_snapshot(&systime_snapshot);
+ ktime_get_snapshot_id(CLOCK_REALTIME, &systime_snapshot);
if (systime_snapshot.cs_id == CSID_X86_TSC ||
systime_snapshot.cs_id == CSID_X86_KVM_CLK) {
diff --git a/drivers/ras/amd/fmpm.c b/drivers/ras/amd/fmpm.c
index 34ef75a..4ccaaf7 100644
--- a/drivers/ras/amd/fmpm.c
+++ b/drivers/ras/amd/fmpm.c
@@ -52,6 +52,7 @@
#include <acpi/apei.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/mce.h>
#include "../debugfs.h"
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index e800252..78076ac 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -757,7 +757,7 @@
select REGMAP_I2C
help
This driver controls a Maxim MAX20086-MAX20089 camera power
- protectorvia I2C bus. The regulator has 2 or 4 outputs depending on
+ protector via I2C bus. The regulator has 2 or 4 outputs depending on
the device model. This driver is only capable to turn on/off them.
config REGULATOR_MAX20411
@@ -1231,6 +1231,7 @@
tristate "Raspberry Pi 7-inch touchscreen panel ATTINY regulator"
depends on ARM || ARM64 || COMPILE_TEST
depends on BACKLIGHT_CLASS_DEVICE
+ depends on GPIOLIB
depends on I2C
select REGMAP_I2C
help
diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c
index 6e4cb28..0dcb50b 100644
--- a/drivers/regulator/qcom-rpmh-regulator.c
+++ b/drivers/regulator/qcom-rpmh-regulator.c
@@ -1512,7 +1512,7 @@ static const struct rpmh_vreg_init_data pmh0101_vreg_data[] = {
RPMH_VREG("ldo13", LDO, 13, &pmic5_pldo530_mvp150, "vdd-l2-l13-l14"),
RPMH_VREG("ldo14", LDO, 14, &pmic5_pldo530_mvp150, "vdd-l2-l13-l14"),
RPMH_VREG("ldo15", LDO, 15, &pmic5_nldo530, "vdd-l15"),
- RPMH_VREG("ldo16", LDO, 15, &pmic5_pldo530_mvp600, "vdd-l5-l16"),
+ RPMH_VREG("ldo16", LDO, 16, &pmic5_pldo530_mvp600, "vdd-l5-l16"),
RPMH_VREG("ldo17", LDO, 17, &pmic5_pldo515_mv, "vdd-l17"),
RPMH_VREG("ldo18", LDO, 18, &pmic5_nldo530, "vdd-l18"),
RPMH_VREG("bob1", BOB, 1, &pmic5_bob, "vdd-bob1"),
diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c
index d77ca48..324c3a33 100644
--- a/drivers/regulator/tps65219-regulator.c
+++ b/drivers/regulator/tps65219-regulator.c
@@ -346,8 +346,9 @@ static irqreturn_t tps65219_regulator_irq_handler(int irq, void *data)
return IRQ_HANDLED;
}
- regulator_notifier_call_chain(irq_data->rdev,
- irq_data->type->event, NULL);
+ if (irq_data->rdev)
+ regulator_notifier_call_chain(irq_data->rdev,
+ irq_data->type->event, NULL);
dev_err(irq_data->dev, "Error IRQ trap %s for %s\n",
irq_data->type->event_name, irq_data->type->regulator_name);
@@ -398,14 +399,65 @@ static struct tps65219_chip_data chip_info_table[] = {
},
};
-static int tps65219_regulator_probe(struct platform_device *pdev)
+static bool tps65219_is_regulator_name(const struct tps65219_chip_data *pmic,
+ const char *name)
+{
+ int i;
+
+ for (i = 0; i < pmic->common_rdesc_size; i++)
+ if (!strcmp(pmic->common_rdesc[i].name, name))
+ return true;
+ for (i = 0; i < pmic->rdesc_size; i++)
+ if (!strcmp(pmic->rdesc[i].name, name))
+ return true;
+ return false;
+}
+
+static int tps65219_register_irqs(struct platform_device *pdev,
+ struct tps65219 *tps,
+ struct regulator_dev *rdev,
+ struct tps65219_regulator_irq_type *irq_types,
+ int nirqs,
+ const char *regulator_name)
{
struct tps65219_regulator_irq_data *irq_data;
+ int i, irq, error;
+
+ for (i = 0; i < nirqs; i++) {
+ if (strcmp(irq_types[i].regulator_name, regulator_name))
+ continue;
+
+ irq = platform_get_irq_byname(pdev, irq_types[i].irq_name);
+ if (irq < 0)
+ return -EINVAL;
+
+ irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
+ if (!irq_data)
+ return -ENOMEM;
+
+ irq_data->dev = tps->dev;
+ irq_data->type = &irq_types[i];
+ irq_data->rdev = rdev;
+
+ error = devm_request_threaded_irq(tps->dev, irq, NULL,
+ tps65219_regulator_irq_handler,
+ IRQF_ONESHOT,
+ irq_types[i].irq_name,
+ irq_data);
+ if (error)
+ return dev_err_probe(tps->dev, error,
+ "Failed to request %s IRQ %d\n",
+ irq_types[i].irq_name, irq);
+ }
+ return 0;
+}
+
+static int tps65219_regulator_probe(struct platform_device *pdev)
+{
struct tps65219_regulator_irq_type *irq_type;
struct tps65219_chip_data *pmic;
struct regulator_dev *rdev;
int error;
- int irq;
int i;
struct tps65219 *tps = dev_get_drvdata(pdev->dev.parent);
@@ -425,6 +477,19 @@ static int tps65219_regulator_probe(struct platform_device *pdev)
return dev_err_probe(tps->dev, PTR_ERR(rdev),
"Failed to register %s regulator\n",
pmic->common_rdesc[i].name);
+
+ error = tps65219_register_irqs(pdev, tps, rdev,
+ pmic->common_irq_types,
+ pmic->common_irq_size,
+ pmic->common_rdesc[i].name);
+ if (error)
+ return error;
+ error = tps65219_register_irqs(pdev, tps, rdev,
+ pmic->irq_types,
+ pmic->dev_irq_size,
+ pmic->common_rdesc[i].name);
+ if (error)
+ return error;
}
for (i = 0; i < pmic->rdesc_size; i++) {
@@ -434,52 +499,42 @@ static int tps65219_regulator_probe(struct platform_device *pdev)
return dev_err_probe(tps->dev, PTR_ERR(rdev),
"Failed to register %s regulator\n",
pmic->rdesc[i].name);
+
+ error = tps65219_register_irqs(pdev, tps, rdev,
+ pmic->common_irq_types,
+ pmic->common_irq_size,
+ pmic->rdesc[i].name);
+ if (error)
+ return error;
+ error = tps65219_register_irqs(pdev, tps, rdev,
+ pmic->irq_types,
+ pmic->dev_irq_size,
+ pmic->rdesc[i].name);
+ if (error)
+ return error;
}
+ /* Register non-regulator IRQs (TIMEOUT, SENSOR) with rdev=NULL */
for (i = 0; i < pmic->common_irq_size; ++i) {
irq_type = &pmic->common_irq_types[i];
- irq = platform_get_irq_byname(pdev, irq_type->irq_name);
- if (irq < 0)
- return -EINVAL;
-
- irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
- if (!irq_data)
- return -ENOMEM;
-
- irq_data->dev = tps->dev;
- irq_data->type = irq_type;
- error = devm_request_threaded_irq(tps->dev, irq, NULL,
- tps65219_regulator_irq_handler,
- IRQF_ONESHOT,
- irq_type->irq_name,
- irq_data);
+ if (tps65219_is_regulator_name(pmic, irq_type->regulator_name))
+ continue;
+ error = tps65219_register_irqs(pdev, tps, NULL,
+ irq_type, 1,
+ irq_type->regulator_name);
if (error)
- return dev_err_probe(tps->dev, error,
- "Failed to request %s IRQ %d\n",
- irq_type->irq_name, irq);
+ return error;
}
for (i = 0; i < pmic->dev_irq_size; ++i) {
irq_type = &pmic->irq_types[i];
- irq = platform_get_irq_byname(pdev, irq_type->irq_name);
- if (irq < 0)
- return -EINVAL;
-
- irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
- if (!irq_data)
- return -ENOMEM;
-
- irq_data->dev = tps->dev;
- irq_data->type = irq_type;
- error = devm_request_threaded_irq(tps->dev, irq, NULL,
- tps65219_regulator_irq_handler,
- IRQF_ONESHOT,
- irq_type->irq_name,
- irq_data);
+ if (tps65219_is_regulator_name(pmic, irq_type->regulator_name))
+ continue;
+ error = tps65219_register_irqs(pdev, tps, NULL,
+ irq_type, 1,
+ irq_type->regulator_name);
if (error)
- return dev_err_probe(tps->dev, error,
- "Failed to request %s IRQ %d\n",
- irq_type->irq_name, irq);
+ return error;
}
return 0;
diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c
index 41b1434..988fc29 100644
--- a/drivers/resctrl/mpam_devices.c
+++ b/drivers/resctrl/mpam_devices.c
@@ -164,11 +164,17 @@ static void mpam_free_garbage(void)
/*
* Once mpam is enabled, new requestors cannot further reduce the available
* partid. Assert that the size is fixed, and new requestors will be turned
- * away.
+ * away. This is needed when walking over structures sized by PARTID.
+ *
+ * During mpam_disable() these structures are not fixed, but the MSC state
+ * is still reset using whatever sizes have been discovered so far. As only
+ * PARTID 0 will be used after mpam_disable(), any race would be benign.
+ * Skip the check if a mpam_disable_reason has been set.
*/
static void mpam_assert_partid_sizes_fixed(void)
{
- WARN_ON_ONCE(!partid_max_published);
+ if (!mpam_disable_reason)
+ WARN_ON_ONCE(!partid_max_published);
}
static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg)
@@ -728,10 +734,9 @@ static void mpam_enable_quirks(struct mpam_msc *msc)
* Try and see what values stick in this bit. If we can write either value,
* its probably not implemented by hardware.
*/
-static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg)
+static bool mpam_ris_hw_probe_csu_nrdy(struct mpam_msc_ris *ris)
{
- u32 now;
- u64 mon_sel;
+ u32 now, mon_sel, ctl_val;
bool can_set, can_clear;
struct mpam_msc *msc = ris->vmsc->msc;
@@ -740,23 +745,30 @@ static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg)
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) |
FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
- _mpam_write_monsel_reg(msc, mon_reg, mon_sel);
+ mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
- _mpam_write_monsel_reg(msc, mon_reg, MSMON___NRDY);
- now = _mpam_read_monsel_reg(msc, mon_reg);
+ /* Hardware might ignore nrdy if it's not enabled */
+ ctl_val = MSMON_CFG_CSU_CTL_TYPE_CSU;
+ ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID;
+ ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG;
+ ctl_val |= MSMON_CFG_x_CTL_EN;
+ mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0);
+ mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
+
+ _mpam_write_monsel_reg(msc, MSMON_CSU, MSMON___NRDY);
+ now = _mpam_read_monsel_reg(msc, MSMON_CSU);
can_set = now & MSMON___NRDY;
- _mpam_write_monsel_reg(msc, mon_reg, 0);
- now = _mpam_read_monsel_reg(msc, mon_reg);
+ _mpam_write_monsel_reg(msc, MSMON_CSU, 0);
+ /* Configuration change to try and coax hardware into setting nrdy */
+ mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0x1);
+ now = _mpam_read_monsel_reg(msc, MSMON_CSU);
can_clear = !(now & MSMON___NRDY);
mpam_mon_sel_unlock(msc);
return (!can_set || !can_clear);
}
-#define mpam_ris_hw_probe_hw_nrdy(_ris, _mon_reg) \
- _mpam_ris_hw_probe_hw_nrdy(_ris, MSMON_##_mon_reg)
-
static void mpam_ris_hw_probe(struct mpam_msc_ris *ris)
{
int err;
@@ -873,20 +885,18 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris)
mpam_set_feature(mpam_feat_msmon_csu_xcl, props);
/* Is NRDY hardware managed? */
- hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, CSU);
- if (hw_managed)
- mpam_set_feature(mpam_feat_msmon_csu_hw_nrdy, props);
- }
+ hw_managed = mpam_ris_hw_probe_csu_nrdy(ris);
- /*
- * Accept the missing firmware property if NRDY appears
- * un-implemented.
- */
- if (err && mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, props))
- dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.");
+ /*
+ * Accept the missing firmware property if NRDY appears
+ * un-implemented.
+ */
+ if (err && hw_managed)
+ dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.");
+ }
}
if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) {
- bool has_long, hw_managed;
+ bool has_long;
u32 mbwumon_idr = mpam_read_partsel_reg(msc, MBWUMON_IDR);
props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr);
@@ -905,16 +915,6 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris)
} else {
mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props);
}
-
- /* Is NRDY hardware managed? */
- hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, MBWU);
- if (hw_managed)
- mpam_set_feature(mpam_feat_msmon_mbwu_hw_nrdy, props);
-
- /*
- * Don't warn about any missing firmware property for
- * MBWU NRDY - it doesn't make any sense!
- */
}
}
}
@@ -1197,7 +1197,6 @@ static void __ris_msmon_read(void *arg)
bool reset_on_next_read = false;
struct mpam_msc_ris *ris = m->ris;
struct msmon_mbwu_state *mbwu_state;
- struct mpam_props *rprops = &ris->props;
struct mpam_msc *msc = m->ris->vmsc->msc;
u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;
@@ -1253,8 +1252,7 @@ static void __ris_msmon_read(void *arg)
switch (m->type) {
case mpam_feat_msmon_csu:
now = mpam_read_monsel_reg(msc, CSU);
- if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops))
- nrdy = now & MSMON___NRDY;
+ nrdy = now & MSMON___NRDY;
now = FIELD_GET(MSMON___VALUE, now);
if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout)
@@ -1266,8 +1264,7 @@ static void __ris_msmon_read(void *arg)
case mpam_feat_msmon_mbwu_63counter:
if (m->type != mpam_feat_msmon_mbwu_31counter) {
now = mpam_msc_read_mbwu_l(msc);
- if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops))
- nrdy = now & MSMON___L_NRDY;
+ nrdy = now & MSMON___L_NRDY;
if (m->type == mpam_feat_msmon_mbwu_63counter)
now = FIELD_GET(MSMON___LWD_VALUE, now);
@@ -1275,8 +1272,7 @@ static void __ris_msmon_read(void *arg)
now = FIELD_GET(MSMON___L_VALUE, now);
} else {
now = mpam_read_monsel_reg(msc, MBWU);
- if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops))
- nrdy = now & MSMON___NRDY;
+ nrdy = now & MSMON___NRDY;
now = FIELD_GET(MSMON___VALUE, now);
}
@@ -2585,6 +2581,9 @@ static void __destroy_component_cfg(struct mpam_component *comp)
lockdep_assert_held(&mpam_list_lock);
+ if (!comp->cfg)
+ return;
+
add_to_garbage(comp->cfg);
list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
msc = vmsc->msc;
diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h
index 1914aef..04d1a59 100644
--- a/drivers/resctrl/mpam_internal.h
+++ b/drivers/resctrl/mpam_internal.h
@@ -181,14 +181,12 @@ enum mpam_device_features {
mpam_feat_msmon_csu,
mpam_feat_msmon_csu_capture,
mpam_feat_msmon_csu_xcl,
- mpam_feat_msmon_csu_hw_nrdy,
mpam_feat_msmon_mbwu,
mpam_feat_msmon_mbwu_31counter,
mpam_feat_msmon_mbwu_44counter,
mpam_feat_msmon_mbwu_63counter,
mpam_feat_msmon_mbwu_capture,
mpam_feat_msmon_mbwu_rwbw,
- mpam_feat_msmon_mbwu_hw_nrdy,
mpam_feat_partid_nrw,
MPAM_FEATURE_LAST
};
diff --git a/drivers/reset/reset-eyeq.c b/drivers/reset/reset-eyeq.c
index 791b728..1a38579 100644
--- a/drivers/reset/reset-eyeq.c
+++ b/drivers/reset/reset-eyeq.c
@@ -422,13 +422,6 @@ static int eqr_of_xlate_twocells(struct reset_controller_dev *rcdev,
return eqr_of_xlate_internal(rcdev, reset_spec->args[0], reset_spec->args[1]);
}
-static void eqr_of_node_put(void *_dev)
-{
- struct device *dev = _dev;
-
- of_node_put(dev->of_node);
-}
-
static int eqr_probe(struct auxiliary_device *adev,
const struct auxiliary_device_id *id)
{
@@ -439,21 +432,8 @@ static int eqr_probe(struct auxiliary_device *adev,
int ret;
/*
- * We are an auxiliary device of clk-eyeq. We do not have an OF node by
- * default; let's reuse our parent's OF node.
- */
- WARN_ON(dev->of_node);
- device_set_of_node_from_dev(dev, dev->parent);
- if (!dev->of_node)
- return -ENODEV;
-
- ret = devm_add_action_or_reset(dev, eqr_of_node_put, dev);
- if (ret)
- return ret;
-
- /*
- * Using our newfound OF node, we can get match data. We cannot use
- * device_get_match_data() because it does not match reused OF nodes.
+ * Get match data. We cannot use device_get_match_data() because it does
+ * not accept reused OF nodes; see device_set_of_node_from_dev().
*/
match = of_match_node(dev->driver->of_match_table, dev->of_node);
if (!match || !match->data)
diff --git a/drivers/s390/char/Kconfig b/drivers/s390/char/Kconfig
index 4d8f099..7416f94 100644
--- a/drivers/s390/char/Kconfig
+++ b/drivers/s390/char/Kconfig
@@ -85,14 +85,6 @@
transfer cache size from its default value 0.5MB to N bytes. If N
is zero, then no caching is performed.
-config SCLP_OFB
- def_bool n
- prompt "Support for Open-for-Business SCLP Event"
- depends on S390
- help
- This option enables the Open-for-Business interface to the s390
- Service Element.
-
config S390_UV_UAPI
def_tristate m
prompt "Ultravisor userspace API"
diff --git a/drivers/s390/char/sclp_config.c b/drivers/s390/char/sclp_config.c
index 9cfbe3f..8c77e8c 100644
--- a/drivers/s390/char/sclp_config.c
+++ b/drivers/s390/char/sclp_config.c
@@ -80,14 +80,11 @@ static void sclp_conf_receiver_fn(struct evbuf_header *evbuf)
static struct sclp_register sclp_conf_register =
{
-#ifdef CONFIG_SCLP_OFB
.send_mask = EVTYP_CONFMGMDATA_MASK,
-#endif
.receive_mask = EVTYP_CONFMGMDATA_MASK,
.receiver_fn = sclp_conf_receiver_fn,
};
-#ifdef CONFIG_SCLP_OFB
static int sclp_ofb_send_req(char *ev_data, size_t len)
{
static DEFINE_MUTEX(send_mutex);
@@ -143,11 +140,9 @@ static const struct bin_attribute ofb_bin_attr = {
},
.write = sysfs_ofb_data_write,
};
-#endif
static int __init sclp_ofb_setup(void)
{
-#ifdef CONFIG_SCLP_OFB
struct kset *ofb_kset;
int rc;
@@ -159,7 +154,6 @@ static int __init sclp_ofb_setup(void)
kset_unregister(ofb_kset);
return rc;
}
-#endif
return 0;
}
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index fbb58ed..9689f72 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -1142,8 +1142,8 @@ int __init chsc_init(void)
{
int ret;
- sei_page = (void *)get_zeroed_page(GFP_KERNEL);
- chsc_page = (void *)get_zeroed_page(GFP_KERNEL);
+ sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+ chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!sei_page || !chsc_page) {
ret = -ENOMEM;
goto out_err;
diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c
index 7341341..b6cb8bb 100644
--- a/drivers/s390/cio/chsc_sch.c
+++ b/drivers/s390/cio/chsc_sch.c
@@ -292,7 +292,7 @@ static int chsc_ioctl_start(void __user *user_area)
if (!css_general_characteristics.dynio)
/* It makes no sense to try. */
return -EOPNOTSUPP;
- chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+ chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
if (!chsc_area)
return -ENOMEM;
request = kzalloc_obj(*request);
@@ -340,7 +340,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area)
ret = -ENOMEM;
goto out_unlock;
}
- on_close_chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+ on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
if (!on_close_chsc_area) {
ret = -ENOMEM;
goto out_free_request;
@@ -392,7 +392,7 @@ static int chsc_ioctl_start_sync(void __user *user_area)
struct chsc_sync_area *chsc_area;
int ret, ccode;
- chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+ chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!chsc_area)
return -ENOMEM;
if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) {
@@ -438,7 +438,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd)
u8 data[PAGE_SIZE - 20];
} __attribute__ ((packed)) *scpcd_area;
- scpcd_area = (void *)get_zeroed_page(GFP_KERNEL);
+ scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!scpcd_area)
return -ENOMEM;
cd = kzalloc_obj(*cd);
@@ -500,7 +500,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd)
u8 data[PAGE_SIZE - 20];
} __attribute__ ((packed)) *scucd_area;
- scucd_area = (void *)get_zeroed_page(GFP_KERNEL);
+ scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!scucd_area)
return -ENOMEM;
cd = kzalloc_obj(*cd);
@@ -563,7 +563,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud)
u8 data[PAGE_SIZE - 20];
} __attribute__ ((packed)) *sscud_area;
- sscud_area = (void *)get_zeroed_page(GFP_KERNEL);
+ sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!sscud_area)
return -ENOMEM;
cud = kzalloc_obj(*cud);
@@ -625,7 +625,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci)
u8 data[PAGE_SIZE - 20];
} __attribute__ ((packed)) *sci_area;
- sci_area = (void *)get_zeroed_page(GFP_KERNEL);
+ sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!sci_area)
return -ENOMEM;
ci = kzalloc_obj(*ci);
@@ -696,7 +696,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
u32 res;
} __attribute__ ((packed)) *cssids_parm;
- sccl_area = (void *)get_zeroed_page(GFP_KERNEL);
+ sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!sccl_area)
return -ENOMEM;
ccl = kzalloc_obj(*ccl);
@@ -756,7 +756,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd)
int ret;
chpd = kzalloc_obj(*chpd);
- scpd_area = (void *)get_zeroed_page(GFP_KERNEL);
+ scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!scpd_area || !chpd) {
ret = -ENOMEM;
goto out_free;
@@ -796,7 +796,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal)
u8 data[PAGE_SIZE - 36];
} __attribute__ ((packed)) *sdcal_area;
- sdcal_area = (void *)get_zeroed_page(GFP_KERNEL);
+ sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
if (!sdcal_area)
return -ENOMEM;
dcal = kzalloc_obj(*dcal);
diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c
index d13ed10..171212a 100644
--- a/drivers/s390/cio/scm.c
+++ b/drivers/s390/cio/scm.c
@@ -229,7 +229,7 @@ int scm_update_information(void)
size_t num;
int ret;
- scm_info = (void *)__get_free_page(GFP_KERNEL);
+ scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA);
if (!scm_info)
return -ENOMEM;
diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index efb08b9..80ab0ff 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -37,7 +37,7 @@
#define TPGS_MODE_EXPLICIT 0x2
#define ALUA_RTPG_SIZE 128
-#define ALUA_FAILOVER_TIMEOUT 60
+#define ALUA_FAILOVER_TIMEOUT 255 /* max 255 (8-bit value) */
#define ALUA_FAILOVER_RETRIES 5
#define ALUA_RTPG_DELAY_MSECS 5
#define ALUA_RTPG_RETRY_DELAY 2
diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c
index 02cd441..496ddd4 100644
--- a/drivers/scsi/fcoe/fcoe_ctlr.c
+++ b/drivers/scsi/fcoe/fcoe_ctlr.c
@@ -1385,7 +1385,7 @@ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip,
while (rlen >= sizeof(*desc)) {
dlen = desc->fip_dlen * FIP_BPW;
- if (dlen > rlen)
+ if (dlen < sizeof(*desc) || dlen > rlen)
goto err;
/* Drop CVL if there are duplicate critical descriptors */
if ((desc->fip_dtype < 32) &&
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index fda07b1..14d563e 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -1491,7 +1491,7 @@ static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
phy_id = device->phy->identify.phy_identifier;
hdr->dw0 |= cpu_to_le32((1U << phy_id)
<< CMD_HDR_PHY_ID_OFF);
- hdr->dw0 |= CMD_HDR_FORCE_PHY_MSK;
+ hdr->dw0 |= cpu_to_le32(CMD_HDR_FORCE_PHY_MSK);
hdr->dw0 |= cpu_to_le32(4U << CMD_HDR_CMD_OFF);
}
diff --git a/drivers/scsi/isci/host.c b/drivers/scsi/isci/host.c
index 6d2f4c8..ff199ba 100644
--- a/drivers/scsi/isci/host.c
+++ b/drivers/scsi/isci/host.c
@@ -1252,6 +1252,9 @@ void isci_host_deinit(struct isci_host *ihost)
wait_for_stop(ihost);
+ /* No further IRQ-driven scheduling can happen past wait_for_stop(). */
+ tasklet_kill(&ihost->completion_tasklet);
+
/* phy stop is after controller stop to allow port and device to
* go idle before shutting down the phys, but the expectation is
* that i/o has been shut off well before we reach this
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index 2699e4e..056cbe5 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -3612,6 +3612,15 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
complete(&cmd_fusion->done);
break;
case MPI2_FUNCTION_SCSI_IO_REQUEST: /*Fast Path IO.*/
+ /*
+ * Firmware can send stale/duplicate completions for
+ * commands already returned to the pool. scmd_local
+ * would be NULL for such cases. Skip processing to
+ * avoid NULL pointer access.
+ */
+ if (!scmd_local)
+ break;
+
/* Update load balancing info */
if (fusion->load_balance_info &&
(megasas_priv(cmd_fusion->scmd)->status &
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 6ff7885..12caffe 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -2738,8 +2738,20 @@ scsih_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim)
pcie_device->enclosure_level,
pcie_device->connector_name);
+ /*
+ * The HBA firmware passes the NVMe drive's MDTS
+ * (Maximum Data Transfer Size) up to the driver. However,
+ * the driver hardcodes a 4K buffer size for the PRP list,
+ * accommodating at most 512 entries. This strictly limits
+ * the maximum supported NVMe I/O transfer to 2 MiB.
+ *
+ * Cap max_hw_sectors to the smaller of the drive's reported
+ * MDTS or the 2 MiB driver limit to prevent kernel oopses.
+ */
+ lim->max_hw_sectors = SZ_2M >> SECTOR_SHIFT;
if (pcie_device->nvme_mdts)
- lim->max_hw_sectors = pcie_device->nvme_mdts / 512;
+ lim->max_hw_sectors = min(lim->max_hw_sectors,
+ pcie_device->nvme_mdts >> SECTOR_SHIFT);
pcie_device_put(pcie_device);
spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
diff --git a/drivers/scsi/pmcraid.h b/drivers/scsi/pmcraid.h
index 9f59930..cd059b7 100644
--- a/drivers/scsi/pmcraid.h
+++ b/drivers/scsi/pmcraid.h
@@ -657,7 +657,7 @@ struct pmcraid_hostrcb {
*/
struct pmcraid_instance {
/* Array of allowed-to-be-exposed resources, initialized from
- * Configutation Table, later updated with CCNs
+ * Configuration Table, later updated with CCNs
*/
struct pmcraid_resource_entry *res_entries;
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 1515495..040c5e1 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -6953,7 +6953,7 @@ static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt)
++num_dev_resets;
if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
- sdev_printk(KERN_INFO, sdp, "doing device reset");
+ sdev_printk(KERN_INFO, sdp, "doing device reset\n");
scsi_debug_stop_all_queued(sdp);
if (devip) {
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index 68a9924..c6defe1 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -218,6 +218,7 @@ static struct {
{"PIONEER", "CD-ROM DRM-602X", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
{"PIONEER", "CD-ROM DRM-604X", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
{"PIONEER", "CD-ROM DRM-624X", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
+ {"Promise", "VTrak E310f", NULL, BLIST_SPARSELUN | BLIST_NO_RSOC},
{"Promise", "VTrak E610f", NULL, BLIST_SPARSELUN | BLIST_NO_RSOC},
{"Promise", "", NULL, BLIST_SPARSELUN},
{"QEMU", "QEMU CD-ROM", NULL, BLIST_SKIP_VPD_PAGES},
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 6e8c7a4..85eef40 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -575,10 +575,33 @@ void scsi_requeue_run_queue(struct work_struct *work)
void scsi_run_host_queues(struct Scsi_Host *shost)
{
- struct scsi_device *sdev;
+ struct scsi_device *sdev, *prev = NULL;
+ unsigned long flags;
- shost_for_each_device(sdev, shost)
+ spin_lock_irqsave(shost->host_lock, flags);
+ __shost_for_each_device(sdev, shost) {
+ /*
+ * Only skip devices so deep into removal they will never need
+ * another kick to their queues. Thus scsi_device_get() cannot
+ * be used as it would skip devices in SDEV_CANCEL state which
+ * may need a queue kick.
+ */
+ if (sdev->sdev_state == SDEV_DEL ||
+ !get_device(&sdev->sdev_gendev))
+ continue;
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ if (prev)
+ put_device(&prev->sdev_gendev);
scsi_run_queue(sdev->request_queue);
+
+ prev = sdev;
+
+ spin_lock_irqsave(shost->host_lock, flags);
+ }
+ spin_unlock_irqrestore(shost->host_lock, flags);
+ if (prev)
+ put_device(&prev->sdev_gendev);
}
static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index dce95e3..173ed63 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -737,6 +737,37 @@ fc_cn_stats_update(u16 event_type, struct fc_fpin_stats *stats)
}
}
+static void
+fc_fpin_pname_stats_update(struct Scsi_Host *shost,
+ struct fc_rport *attach_rport, u16 event_type,
+ u32 desc_len, u32 fixed_len, u32 pname_count,
+ __be64 *pname_list,
+ void (*stats_update)(u16 event_type,
+ struct fc_fpin_stats *stats))
+{
+ u32 i;
+ struct fc_rport *rport;
+ u64 wwpn;
+
+ if (desc_len < fixed_len)
+ pname_count = 0;
+ else
+ pname_count = min(pname_count, (desc_len - fixed_len) /
+ sizeof(pname_list[0]));
+
+ for (i = 0; i < pname_count; i++) {
+ wwpn = be64_to_cpu(pname_list[i]);
+ rport = fc_find_rport_by_wwpn(shost, wwpn);
+ if (rport &&
+ (rport->roles & FC_PORT_ROLE_FCP_TARGET ||
+ rport->roles & FC_PORT_ROLE_NVME_TARGET)) {
+ if (rport == attach_rport)
+ continue;
+ stats_update(event_type, &rport->fpin_stats);
+ }
+ }
+}
+
/*
* fc_fpin_li_stats_update - routine to update Link Integrity
* event statistics.
@@ -747,13 +778,11 @@ fc_cn_stats_update(u16 event_type, struct fc_fpin_stats *stats)
static void
fc_fpin_li_stats_update(struct Scsi_Host *shost, struct fc_tlv_desc *tlv)
{
- u8 i;
struct fc_rport *rport = NULL;
struct fc_rport *attach_rport = NULL;
struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
struct fc_fn_li_desc *li_desc = (struct fc_fn_li_desc *)tlv;
u16 event_type = be16_to_cpu(li_desc->event_type);
- u64 wwpn;
rport = fc_find_rport_by_wwpn(shost,
be64_to_cpu(li_desc->attached_wwpn));
@@ -764,22 +793,11 @@ fc_fpin_li_stats_update(struct Scsi_Host *shost, struct fc_tlv_desc *tlv)
fc_li_stats_update(event_type, &attach_rport->fpin_stats);
}
- if (be32_to_cpu(li_desc->pname_count) > 0) {
- for (i = 0;
- i < be32_to_cpu(li_desc->pname_count);
- i++) {
- wwpn = be64_to_cpu(li_desc->pname_list[i]);
- rport = fc_find_rport_by_wwpn(shost, wwpn);
- if (rport &&
- (rport->roles & FC_PORT_ROLE_FCP_TARGET ||
- rport->roles & FC_PORT_ROLE_NVME_TARGET)) {
- if (rport == attach_rport)
- continue;
- fc_li_stats_update(event_type,
- &rport->fpin_stats);
- }
- }
- }
+ fc_fpin_pname_stats_update(shost, attach_rport, event_type,
+ be32_to_cpu(li_desc->desc_len),
+ FC_TLV_DESC_LENGTH_FROM_SZ(*li_desc),
+ be32_to_cpu(li_desc->pname_count),
+ li_desc->pname_list, fc_li_stats_update);
if (fc_host->port_name == be64_to_cpu(li_desc->attached_wwpn))
fc_li_stats_update(event_type, &fc_host->fpin_stats);
@@ -827,13 +845,11 @@ static void
fc_fpin_peer_congn_stats_update(struct Scsi_Host *shost,
struct fc_tlv_desc *tlv)
{
- u8 i;
struct fc_rport *rport = NULL;
struct fc_rport *attach_rport = NULL;
struct fc_fn_peer_congn_desc *pc_desc =
(struct fc_fn_peer_congn_desc *)tlv;
u16 event_type = be16_to_cpu(pc_desc->event_type);
- u64 wwpn;
rport = fc_find_rport_by_wwpn(shost,
be64_to_cpu(pc_desc->attached_wwpn));
@@ -844,22 +860,11 @@ fc_fpin_peer_congn_stats_update(struct Scsi_Host *shost,
fc_cn_stats_update(event_type, &attach_rport->fpin_stats);
}
- if (be32_to_cpu(pc_desc->pname_count) > 0) {
- for (i = 0;
- i < be32_to_cpu(pc_desc->pname_count);
- i++) {
- wwpn = be64_to_cpu(pc_desc->pname_list[i]);
- rport = fc_find_rport_by_wwpn(shost, wwpn);
- if (rport &&
- (rport->roles & FC_PORT_ROLE_FCP_TARGET ||
- rport->roles & FC_PORT_ROLE_NVME_TARGET)) {
- if (rport == attach_rport)
- continue;
- fc_cn_stats_update(event_type,
- &rport->fpin_stats);
- }
- }
- }
+ fc_fpin_pname_stats_update(shost, attach_rport, event_type,
+ be32_to_cpu(pc_desc->desc_len),
+ FC_TLV_DESC_LENGTH_FROM_SZ(*pc_desc),
+ be32_to_cpu(pc_desc->pname_count),
+ pc_desc->pname_list, fc_cn_stats_update);
}
/*
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index adc3fa5..599e75f 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -2476,8 +2476,7 @@ sd_spinup_disk(struct scsi_disk *sdkp)
{
static const u8 cmd[10] = { TEST_UNIT_READY };
unsigned long spintime_expire = 0;
- int spintime, sense_valid = 0;
- unsigned int the_result;
+ int the_result, spintime, sense_valid = 0;
struct scsi_sense_hdr sshdr;
struct scsi_failure failure_defs[] = {
/* Do not retry Medium Not Present */
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 2b4b2a1..74cd4e8 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -1801,7 +1801,7 @@ sg_start_req(Sg_request *srp, unsigned char *cmd)
}
res = blk_rq_map_user_io(rq, md, hp->dxferp, hp->dxfer_len,
- GFP_ATOMIC, iov_count, iov_count, 1, rw);
+ GFP_KERNEL, iov_count, iov_count, 1, rw);
if (!res) {
srp->bio = rq->bio;
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index b4ed991..2026ac6 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -9427,6 +9427,7 @@ static void pqi_shutdown(struct pci_dev *pci_dev)
pqi_crash_if_pending_command(ctrl_info);
pqi_reset(ctrl_info);
+ pqi_ctrl_unblock_device_reset(ctrl_info);
}
static void pqi_process_lockup_action_param(void)
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index 7adb257..c36c54e 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -395,7 +395,7 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt)
switch (req_op(rq)) {
case REQ_OP_WRITE:
- if (!cd->writeable)
+ if (get_disk_ro(cd->disk))
goto out;
SCpnt->cmnd[0] = WRITE_10;
cd->cdi.media_written = 1;
@@ -681,6 +681,7 @@ static int sr_probe(struct scsi_device *sdev)
error = -ENOMEM;
if (get_capabilities(cd))
goto fail_minor;
+ cdrom_probe_write_features(&cd->cdi);
sr_vendor_init(cd);
set_capacity(disk, cd->capacity);
@@ -899,14 +900,6 @@ static int get_capabilities(struct scsi_cd *cd)
/*else I don't think it can close its tray
cd->cdi.mask |= CDC_CLOSE_TRAY; */
- /*
- * if DVD-RAM, MRW-W or CD-RW, we are randomly writable
- */
- if ((cd->cdi.mask & (CDC_DVD_RAM | CDC_MRW_W | CDC_RAM | CDC_CD_RW)) !=
- (CDC_DVD_RAM | CDC_MRW_W | CDC_RAM | CDC_CD_RW)) {
- cd->writeable = 1;
- }
-
kfree(buffer);
return 0;
}
diff --git a/drivers/scsi/sr.h b/drivers/scsi/sr.h
index dc89927..2d92f9c 100644
--- a/drivers/scsi/sr.h
+++ b/drivers/scsi/sr.h
@@ -35,7 +35,6 @@ typedef struct scsi_cd {
struct scsi_device *device;
unsigned int vendor; /* vendor code, see sr_vendor.c */
unsigned long ms_offset; /* for reading multisession-CD's */
- unsigned writeable : 1;
unsigned use:1; /* is this device still supportable */
unsigned xa_flag:1; /* CD has XA sectors ? */
unsigned readcd_known:1; /* drive supports READ_CD (0xbe) */
diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c
index 77763a1..fc080e5 100644
--- a/drivers/soc/imx/soc-imx8m.c
+++ b/drivers/soc/imx/soc-imx8m.c
@@ -247,7 +247,7 @@ static int imx8m_soc_probe(struct platform_device *pdev)
if (ret)
return ret;
- data = device_get_match_data(dev);
+ data = of_machine_get_match_data(imx8_soc_match);
if (data) {
soc_dev_attr->soc_id = data->name;
ret = imx8m_soc_prepare(pdev, data->ocotp_compatible);
diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c
index b203bc6..5f20108 100644
--- a/drivers/soc/qcom/ice.c
+++ b/drivers/soc/qcom/ice.c
@@ -16,6 +16,7 @@
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
+#include <linux/xarray.h>
#include <linux/firmware/qcom/qcom_scm.h>
@@ -108,11 +109,15 @@ struct qcom_ice {
void __iomem *base;
struct clk *core_clk;
+ struct clk *iface_clk;
bool use_hwkm;
bool hwkm_init_complete;
u8 hwkm_version;
};
+static DEFINE_XARRAY(ice_handles);
+static DEFINE_MUTEX(ice_mutex);
+
static bool qcom_ice_check_supported(struct qcom_ice *ice)
{
u32 regval = qcom_ice_readl(ice, QCOM_ICE_REG_VERSION);
@@ -312,8 +317,13 @@ int qcom_ice_resume(struct qcom_ice *ice)
err = clk_prepare_enable(ice->core_clk);
if (err) {
- dev_err(dev, "failed to enable core clock (%d)\n",
- err);
+ dev_err(dev, "Failed to enable core clock: %d\n", err);
+ return err;
+ }
+
+ err = clk_prepare_enable(ice->iface_clk);
+ if (err) {
+ dev_err(dev, "Failed to enable iface clock: %d\n", err);
return err;
}
qcom_ice_hwkm_init(ice);
@@ -323,6 +333,7 @@ EXPORT_SYMBOL_GPL(qcom_ice_resume);
int qcom_ice_suspend(struct qcom_ice *ice)
{
+ clk_disable_unprepare(ice->iface_clk);
clk_disable_unprepare(ice->core_clk);
ice->hwkm_init_complete = false;
@@ -559,7 +570,7 @@ static struct qcom_ice *qcom_ice_create(struct device *dev,
if (!qcom_scm_ice_available()) {
dev_warn(dev, "ICE SCM interface not found\n");
- return NULL;
+ return ERR_PTR(-EOPNOTSUPP);
}
engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
@@ -580,10 +591,16 @@ static struct qcom_ice *qcom_ice_create(struct device *dev,
if (!engine->core_clk)
engine->core_clk = devm_clk_get_optional_enabled(dev, "ice");
if (!engine->core_clk)
+ engine->core_clk = devm_clk_get_optional_enabled(dev, "core");
+ if (!engine->core_clk)
engine->core_clk = devm_clk_get_enabled(dev, NULL);
if (IS_ERR(engine->core_clk))
return ERR_CAST(engine->core_clk);
+ engine->iface_clk = devm_clk_get_optional_enabled(dev, "iface");
+ if (IS_ERR(engine->iface_clk))
+ return ERR_CAST(engine->iface_clk);
+
if (!qcom_ice_check_supported(engine))
return ERR_PTR(-EOPNOTSUPP);
@@ -631,6 +648,8 @@ static struct qcom_ice *of_qcom_ice_get(struct device *dev)
return qcom_ice_create(&pdev->dev, base);
}
+ guard(mutex)(&ice_mutex);
+
/*
* If the consumer node does not provider an 'ice' reg range
* (legacy DT binding), then it must at least provide a phandle
@@ -639,20 +658,21 @@ static struct qcom_ice *of_qcom_ice_get(struct device *dev)
struct device_node *node __free(device_node) = of_parse_phandle(dev->of_node,
"qcom,ice", 0);
if (!node)
- return NULL;
+ return ERR_PTR(-EOPNOTSUPP);
pdev = of_find_device_by_node(node);
if (!pdev) {
dev_err(dev, "Cannot find device node %s\n", node->name);
- return ERR_PTR(-EPROBE_DEFER);
+ return ERR_PTR(-ENODEV);
}
- ice = platform_get_drvdata(pdev);
- if (!ice) {
- dev_err(dev, "Cannot get ice instance from %s\n",
- dev_name(&pdev->dev));
+ ice = xa_load(&ice_handles, pdev->dev.of_node->phandle);
+ if (IS_ERR_OR_NULL(ice)) {
platform_device_put(pdev);
- return ERR_PTR(-EPROBE_DEFER);
+ if (!ice)
+ return ERR_PTR(-EPROBE_DEFER);
+ else
+ return ice;
}
link = device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER);
@@ -691,8 +711,7 @@ static void devm_of_qcom_ice_put(struct device *dev, void *res)
* phandle via 'qcom,ice' property to an ICE DT, the ICE instance will already
* be created and so this function will return that instead.
*
- * Return: ICE pointer on success, NULL if there is no ICE data provided by the
- * consumer or ERR_PTR() on error.
+ * Return: ICE pointer on success, ERR_PTR() on error.
*/
struct qcom_ice *devm_of_qcom_ice_get(struct device *dev)
{
@@ -703,7 +722,7 @@ struct qcom_ice *devm_of_qcom_ice_get(struct device *dev)
return ERR_PTR(-ENOMEM);
ice = of_qcom_ice_get(dev);
- if (!IS_ERR_OR_NULL(ice)) {
+ if (!IS_ERR(ice)) {
*dr = ice;
devres_add(dev, dr);
} else {
@@ -716,24 +735,40 @@ EXPORT_SYMBOL_GPL(devm_of_qcom_ice_get);
static int qcom_ice_probe(struct platform_device *pdev)
{
+ unsigned long phandle = pdev->dev.of_node->phandle;
struct qcom_ice *engine;
void __iomem *base;
+ guard(mutex)(&ice_mutex);
+
base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(base)) {
dev_warn(&pdev->dev, "ICE registers not found\n");
+ /* Store the error pointer for devm_of_qcom_ice_get() */
+ xa_store(&ice_handles, phandle, (__force void *)base, GFP_KERNEL);
return PTR_ERR(base);
}
engine = qcom_ice_create(&pdev->dev, base);
- if (IS_ERR(engine))
+ if (IS_ERR(engine)) {
+ /* Store the error pointer for devm_of_qcom_ice_get() */
+ xa_store(&ice_handles, phandle, engine, GFP_KERNEL);
return PTR_ERR(engine);
+ }
- platform_set_drvdata(pdev, engine);
+ xa_store(&ice_handles, phandle, engine, GFP_KERNEL);
return 0;
}
+static void qcom_ice_remove(struct platform_device *pdev)
+{
+ unsigned long phandle = pdev->dev.of_node->phandle;
+
+ guard(mutex)(&ice_mutex);
+ xa_store(&ice_handles, phandle, NULL, GFP_KERNEL);
+}
+
static const struct of_device_id qcom_ice_of_match_table[] = {
{ .compatible = "qcom,inline-crypto-engine" },
{ },
@@ -742,6 +777,7 @@ MODULE_DEVICE_TABLE(of, qcom_ice_of_match_table);
static struct platform_driver qcom_ice_driver = {
.probe = qcom_ice_probe,
+ .remove = qcom_ice_remove,
.driver = {
.name = "qcom-ice",
.of_match_table = qcom_ice_of_match_table,
diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c
index 4d1dce4..71a6e5c 100644
--- a/drivers/spi/spi-amd.c
+++ b/drivers/spi/spi-amd.c
@@ -868,7 +868,7 @@ static int amd_spi_probe(struct platform_device *pdev)
dev_dbg(dev, "io_remap_address: %p\n", amd_spi->io_remap_addr);
amd_spi->version = (uintptr_t)device_get_match_data(dev);
- host->bus_num = 0;
+ host->bus_num = (amd_spi->version == AMD_HID2_SPI) ? 2 : 0;
return amd_spi_probe_common(dev, host);
}
diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c
index 19c5eba..f9de2d2c 100644
--- a/drivers/spi/spi-amlogic-spisg.c
+++ b/drivers/spi/spi-amlogic-spisg.c
@@ -794,6 +794,7 @@ static int aml_spisg_probe(struct platform_device *pdev)
dma_set_max_seg_size(&pdev->dev, SPISG_BLOCK_MAX);
+ init_completion(&spisg->completion);
ret = devm_request_irq(&pdev->dev, irq, aml_spisg_irq, 0, NULL, spisg);
if (ret) {
dev_err(&pdev->dev, "irq request failed\n");
@@ -806,8 +807,6 @@ static int aml_spisg_probe(struct platform_device *pdev)
goto out_clk;
}
- init_completion(&spisg->completion);
-
pm_runtime_put(&spisg->pdev->dev);
return 0;
diff --git a/drivers/spi/spi-axiado.c b/drivers/spi/spi-axiado.c
index 9057a0a..649f149 100644
--- a/drivers/spi/spi-axiado.c
+++ b/drivers/spi/spi-axiado.c
@@ -201,7 +201,7 @@ static void ax_spi_fill_tx_fifo(struct ax_spi *xspi)
* then spi control did't work thoroughly, add one byte delay
*/
if (ax_spi_read(xspi, AX_SPI_IVR) & AX_SPI_IVR_TFOV)
- usleep_range(10, 10);
+ udelay(10);
if (xspi->tx_buf)
ax_spi_write_b(xspi, AX_SPI_TXFIFO, *xspi->tx_buf++);
else
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index 65aff2e..057381e 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -1860,14 +1860,10 @@ static int cqspi_probe(struct platform_device *pdev)
if (irq < 0)
return -ENXIO;
- ret = pm_runtime_set_active(dev);
- if (ret)
- return ret;
-
ret = clk_bulk_prepare_enable(CLK_QSPI_NUM, cqspi->clks);
if (ret) {
dev_err(dev, "Cannot enable QSPI clocks.\n");
- goto disable_rpm;
+ return ret;
}
/* Obtain QSPI reset control */
@@ -1962,10 +1958,11 @@ static int cqspi_probe(struct platform_device *pdev)
cqspi->sclk = 0;
if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
- pm_runtime_enable(dev);
pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT);
pm_runtime_use_autosuspend(dev);
pm_runtime_get_noresume(dev);
+ pm_runtime_set_active(dev);
+ pm_runtime_enable(dev);
}
host->num_chipselect = cqspi->num_chipselect;
@@ -1977,7 +1974,7 @@ static int cqspi_probe(struct platform_device *pdev)
ret = cqspi_request_mmap_dma(cqspi);
if (ret == -EPROBE_DEFER) {
dev_err_probe(&pdev->dev, ret, "Failed to request mmap DMA\n");
- goto disable_controller;
+ goto disable_rpm;
}
}
@@ -1995,14 +1992,16 @@ static int cqspi_probe(struct platform_device *pdev)
release_dma_chan:
if (cqspi->rx_chan)
dma_release_channel(cqspi->rx_chan);
-disable_controller:
+disable_rpm:
+ if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
+ pm_runtime_disable(dev);
+ pm_runtime_set_suspended(dev);
+ pm_runtime_put_noidle(dev);
+ pm_runtime_dont_use_autosuspend(dev);
+ }
cqspi_controller_enable(cqspi, 0);
disable_clks:
- if (pm_runtime_get_sync(&pdev->dev) >= 0)
- clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks);
-disable_rpm:
- if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM)))
- pm_runtime_disable(dev);
+ clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks);
return ret;
}
@@ -2026,18 +2025,19 @@ static void cqspi_remove(struct platform_device *pdev)
if (cqspi->rx_chan)
dma_release_channel(cqspi->rx_chan);
- cqspi_controller_enable(cqspi, 0);
-
-
if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM)))
ret = pm_runtime_get_sync(&pdev->dev);
- if (ret >= 0)
+ if (ret >= 0) {
+ cqspi_controller_enable(cqspi, 0);
clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks);
+ }
if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
- pm_runtime_put_sync(&pdev->dev);
pm_runtime_disable(&pdev->dev);
+ pm_runtime_set_suspended(&pdev->dev);
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_dont_use_autosuspend(&pdev->dev);
}
}
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index 08d7dab..891e2ba 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -741,7 +741,6 @@ static int cdns_spi_probe(struct platform_device *pdev)
/* Set to default valid value */
ctlr->max_speed_hz = xspi->clk_rate / 4;
xspi->speed_hz = ctlr->max_speed_hz;
- pm_runtime_put_autosuspend(&pdev->dev);
} else {
ctlr->mode_bits |= SPI_NO_CS;
ctlr->target_abort = cdns_target_abort;
@@ -752,12 +751,17 @@ static int cdns_spi_probe(struct platform_device *pdev)
goto clk_dis_all;
}
+ if (!spi_controller_is_target(ctlr))
+ pm_runtime_put_autosuspend(&pdev->dev);
+
return ret;
clk_dis_all:
if (!spi_controller_is_target(ctlr)) {
pm_runtime_disable(&pdev->dev);
pm_runtime_set_suspended(&pdev->dev);
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_dont_use_autosuspend(&pdev->dev);
}
remove_ctlr:
spi_controller_put(ctlr);
@@ -776,16 +780,23 @@ static void cdns_spi_remove(struct platform_device *pdev)
{
struct spi_controller *ctlr = platform_get_drvdata(pdev);
struct cdns_spi *xspi = spi_controller_get_devdata(ctlr);
+ int ret = 0;
+
+ if (!spi_controller_is_target(ctlr))
+ ret = pm_runtime_get_sync(&pdev->dev);
spi_controller_get(ctlr);
spi_unregister_controller(ctlr);
- cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE);
+ if (ret >= 0)
+ cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE);
if (!spi_controller_is_target(ctlr)) {
pm_runtime_disable(&pdev->dev);
pm_runtime_set_suspended(&pdev->dev);
+ pm_runtime_put_noidle(&pdev->dev);
+ pm_runtime_dont_use_autosuspend(&pdev->dev);
}
spi_controller_put(ctlr);
diff --git a/drivers/spi/spi-ch341.c b/drivers/spi/spi-ch341.c
index 3eaa8f1..6448a44 100644
--- a/drivers/spi/spi-ch341.c
+++ b/drivers/spi/spi-ch341.c
@@ -250,5 +250,5 @@ static struct usb_driver ch341a_usb_driver = {
module_usb_driver(ch341a_usb_driver);
MODULE_AUTHOR("Johannes Thumshirn <jth@kernel.org>");
-MODULE_DESCRIPTION("QiHeng Electronics ch341 USB2SPI");
+MODULE_DESCRIPTION("Nanjing Qinheng Microelectronics CH341 USB2SPI driver");
MODULE_LICENSE("GPL v2");
diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c
index db50018..f716c96 100644
--- a/drivers/spi/spi-ep93xx.c
+++ b/drivers/spi/spi-ep93xx.c
@@ -582,12 +582,14 @@ static int ep93xx_spi_setup_dma(struct device *dev, struct ep93xx_spi *espi)
espi->dma_rx = dma_request_chan(dev, "rx");
if (IS_ERR(espi->dma_rx)) {
ret = dev_err_probe(dev, PTR_ERR(espi->dma_rx), "rx DMA setup failed");
+ espi->dma_rx = NULL;
goto fail_free_page;
}
espi->dma_tx = dma_request_chan(dev, "tx");
if (IS_ERR(espi->dma_tx)) {
ret = dev_err_probe(dev, PTR_ERR(espi->dma_tx), "tx DMA setup failed");
+ espi->dma_tx = NULL;
goto fail_release_rx;
}
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index e5c907c..480d1e8 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -1382,9 +1382,7 @@ static int spi_imx_setupxfer(struct spi_device *spi,
spi_imx->target_burst = t->len;
}
- spi_imx->devtype_data->prepare_transfer(spi_imx, spi, t);
-
- return 0;
+ return spi_imx->devtype_data->prepare_transfer(spi_imx, spi, t);
}
static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
@@ -1709,6 +1707,7 @@ static int spi_imx_dma_data_prepare(struct spi_imx_data *spi_imx,
kfree(spi_imx->dma_data[0].dma_tx_buf);
kfree(spi_imx->dma_data[0].dma_rx_buf);
kfree(spi_imx->dma_data);
+ return ret;
}
}
@@ -1836,7 +1835,7 @@ static void spi_imx_dma_max_wml_find(struct spi_imx_data *spi_imx,
unsigned int i;
for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
- if (!dma_data->dma_len % (i * bytes_per_word))
+ if (!(dma_data->dma_len % (i * bytes_per_word)))
break;
}
/* Use 1 as wml in case no available burst length got */
diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
index a09371a..9326684 100644
--- a/drivers/spi/spi-mem.c
+++ b/drivers/spi/spi-mem.c
@@ -279,13 +279,20 @@ static bool spi_mem_internal_supports_op(struct spi_mem *mem,
*/
bool spi_mem_supports_op(struct spi_mem *mem, const struct spi_mem_op *op)
{
- /* Make sure the operation frequency is correct before going futher */
- spi_mem_adjust_op_freq(mem, (struct spi_mem_op *)op);
+ struct spi_mem_op eval_op = *op;
- if (spi_mem_check_op(op))
+ /*
+ * Work on a local copy; this is a pure capability check and must
+ * not modify the caller's op. Stored templates with max_freq == 0
+ * must remain unset so their frequency is always re-capped to the
+ * current device maximum at execution time.
+ */
+ spi_mem_adjust_op_freq(mem, &eval_op);
+
+ if (spi_mem_check_op(&eval_op))
return false;
- return spi_mem_internal_supports_op(mem, op);
+ return spi_mem_internal_supports_op(mem, &eval_op);
}
EXPORT_SYMBOL_GPL(spi_mem_supports_op);
diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c
index eab059f..4dee0fe 100644
--- a/drivers/spi/spi-microchip-core-qspi.c
+++ b/drivers/spi/spi-microchip-core-qspi.c
@@ -74,6 +74,13 @@
#define STATUS_FLAGSX4 BIT(8)
#define STATUS_MASK GENMASK(8, 0)
+/*
+ * QSPI Direct Access register defines
+ */
+#define DIRECT_ACCESS_EN_SSEL BIT(0)
+#define DIRECT_ACCESS_OP_SSEL BIT(1)
+#define DIRECT_ACCESS_OP_SSEL_SHIFT 1
+
#define BYTESUPPER_MASK GENMASK(31, 16)
#define BYTESLOWER_MASK GENMASK(15, 0)
@@ -158,7 +165,39 @@ static int mchp_coreqspi_set_mode(struct mchp_coreqspi *qspi, const struct spi_m
return 0;
}
-static inline void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi)
+static void mchp_coreqspi_set_cs(struct spi_device *spi, bool enable)
+{
+ struct mchp_coreqspi *qspi = spi_controller_get_devdata(spi->controller);
+ u32 val;
+
+ val = readl(qspi->regs + REG_DIRECT_ACCESS);
+
+ val &= ~DIRECT_ACCESS_OP_SSEL;
+ val |= !enable << DIRECT_ACCESS_OP_SSEL_SHIFT;
+
+ writel(val, qspi->regs + REG_DIRECT_ACCESS);
+}
+
+static int mchp_coreqspi_setup(struct spi_device *spi)
+{
+ struct mchp_coreqspi *qspi = spi_controller_get_devdata(spi->controller);
+ u32 val;
+
+ /*
+ * Active low devices need to be specifically set to their inactive
+ * states during probe.
+ */
+ if (spi->mode & SPI_CS_HIGH)
+ return 0;
+
+ val = readl(qspi->regs + REG_DIRECT_ACCESS);
+ val |= DIRECT_ACCESS_OP_SSEL;
+ writel(val, qspi->regs + REG_DIRECT_ACCESS);
+
+ return 0;
+}
+
+static void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi)
{
u32 control, data;
@@ -194,7 +233,7 @@ static inline void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi)
}
}
-static inline void mchp_coreqspi_write_op(struct mchp_coreqspi *qspi)
+static void mchp_coreqspi_write_op(struct mchp_coreqspi *qspi)
{
u32 control, data;
@@ -222,7 +261,7 @@ static inline void mchp_coreqspi_write_op(struct mchp_coreqspi *qspi)
}
}
-static inline void mchp_coreqspi_write_read_op(struct mchp_coreqspi *qspi)
+static void mchp_coreqspi_write_read_op(struct mchp_coreqspi *qspi)
{
u32 control, data;
@@ -380,20 +419,7 @@ static int mchp_coreqspi_setup_clock(struct mchp_coreqspi *qspi, struct spi_devi
return 0;
}
-static int mchp_coreqspi_setup_op(struct spi_device *spi_dev)
-{
- struct spi_controller *ctlr = spi_dev->controller;
- struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr);
- u32 control = readl_relaxed(qspi->regs + REG_CONTROL);
-
- control |= (CONTROL_MASTER | CONTROL_ENABLE);
- control &= ~CONTROL_CLKIDLE;
- writel_relaxed(control, qspi->regs + REG_CONTROL);
-
- return 0;
-}
-
-static inline void mchp_coreqspi_config_op(struct mchp_coreqspi *qspi, const struct spi_mem_op *op)
+static void mchp_coreqspi_config_op(struct mchp_coreqspi *qspi, const struct spi_mem_op *op)
{
u32 idle_cycles = 0;
int total_bytes, cmd_bytes, frames, ctrl;
@@ -483,6 +509,7 @@ static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *o
reinit_completion(&qspi->data_completion);
mchp_coreqspi_config_op(qspi, op);
+ mchp_coreqspi_set_cs(mem->spi, true);
if (op->cmd.opcode) {
qspi->txbuf = &opcode;
qspi->rxbuf = NULL;
@@ -523,6 +550,7 @@ static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *o
err = -ETIMEDOUT;
error:
+ mchp_coreqspi_set_cs(mem->spi, false);
mutex_unlock(&qspi->op_lock);
mchp_coreqspi_disable_ints(qspi);
@@ -662,18 +690,28 @@ static int mchp_coreqspi_transfer_one(struct spi_controller *ctlr, struct spi_de
struct spi_transfer *t)
{
struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr);
+ bool dual_quad = false;
qspi->tx_len = t->len;
+ if (t->tx_nbits == SPI_NBITS_QUAD || t->rx_nbits == SPI_NBITS_QUAD ||
+ t->tx_nbits == SPI_NBITS_DUAL ||
+ t->rx_nbits == SPI_NBITS_DUAL)
+ dual_quad = true;
+
if (t->tx_buf)
qspi->txbuf = (u8 *)t->tx_buf;
if (!t->rx_buf) {
mchp_coreqspi_write_op(qspi);
- } else {
+ } else if (!dual_quad) {
qspi->rxbuf = (u8 *)t->rx_buf;
qspi->rx_len = t->len;
mchp_coreqspi_write_read_op(qspi);
+ } else {
+ qspi->rxbuf = (u8 *)t->rx_buf;
+ qspi->rx_len = t->len;
+ mchp_coreqspi_read_op(qspi);
}
return 0;
@@ -686,6 +724,7 @@ static int mchp_coreqspi_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct device_node *np = dev->of_node;
int ret;
+ u32 num_cs, val;
ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*qspi));
if (!ctlr)
@@ -718,10 +757,18 @@ static int mchp_coreqspi_probe(struct platform_device *pdev)
return ret;
}
+ /*
+ * The IP core only has a single CS, any more have to be provided via
+ * gpios
+ */
+ if (of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs))
+ num_cs = 1;
+
+ ctlr->num_chipselect = num_cs;
+
ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
ctlr->mem_ops = &mchp_coreqspi_mem_ops;
ctlr->mem_caps = &mchp_coreqspi_mem_caps;
- ctlr->setup = mchp_coreqspi_setup_op;
ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD |
SPI_TX_DUAL | SPI_TX_QUAD;
ctlr->dev.of_node = np;
@@ -729,9 +776,21 @@ static int mchp_coreqspi_probe(struct platform_device *pdev)
ctlr->prepare_message = mchp_coreqspi_prepare_message;
ctlr->unprepare_message = mchp_coreqspi_unprepare_message;
ctlr->transfer_one = mchp_coreqspi_transfer_one;
- ctlr->num_chipselect = 2;
+ ctlr->setup = mchp_coreqspi_setup;
+ ctlr->set_cs = mchp_coreqspi_set_cs;
ctlr->use_gpio_descriptors = true;
+ val = readl_relaxed(qspi->regs + REG_CONTROL);
+ val |= (CONTROL_MASTER | CONTROL_ENABLE);
+ writel_relaxed(val, qspi->regs + REG_CONTROL);
+
+ /*
+ * Put cs into software controlled mode
+ */
+ val = readl_relaxed(qspi->regs + REG_DIRECT_ACCESS);
+ val |= DIRECT_ACCESS_EN_SSEL;
+ writel(val, qspi->regs + REG_DIRECT_ACCESS);
+
ret = spi_register_controller(ctlr);
if (ret)
return dev_err_probe(&pdev->dev, ret,
diff --git a/drivers/spi/spi-mtk-snfi.c b/drivers/spi/spi-mtk-snfi.c
index e616e68..6e96e50f 100644
--- a/drivers/spi/spi-mtk-snfi.c
+++ b/drivers/spi/spi-mtk-snfi.c
@@ -961,7 +961,7 @@ static int mtk_snand_read_page_cache(struct mtk_snand *snf,
&snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
dev_err(snf->dev, "DMA timed out for reading from cache.\n");
ret = -ETIMEDOUT;
- goto cleanup;
+ goto cleanup2;
}
// Wait for BUS_SEC_CNTR returning expected value
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index 45d9b4c..50bb770 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -996,8 +996,11 @@ static int spi_qup_init_dma(struct spi_controller *host, resource_size_t base)
err:
dma_release_channel(host->dma_tx);
+ host->dma_tx = NULL;
err_tx:
dma_release_channel(host->dma_rx);
+ host->dma_rx = NULL;
+
return ret;
}
diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c
index 14cd1b9..231fbcf 100644
--- a/drivers/spi/spi-rockchip.c
+++ b/drivers/spi/spi-rockchip.c
@@ -98,7 +98,6 @@
#define CR0_FRF_MICROWIRE 0x2
#define CR0_XFM_OFFSET 18
-#define CR0_XFM_MASK (0x03 << SPI_XFM_OFFSET)
#define CR0_XFM_TR 0x0
#define CR0_XFM_TO 0x1
#define CR0_XFM_RO 0x2
@@ -109,8 +108,6 @@
#define CR0_SOI_OFFSET 23
-#define CR0_MTM_OFFSET 0x21
-
/* Bit fields in SER, 2bit */
#define SER_MASK 0x3
@@ -357,7 +354,8 @@ static irqreturn_t rockchip_spi_isr(int irq, void *dev_id)
struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);
/* When int_cs_inactive comes, spi target abort */
- if (rs->cs_inactive && readl_relaxed(rs->regs + ROCKCHIP_SPI_IMR) & INT_CS_INACTIVE) {
+ if (rs->cs_inactive &&
+ (readl_relaxed(rs->regs + ROCKCHIP_SPI_ISR) & INT_CS_INACTIVE)) {
ctlr->target_abort(ctlr);
writel_relaxed(0, rs->regs + ROCKCHIP_SPI_IMR);
writel_relaxed(0xffffffff, rs->regs + ROCKCHIP_SPI_ICR);
diff --git a/drivers/spi/spi-rzv2h-rspi.c b/drivers/spi/spi-rzv2h-rspi.c
index f45af58..1655efd 100644
--- a/drivers/spi/spi-rzv2h-rspi.c
+++ b/drivers/spi/spi-rzv2h-rspi.c
@@ -579,7 +579,7 @@ static u32 rzv2h_rspi_setup_clock(struct rzv2h_rspi_priv *rspi, u32 hz)
rspi->info->find_pclk_rate(rspi->pclk, hz, &best_clock);
if (!best_clock.clk_rate)
- return -EINVAL;
+ return 0;
ret = clk_set_rate(best_clock.clk, best_clock.clk_rate);
if (ret)
diff --git a/drivers/spi/spi-sprd.c b/drivers/spi/spi-sprd.c
index fd3fd0c..acebf9c 100644
--- a/drivers/spi/spi-sprd.c
+++ b/drivers/spi/spi-sprd.c
@@ -991,7 +991,8 @@ static int sprd_spi_probe(struct platform_device *pdev)
disable_clk:
clk_disable_unprepare(ss->clk);
release_dma:
- sprd_spi_dma_release(ss);
+ if (ss->dma.enable)
+ sprd_spi_dma_release(ss);
free_controller:
spi_controller_put(sctlr);
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index 1fbd710..e3b413b 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -867,6 +867,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
dev_err(qspi->dev,
"dma_alloc_coherent failed, using PIO mode\n");
dma_release_channel(qspi->rx_chan);
+ qspi->rx_chan = NULL;
goto no_dma;
}
host->dma_rx = qspi->rx_chan;
diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c
index 1f58c90..f1f9f6f 100644
--- a/drivers/staging/greybus/hid.c
+++ b/drivers/staging/greybus/hid.c
@@ -201,7 +201,7 @@ static void gb_hid_init_report(struct gb_hid *ghid, struct hid_report *report)
* we just need to setup the input fields, so using
* hid_report_raw_event is safe.
*/
- hid_report_raw_event(ghid->hid, report->type, ghid->inbuf, size, 1);
+ hid_report_raw_event(ghid->hid, report->type, ghid->inbuf, ghid->bufsize, size, 1);
}
static void gb_hid_init_reports(struct gb_hid *ghid)
diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c
index 7959dae..4cfdf7c 100644
--- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c
+++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c
@@ -194,7 +194,8 @@ struct rtw_cbuf *rtw_cbuf_alloc(u32 size)
struct rtw_cbuf *cbuf;
cbuf = kzalloc_flex(*cbuf, bufs, size);
- cbuf->size = size;
+ if (cbuf)
+ cbuf->size = size;
return cbuf;
}
diff --git a/drivers/staging/vme_user/vme_fake.c b/drivers/staging/vme_user/vme_fake.c
index be4ad47..8abaa31 100644
--- a/drivers/staging/vme_user/vme_fake.c
+++ b/drivers/staging/vme_user/vme_fake.c
@@ -1230,6 +1230,8 @@ static int __init fake_init(void)
err_driver:
kfree(fake_bridge);
err_struct:
+ root_device_unregister(vme_root);
+
return retval;
}
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index e80449f..62ada3a 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -995,6 +995,7 @@ int iscsit_setup_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
int data_direction, payload_length;
struct iscsi_ecdb_ahdr *ecdb_ahdr;
struct iscsi_scsi_req *hdr;
+ u16 ahslength, cdb_length;
int iscsi_task_attr;
unsigned char *cdb;
int sam_task_attr;
@@ -1108,14 +1109,27 @@ int iscsit_setup_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
ISCSI_REASON_CMD_NOT_SUPPORTED, buf);
}
- cdb = kmalloc(be16_to_cpu(ecdb_ahdr->ahslength) + 15,
- GFP_KERNEL);
+ ahslength = be16_to_cpu(ecdb_ahdr->ahslength);
+ if (!ahslength) {
+ pr_err("Extended CDB AHS with zero length, protocol error.\n");
+ return iscsit_add_reject_cmd(cmd,
+ ISCSI_REASON_PROTOCOL_ERROR, buf);
+ }
+ if (ahslength > (hdr->hlength * 4) - 3) {
+ pr_err("Extended CDB AHS length %u exceeds available PDU buffer.\n",
+ ahslength);
+ return iscsit_add_reject_cmd(cmd,
+ ISCSI_REASON_PROTOCOL_ERROR, buf);
+ }
+
+ cdb_length = ahslength - 1 + ISCSI_CDB_SIZE;
+
+ cdb = kmalloc(cdb_length, GFP_KERNEL);
if (cdb == NULL)
return iscsit_add_reject_cmd(cmd,
ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
memcpy(cdb, hdr->cdb, ISCSI_CDB_SIZE);
- memcpy(cdb + ISCSI_CDB_SIZE, ecdb_ahdr->ecdb,
- be16_to_cpu(ecdb_ahdr->ahslength) - 1);
+ memcpy(cdb + ISCSI_CDB_SIZE, ecdb_ahdr->ecdb, cdb_length - ISCSI_CDB_SIZE);
}
data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
@@ -2281,7 +2295,9 @@ iscsit_handle_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
goto reject;
if (conn->conn_ops->DataDigest) {
- data_crc = iscsit_crc_buf(text_in, rx_size, 0, NULL);
+ data_crc = iscsit_crc_buf(text_in,
+ ALIGN(payload_length, 4),
+ 0, NULL);
if (checksum != data_crc) {
pr_err("Text data CRC32C DataDigest"
" 0x%08x does not match computed"
@@ -2300,6 +2316,7 @@ iscsit_handle_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
" Command CmdSN: 0x%08x due to"
" DataCRC error.\n", hdr->cmdsn);
kfree(text_in);
+ cmd->text_in_ptr = NULL;
return 0;
}
} else {
diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c
index c46c69a..a3ad2d2 100644
--- a/drivers/target/iscsi/iscsi_target_auth.c
+++ b/drivers/target/iscsi/iscsi_target_auth.c
@@ -340,13 +340,22 @@ static int chap_server_compute_hash(
goto out;
}
break;
- case BASE64:
+ case BASE64: {
+ size_t r_len = strlen(chap_r);
+
+ while (r_len > 0 && chap_r[r_len - 1] == '=')
+ r_len--;
+ if (r_len > DIV_ROUND_UP(chap->digest_size * 4, 3)) {
+ pr_err("Malformed CHAP_R: base64 payload too long\n");
+ goto out;
+ }
if (chap_base64_decode(client_digest, chap_r, strlen(chap_r)) !=
chap->digest_size) {
pr_err("Malformed CHAP_R: invalid BASE64\n");
goto out;
}
break;
+ }
default:
pr_err("Could not find CHAP_R\n");
goto out;
@@ -473,6 +482,14 @@ static int chap_server_compute_hash(
}
break;
case BASE64:
+ /*
+ * No overflow check needed: initiatorchg_binhex is
+ * CHAP_CHALLENGE_STR_LEN bytes and extract_param() caps
+ * initiatorchg at CHAP_CHALLENGE_STR_LEN characters, so
+ * the decoded output is at most DIV_ROUND_UP(
+ * (CHAP_CHALLENGE_STR_LEN - 1) * 3, 4) bytes, which is
+ * less than CHAP_CHALLENGE_STR_LEN.
+ */
initiatorchg_len = chap_base64_decode(initiatorchg_binhex,
initiatorchg,
strlen(initiatorchg));
diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c
index 832588f..b03ed154 100644
--- a/drivers/target/iscsi/iscsi_target_nego.c
+++ b/drivers/target/iscsi/iscsi_target_nego.c
@@ -899,10 +899,14 @@ static int iscsi_target_handle_csg_zero(
SENDER_TARGET,
login->rsp_buf,
&login->rsp_length,
+ MAX_KEY_VALUE_PAIRS,
conn->param_list,
conn->tpg->tpg_attrib.login_keys_workaround);
- if (ret < 0)
+ if (ret < 0) {
+ iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_INITIATOR_ERR,
+ ISCSI_LOGIN_STATUS_INIT_ERR);
return -1;
+ }
if (!iscsi_check_negotiated_keys(conn->param_list)) {
bool auth_required = iscsi_conn_auth_required(conn);
@@ -986,6 +990,7 @@ static int iscsi_target_handle_csg_one(struct iscsit_conn *conn, struct iscsi_lo
SENDER_TARGET,
login->rsp_buf,
&login->rsp_length,
+ MAX_KEY_VALUE_PAIRS,
conn->param_list,
conn->tpg->tpg_attrib.login_keys_workaround);
if (ret < 0) {
diff --git a/drivers/target/iscsi/iscsi_target_parameters.c b/drivers/target/iscsi/iscsi_target_parameters.c
index 4ed578c..2b318b1 100644
--- a/drivers/target/iscsi/iscsi_target_parameters.c
+++ b/drivers/target/iscsi/iscsi_target_parameters.c
@@ -1371,19 +1371,42 @@ int iscsi_decode_text_input(
return -1;
}
+/*
+ * Append "key=value" plus a trailing NUL into @textbuf at *@length.
+ * Returns 0 on success and advances *@length, or -EMSGSIZE if the
+ * record (including the NUL) would not fit in the remaining buffer.
+ */
+static int iscsi_encode_text_record(char *textbuf, u32 *length,
+ u32 textbuf_size,
+ const char *key, const char *value)
+{
+ int n;
+ u32 avail;
+
+ if (*length >= textbuf_size)
+ return -EMSGSIZE;
+
+ avail = textbuf_size - *length;
+ n = snprintf(textbuf + *length, avail, "%s=%s", key, value);
+ if (n < 0 || (u32)n + 1 > avail)
+ return -EMSGSIZE;
+
+ *length += n + 1;
+ return 0;
+}
+
int iscsi_encode_text_output(
u8 phase,
u8 sender,
char *textbuf,
u32 *length,
+ u32 textbuf_size,
struct iscsi_param_list *param_list,
bool keys_workaround)
{
- char *output_buf = NULL;
struct iscsi_extra_response *er;
struct iscsi_param *param;
-
- output_buf = textbuf + *length;
+ int ret;
if (iscsi_enforce_integrity_rules(phase, param_list) < 0)
return -1;
@@ -1395,10 +1418,12 @@ int iscsi_encode_text_output(
!IS_PSTATE_RESPONSE_SENT(param) &&
!IS_PSTATE_REPLY_OPTIONAL(param) &&
(param->phase & phase)) {
- *length += sprintf(output_buf, "%s=%s",
- param->name, param->value);
- *length += 1;
- output_buf = textbuf + *length;
+ ret = iscsi_encode_text_record(textbuf, length,
+ textbuf_size,
+ param->name,
+ param->value);
+ if (ret < 0)
+ goto err_overflow;
SET_PSTATE_RESPONSE_SENT(param);
pr_debug("Sending key: %s=%s\n",
param->name, param->value);
@@ -1408,10 +1433,12 @@ int iscsi_encode_text_output(
!IS_PSTATE_ACCEPTOR(param) &&
!IS_PSTATE_PROPOSER(param) &&
(param->phase & phase)) {
- *length += sprintf(output_buf, "%s=%s",
- param->name, param->value);
- *length += 1;
- output_buf = textbuf + *length;
+ ret = iscsi_encode_text_record(textbuf, length,
+ textbuf_size,
+ param->name,
+ param->value);
+ if (ret < 0)
+ goto err_overflow;
SET_PSTATE_PROPOSER(param);
iscsi_check_proposer_for_optional_reply(param,
keys_workaround);
@@ -1421,14 +1448,21 @@ int iscsi_encode_text_output(
}
list_for_each_entry(er, ¶m_list->extra_response_list, er_list) {
- *length += sprintf(output_buf, "%s=%s", er->key, er->value);
- *length += 1;
- output_buf = textbuf + *length;
+ ret = iscsi_encode_text_record(textbuf, length, textbuf_size,
+ er->key, er->value);
+ if (ret < 0)
+ goto err_overflow;
pr_debug("Sending key: %s=%s\n", er->key, er->value);
}
iscsi_release_extra_responses(param_list);
return 0;
+
+err_overflow:
+ pr_err("iSCSI login response buffer (%u bytes) exhausted, dropping login.\n",
+ textbuf_size);
+ iscsi_release_extra_responses(param_list);
+ return -1;
}
int iscsi_check_negotiated_keys(struct iscsi_param_list *param_list)
diff --git a/drivers/target/iscsi/iscsi_target_parameters.h b/drivers/target/iscsi/iscsi_target_parameters.h
index c672a97..38d2238 100644
--- a/drivers/target/iscsi/iscsi_target_parameters.h
+++ b/drivers/target/iscsi/iscsi_target_parameters.h
@@ -43,7 +43,7 @@ extern struct iscsi_param *iscsi_find_param_from_key(char *, struct iscsi_param_
extern int iscsi_extract_key_value(char *, char **, char **);
extern int iscsi_update_param_value(struct iscsi_param *, char *);
extern int iscsi_decode_text_input(u8, u8, char *, u32, struct iscsit_conn *);
-extern int iscsi_encode_text_output(u8, u8, char *, u32 *,
+extern int iscsi_encode_text_output(u8, u8, char *, u32 *, u32,
struct iscsi_param_list *, bool);
extern int iscsi_check_negotiated_keys(struct iscsi_param_list *);
extern void iscsi_set_connection_parameters(struct iscsi_conn_ops *,
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index a25fd82..1102973 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -393,6 +393,7 @@ static int tcm_loop_driver_probe(struct device *dev)
if (error) {
pr_err("%s: scsi_add_host failed\n", __func__);
scsi_host_put(sh);
+ tl_hba->sh = NULL;
return -ENODEV;
}
return 0;
@@ -406,8 +407,10 @@ static void tcm_loop_driver_remove(struct device *dev)
tl_hba = to_tcm_loop_hba(dev);
sh = tl_hba->sh;
- scsi_remove_host(sh);
- scsi_host_put(sh);
+ if (sh) {
+ scsi_remove_host(sh);
+ scsi_host_put(sh);
+ }
}
static void tcm_loop_release_adapter(struct device *dev)
@@ -436,6 +439,11 @@ static int tcm_loop_setup_hba_bus(struct tcm_loop_hba *tl_hba, int tcm_loop_host
return -ENODEV;
}
+ if (!tl_hba->sh) {
+ device_unregister(&tl_hba->dev);
+ return -ENODEV;
+ }
+
return 0;
}
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index d93773b..2b19a95 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -3249,7 +3249,7 @@ static ssize_t target_tg_pt_gp_members_show(struct config_item *item,
config_item_name(&lun->lun_group.cg_item));
cur_len++; /* Extra byte for NULL terminator */
- if ((cur_len + len) > PAGE_SIZE) {
+ if (cur_len > TG_PT_GROUP_NAME_BUF || (cur_len + len) > PAGE_SIZE) {
pr_warn("Ran out of lu_gp_show_attr"
"_members buffer\n");
break;
diff --git a/drivers/tee/optee/supp.c b/drivers/tee/optee/supp.c
index a3d11b1..06747e9 100644
--- a/drivers/tee/optee/supp.c
+++ b/drivers/tee/optee/supp.c
@@ -10,7 +10,11 @@
struct optee_supp_req {
struct list_head link;
+ int id;
+
bool in_queue;
+ bool processed;
+
u32 func;
u32 ret;
size_t num_params;
@@ -19,6 +23,9 @@ struct optee_supp_req {
struct completion c;
};
+/* It is temporary request used for revoked pending request in supp->idr. */
+#define INVALID_REQ_PTR ((struct optee_supp_req *)ERR_PTR(-EBADF))
+
void optee_supp_init(struct optee_supp *supp)
{
memset(supp, 0, sizeof(*supp));
@@ -39,21 +46,23 @@ void optee_supp_release(struct optee_supp *supp)
{
int id;
struct optee_supp_req *req;
- struct optee_supp_req *req_tmp;
mutex_lock(&supp->mutex);
- /* Abort all request retrieved by supplicant */
+ /* Abort all request */
idr_for_each_entry(&supp->idr, req, id) {
idr_remove(&supp->idr, id);
- req->ret = TEEC_ERROR_COMMUNICATION;
- complete(&req->c);
- }
+ /* Skip if request was already marked invalid */
+ if (IS_ERR(req))
+ continue;
- /* Abort all queued requests */
- list_for_each_entry_safe(req, req_tmp, &supp->reqs, link) {
- list_del(&req->link);
- req->in_queue = false;
+ /* For queued requests where supplicant has not seen it */
+ if (req->in_queue) {
+ list_del(&req->link);
+ req->in_queue = false;
+ }
+
+ req->processed = true;
req->ret = TEEC_ERROR_COMMUNICATION;
complete(&req->c);
}
@@ -100,8 +109,16 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
/* Insert the request in the request list */
mutex_lock(&supp->mutex);
+ req->id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
+ if (req->id < 0) {
+ mutex_unlock(&supp->mutex);
+ kfree(req);
+ return TEEC_ERROR_OUT_OF_MEMORY;
+ }
+
list_add_tail(&req->link, &supp->reqs);
req->in_queue = true;
+ req->processed = false;
mutex_unlock(&supp->mutex);
/* Tell an eventual waiter there's a new request */
@@ -117,21 +134,43 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
if (wait_for_completion_killable(&req->c)) {
mutex_lock(&supp->mutex);
if (req->in_queue) {
+ /* Supplicant has not seen this request yet. */
+ idr_remove(&supp->idr, req->id);
list_del(&req->link);
req->in_queue = false;
+
+ ret = TEEC_ERROR_COMMUNICATION;
+ } else if (req->processed) {
+ /*
+ * Supplicant has processed this request. Ignore the
+ * kill signal for now and submit the result. req is not
+ * in supp->reqs (removed by supp_pop_entry()) nor in
+ * supp->idr (removed by supp_pop_req()).
+ */
+ ret = req->ret;
+ } else {
+ /*
+ * Supplicant is in the middle of processing this
+ * request. Replace req with INVALID_REQ_PTR so that
+ * the ID remains busy, causing optee_supp_send() to
+ * fail on the next call to supp_pop_req() with this ID.
+ */
+ idr_replace(&supp->idr, INVALID_REQ_PTR, req->id);
+ ret = TEEC_ERROR_COMMUNICATION;
}
+
mutex_unlock(&supp->mutex);
- req->ret = TEEC_ERROR_COMMUNICATION;
+ } else {
+ ret = req->ret;
}
- ret = req->ret;
kfree(req);
return ret;
}
static struct optee_supp_req *supp_pop_entry(struct optee_supp *supp,
- int num_params, int *id)
+ int num_params)
{
struct optee_supp_req *req;
@@ -153,10 +192,6 @@ static struct optee_supp_req *supp_pop_entry(struct optee_supp *supp,
return ERR_PTR(-EINVAL);
}
- *id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
- if (*id < 0)
- return ERR_PTR(-ENOMEM);
-
list_del(&req->link);
req->in_queue = false;
@@ -214,7 +249,6 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
struct optee *optee = tee_get_drvdata(teedev);
struct optee_supp *supp = &optee->supp;
struct optee_supp_req *req = NULL;
- int id;
size_t num_meta;
int rc;
@@ -224,15 +258,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
while (true) {
mutex_lock(&supp->mutex);
- req = supp_pop_entry(supp, *num_params - num_meta, &id);
+ req = supp_pop_entry(supp, *num_params - num_meta);
+ if (req)
+ break; /* Keep mutex held. */
mutex_unlock(&supp->mutex);
- if (req) {
- if (IS_ERR(req))
- return PTR_ERR(req);
- break;
- }
-
/*
* If we didn't get a request we'll block in
* wait_for_completion() to avoid needless spinning.
@@ -245,6 +275,13 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
return -ERESTARTSYS;
}
+ /* supp->mutex held and req != NULL. */
+
+ if (IS_ERR(req)) {
+ mutex_unlock(&supp->mutex);
+ return PTR_ERR(req);
+ }
+
if (num_meta) {
/*
* tee-supplicant support meta parameters -> requsts can be
@@ -252,13 +289,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
*/
param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT |
TEE_IOCTL_PARAM_ATTR_META;
- param->u.value.a = id;
+ param->u.value.a = req->id;
param->u.value.b = 0;
param->u.value.c = 0;
} else {
- mutex_lock(&supp->mutex);
- supp->req_id = id;
- mutex_unlock(&supp->mutex);
+ supp->req_id = req->id;
}
*func = req->func;
@@ -266,6 +301,7 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
memcpy(param + num_meta, req->param,
sizeof(struct tee_param) * req->num_params);
+ mutex_unlock(&supp->mutex);
return 0;
}
@@ -297,12 +333,17 @@ static struct optee_supp_req *supp_pop_req(struct optee_supp *supp,
if (!req)
return ERR_PTR(-ENOENT);
+ /* optee_supp_thrd_req() already returned to optee. */
+ if (IS_ERR(req))
+ goto failed_req;
+
if ((num_params - nm) != req->num_params)
return ERR_PTR(-EINVAL);
+ *num_meta = nm;
+failed_req:
idr_remove(&supp->idr, id);
supp->req_id = -1;
- *num_meta = nm;
return req;
}
@@ -328,10 +369,9 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
mutex_lock(&supp->mutex);
req = supp_pop_req(supp, num_params, param, &num_meta);
- mutex_unlock(&supp->mutex);
-
if (IS_ERR(req)) {
- /* Something is wrong, let supplicant restart. */
+ mutex_unlock(&supp->mutex);
+ /* Something is wrong, let supplicant handel it. */
return PTR_ERR(req);
}
@@ -355,9 +395,10 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
}
}
req->ret = ret;
-
+ req->processed = true;
/* Let the requesting thread continue */
complete(&req->c);
+ mutex_unlock(&supp->mutex);
return 0;
}
diff --git a/drivers/tee/qcomtee/core.c b/drivers/tee/qcomtee/core.c
index b1cb50e..60fe3b5 100644
--- a/drivers/tee/qcomtee/core.c
+++ b/drivers/tee/qcomtee/core.c
@@ -306,8 +306,10 @@ int qcomtee_object_user_init(struct qcomtee_object *object,
break;
case QCOMTEE_OBJECT_TYPE_CB:
object->ops = ops;
- if (!object->ops->dispatch)
- return -EINVAL;
+ if (!object->ops->dispatch) {
+ ret = -EINVAL;
+ break;
+ }
/* If failed, "no-name". */
object->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index ef9642d7..1aac50c 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -530,11 +530,24 @@ static int params_to_user(struct tee_ioctl_param __user *uparams,
return 0;
}
+static void free_params(struct tee_param *params, size_t num_params)
+{
+ size_t n;
+
+ if (!params)
+ return;
+
+ for (n = 0; n < num_params; n++)
+ if (tee_param_is_memref(params + n) && params[n].u.memref.shm)
+ tee_shm_put(params[n].u.memref.shm);
+
+ kfree(params);
+}
+
static int tee_ioctl_open_session(struct tee_context *ctx,
struct tee_ioctl_buf_data __user *ubuf)
{
int rc;
- size_t n;
struct tee_ioctl_buf_data buf;
struct tee_ioctl_open_session_arg __user *uarg;
struct tee_ioctl_open_session_arg arg;
@@ -595,16 +608,7 @@ static int tee_ioctl_open_session(struct tee_context *ctx,
*/
if (rc && have_session && ctx->teedev->desc->ops->close_session)
ctx->teedev->desc->ops->close_session(ctx, arg.session);
-
- if (params) {
- /* Decrease ref count for all valid shared memory pointers */
- for (n = 0; n < arg.num_params; n++)
- if (tee_param_is_memref(params + n) &&
- params[n].u.memref.shm)
- tee_shm_put(params[n].u.memref.shm);
- kfree(params);
- }
-
+ free_params(params, arg.num_params);
return rc;
}
@@ -612,7 +616,6 @@ static int tee_ioctl_invoke(struct tee_context *ctx,
struct tee_ioctl_buf_data __user *ubuf)
{
int rc;
- size_t n;
struct tee_ioctl_buf_data buf;
struct tee_ioctl_invoke_arg __user *uarg;
struct tee_ioctl_invoke_arg arg;
@@ -657,14 +660,7 @@ static int tee_ioctl_invoke(struct tee_context *ctx,
}
rc = params_to_user(uparams, arg.num_params, params);
out:
- if (params) {
- /* Decrease ref count for all valid shared memory pointers */
- for (n = 0; n < arg.num_params; n++)
- if (tee_param_is_memref(params + n) &&
- params[n].u.memref.shm)
- tee_shm_put(params[n].u.memref.shm);
- kfree(params);
- }
+ free_params(params, arg.num_params);
return rc;
}
@@ -672,7 +668,6 @@ static int tee_ioctl_object_invoke(struct tee_context *ctx,
struct tee_ioctl_buf_data __user *ubuf)
{
int rc;
- size_t n;
struct tee_ioctl_buf_data buf;
struct tee_ioctl_object_invoke_arg __user *uarg;
struct tee_ioctl_object_invoke_arg arg;
@@ -716,14 +711,7 @@ static int tee_ioctl_object_invoke(struct tee_context *ctx,
}
rc = params_to_user(uparams, arg.num_params, params);
out:
- if (params) {
- /* Decrease ref count for all valid shared memory pointers */
- for (n = 0; n < arg.num_params; n++)
- if (tee_param_is_memref(params + n) &&
- params[n].u.memref.shm)
- tee_shm_put(params[n].u.memref.shm);
- kfree(params);
- }
+ free_params(params, arg.num_params);
return rc;
}
@@ -846,9 +834,15 @@ static int tee_ioctl_supp_recv(struct tee_context *ctx,
return -ENOMEM;
rc = params_from_user(ctx, params, num_params, uarg->params);
- if (rc)
- goto out;
+ if (rc) {
+ free_params(params, num_params);
+ return rc;
+ }
+ /*
+ * supp_recv() may consume and replace the supplied parameters, so the
+ * final cleanup cannot use free_params() like the other ioctl paths.
+ */
rc = ctx->teedev->desc->ops->supp_recv(ctx, &func, &num_params, params);
if (rc)
goto out;
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index e9ea9f8..6742b35 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -435,7 +435,7 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags,
num_pages = iov_iter_npages(iter, INT_MAX);
if (!num_pages) {
ret = ERR_PTR(-ENOMEM);
- goto err_ctx_put;
+ goto err_free_shm;
}
shm->pages = kzalloc_objs(*shm->pages, num_pages);
diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c
index 8c4ae75..3273b8f 100644
--- a/drivers/thermal/intel/intel_hfi.c
+++ b/drivers/thermal/intel/intel_hfi.c
@@ -41,6 +41,7 @@
#include <linux/topology.h>
#include <linux/workqueue.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include "intel_hfi.h"
diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c
index 5401097..d1dd2f5 100644
--- a/drivers/thermal/intel/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c
@@ -20,6 +20,7 @@
#include <linux/debugfs.h>
#include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
#include <asm/msr.h>
#include "thermal_interrupt.h"
diff --git a/drivers/thunderbolt/property.c b/drivers/thunderbolt/property.c
index 50cbfc9..da2c59a 100644
--- a/drivers/thunderbolt/property.c
+++ b/drivers/thunderbolt/property.c
@@ -8,6 +8,7 @@
*/
#include <linux/err.h>
+#include <linux/overflow.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/uuid.h>
@@ -34,10 +35,11 @@ struct tb_property_dir_entry {
};
#define TB_PROPERTY_ROOTDIR_MAGIC 0x55584401
+#define TB_PROPERTY_MAX_DEPTH 8
static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
size_t block_len, unsigned int dir_offset, size_t dir_len,
- bool is_root);
+ bool is_root, unsigned int depth);
static inline void parse_dwdata(void *dst, const void *src, size_t dwords)
{
@@ -52,13 +54,16 @@ static inline void format_dwdata(void *dst, const void *src, size_t dwords)
static bool tb_property_entry_valid(const struct tb_property_entry *entry,
size_t block_len)
{
+ u32 end;
+
switch (entry->type) {
case TB_PROPERTY_TYPE_DIRECTORY:
case TB_PROPERTY_TYPE_DATA:
case TB_PROPERTY_TYPE_TEXT:
if (entry->length > block_len)
return false;
- if (entry->value + entry->length > block_len)
+ if (check_add_overflow(entry->value, entry->length, &end) ||
+ end > block_len)
return false;
break;
@@ -93,7 +98,8 @@ tb_property_alloc(const char *key, enum tb_property_type type)
}
static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
- const struct tb_property_entry *entry)
+ const struct tb_property_entry *entry,
+ unsigned int depth)
{
char key[TB_PROPERTY_KEY_SIZE + 1];
struct tb_property *property;
@@ -114,7 +120,7 @@ static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
switch (property->type) {
case TB_PROPERTY_TYPE_DIRECTORY:
dir = __tb_property_parse_dir(block, block_len, entry->value,
- entry->length, false);
+ entry->length, false, depth + 1);
if (!dir) {
kfree(property);
return NULL;
@@ -159,21 +165,31 @@ static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
}
static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
- size_t block_len, unsigned int dir_offset, size_t dir_len, bool is_root)
+ size_t block_len, unsigned int dir_offset, size_t dir_len, bool is_root,
+ unsigned int depth)
{
const struct tb_property_entry *entries;
size_t i, content_len, nentries;
unsigned int content_offset;
struct tb_property_dir *dir;
+ if (depth > TB_PROPERTY_MAX_DEPTH)
+ return NULL;
+
dir = kzalloc_obj(*dir);
if (!dir)
return NULL;
+ INIT_LIST_HEAD(&dir->properties);
+
if (is_root) {
content_offset = dir_offset + 2;
content_len = dir_len;
} else {
+ if (dir_len < 4) {
+ tb_property_free_dir(dir);
+ return NULL;
+ }
dir->uuid = kmemdup(&block[dir_offset], sizeof(*dir->uuid),
GFP_KERNEL);
if (!dir->uuid) {
@@ -187,12 +203,10 @@ static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
entries = (const struct tb_property_entry *)&block[content_offset];
nentries = content_len / (sizeof(*entries) / 4);
- INIT_LIST_HEAD(&dir->properties);
-
for (i = 0; i < nentries; i++) {
struct tb_property *property;
- property = tb_property_parse(block, block_len, &entries[i]);
+ property = tb_property_parse(block, block_len, &entries[i], depth);
if (!property) {
tb_property_free_dir(dir);
return NULL;
@@ -231,7 +245,7 @@ struct tb_property_dir *tb_property_parse_dir(const u32 *block,
return NULL;
return __tb_property_parse_dir(block, block_len, 0, rootdir->length,
- true);
+ true, 0);
}
/**
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 94beadb..2af0c4d 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -427,7 +427,7 @@ static int dw8250_handle_irq(struct uart_port *p)
unsigned int quirks = d->pdata->quirks;
unsigned int status;
- guard(uart_port_lock_irqsave)(p);
+ guard(uart_port_lock_check_sysrq_irqsave)(p);
switch (FIELD_GET(DW_UART_IIR_IID, iir)) {
case UART_IIR_NO_INT:
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index af78cc0..c66ba71 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -1784,7 +1784,10 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
}
/*
- * Context: port's lock must be held by the caller.
+ * Context: port's lock must be held by the caller. The caller must
+ * release it via guard(uart_port_lock_check_sysrq_irqsave) or
+ * uart_unlock_and_check_sysrq_irqrestore(), which captures SysRq
+ * character on unlock.
*/
void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir)
{
@@ -1837,7 +1840,7 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
if (iir & UART_IIR_NO_INT)
return 0;
- guard(uart_port_lock_irqsave)(port);
+ guard(uart_port_lock_check_sysrq_irqsave)(port);
serial8250_handle_irq_locked(port, iir);
return 1;
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 9aa61c9..ec284ac 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -334,7 +334,7 @@
Say Y here if you want to support this ICs.
config SERIAL_DZ
- bool "DECstation DZ serial driver"
+ tristate "DECstation DZ serial driver"
depends on MACH_DECSTATION && 32BIT
select SERIAL_CORE
default y
diff --git a/drivers/tty/serial/altera_jtaguart.c b/drivers/tty/serial/altera_jtaguart.c
index d47a62d..20f079f 100644
--- a/drivers/tty/serial/altera_jtaguart.c
+++ b/drivers/tty/serial/altera_jtaguart.c
@@ -379,6 +379,7 @@ static int altera_jtaguart_probe(struct platform_device *pdev)
struct resource *res_mem;
int i = pdev->id;
int irq;
+ int ret;
/* -1 emphasizes that the platform must have one port, no .N suffix */
if (i == -1)
@@ -418,7 +419,11 @@ static int altera_jtaguart_probe(struct platform_device *pdev)
port->flags = UPF_BOOT_AUTOCONF;
port->dev = &pdev->dev;
- uart_add_one_port(&altera_jtaguart_driver, port);
+ ret = uart_add_one_port(&altera_jtaguart_driver, port);
+ if (ret) {
+ iounmap(port->membase);
+ return ret;
+ }
return 0;
}
diff --git a/drivers/tty/serial/dz.c b/drivers/tty/serial/dz.c
index e53c543..39d93e9 100644
--- a/drivers/tty/serial/dz.c
+++ b/drivers/tty/serial/dz.c
@@ -40,6 +40,7 @@
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/module.h>
+#include <linux/platform_device.h>
#include <linux/serial.h>
#include <linux/serial_core.h>
#include <linux/sysrq.h>
@@ -48,14 +49,6 @@
#include <linux/atomic.h>
#include <linux/io.h>
-#include <asm/bootinfo.h>
-
-#include <asm/dec/interrupts.h>
-#include <asm/dec/kn01.h>
-#include <asm/dec/kn02.h>
-#include <asm/dec/machtype.h>
-#include <asm/dec/prom.h>
-#include <asm/dec/system.h>
#include "dz.h"
@@ -65,7 +58,9 @@ MODULE_LICENSE("GPL");
static char dz_name[] __initdata = "DECstation DZ serial driver version ";
-static char dz_version[] __initdata = "1.04";
+static char dz_version[] __initdata = "1.05";
+
+#define DZ_IO_SIZE 0x20 /* IOMEM space size. */
struct dz_port {
struct dz_mux *mux;
@@ -81,6 +76,7 @@ struct dz_mux {
};
static struct dz_mux dz_mux;
+static struct uart_driver dz_reg;
static inline struct dz_port *to_dport(struct uart_port *uport)
{
@@ -542,14 +538,47 @@ static int dz_encode_baud_rate(unsigned int baud)
static void dz_reset(struct dz_port *dport)
{
struct dz_mux *mux = dport->mux;
+ unsigned short tcr;
+ int loops = 10000;
if (mux->initialised)
return;
+ tcr = dz_in(dport, DZ_TCR);
+
+ /* Do not disturb any ongoing transmissions. */
+ if (dz_in(dport, DZ_CSR) & DZ_MSE) {
+ unsigned short csr, mask;
+
+ mask = tcr;
+ while ((mask & DZ_LNENB) && loops--) {
+ csr = dz_in(dport, DZ_CSR);
+ if (!(csr & DZ_TRDY))
+ continue;
+ mask &= ~(1 << ((csr & DZ_TLINE) >> 8));
+ dz_out(dport, DZ_TCR, mask);
+ iob();
+ udelay(2); /* 1.4us TRDY recovery. */
+ }
+ fsleep(1200); /* Transmitter drain. */
+ }
+
dz_out(dport, DZ_CSR, DZ_CLR);
while (dz_in(dport, DZ_CSR) & DZ_CLR);
iob();
+ /*
+ * Set parameters across all lines such as not to interfere
+ * with the initial PROM-based console. Otherwise any output
+ * produced before the console handover would cause the system
+ * firmware to produce rubbish.
+ */
+ for (int line = 0; line < DZ_NB_PORT; line++)
+ dz_out(dport, DZ_LPR, DZ_B9600 | DZ_CS8 | line);
+
+ /* Re-enable transmission for the initial PROM-based console. */
+ dz_out(dport, DZ_TCR, tcr);
+
/* Enable scanning. */
dz_out(dport, DZ_CSR, DZ_MSE);
@@ -633,26 +662,6 @@ static void dz_set_termios(struct uart_port *uport, struct ktermios *termios,
uart_port_unlock_irqrestore(&dport->port, flags);
}
-/*
- * Hack alert!
- * Required solely so that the initial PROM-based console
- * works undisturbed in parallel with this one.
- */
-static void dz_pm(struct uart_port *uport, unsigned int state,
- unsigned int oldstate)
-{
- struct dz_port *dport = to_dport(uport);
- unsigned long flags;
-
- uart_port_lock_irqsave(&dport->port, &flags);
- if (state < 3)
- dz_start_tx(&dport->port);
- else
- dz_stop_tx(&dport->port);
- uart_port_unlock_irqrestore(&dport->port, flags);
-}
-
-
static const char *dz_type(struct uart_port *uport)
{
return "DZ";
@@ -668,14 +677,13 @@ static void dz_release_port(struct uart_port *uport)
map_guard = atomic_add_return(-1, &mux->map_guard);
if (!map_guard)
- release_mem_region(uport->mapbase, dec_kn_slot_size);
+ release_mem_region(uport->mapbase, DZ_IO_SIZE);
}
static int dz_map_port(struct uart_port *uport)
{
if (!uport->membase)
- uport->membase = ioremap(uport->mapbase,
- dec_kn_slot_size);
+ uport->membase = ioremap(uport->mapbase, DZ_IO_SIZE);
if (!uport->membase) {
printk(KERN_ERR "dz: Cannot map MMIO\n");
return -ENOMEM;
@@ -691,8 +699,7 @@ static int dz_request_port(struct uart_port *uport)
map_guard = atomic_add_return(1, &mux->map_guard);
if (map_guard == 1) {
- if (!request_mem_region(uport->mapbase, dec_kn_slot_size,
- "dz")) {
+ if (!request_mem_region(uport->mapbase, DZ_IO_SIZE, "dz")) {
atomic_add(-1, &mux->map_guard);
printk(KERN_ERR
"dz: Unable to reserve MMIO resource\n");
@@ -703,7 +710,7 @@ static int dz_request_port(struct uart_port *uport)
if (ret) {
map_guard = atomic_add_return(-1, &mux->map_guard);
if (!map_guard)
- release_mem_region(uport->mapbase, dec_kn_slot_size);
+ release_mem_region(uport->mapbase, DZ_IO_SIZE);
return ret;
}
return 0;
@@ -748,7 +755,6 @@ static const struct uart_ops dz_ops = {
.startup = dz_startup,
.shutdown = dz_shutdown,
.set_termios = dz_set_termios,
- .pm = dz_pm,
.type = dz_type,
.release_port = dz_release_port,
.request_port = dz_request_port,
@@ -756,20 +762,15 @@ static const struct uart_ops dz_ops = {
.verify_port = dz_verify_port,
};
-static void __init dz_init_ports(void)
+static int __init dz_probe(struct platform_device *pdev)
{
- static int first = 1;
- unsigned long base;
+ struct resource *mem_resource, *irq_resource;
int line;
- if (!first)
- return;
- first = 0;
-
- if (mips_machtype == MACH_DS23100 || mips_machtype == MACH_DS5100)
- base = dec_kn_slot_base + KN01_DZ11;
- else
- base = dec_kn_slot_base + KN02_DZ11;
+ mem_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ irq_resource = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!mem_resource || !irq_resource)
+ return -ENODEV;
for (line = 0; line < DZ_NB_PORT; line++) {
struct dz_port *dport = &dz_mux.dport[line];
@@ -777,14 +778,33 @@ static void __init dz_init_ports(void)
dport->mux = &dz_mux;
- uport->irq = dec_interrupt[DEC_IRQ_DZ11];
+ uport->dev = &pdev->dev;
+ uport->irq = irq_resource->start;
uport->fifosize = 1;
uport->iotype = UPIO_MEM;
uport->flags = UPF_BOOT_AUTOCONF;
uport->ops = &dz_ops;
uport->line = line;
- uport->mapbase = base;
+ uport->mapbase = mem_resource->start;
uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_DZ_CONSOLE);
+
+ if (uart_add_one_port(&dz_reg, uport))
+ uport->dev = NULL;
+ }
+
+ return 0;
+}
+
+static void __exit dz_remove(struct platform_device *pdev)
+{
+ int line;
+
+ for (line = DZ_NB_PORT - 1; line >= 0; line--) {
+ struct dz_port *dport = &dz_mux.dport[line];
+ struct uart_port *uport = &dport->port;
+
+ if (uport->dev)
+ uart_remove_one_port(&dz_reg, uport);
}
}
@@ -867,24 +887,14 @@ static int __init dz_console_setup(struct console *co, char *options)
int bits = 8;
int parity = 'n';
int flow = 'n';
- int ret;
- ret = dz_map_port(uport);
- if (ret)
- return ret;
-
- spin_lock_init(&dport->port.lock); /* For dz_pm(). */
-
- dz_reset(dport);
- dz_pm(uport, 0, -1);
-
+ if (!dport->mux)
+ return -ENODEV;
if (options)
uart_parse_options(options, &baud, &parity, &bits, &flow);
-
- return uart_set_options(&dport->port, co, baud, parity, bits, flow);
+ return uart_set_options(uport, co, baud, parity, bits, flow);
}
-static struct uart_driver dz_reg;
static struct console dz_console = {
.name = "ttyS",
.write = dz_console_print,
@@ -895,18 +905,6 @@ static struct console dz_console = {
.data = &dz_reg,
};
-static int __init dz_serial_console_init(void)
-{
- if (!IOASIC) {
- dz_init_ports();
- register_console(&dz_console);
- return 0;
- } else
- return -ENXIO;
-}
-
-console_initcall(dz_serial_console_init);
-
#define SERIAL_DZ_CONSOLE &dz_console
#else
#define SERIAL_DZ_CONSOLE NULL
@@ -922,25 +920,32 @@ static struct uart_driver dz_reg = {
.cons = SERIAL_DZ_CONSOLE,
};
+static struct platform_driver dz_driver = {
+ .remove = __exit_p(dz_remove),
+ .driver = { .name = "dz" },
+};
+
static int __init dz_init(void)
{
- int ret, i;
-
- if (IOASIC)
- return -ENXIO;
+ int ret;
printk("%s%s\n", dz_name, dz_version);
- dz_init_ports();
-
ret = uart_register_driver(&dz_reg);
if (ret)
return ret;
+ ret = platform_driver_probe(&dz_driver, dz_probe);
+ if (ret)
+ uart_unregister_driver(&dz_reg);
- for (i = 0; i < DZ_NB_PORT; i++)
- uart_add_one_port(&dz_reg, &dz_mux.dport[i].port);
+ return ret;
+}
- return 0;
+static void __exit dz_exit(void)
+{
+ platform_driver_unregister(&dz_driver);
+ uart_unregister_driver(&dz_reg);
}
module_init(dz_init);
+module_exit(dz_exit);
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 1bd7ec9..b7919c0 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -1379,7 +1379,8 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
if (!nent) {
dev_err(sport->port.dev, "DMA Rx mapping error\n");
- return -EINVAL;
+ ret = -EINVAL;
+ goto err_free_buf;
}
dma_rx_sconfig.src_addr = lpuart_dma_datareg_addr(sport);
@@ -1391,7 +1392,7 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
if (ret < 0) {
dev_err(sport->port.dev,
"DMA Rx slave config failed, err = %d\n", ret);
- return ret;
+ goto err_unmap_sg;
}
sport->dma_rx_desc = dmaengine_prep_dma_cyclic(chan,
@@ -1402,7 +1403,8 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
DMA_PREP_INTERRUPT);
if (!sport->dma_rx_desc) {
dev_err(sport->port.dev, "Cannot prepare cyclic DMA\n");
- return -EFAULT;
+ ret = -ENOMEM;
+ goto err_unmap_sg;
}
sport->dma_rx_desc->callback = lpuart_dma_rx_complete;
@@ -1426,6 +1428,13 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
}
return 0;
+
+err_unmap_sg:
+ dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+err_free_buf:
+ kfree(ring->buf);
+ ring->buf = NULL;
+ return ret;
}
static void lpuart_dma_rx_free(struct uart_port *port)
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index 6729d8e..ba1fcd6 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -689,8 +689,7 @@ static void pch_request_dma(struct uart_port *port)
if (!chan) {
dev_err(priv->port.dev, "%s:dma_request_channel FAILS(Tx)\n",
__func__);
- pci_dev_put(dma_dev);
- return;
+ goto err_pci_get;
}
priv->chan_tx = chan;
@@ -704,18 +703,26 @@ static void pch_request_dma(struct uart_port *port)
if (!chan) {
dev_err(priv->port.dev, "%s:dma_request_channel FAILS(Rx)\n",
__func__);
- dma_release_channel(priv->chan_tx);
- priv->chan_tx = NULL;
- pci_dev_put(dma_dev);
- return;
+ goto err_req_tx;
}
/* Get Consistent memory for DMA */
priv->rx_buf_virt = dma_alloc_coherent(port->dev, port->fifosize,
&priv->rx_buf_dma, GFP_KERNEL);
+ if (!priv->rx_buf_virt)
+ goto err_req_rx;
priv->chan_rx = chan;
pci_dev_put(dma_dev);
+ return;
+
+err_req_rx:
+ dma_release_channel(chan);
+err_req_tx:
+ dma_release_channel(priv->chan_tx);
+ priv->chan_tx = NULL;
+err_pci_get:
+ pci_dev_put(dma_dev);
}
static void pch_dma_rx_complete(void *arg)
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index b365dd5..17da115 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -50,7 +50,7 @@
#define TX_STOP_BIT_LEN_2 2
/* SE_UART_RX_TRANS_CFG */
-#define UART_RX_PAR_EN BIT(3)
+#define UART_RX_PAR_EN BIT(4)
/* SE_UART_RX_WORD_LEN */
#define RX_WORD_LEN_MASK GENMASK(9, 0)
@@ -1031,8 +1031,20 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
{
struct qcom_geni_serial_port *port = to_dev_port(uport);
struct tty_port *tport = &uport->state->port;
+ unsigned int fifo_len = kfifo_len(&tport->xmit_fifo);
- uart_xmit_advance(uport, port->tx_remaining);
+ /*
+ * Only advance the kfifo if it still contains the bytes that were
+ * transferred. uart_flush_buffer() may have run before this IRQ
+ * fired: it calls kfifo_reset() under the port lock, making
+ * fifo_len = 0 while tx_remaining remains non-zero. Calling
+ * uart_xmit_advance() in that case would underflow kfifo->out past
+ * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining
+ * and triggering a spurious large DMA transfer of stale data.
+ */
+ if (fifo_len >= port->tx_remaining)
+ uart_xmit_advance(uport, port->tx_remaining);
+
geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
port->tx_dma_addr = 0;
port->tx_remaining = 0;
diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c
index e27806b..17cd5bb 100644
--- a/drivers/tty/serial/samsung_tty.c
+++ b/drivers/tty/serial/samsung_tty.c
@@ -245,12 +245,9 @@ static bool s3c24xx_serial_txempty_nofifo(const struct uart_port *port)
static void s3c24xx_serial_rx_enable(struct uart_port *port)
{
struct s3c24xx_uart_port *ourport = to_ourport(port);
- unsigned long flags;
int count = 10000;
u32 ucon, ufcon;
- uart_port_lock_irqsave(port, &flags);
-
while (--count && !s3c24xx_serial_txempty_nofifo(port))
udelay(100);
@@ -263,23 +260,18 @@ static void s3c24xx_serial_rx_enable(struct uart_port *port)
wr_regl(port, S3C2410_UCON, ucon);
ourport->rx_enabled = 1;
- uart_port_unlock_irqrestore(port, flags);
}
static void s3c24xx_serial_rx_disable(struct uart_port *port)
{
struct s3c24xx_uart_port *ourport = to_ourport(port);
- unsigned long flags;
u32 ucon;
- uart_port_lock_irqsave(port, &flags);
-
ucon = rd_regl(port, S3C2410_UCON);
ucon &= ~S3C2410_UCON_RXIRQMODE;
wr_regl(port, S3C2410_UCON, ucon);
ourport->rx_enabled = 0;
- uart_port_unlock_irqrestore(port, flags);
}
static void s3c24xx_serial_stop_tx(struct uart_port *port)
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index 6c819b6..54db019 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -3025,7 +3025,7 @@ int sci_request_port(struct uart_port *port)
ret = sci_remap_port(port);
if (unlikely(ret != 0)) {
- release_resource(res);
+ release_mem_region(port->mapbase, sport->reg_size);
return ret;
}
diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c
index 72a3c0d..8f92b41 100644
--- a/drivers/tty/serial/zs.c
+++ b/drivers/tty/serial/zs.c
@@ -56,6 +56,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/major.h>
+#include <linux/platform_device.h>
#include <linux/serial.h>
#include <linux/serial_core.h>
#include <linux/spinlock.h>
@@ -66,10 +67,6 @@
#include <linux/atomic.h>
-#include <asm/dec/interrupts.h>
-#include <asm/dec/ioasic_addrs.h>
-#include <asm/dec/system.h>
-
#include "zs.h"
@@ -79,7 +76,7 @@ MODULE_LICENSE("GPL");
static char zs_name[] __initdata = "DECstation Z85C30 serial driver version ";
-static char zs_version[] __initdata = "0.10";
+static char zs_version[] __initdata = "0.11";
/*
* It would be nice to dynamically allocate everything that
@@ -98,25 +95,27 @@ static char zs_version[] __initdata = "0.10";
#define to_zport(uport) container_of(uport, struct zs_port, port)
-struct zs_parms {
- resource_size_t scc[ZS_NUM_SCCS];
- int irq[ZS_NUM_SCCS];
-};
-
static struct zs_scc zs_sccs[ZS_NUM_SCCS];
+static struct uart_driver zs_reg;
+/*
+ * Set parameters in WR5, WR12, WR13 such as not to interfere
+ * with the initial PROM-based console. Otherwise any output
+ * produced before the console handover would cause the system
+ * firmware to hang (TxENAB) or produce rubbish (Tx8, B9600).
+ */
static u8 zs_init_regs[ZS_NUM_REGS] __initdata = {
0, /* write 0 */
PAR_SPEC, /* write 1 */
0, /* write 2 */
0, /* write 3 */
X16CLK | SB1, /* write 4 */
- 0, /* write 5 */
+ Tx8 | TxENAB, /* write 5 */
0, 0, 0, /* write 6, 7, 8 */
MIE | DLC | NV, /* write 9 */
NRZ, /* write 10 */
TCBR | RCBR, /* write 11 */
- 0, 0, /* BRG time constant, write 12 + 13 */
+ 0x16, 0x00, /* BRG time constant, write 12 + 13 */
BRSRC | BRENABL, /* write 14 */
0, /* write 15 */
};
@@ -680,9 +679,9 @@ static void zs_status_handle(struct zs_port *zport, struct zs_port *zport_a)
uart_handle_dcd_change(uport,
zport->mctrl & TIOCM_CAR);
if (delta & TIOCM_RNG)
- uport->icount.dsr++;
- if (delta & TIOCM_DSR)
uport->icount.rng++;
+ if (delta & TIOCM_DSR)
+ uport->icount.dsr++;
if (delta)
wake_up_interruptible(&uport->state->port.delta_msr_wait);
@@ -826,22 +825,22 @@ static void zs_shutdown(struct uart_port *uport)
static void zs_reset(struct zs_port *zport)
{
+ struct zs_port *zport_a = &zport->scc->zport[ZS_CHAN_A];
struct zs_scc *scc = zport->scc;
int irq;
unsigned long flags;
spin_lock_irqsave(&scc->zlock, flags);
irq = !irqs_disabled_flags(flags);
- if (!scc->initialised) {
- /* Reset the pointer first, just in case... */
- read_zsreg(zport, R0);
- /* And let the current transmission finish. */
- zs_line_drain(zport, irq);
- write_zsreg(zport, R9, FHWRES);
- udelay(10);
- write_zsreg(zport, R9, 0);
- scc->initialised = 1;
- }
+
+ /* Reset the pointer first, just in case... */
+ read_zsreg(zport, R0);
+ /* And let the current transmission finish. */
+ zs_line_drain(zport, irq);
+ write_zsreg(zport, R9, zport == zport_a ? CHRA : CHRB);
+ udelay(10);
+ write_zsreg(zport, R9, 0);
+
load_zsregs(zport, zport->regs, irq);
spin_unlock_irqrestore(&scc->zlock, flags);
}
@@ -956,23 +955,6 @@ static void zs_set_termios(struct uart_port *uport, struct ktermios *termios,
spin_unlock_irqrestore(&scc->zlock, flags);
}
-/*
- * Hack alert!
- * Required solely so that the initial PROM-based console
- * works undisturbed in parallel with this one.
- */
-static void zs_pm(struct uart_port *uport, unsigned int state,
- unsigned int oldstate)
-{
- struct zs_port *zport = to_zport(uport);
-
- if (state < 3)
- zport->regs[5] |= TxENAB;
- else
- zport->regs[5] &= ~TxENAB;
- write_zsreg(zport, R5, zport->regs[5]);
-}
-
static const char *zs_type(struct uart_port *uport)
{
@@ -1055,7 +1037,6 @@ static const struct uart_ops zs_ops = {
.startup = zs_startup,
.shutdown = zs_shutdown,
.set_termios = zs_set_termios,
- .pm = zs_pm,
.type = zs_type,
.release_port = zs_release_port,
.request_port = zs_request_port,
@@ -1066,63 +1047,62 @@ static const struct uart_ops zs_ops = {
/*
* Initialize Z85C30 port structures.
*/
-static int __init zs_probe_sccs(void)
+static int __init zs_probe(struct platform_device *pdev)
{
- static int probed;
- struct zs_parms zs_parms;
- int chip, side, irq;
- int n_chips = 0;
+ struct resource *mem_resource, *irq_resource;
+ int chip, side;
int i;
- if (probed)
- return 0;
+ mem_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ irq_resource = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!mem_resource || !irq_resource)
+ return -ENODEV;
- irq = dec_interrupt[DEC_IRQ_SCC0];
- if (irq >= 0) {
- zs_parms.scc[n_chips] = IOASIC_SCC0;
- zs_parms.irq[n_chips] = dec_interrupt[DEC_IRQ_SCC0];
- n_chips++;
- }
- irq = dec_interrupt[DEC_IRQ_SCC1];
- if (irq >= 0) {
- zs_parms.scc[n_chips] = IOASIC_SCC1;
- zs_parms.irq[n_chips] = dec_interrupt[DEC_IRQ_SCC1];
- n_chips++;
- }
- if (!n_chips)
- return -ENXIO;
+ chip = pdev->id;
+ spin_lock_init(&zs_sccs[chip].zlock);
+ for (side = 0; side < ZS_NUM_CHAN; side++) {
+ struct zs_port *zport = &zs_sccs[chip].zport[side];
+ struct uart_port *uport = &zport->port;
- probed = 1;
+ zport->scc = &zs_sccs[chip];
+ zport->clk_mode = 16;
- for (chip = 0; chip < n_chips; chip++) {
- spin_lock_init(&zs_sccs[chip].zlock);
- for (side = 0; side < ZS_NUM_CHAN; side++) {
- struct zs_port *zport = &zs_sccs[chip].zport[side];
- struct uart_port *uport = &zport->port;
+ uport->dev = &pdev->dev;
+ uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_ZS_CONSOLE);
+ uport->irq = irq_resource->start;
+ uport->uartclk = ZS_CLOCK;
+ uport->fifosize = 1;
+ uport->iotype = UPIO_MEM;
+ uport->flags = UPF_BOOT_AUTOCONF;
+ uport->ops = &zs_ops;
+ uport->line = chip * ZS_NUM_CHAN + side;
+ uport->mapbase = mem_resource->start +
+ (side ^ ZS_CHAN_B) * ZS_CHAN_IO_SIZE;
- zport->scc = &zs_sccs[chip];
- zport->clk_mode = 16;
+ for (i = 0; i < ZS_NUM_REGS; i++)
+ zport->regs[i] = zs_init_regs[i];
- uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_ZS_CONSOLE);
- uport->irq = zs_parms.irq[chip];
- uport->uartclk = ZS_CLOCK;
- uport->fifosize = 1;
- uport->iotype = UPIO_MEM;
- uport->flags = UPF_BOOT_AUTOCONF;
- uport->ops = &zs_ops;
- uport->line = chip * ZS_NUM_CHAN + side;
- uport->mapbase = dec_kn_slot_base +
- zs_parms.scc[chip] +
- (side ^ ZS_CHAN_B) * ZS_CHAN_IO_SIZE;
-
- for (i = 0; i < ZS_NUM_REGS; i++)
- zport->regs[i] = zs_init_regs[i];
- }
+ if (uart_add_one_port(&zs_reg, uport))
+ uport->dev = NULL;
}
return 0;
}
+static void __exit zs_remove(struct platform_device *pdev)
+{
+ int chip, side;
+
+ chip = pdev->id;
+ for (side = ZS_NUM_CHAN - 1; side >= 0; side--) {
+ struct zs_port *zport = &zs_sccs[chip].zport[side];
+ struct uart_port *uport = &zport->port;
+
+ if (uport->dev)
+ uart_remove_one_port(&zs_reg, uport);
+ }
+}
+
#ifdef CONFIG_SERIAL_ZS_CONSOLE
static void zs_console_putchar(struct uart_port *uport, unsigned char ch)
@@ -1203,21 +1183,14 @@ static int __init zs_console_setup(struct console *co, char *options)
int bits = 8;
int parity = 'n';
int flow = 'n';
- int ret;
- ret = zs_map_port(uport);
- if (ret)
- return ret;
-
- zs_reset(zport);
- zs_pm(uport, 0, -1);
-
+ if (!zport->scc)
+ return -ENODEV;
if (options)
uart_parse_options(options, &baud, &parity, &bits, &flow);
return uart_set_options(uport, co, baud, parity, bits, flow);
}
-static struct uart_driver zs_reg;
static struct console zs_console = {
.name = "ttyS",
.write = zs_console_write,
@@ -1228,23 +1201,6 @@ static struct console zs_console = {
.data = &zs_reg,
};
-/*
- * Register console.
- */
-static int __init zs_serial_console_init(void)
-{
- int ret;
-
- ret = zs_probe_sccs();
- if (ret)
- return ret;
- register_console(&zs_console);
-
- return 0;
-}
-
-console_initcall(zs_serial_console_init);
-
#define SERIAL_ZS_CONSOLE &zs_console
#else
#define SERIAL_ZS_CONSOLE NULL
@@ -1260,47 +1216,31 @@ static struct uart_driver zs_reg = {
.cons = SERIAL_ZS_CONSOLE,
};
+static struct platform_driver zs_driver = {
+ .remove = __exit_p(zs_remove),
+ .driver = { .name = "zs" },
+};
+
/* zs_init inits the driver. */
static int __init zs_init(void)
{
- int i, ret;
+ int ret;
pr_info("%s%s\n", zs_name, zs_version);
- /* Find out how many Z85C30 SCCs we have. */
- ret = zs_probe_sccs();
- if (ret)
- return ret;
-
ret = uart_register_driver(&zs_reg);
if (ret)
return ret;
+ ret = platform_driver_probe(&zs_driver, zs_probe);
+ if (ret)
+ uart_unregister_driver(&zs_reg);
- for (i = 0; i < ZS_NUM_SCCS * ZS_NUM_CHAN; i++) {
- struct zs_scc *scc = &zs_sccs[i / ZS_NUM_CHAN];
- struct zs_port *zport = &scc->zport[i % ZS_NUM_CHAN];
- struct uart_port *uport = &zport->port;
-
- if (zport->scc)
- uart_add_one_port(&zs_reg, uport);
- }
-
- return 0;
+ return ret;
}
static void __exit zs_exit(void)
{
- int i;
-
- for (i = ZS_NUM_SCCS * ZS_NUM_CHAN - 1; i >= 0; i--) {
- struct zs_scc *scc = &zs_sccs[i / ZS_NUM_CHAN];
- struct zs_port *zport = &scc->zport[i % ZS_NUM_CHAN];
- struct uart_port *uport = &zport->port;
-
- if (zport->scc)
- uart_remove_one_port(&zs_reg, uport);
- }
-
+ platform_driver_unregister(&zs_driver);
uart_unregister_driver(&zs_reg);
}
diff --git a/drivers/tty/serial/zs.h b/drivers/tty/serial/zs.h
index 26ef8ea..e0d3c18 100644
--- a/drivers/tty/serial/zs.h
+++ b/drivers/tty/serial/zs.h
@@ -41,7 +41,6 @@ struct zs_scc {
struct zs_port zport[2];
spinlock_t zlock;
atomic_t irq_guard;
- int initialised;
};
#endif /* __KERNEL__ */
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 4805e40..c3f0895 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -9259,6 +9259,30 @@ static void ufshcd_config_mcq(struct ufs_hba *hba)
hba->nutrs);
}
+/**
+ * ufshcd_get_op_mode - get UFS operating mode.
+ * @hba: per-adapter instance
+ *
+ * Use the PA_PWRMODE value to represent the operating mode of UFS.
+ *
+ */
+static enum ufs_op_mode ufshcd_get_op_mode(struct ufs_hba *hba)
+{
+ u32 mode;
+ u8 rx_mode;
+ u8 tx_mode;
+
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
+ rx_mode = (mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK;
+ tx_mode = mode & PWRMODE_MASK;
+
+ if ((rx_mode == SLOW_MODE || rx_mode == SLOWAUTO_MODE) &&
+ (tx_mode == SLOW_MODE || tx_mode == SLOWAUTO_MODE))
+ return LS_MODE;
+
+ return HS_MODE;
+}
+
static int ufshcd_post_device_init(struct ufs_hba *hba)
{
int ret;
@@ -9281,11 +9305,13 @@ static int ufshcd_post_device_init(struct ufs_hba *hba)
return 0;
/*
- * Set the right value to bRefClkFreq before attempting to
+ * Set the right value to bRefClkFreq in LS_MODE before attempting to
* switch to HS gears.
*/
- if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
+ if (ufshcd_get_op_mode(hba) == LS_MODE &&
+ hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
ufshcd_set_dev_ref_clk(hba);
+
/* Gear up to HS gear. */
ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info,
UFSHCD_PMC_POLICY_DONT_FORCE);
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index bc037db..9c0973a 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -177,13 +177,13 @@ static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
int i;
ice = devm_of_qcom_ice_get(dev);
- if (ice == ERR_PTR(-EOPNOTSUPP)) {
- dev_warn(dev, "Disabling inline encryption support\n");
- ice = NULL;
- }
+ if (IS_ERR(ice)) {
+ if (ice != ERR_PTR(-EOPNOTSUPP))
+ return PTR_ERR(ice);
- if (IS_ERR_OR_NULL(ice))
- return PTR_ERR_OR_ZERO(ice);
+ dev_warn(dev, "Disabling inline encryption support\n");
+ return 0;
+ }
host->ice = ice;
diff --git a/drivers/uio/uio_pci_generic_sva.c b/drivers/uio/uio_pci_generic_sva.c
index 4a46acd..d05ef77 100644
--- a/drivers/uio/uio_pci_generic_sva.c
+++ b/drivers/uio/uio_pci_generic_sva.c
@@ -129,15 +129,13 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *id)
ret = devm_uio_register_device(&pdev->dev, &udev->info);
if (ret) {
dev_err(&pdev->dev, "Failed to register uio device\n");
- goto out_free;
+ goto out_disable;
}
pci_set_drvdata(pdev, udev);
return 0;
-out_free:
- kfree(udev);
out_disable:
pci_disable_device(pdev);
@@ -146,11 +144,8 @@ static int probe(struct pci_dev *pdev, const struct pci_device_id *id)
static void remove(struct pci_dev *pdev)
{
- struct uio_pci_sva_dev *udev = pci_get_drvdata(pdev);
-
pci_release_regions(pdev);
pci_disable_device(pdev);
- kfree(udev);
}
static ssize_t pasid_show(struct device *dev,
diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c
index 8382231..1db8db1 100644
--- a/drivers/usb/cdns3/cdns3-gadget.c
+++ b/drivers/usb/cdns3/cdns3-gadget.c
@@ -2817,9 +2817,19 @@ int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
if (request) {
- if (trb)
+ if (trb) {
*trb = trb_tmp;
+ /*
+ * Per datasheet, EPRST causes DMA to reposition to the next TD.
+ * Manually reset EP_TRADDR to the current TRB to prevent
+ * the hardware from skipping the interrupted request.
+ */
+ writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
+ priv_req->start_trb * TRB_SIZE),
+ &priv_dev->regs->ep_traddr);
+ }
+
cdns3_rearm_transfer(priv_ep, 1);
}
diff --git a/drivers/usb/cdns3/cdns3-plat.c b/drivers/usb/cdns3/cdns3-plat.c
index 735df88..94e9706 100644
--- a/drivers/usb/cdns3/cdns3-plat.c
+++ b/drivers/usb/cdns3/cdns3-plat.c
@@ -126,15 +126,15 @@ static int cdns3_plat_probe(struct platform_device *pdev)
return dev_err_probe(dev, PTR_ERR(cdns->usb2_phy),
"Failed to get cdn3,usb2-phy\n");
- ret = phy_init(cdns->usb2_phy);
- if (ret)
- return ret;
-
cdns->usb3_phy = devm_phy_optional_get(dev, "cdns3,usb3-phy");
if (IS_ERR(cdns->usb3_phy))
return dev_err_probe(dev, PTR_ERR(cdns->usb3_phy),
"Failed to get cdn3,usb3-phy\n");
+ ret = phy_init(cdns->usb2_phy);
+ if (ret)
+ return ret;
+
ret = phy_init(cdns->usb3_phy);
if (ret)
goto err_phy3_init;
@@ -186,6 +186,9 @@ static void cdns3_plat_remove(struct platform_device *pdev)
struct device *dev = cdns->dev;
pm_runtime_get_sync(dev);
+ if (!(cdns->pdata && (cdns->pdata->quirks & CDNS3_DEFAULT_PM_RUNTIME_ALLOW)))
+ pm_runtime_allow(dev);
+
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);
cdns_remove(cdns);
diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
index 7cfabb0..2ab3db3 100644
--- a/drivers/usb/chipidea/core.c
+++ b/drivers/usb/chipidea/core.c
@@ -655,12 +655,6 @@ static enum ci_role ci_get_role(struct ci_hdrc *ci)
return role;
}
-static struct usb_role_switch_desc ci_role_switch = {
- .set = ci_usb_role_switch_set,
- .get = ci_usb_role_switch_get,
- .allow_userspace_control = true,
-};
-
static int ci_get_platdata(struct device *dev,
struct ci_hdrc_platform_data *platdata)
{
@@ -787,9 +781,6 @@ static int ci_get_platdata(struct device *dev,
cable->connected = false;
}
- if (device_property_read_bool(dev, "usb-role-switch"))
- ci_role_switch.fwnode = dev->fwnode;
-
platdata->pctl = devm_pinctrl_get(dev);
if (!IS_ERR(platdata->pctl)) {
struct pinctrl_state *p;
@@ -1033,6 +1024,7 @@ ATTRIBUTE_GROUPS(ci);
static int ci_hdrc_probe(struct platform_device *pdev)
{
+ struct usb_role_switch_desc ci_role_switch = {};
struct device *dev = &pdev->dev;
struct ci_hdrc *ci;
struct resource *res;
@@ -1179,7 +1171,11 @@ static int ci_hdrc_probe(struct platform_device *pdev)
}
}
- if (ci_role_switch.fwnode) {
+ if (device_property_read_bool(dev, "usb-role-switch")) {
+ ci_role_switch.set = ci_usb_role_switch_set;
+ ci_role_switch.get = ci_usb_role_switch_get;
+ ci_role_switch.allow_userspace_control = true;
+ ci_role_switch.fwnode = dev_fwnode(dev);
ci_role_switch.driver_data = ci;
ci->role_switch = usb_role_switch_register(dev,
&ci_role_switch);
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index 54059e4..ddf0b59 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -114,8 +114,6 @@ static int acm_ctrl_msg(struct acm *acm, int request, int value,
int retval;
retval = usb_autopm_get_interface(acm->control);
-#define VENDOR_CLASS_DATA_IFACE BIT(9) /* data interface uses vendor-specific class */
-#define ALWAYS_POLL_CTRL BIT(10) /* keep ctrl URB active even without an open TTY */
if (retval)
return retval;
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 25fd532..01f448a 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -115,3 +115,5 @@ struct acm {
#define DISABLE_ECHO BIT(7)
#define MISSING_CAP_BRK BIT(8)
#define NO_UNION_12 BIT(9)
+#define VENDOR_CLASS_DATA_IFACE BIT(10) /* data interface uses vendor-specific class */
+#define ALWAYS_POLL_CTRL BIT(11) /* keep ctrl URB active even without an open TTY */
diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c
index 669b9e6..7464147 100644
--- a/drivers/usb/class/usblp.c
+++ b/drivers/usb/class/usblp.c
@@ -1178,7 +1178,7 @@ static int usblp_probe(struct usb_interface *intf,
}
/* Allocate buffer for printer status */
- usblp->statusbuf = kmalloc(STATUS_BUF_SIZE, GFP_KERNEL);
+ usblp->statusbuf = kzalloc(STATUS_BUF_SIZE, GFP_KERNEL);
if (!usblp->statusbuf) {
retval = -ENOMEM;
goto abort;
@@ -1377,6 +1377,7 @@ static int usblp_cache_device_id_string(struct usblp *usblp)
{
int err, length;
+ memset(usblp->device_id_string, 0, USBLP_DEVICE_ID_SIZE);
err = usblp_get_id(usblp, 0, usblp->device_id_string, USBLP_DEVICE_ID_SIZE - 1);
if (err < 0) {
dev_dbg(&usblp->intf->dev,
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index bd93478..af9ae55 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -2306,6 +2306,14 @@ static void usbtmc_interrupt(struct urb *urb)
switch (status) {
case 0: /* SUCCESS */
+ /* ensure at least two bytes of headers were transferred */
+ if (urb->actual_length < 2) {
+ dev_warn(dev,
+ "actual length %d not sufficient for interrupt headers\n",
+ urb->actual_length);
+ goto exit;
+ }
+
/* check for valid STB notification */
if (data->iin_buffer[0] > 0x81) {
data->bNotify1 = data->iin_buffer[0];
@@ -2432,6 +2440,12 @@ static int usbtmc_probe(struct usb_interface *intf,
data->iin_ep = int_in->bEndpointAddress;
data->iin_wMaxPacketSize = usb_endpoint_maxp(int_in);
data->iin_interval = int_in->bInterval;
+ /* wMaxPacketSize should be 0x02 or more as per USB488 Table 22 */
+ if (iface_desc->desc.bInterfaceProtocol == 1 &&
+ data->iin_wMaxPacketSize < 2) {
+ retcode = -EINVAL;
+ goto err_put;
+ }
dev_dbg(&intf->dev, "Found Int in endpoint at %u\n",
data->iin_ep);
}
diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
index b34fb65..9b69148 100644
--- a/drivers/usb/common/ulpi.c
+++ b/drivers/usb/common/ulpi.c
@@ -286,12 +286,15 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi)
ACPI_COMPANION_SET(&ulpi->dev, ACPI_COMPANION(dev));
ret = ulpi_of_register(ulpi);
- if (ret)
+ if (ret) {
+ kfree(ulpi);
return ret;
+ }
ret = ulpi_read_id(ulpi);
if (ret) {
of_node_put(ulpi->dev.of_node);
+ kfree(ulpi);
return ret;
}
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index 417140b..45e20c6 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -56,8 +56,7 @@ static void usb_parse_ssp_isoc_endpoint_companion(struct device *ddev,
desc = (struct usb_ssp_isoc_ep_comp_descriptor *) buffer;
if (size < USB_DT_SSP_ISOC_EP_COMP_SIZE ||
desc->bDescriptorType != USB_DT_SSP_ISOC_ENDPOINT_COMP) {
- dev_notice(ddev, "Invalid SuperSpeedPlus isoc endpoint companion"
- "for config %d interface %d altsetting %d ep %d.\n",
+ dev_notice(ddev, "Invalid SuperSpeedPlus isoc endpoint companion for config %d interface %d altsetting %d ep 0x%X.\n",
cfgno, inum, asnum, ep->desc.bEndpointAddress);
return;
}
@@ -91,7 +90,7 @@ static void usb_parse_eusb2_isoc_endpoint_companion(struct device *ddev,
size -= h->bLength;
}
- dev_notice(ddev, "No eUSB2 isoc ep %d companion for config %d interface %d altsetting %d\n",
+ dev_notice(ddev, "No eUSB2 isoc ep 0x%X companion for config %d interface %d altsetting %d\n",
ep->desc.bEndpointAddress, cfgno, inum, asnum);
}
@@ -115,9 +114,7 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
}
if (desc->bDescriptorType != USB_DT_SS_ENDPOINT_COMP) {
- dev_notice(ddev, "No SuperSpeed endpoint companion for config %d "
- " interface %d altsetting %d ep %d: "
- "using minimum values\n",
+ dev_notice(ddev, "No SuperSpeed endpoint companion for config %d interface %d altsetting %d ep 0x%X: using minimum values\n",
cfgno, inum, asnum, ep->desc.bEndpointAddress);
/* Fill in some default values.
@@ -141,42 +138,32 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
/* Check the various values */
if (usb_endpoint_xfer_control(&ep->desc) && desc->bMaxBurst != 0) {
- dev_notice(ddev, "Control endpoint with bMaxBurst = %d in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to zero\n", desc->bMaxBurst,
- cfgno, inum, asnum, ep->desc.bEndpointAddress);
+ dev_notice(ddev, "Control endpoint with bMaxBurst = %d in config %d interface %d altsetting %d ep 0x%X: setting to zero\n",
+ desc->bMaxBurst, cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bMaxBurst = 0;
} else if (desc->bMaxBurst > 15) {
- dev_notice(ddev, "Endpoint with bMaxBurst = %d in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to 15\n", desc->bMaxBurst,
- cfgno, inum, asnum, ep->desc.bEndpointAddress);
+ dev_notice(ddev, "Endpoint with bMaxBurst = %d in config %d interface %d altsetting %d ep 0x%X: setting to 15\n",
+ desc->bMaxBurst, cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bMaxBurst = 15;
}
if ((usb_endpoint_xfer_control(&ep->desc) ||
usb_endpoint_xfer_int(&ep->desc)) &&
desc->bmAttributes != 0) {
- dev_notice(ddev, "%s endpoint with bmAttributes = %d in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to zero\n",
+ dev_notice(ddev, "%s endpoint with bmAttributes = %d in config %d interface %d altsetting %d ep 0x%X: setting to zero\n",
usb_endpoint_xfer_control(&ep->desc) ? "Control" : "Bulk",
desc->bmAttributes,
cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bmAttributes = 0;
} else if (usb_endpoint_xfer_bulk(&ep->desc) &&
desc->bmAttributes > 16) {
- dev_notice(ddev, "Bulk endpoint with more than 65536 streams in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to max\n",
+ dev_notice(ddev, "Bulk endpoint with more than 65536 streams in config %d interface %d altsetting %d ep 0x%X: setting to max\n",
cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bmAttributes = 16;
} else if (usb_endpoint_xfer_isoc(&ep->desc) &&
!USB_SS_SSP_ISOC_COMP(desc->bmAttributes) &&
USB_SS_MULT(desc->bmAttributes) > 3) {
- dev_notice(ddev, "Isoc endpoint has Mult of %d in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to 3\n",
+ dev_notice(ddev, "Isoc endpoint has Mult of %d in config %d interface %d altsetting %d ep 0x%X: setting to 3\n",
USB_SS_MULT(desc->bmAttributes),
cfgno, inum, asnum, ep->desc.bEndpointAddress);
ep->ss_ep_comp.bmAttributes = 2;
@@ -191,10 +178,15 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
(desc->bMaxBurst + 1);
else
max_tx = 999999;
- if (le16_to_cpu(desc->wBytesPerInterval) > max_tx) {
- dev_notice(ddev, "%s endpoint with wBytesPerInterval of %d in "
- "config %d interface %d altsetting %d ep %d: "
- "setting to %d\n",
+ /*
+ * wBytesPerInterval > max_tx is bogus, but USB3 spec doesn't forbid the opposite.
+ * Experience shows that wBytesPerInterval < wMaxPacketSize on common interrupt IN
+ * endpoints is usually bogus too, and recent HCs enforce interrupt BW limits.
+ */
+ if (le16_to_cpu(desc->wBytesPerInterval) > max_tx ||
+ (le16_to_cpu(desc->wBytesPerInterval) < usb_endpoint_maxp(&ep->desc) &&
+ usb_endpoint_is_int_in(&ep->desc))) {
+ dev_notice(ddev, "%s endpoint with wBytesPerInterval of %d in config %d interface %d altsetting %d ep 0x%X: setting to %d\n",
usb_endpoint_xfer_isoc(&ep->desc) ? "Isoc" : "Int",
le16_to_cpu(desc->wBytesPerInterval),
cfgno, inum, asnum, ep->desc.bEndpointAddress,
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 89221f1..b181b43 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -328,9 +328,7 @@ static const u8 ss_rh_config_descriptor[] = {
USB_DT_ENDPOINT, /* __u8 ep_bDescriptorType; Endpoint */
0x81, /* __u8 ep_bEndpointAddress; IN Endpoint 1 */
0x03, /* __u8 ep_bmAttributes; Interrupt */
- /* __le16 ep_wMaxPacketSize; 1 + (MAX_ROOT_PORTS / 8)
- * see hub.c:hub_configure() for details. */
- (USB_MAXCHILDREN + 1 + 7) / 8, 0x00,
+ 0x02, 0x00, /* __le16 ep_wMaxPacketSize; 2 bytes per USB3 10.15.1 */
0x0c, /* __u8 ep_bInterval; (256ms -- usb 2.0 spec) */
/* one SuperSpeed endpoint companion descriptor */
diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c
index 0ffdaef..87810ef 100644
--- a/drivers/usb/core/quirks.c
+++ b/drivers/usb/core/quirks.c
@@ -513,6 +513,10 @@ static const struct usb_device_id usb_quirk_list[] = {
/* Lenovo ThinkPad USB-C Dock Gen2 Ethernet (RTL8153 GigE) */
{ USB_DEVICE(0x17ef, 0xa387), .driver_info = USB_QUIRK_NO_LPM },
+ /* Lenovo ThinkPad USB-C Dock Gen2 USB 3.1 and USB 2.0 hub controllers */
+ { USB_DEVICE(0x17ef, 0xa391), .driver_info = USB_QUIRK_NO_LPM },
+ { USB_DEVICE(0x17ef, 0xa392), .driver_info = USB_QUIRK_NO_LPM },
+
/* BUILDWIN Photo Frame */
{ USB_DEVICE(0x1908, 0x1315), .driver_info =
USB_QUIRK_HONOR_BNUMINTERFACES },
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index 1a763ad..2414291 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -4804,6 +4804,7 @@ static int _dwc2_hcd_urb_dequeue(struct usb_hcd *hcd, struct urb *urb,
struct dwc2_hsotg *hsotg = dwc2_hcd_to_hsotg(hcd);
int rc;
unsigned long flags;
+ int urb_status;
dev_dbg(hsotg->dev, "DWC OTG HCD URB Dequeue\n");
dwc2_dump_urb_info(hcd, urb, "urb_dequeue");
@@ -4828,11 +4829,12 @@ static int _dwc2_hcd_urb_dequeue(struct usb_hcd *hcd, struct urb *urb,
/* Higher layer software sets URB status */
spin_unlock(&hsotg->lock);
+ urb_status = urb->status;
usb_hcd_giveback_urb(hcd, urb, status);
spin_lock(&hsotg->lock);
dev_dbg(hsotg->dev, "Called usb_hcd_giveback_urb()\n");
- dev_dbg(hsotg->dev, " urb->status = %d\n", urb->status);
+ dev_dbg(hsotg->dev, " urb->status = %d\n", urb_status);
out:
spin_unlock_irqrestore(&hsotg->lock, flags);
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index 58899b1..6521389 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -1359,12 +1359,6 @@ int dwc3_core_init(struct dwc3 *dwc)
hw_mode = DWC3_GHWPARAMS0_MODE(dwc->hwparams.hwparams0);
- /*
- * Write Linux Version Code to our GUID register so it's easy to figure
- * out which kernel version a bug was found.
- */
- dwc3_writel(dwc, DWC3_GUID, LINUX_VERSION_CODE);
-
ret = dwc3_phy_setup(dwc);
if (ret)
return ret;
@@ -1398,6 +1392,12 @@ int dwc3_core_init(struct dwc3 *dwc)
if (ret)
goto err_exit_phy;
+ /*
+ * Write Linux Version Code to our GUID register so it's easy to figure
+ * out which kernel version a bug was found.
+ */
+ dwc3_writel(dwc, DWC3_GUID, LINUX_VERSION_CODE);
+
dwc3_core_setup_global_control(dwc);
dwc3_core_num_eps(dwc);
diff --git a/drivers/usb/dwc3/dwc3-xilinx.c b/drivers/usb/dwc3/dwc3-xilinx.c
index f41b0da..9b95255 100644
--- a/drivers/usb/dwc3/dwc3-xilinx.c
+++ b/drivers/usb/dwc3/dwc3-xilinx.c
@@ -184,15 +184,13 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
}
ret = phy_init(priv_data->usb3_phy);
- if (ret < 0) {
- phy_exit(priv_data->usb3_phy);
+ if (ret < 0)
goto err;
- }
ret = reset_control_deassert(apbrst);
if (ret < 0) {
dev_err(dev, "Failed to release APB reset\n");
- goto err;
+ goto err_phy_exit;
}
if (priv_data->usb3_phy) {
@@ -208,26 +206,24 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
ret = reset_control_deassert(crst);
if (ret < 0) {
dev_err(dev, "Failed to release core reset\n");
- goto err;
+ goto err_phy_exit;
}
ret = reset_control_deassert(hibrst);
if (ret < 0) {
dev_err(dev, "Failed to release hibernation reset\n");
- goto err;
+ goto err_phy_exit;
}
ret = phy_power_on(priv_data->usb3_phy);
- if (ret < 0) {
- phy_exit(priv_data->usb3_phy);
- goto err;
- }
+ if (ret < 0)
+ goto err_phy_exit;
/* ulpi reset via gpio-modepin or gpio-framework driver */
reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
if (IS_ERR(reset_gpio)) {
- return dev_err_probe(dev, PTR_ERR(reset_gpio),
- "Failed to request reset GPIO\n");
+ ret = PTR_ERR(reset_gpio);
+ goto err_phy_power_off;
}
if (reset_gpio) {
@@ -237,6 +233,13 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
}
dwc3_xlnx_set_coherency(priv_data, XLNX_USB_TRAFFIC_ROUTE_CONFIG);
+
+ return 0;
+
+err_phy_power_off:
+ phy_power_off(priv_data->usb3_phy);
+err_phy_exit:
+ phy_exit(priv_data->usb3_phy);
err:
return ret;
}
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index a902184..dc36643 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -2172,7 +2172,10 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
sizeof(url_descriptor->URL)
- WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_offset);
- if (w_length < WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_length)
+ if (w_length < WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH)
+ landing_page_length = landing_page_offset;
+ else if (w_length <
+ WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_length)
landing_page_length = w_length
- WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_offset;
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 002c3441..75912ce 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -150,6 +150,8 @@ struct ffs_dma_fence {
struct dma_fence base;
struct ffs_dmabuf_priv *priv;
struct work_struct work;
+ struct usb_ep *ep;
+ struct usb_request *req;
};
struct ffs_epfile {
@@ -619,7 +621,7 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf,
/* unlocks spinlock */
ret = __ffs_ep0_queue_wait(ffs, data, len);
- if ((ret > 0) && (copy_to_user(buf, data, len)))
+ if ((ret > 0) && (copy_to_user(buf, data, ret)))
ret = -EFAULT;
goto done_mutex;
@@ -1385,6 +1387,21 @@ static void ffs_dmabuf_cleanup(struct work_struct *work)
struct ffs_dmabuf_priv *priv = dma_fence->priv;
struct dma_buf_attachment *attach = priv->attach;
struct dma_fence *fence = &dma_fence->base;
+ struct usb_request *req = dma_fence->req;
+ struct usb_ep *ep = dma_fence->ep;
+
+ /*
+ * eps_lock pairs with the cancel paths so they cannot pass a freed
+ * req to usb_ep_dequeue(). Only clear if priv->req still names ours;
+ * a re-queue on the same attachment may have taken that slot.
+ */
+ spin_lock_irq(&priv->ffs->eps_lock);
+ if (priv->req == req)
+ priv->req = NULL;
+ spin_unlock_irq(&priv->ffs->eps_lock);
+
+ if (ep && req)
+ usb_ep_free_request(ep, req);
ffs_dmabuf_put(attach);
dma_fence_put(fence);
@@ -1414,8 +1431,8 @@ static void ffs_epfile_dmabuf_io_complete(struct usb_ep *ep,
struct usb_request *req)
{
pr_vdebug("FFS: DMABUF transfer complete, status=%d\n", req->status);
+ /* req is freed by ffs_dmabuf_cleanup() under eps_lock. */
ffs_dmabuf_signal_done(req->context, req->status);
- usb_ep_free_request(ep, req);
}
static const char *ffs_dmabuf_get_driver_name(struct dma_fence *fence)
@@ -1699,6 +1716,10 @@ static int ffs_dmabuf_transfer(struct file *file,
usb_req->context = fence;
usb_req->complete = ffs_epfile_dmabuf_io_complete;
+ /* ffs_dmabuf_cleanup() frees usb_req via these two fields. */
+ fence->req = usb_req;
+ fence->ep = ep->ep;
+
cookie = dma_fence_begin_signalling();
ret = usb_ep_queue(ep->ep, usb_req, GFP_ATOMIC);
dma_fence_end_signalling(cookie);
@@ -1708,7 +1729,6 @@ static int ffs_dmabuf_transfer(struct file *file,
} else {
pr_warn("FFS: Failed to queue DMABUF: %d\n", ret);
ffs_dmabuf_signal_done(fence, ret);
- usb_ep_free_request(ep->ep, usb_req);
}
spin_unlock_irq(&epfile->ffs->eps_lock);
diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c
index c5a12a6..3c6b43d 100644
--- a/drivers/usb/gadget/function/f_hid.c
+++ b/drivers/usb/gadget/function/f_hid.c
@@ -1622,7 +1622,7 @@ static struct usb_function *hidg_alloc(struct usb_function_instance *fi)
hidg->dev.devt = MKDEV(major, opts->minor);
ret = dev_set_name(&hidg->dev, "hidg%d", opts->minor);
if (ret)
- goto err_unlock;
+ goto err_put_device;
hidg->bInterfaceSubClass = opts->subclass;
hidg->bInterfaceProtocol = opts->protocol;
@@ -1659,7 +1659,6 @@ static struct usb_function *hidg_alloc(struct usb_function_instance *fi)
err_put_device:
put_device(&hidg->dev);
-err_unlock:
mutex_unlock(&opts->lock);
return ERR_PTR(ret);
}
diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c
index 8d404d8..73dc7e4 100644
--- a/drivers/usb/gadget/function/f_uvc.c
+++ b/drivers/usb/gadget/function/f_uvc.c
@@ -769,6 +769,16 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f)
uvc_ss_streaming_ep.bEndpointAddress = uvc->video.ep->address;
/*
+ * Hold opts->lock across both the XU string-descriptor fixup below and
+ * the descriptor-copy block further down. Without this, configfs
+ * uvcg_extension_drop() (which takes opts->lock) can race with the
+ * list_for_each_entry() walks here and inside uvc_copy_descriptors(),
+ * leading to a UAF on a freed struct uvcg_extension. See
+ * drivers/usb/gadget/function/uvc_configfs.c::uvcg_extension_drop().
+ */
+ mutex_lock(&opts->lock);
+
+ /*
* XUs can have an arbitrary string descriptor describing them. If they
* have one pick up the ID.
*/
@@ -785,7 +795,7 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f)
ARRAY_SIZE(uvc_en_us_strings));
if (IS_ERR(us)) {
ret = PTR_ERR(us);
- goto error;
+ goto error_unlock;
}
uvc_iad.iFunction = opts->iad_index ? cdev->usb_strings[opts->iad_index].id :
@@ -799,14 +809,14 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f)
/* Allocate interface IDs. */
if ((ret = usb_interface_id(c, f)) < 0)
- goto error;
+ goto error_unlock;
uvc_iad.bFirstInterface = ret;
uvc_control_intf.bInterfaceNumber = ret;
uvc->control_intf = ret;
opts->control_interface = ret;
if ((ret = usb_interface_id(c, f)) < 0)
- goto error;
+ goto error_unlock;
uvc_streaming_intf_alt0.bInterfaceNumber = ret;
uvc_streaming_intf_alt1.bInterfaceNumber = ret;
uvc->streaming_intf = ret;
@@ -817,30 +827,32 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f)
if (IS_ERR(f->fs_descriptors)) {
ret = PTR_ERR(f->fs_descriptors);
f->fs_descriptors = NULL;
- goto error;
+ goto error_unlock;
}
f->hs_descriptors = uvc_copy_descriptors(uvc, USB_SPEED_HIGH);
if (IS_ERR(f->hs_descriptors)) {
ret = PTR_ERR(f->hs_descriptors);
f->hs_descriptors = NULL;
- goto error;
+ goto error_unlock;
}
f->ss_descriptors = uvc_copy_descriptors(uvc, USB_SPEED_SUPER);
if (IS_ERR(f->ss_descriptors)) {
ret = PTR_ERR(f->ss_descriptors);
f->ss_descriptors = NULL;
- goto error;
+ goto error_unlock;
}
f->ssp_descriptors = uvc_copy_descriptors(uvc, USB_SPEED_SUPER_PLUS);
if (IS_ERR(f->ssp_descriptors)) {
ret = PTR_ERR(f->ssp_descriptors);
f->ssp_descriptors = NULL;
- goto error;
+ goto error_unlock;
}
+ mutex_unlock(&opts->lock);
+
/* Preallocate control endpoint request. */
uvc->control_req = usb_ep_alloc_request(cdev->gadget->ep0, GFP_KERNEL);
uvc->control_buf = kmalloc(UVC_MAX_REQUEST_SIZE, GFP_KERNEL);
@@ -872,6 +884,8 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f)
return 0;
+error_unlock:
+ mutex_unlock(&opts->lock);
v4l2_error:
v4l2_device_unregister(&uvc->v4l2_dev);
error:
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index f094491..f479034 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -2134,6 +2134,8 @@ static int dummy_hub_control(
case ClearHubFeature:
break;
case ClearPortFeature:
+ if (wIndex != 1)
+ goto error;
switch (wValue) {
case USB_PORT_FEAT_SUSPEND:
if (hcd->speed == HCD_USB3) {
@@ -2248,6 +2250,8 @@ static int dummy_hub_control(
retval = -EPIPE;
break;
case SetPortFeature:
+ if (wIndex != 1)
+ goto error;
switch (wValue) {
case USB_PORT_FEAT_LINK_STATE:
if (hcd->speed != HCD_USB3) {
diff --git a/drivers/usb/gadget/udc/net2280.c b/drivers/usb/gadget/udc/net2280.c
index d02765b..7c5f30c 100644
--- a/drivers/usb/gadget/udc/net2280.c
+++ b/drivers/usb/gadget/udc/net2280.c
@@ -3790,10 +3790,8 @@ static int net2280_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return 0;
done:
- if (dev) {
+ if (dev)
net2280_remove(pdev);
- kfree(dev);
- }
return retval;
}
diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c
index 91139ae..f3ca79c 100644
--- a/drivers/usb/gadget/udc/omap_udc.c
+++ b/drivers/usb/gadget/udc/omap_udc.c
@@ -733,8 +733,6 @@ static void dma_channel_claim(struct omap_ep *ep, unsigned channel)
if (status == 0) {
omap_writew(reg, UDC_TXDMA_CFG);
/* EMIFF or SDRC */
- omap_set_dma_src_burst_mode(ep->lch,
- OMAP_DMA_DATA_BURST_4);
omap_set_dma_src_data_pack(ep->lch, 1);
/* TIPB */
omap_set_dma_dest_params(ep->lch,
@@ -756,8 +754,6 @@ static void dma_channel_claim(struct omap_ep *ep, unsigned channel)
UDC_DATA_DMA,
0, 0);
/* EMIFF or SDRC */
- omap_set_dma_dest_burst_mode(ep->lch,
- OMAP_DMA_DATA_BURST_4);
omap_set_dma_dest_data_pack(ep->lch, 1);
}
}
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index d2214d3..d5637b3 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -247,6 +247,7 @@ struct tegra_xusb_soc {
bool has_ipfs;
bool lpm_support;
bool otg_reset_sspi;
+ bool otg_set_port_power;
bool has_bar2;
};
@@ -1352,12 +1353,13 @@ static void tegra_xhci_id_work(struct work_struct *work)
struct tegra_xusb_mbox_msg msg;
struct phy *phy = tegra_xusb_get_phy(tegra, "usb2",
tegra->otg_usb2_port);
+ bool host_mode = tegra->host_mode;
u32 status;
int ret;
- dev_dbg(tegra->dev, "host mode %s\n", str_on_off(tegra->host_mode));
+ dev_dbg(tegra->dev, "host mode %s\n", str_on_off(host_mode));
- if (tegra->host_mode)
+ if (host_mode)
phy_set_mode_ext(phy, PHY_MODE_USB_OTG, USB_ROLE_HOST);
else
phy_set_mode_ext(phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
@@ -1366,41 +1368,43 @@ static void tegra_xhci_id_work(struct work_struct *work)
tegra->otg_usb2_port);
pm_runtime_get_sync(tegra->dev);
- if (tegra->host_mode) {
- /* switch to host mode */
- if (tegra->otg_usb3_port >= 0) {
- if (tegra->soc->otg_reset_sspi) {
- /* set PP=0 */
- tegra_xhci_hc_driver.hub_control(
- xhci->shared_hcd, GetPortStatus,
- 0, tegra->otg_usb3_port+1,
- (char *) &status, sizeof(status));
- if (status & USB_SS_PORT_STAT_POWER)
- tegra_xhci_set_port_power(tegra, false,
- false);
+ if (tegra->soc->otg_set_port_power) {
+ if (host_mode) {
+ /* switch to host mode */
+ if (tegra->otg_usb3_port >= 0) {
+ if (tegra->soc->otg_reset_sspi) {
+ /* set PP=0 */
+ tegra_xhci_hc_driver.hub_control(
+ xhci->shared_hcd, GetPortStatus,
+ 0, tegra->otg_usb3_port+1,
+ (char *) &status, sizeof(status));
+ if (status & USB_SS_PORT_STAT_POWER)
+ tegra_xhci_set_port_power(tegra, false,
+ false);
- /* reset OTG port SSPI */
- msg.cmd = MBOX_CMD_RESET_SSPI;
- msg.data = tegra->otg_usb3_port+1;
+ /* reset OTG port SSPI */
+ msg.cmd = MBOX_CMD_RESET_SSPI;
+ msg.data = tegra->otg_usb3_port+1;
- ret = tegra_xusb_mbox_send(tegra, &msg);
- if (ret < 0) {
- dev_info(tegra->dev,
- "failed to RESET_SSPI %d\n",
- ret);
+ ret = tegra_xusb_mbox_send(tegra, &msg);
+ if (ret < 0) {
+ dev_info(tegra->dev,
+ "failed to RESET_SSPI %d\n",
+ ret);
+ }
}
+
+ tegra_xhci_set_port_power(tegra, false, true);
}
- tegra_xhci_set_port_power(tegra, false, true);
+ tegra_xhci_set_port_power(tegra, true, true);
+
+ } else {
+ if (tegra->otg_usb3_port >= 0)
+ tegra_xhci_set_port_power(tegra, false, false);
+
+ tegra_xhci_set_port_power(tegra, true, false);
}
-
- tegra_xhci_set_port_power(tegra, true, true);
-
- } else {
- if (tegra->otg_usb3_port >= 0)
- tegra_xhci_set_port_power(tegra, false, false);
-
- tegra_xhci_set_port_power(tegra, true, false);
}
pm_runtime_put_autosuspend(tegra->dev);
}
@@ -2553,6 +2557,7 @@ static const struct tegra_xusb_soc tegra124_soc = {
.scale_ss_clock = true,
.has_ipfs = true,
.otg_reset_sspi = false,
+ .otg_set_port_power = true,
.ops = &tegra124_ops,
.mbox = {
.cmd = 0xe4,
@@ -2593,6 +2598,7 @@ static const struct tegra_xusb_soc tegra210_soc = {
.scale_ss_clock = false,
.has_ipfs = true,
.otg_reset_sspi = true,
+ .otg_set_port_power = true,
.ops = &tegra124_ops,
.mbox = {
.cmd = 0xe4,
@@ -2640,6 +2646,7 @@ static const struct tegra_xusb_soc tegra186_soc = {
.scale_ss_clock = false,
.has_ipfs = false,
.otg_reset_sspi = false,
+ .otg_set_port_power = true,
.ops = &tegra124_ops,
.mbox = {
.cmd = 0xe4,
@@ -2673,6 +2680,7 @@ static const struct tegra_xusb_soc tegra194_soc = {
.scale_ss_clock = false,
.has_ipfs = false,
.otg_reset_sspi = false,
+ .otg_set_port_power = false,
.ops = &tegra124_ops,
.mbox = {
.cmd = 0x68,
@@ -2708,6 +2716,7 @@ static const struct tegra_xusb_soc tegra234_soc = {
.scale_ss_clock = false,
.has_ipfs = false,
.otg_reset_sspi = false,
+ .otg_set_port_power = false,
.ops = &tegra234_ops,
.mbox = {
.cmd = XUSB_BAR2_ARU_MBOX_CMD,
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index 48bb9bf..333ab79 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -337,7 +337,6 @@ static int omap2430_probe(struct platform_device *pdev)
} else {
device_set_of_node_from_dev(&musb->dev, &pdev->dev);
}
- of_node_put(np);
glue->dev = &pdev->dev;
glue->musb = musb;
@@ -455,6 +454,7 @@ static int omap2430_probe(struct platform_device *pdev)
dev_err(&pdev->dev, "failed to register musb device\n");
goto err_disable_rpm;
}
+ of_node_put(np);
return 0;
@@ -464,6 +464,7 @@ static int omap2430_probe(struct platform_device *pdev)
if (!IS_ERR(glue->control_otghs))
put_device(glue->control_otghs);
err_put_musb:
+ of_node_put(np);
platform_device_put(musb);
return ret;
diff --git a/drivers/usb/serial/belkin_sa.c b/drivers/usb/serial/belkin_sa.c
index 38ac910..7bbd952 100644
--- a/drivers/usb/serial/belkin_sa.c
+++ b/drivers/usb/serial/belkin_sa.c
@@ -194,6 +194,9 @@ static void belkin_sa_read_int_callback(struct urb *urb)
usb_serial_debug_data(&port->dev, __func__, urb->actual_length, data);
+ if (urb->actual_length < BELKIN_SA_MSR_INDEX + 1)
+ goto exit;
+
/* Handle known interrupt data */
/* ignore data[0] and data[1] */
diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c
index afff1a0..bcf302e 100644
--- a/drivers/usb/serial/cypress_m8.c
+++ b/drivers/usb/serial/cypress_m8.c
@@ -445,6 +445,14 @@ static int cypress_generic_port_probe(struct usb_serial_port *port)
return -ENODEV;
}
+ /*
+ * The buffer must be large enough for the one or two-byte header (and
+ * following data), but assume anything smaller than eight bytes is
+ * broken.
+ */
+ if (port->interrupt_out_size < 8)
+ return -EINVAL;
+
priv = kzalloc_obj(struct cypress_private);
if (!priv)
return -ENOMEM;
@@ -1017,8 +1025,8 @@ static void cypress_read_int_callback(struct urb *urb)
char tty_flag = TTY_NORMAL;
int bytes = 0;
int result;
- int i = 0;
int status = urb->status;
+ int i;
switch (status) {
case 0: /* success */
@@ -1056,22 +1064,32 @@ static void cypress_read_int_callback(struct urb *urb)
spin_lock_irqsave(&priv->lock, flags);
result = urb->actual_length;
+ i = 0;
switch (priv->pkt_fmt) {
default:
case packet_format_1:
/* This is for the CY7C64013... */
+ if (result < 2)
+ break;
priv->current_status = data[0] & 0xF8;
bytes = data[1] + 2;
i = 2;
break;
case packet_format_2:
/* This is for the CY7C63743... */
+ if (result < 1)
+ break;
priv->current_status = data[0] & 0xF8;
bytes = (data[0] & 0x07) + 1;
i = 1;
break;
}
spin_unlock_irqrestore(&priv->lock, flags);
+ if (i == 0) {
+ dev_dbg(dev, "%s - short packet received: %d bytes\n",
+ __func__, result);
+ goto continue_read;
+ }
if (result < bytes) {
dev_dbg(dev,
"%s - wrong packet size - received %d bytes but packet said %d bytes\n",
diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c
index d515df0..c481208 100644
--- a/drivers/usb/serial/digi_acceleport.c
+++ b/drivers/usb/serial/digi_acceleport.c
@@ -1229,15 +1229,34 @@ static int digi_port_init(struct usb_serial_port *port, unsigned port_num)
static int digi_startup(struct usb_serial *serial)
{
struct digi_serial *serial_priv;
+ int oob_port_num;
int ret;
+ int i;
+
+ /*
+ * The port bulk-out buffers must be large enough for header and
+ * buffered data.
+ */
+ for (i = 0; i < serial->type->num_ports; i++) {
+ if (serial->port[i]->bulk_out_size < DIGI_OUT_BUF_SIZE + 2)
+ return -EINVAL;
+ }
+
+ /*
+ * The OOB port bulk-out buffer must be large enough for the two
+ * commands in digi_set_modem_signals().
+ */
+ oob_port_num = serial->type->num_ports;
+ if (serial->port[oob_port_num]->bulk_out_size < 8)
+ return -EINVAL;
serial_priv = kzalloc_obj(*serial_priv);
if (!serial_priv)
return -ENOMEM;
spin_lock_init(&serial_priv->ds_serial_lock);
- serial_priv->ds_oob_port_num = serial->type->num_ports;
- serial_priv->ds_oob_port = serial->port[serial_priv->ds_oob_port_num];
+ serial_priv->ds_oob_port_num = oob_port_num;
+ serial_priv->ds_oob_port = serial->port[oob_port_num];
ret = digi_port_init(serial_priv->ds_oob_port,
serial_priv->ds_oob_port_num);
diff --git a/drivers/usb/serial/keyspan.c b/drivers/usb/serial/keyspan.c
index 4644884..28b8060 100644
--- a/drivers/usb/serial/keyspan.c
+++ b/drivers/usb/serial/keyspan.c
@@ -1187,6 +1187,10 @@ static void usa49wg_indat_callback(struct urb *urb)
len = 0;
while (i < urb->actual_length) {
+ if (urb->actual_length - i < 3) {
+ dev_warn_ratelimited(&urb->dev->dev, "malformed indat packet\n");
+ break;
+ }
/* Check port number from message */
if (data[i] >= serial->num_ports) {
diff --git a/drivers/usb/serial/mct_u232.c b/drivers/usb/serial/mct_u232.c
index 18844b9..1631618 100644
--- a/drivers/usb/serial/mct_u232.c
+++ b/drivers/usb/serial/mct_u232.c
@@ -378,6 +378,7 @@ static int mct_u232_port_probe(struct usb_serial_port *port)
{
struct usb_serial *serial = port->serial;
struct mct_u232_private *priv;
+ u16 pid;
/* check first to simplify error handling */
if (!serial->port[1] || !serial->port[1]->interrupt_in_urb) {
@@ -385,6 +386,16 @@ static int mct_u232_port_probe(struct usb_serial_port *port)
return -ENODEV;
}
+ /*
+ * Compensate for a hardware bug: although the Sitecom U232-P25
+ * device reports a maximum output packet size of 32 bytes,
+ * it seems to be able to accept only 16 bytes (and that's what
+ * SniffUSB says too...)
+ */
+ pid = le16_to_cpu(serial->dev->descriptor.idProduct);
+ if (pid == MCT_U232_SITECOM_PID)
+ port->bulk_out_size = min(16, port->bulk_out_size);
+
priv = kzalloc_obj(*priv);
if (!priv)
return -ENOMEM;
@@ -410,7 +421,6 @@ static void mct_u232_port_remove(struct usb_serial_port *port)
static int mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port)
{
- struct usb_serial *serial = port->serial;
struct mct_u232_private *priv = usb_get_serial_port_data(port);
int retval = 0;
unsigned int control_state;
@@ -418,15 +428,6 @@ static int mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port)
unsigned char last_lcr;
unsigned char last_msr;
- /* Compensate for a hardware bug: although the Sitecom U232-P25
- * device reports a maximum output packet size of 32 bytes,
- * it seems to be able to accept only 16 bytes (and that's what
- * SniffUSB says too...)
- */
- if (le16_to_cpu(serial->dev->descriptor.idProduct)
- == MCT_U232_SITECOM_PID)
- port->bulk_out_size = 16;
-
/* Do a defined restart: the normal serial device seems to
* always turn on DTR and RTS here, so do the same. I'm not
* sure if this is really necessary. But it should not harm
@@ -543,6 +544,11 @@ static void mct_u232_read_int_callback(struct urb *urb)
goto exit;
}
+ if (urb->actual_length < 2) {
+ dev_warn_ratelimited(&port->dev, "short interrupt-in packet\n");
+ goto exit;
+ }
+
/*
* The interrupt-in pipe signals exceptional conditions (modem line
* signal changes and errors). data[0] holds MSR, data[1] holds LSR.
diff --git a/drivers/usb/serial/mxuport.c b/drivers/usb/serial/mxuport.c
index ad5fdf5..c9b9928 100644
--- a/drivers/usb/serial/mxuport.c
+++ b/drivers/usb/serial/mxuport.c
@@ -962,6 +962,14 @@ static int mxuport_calc_num_ports(struct usb_serial *serial,
*/
BUILD_BUG_ON(ARRAY_SIZE(epds->bulk_out) < 16);
+ /*
+ * The bulk-out buffers must be large enough for the four-byte header
+ * (and following data), but assume anything smaller than eight bytes
+ * is broken.
+ */
+ if (usb_endpoint_maxp(epds->bulk_out[0]) < 8)
+ return -EINVAL;
+
for (i = 1; i < num_ports; ++i)
epds->bulk_out[i] = epds->bulk_out[0];
diff --git a/drivers/usb/serial/omninet.c b/drivers/usb/serial/omninet.c
index aa1e974..b59982e 100644
--- a/drivers/usb/serial/omninet.c
+++ b/drivers/usb/serial/omninet.c
@@ -30,6 +30,10 @@
/* This one seems to be a re-branded ZyXEL device */
#define BT_IGNITIONPRO_ID 0x2000
+#define OMNINET_HEADERLEN 4
+#define OMNINET_BULKOUTSIZE 64
+#define OMNINET_PAYLOADSIZE (OMNINET_BULKOUTSIZE - OMNINET_HEADERLEN)
+
/* function prototypes */
static void omninet_process_read_urb(struct urb *urb);
static int omninet_prepare_write_buffer(struct usb_serial_port *port,
@@ -54,6 +58,7 @@ static struct usb_serial_driver zyxel_omninet_device = {
.description = "ZyXEL - omni.net usb",
.id_table = id_table,
.num_bulk_out = 2,
+ .bulk_out_size = OMNINET_BULKOUTSIZE,
.calc_num_ports = omninet_calc_num_ports,
.port_probe = omninet_port_probe,
.port_remove = omninet_port_remove,
@@ -130,10 +135,6 @@ static void omninet_port_remove(struct usb_serial_port *port)
kfree(od);
}
-#define OMNINET_HEADERLEN 4
-#define OMNINET_BULKOUTSIZE 64
-#define OMNINET_PAYLOADSIZE (OMNINET_BULKOUTSIZE - OMNINET_HEADERLEN)
-
static void omninet_process_read_urb(struct urb *urb)
{
struct usb_serial_port *port = urb->context;
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index c714618..48ae018 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -1513,7 +1513,11 @@ static const struct usb_device_id option_ids[] = {
{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1231, 0xff), /* Telit LE910Cx (RNDIS) */
.driver_info = NCTRL(2) | RSVD(3) },
{ USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x1250, 0xff, 0x00, 0x00) }, /* Telit LE910Cx (rmnet) */
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1251, 0xff) }, /* Telit LE910Cx (RNDIS) */
{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1252, 0xff) }, /* Telit LE910Cx (MBIM) */
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1253, 0xff) }, /* Telit LE910Cx (ECM) */
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1254, 0xff) }, /* Telit LE910Cx */
+ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1255, 0xff) }, /* Telit LE910Cx */
{ USB_DEVICE(TELIT_VENDOR_ID, 0x1260),
.driver_info = NCTRL(0) | RSVD(1) | RSVD(2) },
{ USB_DEVICE(TELIT_VENDOR_ID, 0x1261),
@@ -2446,6 +2450,12 @@ static const struct usb_device_id option_ids[] = {
{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x30) }, /* MeiG Smart SRM825WN (Diag) */
{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x40) }, /* MeiG Smart SRM825WN (AT) */
{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x60) }, /* MeiG Smart SRM825WN (NMEA) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d63, 0xff, 0xff, 0x30) }, /* MeiG SRM813Q (Diag) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d63, 0xff, 0xff, 0x40) }, /* MeiG SRM813Q (AT) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x30) }, /* MeiG SRM813Q (Diag) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x40) }, /* MeiG SRM813Q (AT) */
+ { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x60) }, /* MeiG SRM813Q (NMEA) */
+
{ USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) }, /* LongSung M5710 */
{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */
{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */
@@ -2466,7 +2476,8 @@ static const struct usb_device_id option_ids[] = {
{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) }, /* Rolling RW101R-GL (laptop MBIM) */
{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff), /* Rolling RW350-GL (laptop MBIM) */
.driver_info = RSVD(5) },
- { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff) }, /* Rolling RW135R-GL (laptop MBIM) */
+ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff), /* Rolling RW135R-GL (laptop MBIM) */
+ .driver_info = RSVD(5) },
{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */
{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) },
{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) },
diff --git a/drivers/usb/serial/safe_serial.c b/drivers/usb/serial/safe_serial.c
index 238b549..d267a31 100644
--- a/drivers/usb/serial/safe_serial.c
+++ b/drivers/usb/serial/safe_serial.c
@@ -259,6 +259,7 @@ static int safe_prepare_write_buffer(struct usb_serial_port *port,
static int safe_startup(struct usb_serial *serial)
{
struct usb_interface_descriptor *desc;
+ int bulk_out_size;
if (serial->dev->descriptor.bDeviceClass != CDC_DEVICE_CLASS)
return -ENODEV;
@@ -279,6 +280,16 @@ static int safe_startup(struct usb_serial *serial)
default:
return -EINVAL;
}
+
+ /*
+ * The bulk-out buffer needs to be large enough for the two-byte
+ * trailer in safe mode, but assume anything smaller than eight bytes
+ * is broken.
+ */
+ bulk_out_size = serial->port[0]->bulk_out_size;
+ if (bulk_out_size > 0 && bulk_out_size < 8)
+ return -EINVAL;
+
return 0;
}
diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h
index 939a98c..d6f86d5 100644
--- a/drivers/usb/storage/unusual_uas.h
+++ b/drivers/usb/storage/unusual_uas.h
@@ -132,6 +132,13 @@ UNUSUAL_DEV(0x152d, 0x0583, 0x0000, 0x9999,
USB_SC_DEVICE, USB_PR_DEVICE, NULL,
US_FL_NO_REPORT_OPCODES),
+/* Reported-by: Sam Burkels <sam@1a38.nl> */
+UNUSUAL_DEV(0x154b, 0xf009, 0x0000, 0x9999,
+ "PNY",
+ "PNY ELITE PSSD",
+ USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+ US_FL_NO_ATA_1X | US_FL_NO_REPORT_OPCODES),
+
/* Reported-by: Thinh Nguyen <thinhn@synopsys.com> */
UNUSUAL_DEV(0x154b, 0xf00b, 0x0000, 0x9999,
"PNY",
diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c
index 35d9c30..263a89c 100644
--- a/drivers/usb/typec/altmodes/displayport.c
+++ b/drivers/usb/typec/altmodes/displayport.c
@@ -405,6 +405,8 @@ static int dp_altmode_vdm(struct typec_altmode *alt,
dp->state = DP_STATE_EXIT_PRIME;
break;
case DP_CMD_STATUS_UPDATE:
+ if (count < 2)
+ break;
dp->data.status = *vdo;
ret = dp_altmode_status_update(dp);
break;
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index 889c4c2..9ab1277 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -1751,19 +1751,22 @@ static int fusb302_probe(struct i2c_client *client)
bridge_dev = devm_drm_dp_hpd_bridge_alloc(chip->dev, to_of_node(chip->tcpc_dev.fwnode));
if (IS_ERR(bridge_dev)) {
- ret = PTR_ERR(bridge_dev);
- dev_err_probe(chip->dev, ret, "failed to alloc bridge\n");
- goto destroy_workqueue;
+ ret = dev_err_probe(chip->dev, PTR_ERR(bridge_dev),
+ "failed to alloc bridge\n");
+ goto fwnode_put;
}
chip->tcpm_port = tcpm_register_port(&client->dev, &chip->tcpc_dev);
if (IS_ERR(chip->tcpm_port)) {
- fwnode_handle_put(chip->tcpc_dev.fwnode);
ret = dev_err_probe(dev, PTR_ERR(chip->tcpm_port),
"cannot register tcpm port\n");
- goto destroy_workqueue;
+ goto fwnode_put;
}
+ ret = devm_drm_dp_hpd_bridge_add(chip->dev, bridge_dev);
+ if (ret)
+ goto tcpm_unregister_port;
+
ret = request_threaded_irq(chip->gpio_int_n_irq, NULL, fusb302_irq_intn,
IRQF_ONESHOT | IRQF_TRIGGER_LOW,
"fsc_interrupt_int_n", chip);
@@ -1774,14 +1777,11 @@ static int fusb302_probe(struct i2c_client *client)
enable_irq_wake(chip->gpio_int_n_irq);
i2c_set_clientdata(client, chip);
- ret = devm_drm_dp_hpd_bridge_add(chip->dev, bridge_dev);
- if (ret)
- return ret;
-
- return ret;
+ return 0;
tcpm_unregister_port:
tcpm_unregister_port(chip->tcpm_port);
+fwnode_put:
fwnode_handle_put(chip->tcpc_dev.fwnode);
destroy_workqueue:
fusb302_debugfs_exit(chip);
diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
index c0ee7e69..7324139 100644
--- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c
+++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
@@ -181,6 +181,15 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status)
rx_buf_ptr = rx_buf + TCPC_RECEIVE_BUFFER_RX_BYTE_BUF_OFFSET;
msg.header = cpu_to_le16(*(u16 *)rx_buf_ptr);
rx_buf_ptr = rx_buf_ptr + sizeof(msg.header);
+
+ if (count < TCPC_RECEIVE_BUFFER_RX_BYTE_BUF_OFFSET + sizeof(msg.header) +
+ pd_header_cnt_le(msg.header) * sizeof(msg.payload[0])) {
+ max_tcpci_write16(chip, TCPC_ALERT, TCPC_ALERT_RX_STATUS);
+ dev_err(chip->dev, "Invalid TCPC_RX_BYTE_CNT %d for header cnt %d\n",
+ count, pd_header_cnt_le(msg.header));
+ return;
+ }
+
for (payload_index = 0; payload_index < pd_header_cnt_le(msg.header); payload_index++,
rx_buf_ptr += sizeof(msg.payload[0]))
msg.payload[payload_index] = cpu_to_le32(*(u32 *)rx_buf_ptr);
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index dfbb94d..7ef746a 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -732,9 +732,14 @@ static const char * const pd_rev[] = {
(tcpm_cc_is_source((port)->cc2) && \
!tcpm_cc_is_source((port)->cc1)))
+#define tcpm_port_is_debug_source(port) \
+ (tcpm_cc_is_source((port)->cc1) && tcpm_cc_is_source((port)->cc2))
+
+#define tcpm_port_is_debug_sink(port) \
+ (tcpm_cc_is_sink((port)->cc1) && tcpm_cc_is_sink((port)->cc2))
+
#define tcpm_port_is_debug(port) \
- ((tcpm_cc_is_source((port)->cc1) && tcpm_cc_is_source((port)->cc2)) || \
- (tcpm_cc_is_sink((port)->cc1) && tcpm_cc_is_sink((port)->cc2)))
+ (tcpm_port_is_debug_source(port) || tcpm_port_is_debug_sink(port))
#define tcpm_port_is_audio(port) \
(tcpm_cc_is_audio((port)->cc1) && tcpm_cc_is_audio((port)->cc2))
@@ -1850,6 +1855,9 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt)
u32 vdo = p[VDO_INDEX_IDH];
u32 product = p[VDO_INDEX_PRODUCT];
+ if (cnt <= VDO_INDEX_PRODUCT)
+ return;
+
memset(&port->mode_data, 0, sizeof(port->mode_data));
port->partner_ident.id_header = vdo;
@@ -1870,6 +1878,9 @@ static void svdm_consume_identity_sop_prime(struct tcpm_port *port, const u32 *p
u32 product = p[VDO_INDEX_PRODUCT];
int svdm_version;
+ if (cnt <= VDO_INDEX_CABLE_1)
+ return;
+
/*
* Attempt to consume identity only if cable currently is not set
*/
@@ -1893,7 +1904,7 @@ static void svdm_consume_identity_sop_prime(struct tcpm_port *port, const u32 *p
switch (port->negotiated_rev_prime) {
case PD_REV30:
port->cable_desc.pd_revision = 0x0300;
- if (port->cable_desc.active)
+ if (port->cable_desc.active && cnt > VDO_INDEX_CABLE_2)
port->cable_ident.vdo[1] = p[VDO_INDEX_CABLE_2];
break;
case PD_REV20:
@@ -1981,23 +1992,19 @@ static void svdm_consume_modes(struct tcpm_port *port, const u32 *p, int cnt,
switch (rx_sop_type) {
case TCPC_TX_SOP_PRIME:
pmdata = &port->mode_data_prime;
- if (pmdata->altmodes >= ARRAY_SIZE(port->plug_prime_altmode)) {
- /* Already logged in svdm_consume_svids() */
- return;
- }
break;
case TCPC_TX_SOP:
pmdata = &port->mode_data;
- if (pmdata->altmodes >= ARRAY_SIZE(port->partner_altmode)) {
- /* Already logged in svdm_consume_svids() */
- return;
- }
break;
default:
return;
}
for (i = 1; i < cnt; i++) {
+ if (pmdata->altmodes >= ALTMODE_DISCOVERY_MAX) {
+ /* Already logged in svdm_consume_svids() */
+ return;
+ }
paltmode = &pmdata->altmode_desc[pmdata->altmodes];
memset(paltmode, 0, sizeof(*paltmode));
@@ -2142,6 +2149,55 @@ static bool tcpm_cable_vdm_supported(struct tcpm_port *port)
tcpm_can_communicate_sop_prime(port);
}
+static int tcpm_handle_discover_mode(struct tcpm_port *port, u32 *response,
+ enum tcpm_transmit_type rx_sop_type,
+ enum tcpm_transmit_type *response_tx_sop_type)
+{
+ struct typec_port *typec = port->typec_port;
+ struct pd_mode_data *modep;
+
+ if (rx_sop_type == TCPC_TX_SOP) {
+ modep = &port->mode_data;
+ modep->svid_index++;
+
+ if (modep->svid_index < modep->nsvids) {
+ u16 svid = modep->svids[modep->svid_index];
+ *response_tx_sop_type = TCPC_TX_SOP;
+ response[0] = VDO(svid, 1,
+ typec_get_negotiated_svdm_version(typec),
+ CMD_DISCOVER_MODES);
+ return 1;
+ }
+
+ if (tcpm_cable_vdm_supported(port)) {
+ *response_tx_sop_type = TCPC_TX_SOP_PRIME;
+ response[0] = VDO(USB_SID_PD, 1,
+ typec_get_cable_svdm_version(typec),
+ CMD_DISCOVER_SVID);
+ return 1;
+ }
+
+ tcpm_register_partner_altmodes(port);
+ } else if (rx_sop_type == TCPC_TX_SOP_PRIME) {
+ modep = &port->mode_data_prime;
+ modep->svid_index++;
+
+ if (modep->svid_index < modep->nsvids) {
+ u16 svid = modep->svids[modep->svid_index];
+ *response_tx_sop_type = TCPC_TX_SOP_PRIME;
+ response[0] = VDO(svid, 1,
+ typec_get_cable_svdm_version(typec),
+ CMD_DISCOVER_MODES);
+ return 1;
+ }
+
+ tcpm_register_plug_altmodes(port);
+ tcpm_register_partner_altmodes(port);
+ }
+
+ return 0;
+}
+
static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev,
const u32 *p, int cnt, u32 *response,
enum adev_actions *adev_action,
@@ -2399,41 +2455,11 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev,
}
break;
case CMD_DISCOVER_MODES:
- if (rx_sop_type == TCPC_TX_SOP) {
- /* 6.4.4.3.3 */
- svdm_consume_modes(port, p, cnt, rx_sop_type);
- modep->svid_index++;
- if (modep->svid_index < modep->nsvids) {
- u16 svid = modep->svids[modep->svid_index];
- *response_tx_sop_type = TCPC_TX_SOP;
- response[0] = VDO(svid, 1, svdm_version,
- CMD_DISCOVER_MODES);
- rlen = 1;
- } else if (tcpm_cable_vdm_supported(port)) {
- *response_tx_sop_type = TCPC_TX_SOP_PRIME;
- response[0] = VDO(USB_SID_PD, 1,
- typec_get_cable_svdm_version(typec),
- CMD_DISCOVER_SVID);
- rlen = 1;
- } else {
- tcpm_register_partner_altmodes(port);
- }
- } else if (rx_sop_type == TCPC_TX_SOP_PRIME) {
- /* 6.4.4.3.3 */
- svdm_consume_modes(port, p, cnt, rx_sop_type);
- modep_prime->svid_index++;
- if (modep_prime->svid_index < modep_prime->nsvids) {
- u16 svid = modep_prime->svids[modep_prime->svid_index];
- *response_tx_sop_type = TCPC_TX_SOP_PRIME;
- response[0] = VDO(svid, 1,
- typec_get_cable_svdm_version(typec),
- CMD_DISCOVER_MODES);
- rlen = 1;
- } else {
- tcpm_register_plug_altmodes(port);
- tcpm_register_partner_altmodes(port);
- }
- }
+ /* 6.4.4.3.3 */
+ svdm_consume_modes(port, p, cnt, rx_sop_type);
+ rlen = tcpm_handle_discover_mode(port, response,
+ rx_sop_type,
+ response_tx_sop_type);
break;
case CMD_ENTER_MODE:
*response_tx_sop_type = rx_sop_type;
@@ -2476,9 +2502,15 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev,
switch (cmd) {
case CMD_DISCOVER_IDENT:
case CMD_DISCOVER_SVID:
- case CMD_DISCOVER_MODES:
case VDO_CMD_VENDOR(0) ... VDO_CMD_VENDOR(15):
break;
+ case CMD_DISCOVER_MODES:
+ tcpm_log(port, "Skip SVID 0x%04x (failed to discover mode)",
+ PD_VDO_SVID_SVID0(p[0]));
+ rlen = tcpm_handle_discover_mode(port, response,
+ rx_sop_type,
+ response_tx_sop_type);
+ break;
case CMD_ENTER_MODE:
/* Back to USB Operation */
*adev_action = ADEV_NOTIFY_USB_AND_QUEUE_VDM;
@@ -5176,7 +5208,7 @@ static void run_state_machine(struct tcpm_port *port)
tcpm_set_state(port, SNK_UNATTACHED, PD_T_DRP_SNK);
break;
case SRC_ATTACH_WAIT:
- if (tcpm_port_is_debug(port))
+ if (tcpm_port_is_debug_source(port))
tcpm_set_state(port, DEBUG_ACC_ATTACHED,
port->timings.cc_debounce_time);
else if (tcpm_port_is_audio(port))
@@ -5434,7 +5466,7 @@ static void run_state_machine(struct tcpm_port *port)
tcpm_set_state(port, SRC_UNATTACHED, PD_T_DRP_SRC);
break;
case SNK_ATTACH_WAIT:
- if (tcpm_port_is_debug(port))
+ if (tcpm_port_is_debug_sink(port))
tcpm_set_state(port, DEBUG_ACC_ATTACHED,
PD_T_CC_DEBOUNCE);
else if (tcpm_port_is_audio(port))
@@ -5454,7 +5486,7 @@ static void run_state_machine(struct tcpm_port *port)
if (tcpm_port_is_disconnected(port))
tcpm_set_state(port, SNK_UNATTACHED,
PD_T_PD_DEBOUNCE);
- else if (tcpm_port_is_debug(port))
+ else if (tcpm_port_is_debug_sink(port))
tcpm_set_state(port, DEBUG_ACC_ATTACHED,
PD_T_CC_DEBOUNCE);
else if (tcpm_port_is_audio(port))
@@ -5935,6 +5967,8 @@ static void run_state_machine(struct tcpm_port *port)
/* remove existing capabilities */
tcpm_partner_source_caps_reset(port);
tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP);
+ port->vdm_sm_running = false;
+ port->explicit_contract = false;
tcpm_ams_finish(port);
if (port->pwr_role == TYPEC_SOURCE) {
port->upcoming_state = SRC_SEND_CAPABILITIES;
@@ -6360,10 +6394,10 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
switch (port->state) {
case TOGGLING:
- if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) ||
+ if (tcpm_port_is_debug_source(port) || tcpm_port_is_audio(port) ||
tcpm_port_is_source(port))
tcpm_set_state(port, SRC_ATTACH_WAIT, 0);
- else if (tcpm_port_is_sink(port))
+ else if (tcpm_port_is_debug_sink(port) || tcpm_port_is_sink(port))
tcpm_set_state(port, SNK_ATTACH_WAIT, 0);
break;
case CHECK_CONTAMINANT:
@@ -6371,9 +6405,11 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
break;
case SRC_UNATTACHED:
case ACC_UNATTACHED:
- if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) ||
+ if (tcpm_port_is_debug_source(port) || tcpm_port_is_audio(port) ||
tcpm_port_is_source(port))
tcpm_set_state(port, SRC_ATTACH_WAIT, 0);
+ else if (tcpm_port_is_debug_sink(port))
+ tcpm_set_state(port, SNK_ATTACH_WAIT, 0);
break;
case SRC_ATTACH_WAIT:
if (tcpm_port_is_disconnected(port) ||
@@ -6395,7 +6431,7 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
}
break;
case SNK_UNATTACHED:
- if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) ||
+ if (tcpm_port_is_debug_sink(port) || tcpm_port_is_audio(port) ||
tcpm_port_is_sink(port))
tcpm_set_state(port, SNK_ATTACH_WAIT, 0);
break;
diff --git a/drivers/usb/typec/tcpm/wcove.c b/drivers/usb/typec/tcpm/wcove.c
index 759c982..0e5a3e2 100644
--- a/drivers/usb/typec/tcpm/wcove.c
+++ b/drivers/usb/typec/tcpm/wcove.c
@@ -444,9 +444,11 @@ static int wcove_start_toggling(struct tcpc_dev *tcpc,
return regmap_write(wcove->regmap, USBC_CONTROL1, usbc_ctrl);
}
-static int wcove_read_rx_buffer(struct wcove_typec *wcove, void *msg)
+static int wcove_read_rx_buffer(struct wcove_typec *wcove,
+ struct pd_message *msg)
{
- unsigned int info;
+ unsigned int info, val, len;
+ u8 *buf = (u8 *)msg;
int ret;
int i;
@@ -454,12 +456,13 @@ static int wcove_read_rx_buffer(struct wcove_typec *wcove, void *msg)
if (ret)
return ret;
- /* FIXME: Check that USBC_RXINFO_RXBYTES(info) matches the header */
+ len = min(USBC_RXINFO_RXBYTES(info), sizeof(*msg));
- for (i = 0; i < USBC_RXINFO_RXBYTES(info); i++) {
- ret = regmap_read(wcove->regmap, USBC_RX_DATA + i, msg + i);
+ for (i = 0; i < len; i++) {
+ ret = regmap_read(wcove->regmap, USBC_RX_DATA + i, &val);
if (ret)
return ret;
+ buf[i] = val;
}
return regmap_write(wcove->regmap, USBC_RXSTATUS,
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index 43faec7..d0b7693 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -1835,6 +1835,7 @@ static int tps6598x_probe(struct i2c_client *client)
goto err_role_put;
if (status & TPS_STATUS_PLUG_PRESENT) {
+ ret = -EINVAL;
if (!tps6598x_read_power_status(tps))
goto err_unregister_port;
if (!tps->data->read_data_status(tps))
diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c
index 8aae80b..67a0991 100644
--- a/drivers/usb/typec/ucsi/displayport.c
+++ b/drivers/usb/typec/ucsi/displayport.c
@@ -240,6 +240,10 @@ static int ucsi_displayport_vdm(struct typec_altmode *alt,
dp->header |= VDO_CMDT(CMDT_RSP_ACK);
break;
case DP_CMD_CONFIGURE:
+ if (count < 2) {
+ dp->header |= VDO_CMDT(CMDT_RSP_NAK);
+ break;
+ }
dp->data.conf = *data;
if (ucsi_displayport_configure(dp)) {
dp->header |= VDO_CMDT(CMDT_RSP_NAK);
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 5b7ad9e..61cb24e 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -1277,7 +1277,7 @@ static void ucsi_handle_connector_change(struct work_struct *work)
work);
struct ucsi *ucsi = con->ucsi;
u8 curr_scale, volt_scale;
- enum typec_role role;
+ enum typec_role role, prev_role;
u16 change;
int ret;
u32 val;
@@ -1288,6 +1288,8 @@ static void ucsi_handle_connector_change(struct work_struct *work)
dev_err_once(ucsi->dev, "%s entered without EVENT_PENDING\n",
__func__);
+ prev_role = UCSI_CONSTAT(con, PWR_DIR);
+
ret = ucsi_get_connector_status(con, true);
if (ret) {
dev_err(ucsi->dev, "%s: GET_CONNECTOR_STATUS failed (%d)\n",
@@ -1304,9 +1306,14 @@ static void ucsi_handle_connector_change(struct work_struct *work)
change = UCSI_CONSTAT(con, CHANGE);
role = UCSI_CONSTAT(con, PWR_DIR);
- if (change & UCSI_CONSTAT_POWER_DIR_CHANGE) {
+ if ((change & UCSI_CONSTAT_POWER_DIR_CHANGE) && role != prev_role) {
typec_set_pwr_role(con->port, role);
- ucsi_port_psy_changed(con);
+
+ /* Some power_supply properties vary depending on the power direction when
+ * connected
+ */
+ if (UCSI_CONSTAT(con, CONNECTED))
+ ucsi_port_psy_changed(con);
/* Complete pending power role swap */
if (!completion_done(&con->complete))
@@ -1380,13 +1387,22 @@ static void ucsi_handle_connector_change(struct work_struct *work)
*/
void ucsi_connector_change(struct ucsi *ucsi, u8 num)
{
- struct ucsi_connector *con = &ucsi->connector[num - 1];
+ struct ucsi_connector *con;
if (!(ucsi->ntfy & UCSI_ENABLE_NTFY_CONNECTOR_CHANGE)) {
dev_dbg(ucsi->dev, "Early connector change event\n");
return;
}
+ if (!num || num > ucsi->cap.num_connectors) {
+ dev_warn_ratelimited(ucsi->dev,
+ "Bogus connector change on %u (max %u)\n",
+ num, ucsi->cap.num_connectors);
+ return;
+ }
+
+ con = &ucsi->connector[num - 1];
+
if (!test_and_set_bit(EVENT_PENDING, &ucsi->flags))
schedule_work(&con->work);
}
diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
index 199799b..4463c1a 100644
--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
+++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
@@ -1243,6 +1243,11 @@ static int do_flash(struct ucsi_ccg *uc, enum enum_flash_mode mode)
*****************************************************************/
p = strnchr(fw->data, fw->size, ':');
+ if (!p) {
+ dev_err(dev, "Bad FW format: no ':' record header found\n");
+ err = -EINVAL;
+ goto release_mem;
+ }
while (p < eof) {
s = strnchr(p + 1, eof - p - 1, ':');
diff --git a/drivers/usb/usbip/vudc_dev.c b/drivers/usb/usbip/vudc_dev.c
index 9038310..c5f079c 100644
--- a/drivers/usb/usbip/vudc_dev.c
+++ b/drivers/usb/usbip/vudc_dev.c
@@ -632,6 +632,7 @@ void vudc_remove(struct platform_device *pdev)
{
struct vudc *udc = platform_get_drvdata(pdev);
+ v_stop_timer(udc);
usb_del_gadget_udc(&udc->gadget);
cleanup_vudc_hw(udc);
kfree(udc);
diff --git a/drivers/usb/usbip/vudc_transfer.c b/drivers/usb/usbip/vudc_transfer.c
index a4f02ea..d4ce85c 100644
--- a/drivers/usb/usbip/vudc_transfer.c
+++ b/drivers/usb/usbip/vudc_transfer.c
@@ -490,7 +490,8 @@ void v_stop_timer(struct vudc *udc)
{
struct transfer_timer *t = &udc->tr_timer;
- /* timer itself will take care of stopping */
+ /* Delete the timer synchronously before teardown frees udc. */
dev_dbg(&udc->pdev->dev, "timer stop");
+ timer_delete_sync(&t->timer);
t->state = VUDC_TR_STOPPED;
}
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 3f8d093..050e754 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -483,6 +483,40 @@ static int vfio_pci_core_runtime_resume(struct device *dev)
#endif /* CONFIG_PM */
/*
+ * Eager-request BAR resources, and iomap them. Soft failures are
+ * allowed, and consumers must check the barmap before use in order to
+ * give compatible user-visible behaviour with the previous on-demand
+ * allocation method.
+ */
+static void vfio_pci_core_map_bars(struct vfio_pci_core_device *vdev)
+{
+ struct pci_dev *pdev = vdev->pdev;
+ int i;
+
+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
+ int bar = i + PCI_STD_RESOURCES;
+
+ vdev->barmap[bar] = IOMEM_ERR_PTR(-ENODEV);
+
+ if (!pci_resource_len(pdev, i))
+ continue;
+
+ if (pci_request_selected_regions(pdev, 1 << bar, "vfio")) {
+ pci_dbg(pdev, "Failed to reserve region %d\n", bar);
+ vdev->barmap[bar] = IOMEM_ERR_PTR(-EBUSY);
+ continue;
+ }
+
+ vdev->barmap[bar] = pci_iomap(pdev, bar, 0);
+ if (!vdev->barmap[bar]) {
+ pci_dbg(pdev, "Failed to iomap region %d\n", bar);
+ pci_release_selected_regions(pdev, 1 << bar);
+ vdev->barmap[bar] = IOMEM_ERR_PTR(-ENOMEM);
+ }
+ }
+}
+
+/*
* The pci-driver core runtime PM routines always save the device state
* before going into suspended state. If the device is going into low power
* state with only with runtime PM ops, then no explicit handling is needed
@@ -568,6 +602,7 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
vdev->has_vga = true;
+ vfio_pci_core_map_bars(vdev);
return 0;
@@ -648,7 +683,7 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
for (i = 0; i < PCI_STD_NUM_BARS; i++) {
bar = i + PCI_STD_RESOURCES;
- if (!vdev->barmap[bar])
+ if (IS_ERR_OR_NULL(vdev->barmap[bar]))
continue;
pci_iounmap(pdev, vdev->barmap[bar]);
pci_release_selected_regions(pdev, 1 << bar);
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index f87fd32..1a177ce 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -244,9 +244,11 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
return -EINVAL;
/*
- * For PCI the region_index is the BAR number like everything else.
+ * For PCI the region_index is the BAR number like everything
+ * else. Check that PCI resources have been claimed for it.
*/
- if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX)
+ if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX ||
+ vfio_pci_core_setup_barmap(vdev, get_dma_buf.region_index))
return -ENODEV;
dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges,
@@ -354,19 +356,18 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
if (revoked) {
kref_put(&priv->kref, vfio_pci_dma_buf_done);
wait_for_completion(&priv->comp);
- } else {
/*
- * Kref is initialize again, because when revoke
- * was performed the reference counter was decreased
- * to zero to trigger completion.
+ * Re-arm the registered kref reference and the
+ * completion so the post-revoke state matches the
+ * post-creation state. An un-revoke followed by a
+ * new mapping needs the kref to be non-zero before
+ * kref_get(), and vfio_pci_dma_buf_cleanup()
+ * delegates its drain back through this revoke
+ * path on a possibly-already-revoked dma-buf.
*/
kref_init(&priv->kref);
- /*
- * There is no need to wait as no mapping was
- * performed when the previous status was
- * priv->revoked == true.
- */
reinit_completion(&priv->comp);
+ } else {
dma_resv_lock(priv->dmabuf->resv, NULL);
priv->revoked = false;
dma_resv_unlock(priv->dmabuf->resv);
@@ -382,21 +383,22 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
struct vfio_pci_dma_buf *tmp;
down_write(&vdev->memory_lock);
+
+ /*
+ * Drain any active mappings via the revoke path. The move is
+ * idempotent for dma-bufs already in the revoked state and
+ * leaves every priv with the kref re-armed and the completion
+ * ready, so cleanup itself does not need to participate in kref
+ * bookkeeping.
+ */
+ vfio_pci_dma_buf_move(vdev, true);
+
list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
if (!get_file_active(&priv->dmabuf->file))
continue;
- dma_resv_lock(priv->dmabuf->resv, NULL);
list_del_init(&priv->dmabufs_elm);
priv->vdev = NULL;
- priv->revoked = true;
- dma_buf_invalidate_mappings(priv->dmabuf);
- dma_resv_wait_timeout(priv->dmabuf->resv,
- DMA_RESV_USAGE_BOOKKEEP, false,
- MAX_SCHEDULE_TIMEOUT);
- dma_resv_unlock(priv->dmabuf->resv);
- kref_put(&priv->kref, vfio_pci_dma_buf_done);
- wait_for_completion(&priv->comp);
vfio_device_put_registration(&vdev->vdev);
fput(priv->dmabuf->file);
}
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index 4251ee0..3bfbb87 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -198,27 +198,15 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
}
EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
+/*
+ * The barmap is set up in vfio_pci_core_enable(). Callers use this
+ * function to check that the BAR resources are requested or that the
+ * pci_iomap() was done.
+ */
int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
{
- struct pci_dev *pdev = vdev->pdev;
- int ret;
- void __iomem *io;
-
- if (vdev->barmap[bar])
- return 0;
-
- ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
- if (ret)
- return ret;
-
- io = pci_iomap(pdev, bar, 0);
- if (!io) {
- pci_release_selected_regions(pdev, 1 << bar);
- return -ENOMEM;
- }
-
- vdev->barmap[bar] = io;
-
+ if (IS_ERR(vdev->barmap[bar]))
+ return PTR_ERR(vdev->barmap[bar]);
return 0;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c
index a12dd25a..fd00b86 100644
--- a/drivers/video/fbdev/core/fb_defio.c
+++ b/drivers/video/fbdev/core/fb_defio.c
@@ -14,7 +14,6 @@
#include <linux/export.h>
#include <linux/string.h>
#include <linux/mm.h>
-#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c
index c341d76..fdbb867 100644
--- a/drivers/video/fbdev/udlfb.c
+++ b/drivers/video/fbdev/udlfb.c
@@ -321,12 +321,32 @@ static int dlfb_set_video_mode(struct dlfb_data *dlfb,
return retval;
}
+static void dlfb_vm_open(struct vm_area_struct *vma)
+{
+ struct dlfb_data *dlfb = vma->vm_private_data;
+
+ atomic_inc(&dlfb->mmap_count);
+}
+
+static void dlfb_vm_close(struct vm_area_struct *vma)
+{
+ struct dlfb_data *dlfb = vma->vm_private_data;
+
+ atomic_dec(&dlfb->mmap_count);
+}
+
+static const struct vm_operations_struct dlfb_vm_ops = {
+ .open = dlfb_vm_open,
+ .close = dlfb_vm_close,
+};
+
static int dlfb_ops_mmap(struct fb_info *info, struct vm_area_struct *vma)
{
unsigned long start = vma->vm_start;
unsigned long size = vma->vm_end - vma->vm_start;
unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
unsigned long page, pos;
+ struct dlfb_data *dlfb = info->par;
if (info->fbdefio)
return fb_deferred_io_mmap(info, vma);
@@ -358,6 +378,9 @@ static int dlfb_ops_mmap(struct fb_info *info, struct vm_area_struct *vma)
size = 0;
}
+ vma->vm_ops = &dlfb_vm_ops;
+ vma->vm_private_data = dlfb;
+ atomic_inc(&dlfb->mmap_count);
return 0;
}
@@ -1176,7 +1199,6 @@ static void dlfb_deferred_vfree(struct dlfb_data *dlfb, void *mem)
/*
* Assumes &info->lock held by caller
- * Assumes no active clients have framebuffer open
*/
static int dlfb_realloc_framebuffer(struct dlfb_data *dlfb, struct fb_info *info, u32 new_len)
{
@@ -1188,6 +1210,13 @@ static int dlfb_realloc_framebuffer(struct dlfb_data *dlfb, struct fb_info *info
new_len = PAGE_ALIGN(new_len);
if (new_len > old_len) {
+ if (atomic_read(&dlfb->mmap_count) > 0) {
+ dev_warn(info->dev,
+ "refusing realloc: %d active mmaps\n",
+ atomic_read(&dlfb->mmap_count));
+ return -EBUSY;
+ }
+
/*
* Alloc system memory for virtual framebuffer
*/
diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c
index 74f2086..f170ff4 100644
--- a/drivers/virt/acrn/hsm.c
+++ b/drivers/virt/acrn/hsm.c
@@ -16,6 +16,7 @@
#include <linux/slab.h>
#include <asm/acrn.h>
+#include <asm/cpuid/api.h>
#include <asm/hypervisor.h>
#include "acrn_drv.h"
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index e001e67..d186ae5 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -176,7 +176,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
struct snp_guest_req req = {};
int ret, npages = 0, resp_len;
sockptr_t certs_address;
- struct page *page;
+ u64 pfn;
if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data))
return -EINVAL;
@@ -211,16 +211,16 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
* zeros to indicate that certificate data was not provided.
*/
npages = report_req->certs_len >> PAGE_SHIFT;
- page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO,
- get_order(report_req->certs_len));
- if (!page)
+ req.certs_data = alloc_pages_exact(npages << PAGE_SHIFT,
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ if (!req.certs_data)
return -ENOMEM;
- req.certs_data = page_address(page);
+ pfn = PHYS_PFN(virt_to_phys(req.certs_data));
ret = set_memory_decrypted((unsigned long)req.certs_data, npages);
if (ret) {
pr_err("failed to mark page shared, ret=%d\n", ret);
- __free_pages(page, get_order(report_req->certs_len));
+ snp_leak_pages(pfn, npages);
return -EFAULT;
}
@@ -274,10 +274,12 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
kfree(report_resp);
e_free_data:
if (npages) {
- if (set_memory_encrypted((unsigned long)req.certs_data, npages))
+ if (set_memory_encrypted((unsigned long)req.certs_data, npages)) {
WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
- else
- __free_pages(page, get_order(report_req->certs_len));
+ snp_leak_pages(pfn, npages);
+ } else {
+ free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
+ }
}
return ret;
}
diff --git a/drivers/virtio/virtio_rtc_ptp.c b/drivers/virtio/virtio_rtc_ptp.c
index f845999..ff8d834 100644
--- a/drivers/virtio/virtio_rtc_ptp.c
+++ b/drivers/virtio/virtio_rtc_ptp.c
@@ -139,7 +139,7 @@ static int viortc_ptp_getcrosststamp(struct ptp_clock_info *ptp,
if (ret)
return ret;
- ktime_get_snapshot(&history_begin);
+ ktime_get_snapshot_id(xtstamp->clock_id, &history_begin);
if (history_begin.cs_id != cs_id)
return -EOPNOTSUPP;
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index dc78729..6a9695a 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -1547,7 +1547,7 @@
config RDC321X_WDT
tristate "RDC R-321x SoC watchdog"
- depends on X86_RDC321X || COMPILE_TEST
+ depends on X86_32 || COMPILE_TEST
depends on PCI
help
This is the driver for the built in hardware watchdog
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index bc9a416..6ea9455 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -40,6 +40,7 @@
#include <linux/ktime.h>
#ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
#include <asm/desc.h>
#include <asm/ptrace.h>
#include <asm/idtentry.h>
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index a6abf1cc..35f879d 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -59,6 +59,7 @@
#include <xen/swiotlb-xen.h>
#include <xen/balloon.h>
#ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
#include <asm/xen/cpuid.h>
#endif
#include <xen/mem-reservation.h>
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 15ba592..725a49a 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -1620,6 +1620,12 @@ static void privcmd_close(struct vm_area_struct *vma)
kvfree(pages);
}
+static int privcmd_may_split(struct vm_area_struct *area, unsigned long addr)
+{
+ /* Forbid splitting, avoids double free via privcmd_close(). */
+ return -EINVAL;
+}
+
static vm_fault_t privcmd_fault(struct vm_fault *vmf)
{
printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n",
@@ -1631,6 +1637,7 @@ static vm_fault_t privcmd_fault(struct vm_fault *vmf)
static const struct vm_operations_struct privcmd_vm_ops = {
.close = privcmd_close,
+ .may_split = privcmd_may_split,
.fault = privcmd_fault
};
diff --git a/drivers/xen/sys-hypervisor.c b/drivers/xen/sys-hypervisor.c
index b1bb01ba..9192324 100644
--- a/drivers/xen/sys-hypervisor.c
+++ b/drivers/xen/sys-hypervisor.c
@@ -366,6 +366,8 @@ static ssize_t buildid_show(struct hyp_sysfs_attr *attr, char *buffer)
ret = sprintf(buffer, "<denied>");
return ret;
}
+ if (ret > PAGE_SIZE)
+ return -ENOSPC;
buildid = kmalloc(sizeof(*buildid) + ret, GFP_KERNEL);
if (!buildid)
@@ -373,8 +375,10 @@ static ssize_t buildid_show(struct hyp_sysfs_attr *attr, char *buffer)
buildid->len = ret;
ret = HYPERVISOR_xen_version(XENVER_build_id, buildid);
- if (ret > 0)
- ret = sprintf(buffer, "%s", buildid->buf);
+ if (ret > 0) {
+ /* Build id is binary, not a string. */
+ memcpy(buffer, buildid->buf, ret);
+ }
kfree(buildid);
return ret;
diff --git a/drivers/xen/xen-acpi-pad.c b/drivers/xen/xen-acpi-pad.c
index 75a3986..5b98e0e 100644
--- a/drivers/xen/xen-acpi-pad.c
+++ b/drivers/xen/xen-acpi-pad.c
@@ -110,9 +110,13 @@ static void acpi_pad_notify(acpi_handle handle, u32 event,
static int acpi_pad_probe(struct platform_device *pdev)
{
- struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+ struct acpi_device *device;
acpi_status status;
+ device = ACPI_COMPANION(&pdev->dev);
+ if (!device)
+ return -ENODEV;
+
strcpy(acpi_device_name(device), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
strcpy(acpi_device_class(device), ACPI_PROCESSOR_AGGREGATOR_CLASS);
diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c
index 82b0a34..c202e7c 100644
--- a/drivers/xen/xenbus/xenbus_xs.c
+++ b/drivers/xen/xenbus/xenbus_xs.c
@@ -47,6 +47,9 @@
#include <linux/rwsem.h>
#include <linux/mutex.h>
#include <asm/xen/hypervisor.h>
+#ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
+#endif
#include <xen/xenbus.h>
#include <xen/xen.h>
#include "xenbus.h"
diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h
index d3aefbe..34c115d 100644
--- a/fs/9p/v9fs_vfs.h
+++ b/fs/9p/v9fs_vfs.h
@@ -75,17 +75,4 @@ static inline void v9fs_invalidate_inode_attr(struct inode *inode)
int v9fs_open_to_dotl_flags(int flags);
-static inline void v9fs_i_size_write(struct inode *inode, loff_t i_size)
-{
- /*
- * 32-bit need the lock, concurrent updates could break the
- * sequences and make i_size_read() loop forever.
- * 64-bit updates are atomic and can skip the locking.
- */
- if (sizeof(i_size) > sizeof(long))
- spin_lock(&inode->i_lock);
- i_size_write(inode, i_size);
- if (sizeof(i_size) > sizeof(long))
- spin_unlock(&inode->i_lock);
-}
#endif
diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
index d1508b1..f468acb8 100644
--- a/fs/9p/vfs_inode.c
+++ b/fs/9p/vfs_inode.c
@@ -1141,11 +1141,13 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode,
mode |= inode->i_mode & ~S_IALLUGO;
inode->i_mode = mode;
- v9inode->netfs.remote_i_size = stat->length;
+ spin_lock(&inode->i_lock);
+ netfs_write_remote_i_size(inode, stat->length);
if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE))
- v9fs_i_size_write(inode, stat->length);
+ i_size_write(inode, stat->length);
/* not real number of blocks, but 512 byte ones ... */
inode->i_blocks = (stat->length + 512 - 1) >> 9;
+ spin_unlock(&inode->i_lock);
v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR;
}
diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
index 71796a8..141fb54 100644
--- a/fs/9p/vfs_inode_dotl.c
+++ b/fs/9p/vfs_inode_dotl.c
@@ -634,10 +634,12 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
mode |= inode->i_mode & ~S_IALLUGO;
inode->i_mode = mode;
- v9inode->netfs.remote_i_size = stat->st_size;
+ spin_lock(&inode->i_lock);
+ netfs_write_remote_i_size(inode, stat->st_size);
if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE))
- v9fs_i_size_write(inode, stat->st_size);
+ i_size_write(inode, stat->st_size);
inode->i_blocks = stat->st_blocks;
+ spin_unlock(&inode->i_lock);
} else {
if (stat->st_result_mask & P9_STATS_ATIME) {
inode_set_atime(inode, stat->st_atime_sec,
@@ -662,13 +664,15 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
mode |= inode->i_mode & ~S_IALLUGO;
inode->i_mode = mode;
}
+ spin_lock(&inode->i_lock);
if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) &&
stat->st_result_mask & P9_STATS_SIZE) {
- v9inode->netfs.remote_i_size = stat->st_size;
- v9fs_i_size_write(inode, stat->st_size);
+ netfs_write_remote_i_size(inode, stat->st_size);
+ i_size_write(inode, stat->st_size);
}
if (stat->st_result_mask & P9_STATS_BLOCKS)
inode->i_blocks = stat->st_blocks;
+ spin_unlock(&inode->i_lock);
}
if (stat->st_result_mask & P9_STATS_GEN)
inode->i_generation = stat->st_gen;
diff --git a/fs/afs/Makefile b/fs/afs/Makefile
index b49b8fe..0d8f198 100644
--- a/fs/afs/Makefile
+++ b/fs/afs/Makefile
@@ -30,6 +30,7 @@
server.o \
server_list.o \
super.o \
+ symlink.o \
validation.o \
vlclient.o \
vl_alias.o \
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index aaaa558..498b99c 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -44,6 +44,8 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
struct dentry *old_dentry, struct inode *new_dir,
struct dentry *new_dentry, unsigned int flags);
+static int afs_dir_writepages(struct address_space *mapping,
+ struct writeback_control *wbc);
const struct file_operations afs_dir_file_operations = {
.open = afs_dir_open,
@@ -68,7 +70,7 @@ const struct inode_operations afs_dir_inode_operations = {
};
const struct address_space_operations afs_dir_aops = {
- .writepages = afs_single_writepages,
+ .writepages = afs_dir_writepages,
};
const struct dentry_operations afs_fs_dentry_operations = {
@@ -233,22 +235,13 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file)
struct iov_iter iter;
ssize_t ret;
loff_t i_size;
- bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) &&
- !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags));
i_size = i_size_read(&dvnode->netfs.inode);
- if (is_dir) {
- if (i_size < AFS_DIR_BLOCK_SIZE)
- return afs_bad(dvnode, afs_file_error_dir_small);
- if (i_size > AFS_DIR_BLOCK_SIZE * 1024) {
- trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
- return -EFBIG;
- }
- } else {
- if (i_size > AFSPATHMAX) {
- trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
- return -EFBIG;
- }
+ if (i_size < AFS_DIR_BLOCK_SIZE)
+ return afs_bad(dvnode, afs_file_error_dir_small);
+ if (i_size > AFS_DIR_BLOCK_SIZE * 1024) {
+ trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
+ return -EFBIG;
}
/* Expand the storage. TODO: Shrink the storage too. */
@@ -277,24 +270,18 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file)
* buffer.
*/
ret = -ESTALE;
- } else if (is_dir) {
+ } else {
int ret2 = afs_dir_check(dvnode);
if (ret2 < 0)
ret = ret2;
- } else if (i_size < folioq_folio_size(dvnode->directory, 0)) {
- /* NUL-terminate a symlink. */
- char *symlink = kmap_local_folio(folioq_folio(dvnode->directory, 0), 0);
-
- symlink[i_size] = 0;
- kunmap_local(symlink);
}
}
return ret;
}
-ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file)
+static ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file)
{
ssize_t ret;
@@ -1763,13 +1750,20 @@ static int afs_link(struct dentry *from, struct inode *dir,
return ret;
}
+static void afs_symlink_put(struct afs_operation *op)
+{
+ kfree(op->create.symlink);
+ op->create.symlink = NULL;
+ afs_create_put(op);
+}
+
static const struct afs_operation_ops afs_symlink_operation = {
.issue_afs_rpc = afs_fs_symlink,
.issue_yfs_rpc = yfs_fs_symlink,
.success = afs_create_success,
.aborted = afs_check_for_remote_deletion,
.edit_dir = afs_create_edit_dir,
- .put = afs_create_put,
+ .put = afs_symlink_put,
};
/*
@@ -1779,7 +1773,9 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, const char *content)
{
struct afs_operation *op;
+ struct afs_symlink *symlink;
struct afs_vnode *dvnode = AFS_FS_I(dir);
+ size_t clen = strlen(content);
int ret;
_enter("{%llx:%llu},{%pd},%s",
@@ -1791,12 +1787,20 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
goto error;
ret = -EINVAL;
- if (strlen(content) >= AFSPATHMAX)
+ if (clen >= AFSPATHMAX)
goto error;
+ ret = -ENOMEM;
+ symlink = kmalloc_flex(struct afs_symlink, content, clen + 1, GFP_KERNEL);
+ if (!symlink)
+ goto error;
+ refcount_set(&symlink->ref, 1);
+ memcpy(symlink->content, content, clen + 1);
+
op = afs_alloc_operation(NULL, dvnode->volume);
if (IS_ERR(op)) {
ret = PTR_ERR(op);
+ kfree(symlink);
goto error;
}
@@ -1808,7 +1812,7 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
op->dentry = dentry;
op->ops = &afs_symlink_operation;
op->create.reason = afs_edit_dir_for_symlink;
- op->create.symlink = content;
+ op->create.symlink = symlink;
op->mtime = current_time(dir);
ret = afs_do_sync_operation(op);
afs_dir_unuse_cookie(dvnode, ret);
@@ -2192,28 +2196,33 @@ static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
}
/*
- * Write the file contents to the cache as a single blob.
+ * Write the directory contents to the cache as a single blob.
*/
-int afs_single_writepages(struct address_space *mapping,
- struct writeback_control *wbc)
+static int afs_dir_writepages(struct address_space *mapping,
+ struct writeback_control *wbc)
{
struct afs_vnode *dvnode = AFS_FS_I(mapping->host);
struct iov_iter iter;
- bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) &&
- !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags));
int ret = 0;
/* Need to lock to prevent the folio queue and folios from being thrown
* away.
*/
- down_read(&dvnode->validate_lock);
+ if (!down_read_trylock(&dvnode->validate_lock)) {
+ if (wbc->sync_mode == WB_SYNC_NONE) {
+ /* The VFS will have undirtied the inode. */
+ netfs_single_mark_inode_dirty(&dvnode->netfs.inode);
+ return 0;
+ }
+ down_read(&dvnode->validate_lock);
+ }
- if (is_dir ?
- test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) :
- atomic64_read(&dvnode->cb_expires_at) != AFS_NO_CB_PROMISE) {
+ if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) {
iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0,
i_size_read(&dvnode->netfs.inode));
ret = netfs_writeback_single(mapping, wbc, &iter);
+ if (ret == 1)
+ ret = 0; /* Skipped write due to lock conflict. */
}
up_read(&dvnode->validate_lock);
diff --git a/fs/afs/file.c b/fs/afs/file.c
index 85696ac..0467742 100644
--- a/fs/afs/file.c
+++ b/fs/afs/file.c
@@ -427,21 +427,35 @@ static void afs_free_request(struct netfs_io_request *rreq)
afs_put_wb_key(rreq->netfs_priv2);
}
-static void afs_update_i_size(struct inode *inode, loff_t new_i_size)
+/*
+ * Set the file size and block count, taking ->cb_lock and ->i_lock to maintain
+ * coherency and prevent 64-bit tearing on 32-bit arches.
+ *
+ * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K
+ * for consistency with other AFS clients.
+ */
+void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size)
{
- struct afs_vnode *vnode = AFS_FS_I(inode);
+ struct inode *inode = &vnode->netfs.inode;
loff_t i_size;
write_seqlock(&vnode->cb_lock);
- i_size = i_size_read(&vnode->netfs.inode);
+ spin_lock(&inode->i_lock);
+ i_size = i_size_read(inode);
if (new_i_size > i_size) {
- i_size_write(&vnode->netfs.inode, new_i_size);
- inode_set_bytes(&vnode->netfs.inode, new_i_size);
+ i_size_write(inode, new_i_size);
+ inode_set_bytes(inode, round_up(new_i_size, 1024));
}
+ spin_unlock(&inode->i_lock);
write_sequnlock(&vnode->cb_lock);
fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size);
}
+static void afs_update_i_size(struct inode *inode, loff_t new_i_size)
+{
+ afs_set_i_size(AFS_FS_I(inode), new_i_size);
+}
+
static void afs_netfs_invalidate_cache(struct netfs_io_request *wreq)
{
struct afs_vnode *vnode = AFS_FS_I(wreq->inode);
diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c
index 95494d5..a2ffd60 100644
--- a/fs/afs/fsclient.c
+++ b/fs/afs/fsclient.c
@@ -886,7 +886,7 @@ void afs_fs_symlink(struct afs_operation *op)
namesz = name->len;
padsz = (4 - (namesz & 3)) & 3;
- c_namesz = strlen(op->create.symlink);
+ c_namesz = strlen(op->create.symlink->content);
c_padsz = (4 - (c_namesz & 3)) & 3;
reqsz = (6 * 4) + namesz + padsz + c_namesz + c_padsz + (6 * 4);
@@ -910,7 +910,7 @@ void afs_fs_symlink(struct afs_operation *op)
bp = (void *) bp + padsz;
}
*bp++ = htonl(c_namesz);
- memcpy(bp, op->create.symlink, c_namesz);
+ memcpy(bp, op->create.symlink->content, c_namesz);
bp = (void *) bp + c_namesz;
if (c_padsz > 0) {
memset(bp, 0, c_padsz);
diff --git a/fs/afs/inode.c b/fs/afs/inode.c
index a517343..3f48458 100644
--- a/fs/afs/inode.c
+++ b/fs/afs/inode.c
@@ -25,96 +25,6 @@
#include "internal.h"
#include "afs_fs.h"
-void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op)
-{
- size_t size = strlen(op->create.symlink) + 1;
- size_t dsize = 0;
- char *p;
-
- if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size,
- mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0)
- return;
-
- vnode->directory_size = dsize;
- p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
- memcpy(p, op->create.symlink, size);
- kunmap_local(p);
- set_bit(AFS_VNODE_DIR_READ, &vnode->flags);
- netfs_single_mark_inode_dirty(&vnode->netfs.inode);
-}
-
-static void afs_put_link(void *arg)
-{
- struct folio *folio = virt_to_folio(arg);
-
- kunmap_local(arg);
- folio_put(folio);
-}
-
-const char *afs_get_link(struct dentry *dentry, struct inode *inode,
- struct delayed_call *callback)
-{
- struct afs_vnode *vnode = AFS_FS_I(inode);
- struct folio *folio;
- char *content;
- ssize_t ret;
-
- if (!dentry) {
- /* RCU pathwalk. */
- if (!test_bit(AFS_VNODE_DIR_READ, &vnode->flags) || !afs_check_validity(vnode))
- return ERR_PTR(-ECHILD);
- goto good;
- }
-
- if (test_bit(AFS_VNODE_DIR_READ, &vnode->flags))
- goto fetch;
-
- ret = afs_validate(vnode, NULL);
- if (ret < 0)
- return ERR_PTR(ret);
-
- if (!test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags) &&
- test_bit(AFS_VNODE_DIR_READ, &vnode->flags))
- goto good;
-
-fetch:
- ret = afs_read_single(vnode, NULL);
- if (ret < 0)
- return ERR_PTR(ret);
- set_bit(AFS_VNODE_DIR_READ, &vnode->flags);
-
-good:
- folio = folioq_folio(vnode->directory, 0);
- folio_get(folio);
- content = kmap_local_folio(folio, 0);
- set_delayed_call(callback, afs_put_link, content);
- return content;
-}
-
-int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
-{
- DEFINE_DELAYED_CALL(done);
- const char *content;
- int len;
-
- content = afs_get_link(dentry, d_inode(dentry), &done);
- if (IS_ERR(content)) {
- do_delayed_call(&done);
- return PTR_ERR(content);
- }
-
- len = umin(strlen(content), buflen);
- if (copy_to_user(buffer, content, len))
- len = -EFAULT;
- do_delayed_call(&done);
- return len;
-}
-
-static const struct inode_operations afs_symlink_inode_operations = {
- .get_link = afs_get_link,
- .readlink = afs_readlink,
-};
-
static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *parent_vnode)
{
static unsigned long once_only;
@@ -214,7 +124,7 @@ static int afs_inode_init_from_status(struct afs_operation *op,
inode->i_mode = S_IFLNK | status->mode;
inode->i_op = &afs_symlink_inode_operations;
}
- inode->i_mapping->a_ops = &afs_dir_aops;
+ inode->i_mapping->a_ops = &afs_symlink_aops;
inode_nohighmem(inode);
mapping_set_release_always(inode->i_mapping);
break;
@@ -224,7 +134,8 @@ static int afs_inode_init_from_status(struct afs_operation *op,
return afs_protocol_error(NULL, afs_eproto_file_type);
}
- afs_set_i_size(vnode, status->size);
+ i_size_write(inode, status->size);
+ inode_set_bytes(inode, status->size);
afs_set_netfs_context(vnode);
vnode->invalid_before = status->data_version;
@@ -253,7 +164,8 @@ static void afs_apply_status(struct afs_operation *op,
{
struct afs_file_status *status = &vp->scb.status;
struct afs_vnode *vnode = vp->vnode;
- struct inode *inode = &vnode->netfs.inode;
+ struct netfs_inode *ictx = &vnode->netfs;
+ struct inode *inode = &ictx->inode;
struct timespec64 t;
umode_t mode;
bool unexpected_jump = false;
@@ -336,6 +248,8 @@ static void afs_apply_status(struct afs_operation *op,
}
if (data_changed) {
+ unsigned long long zero_point, size = status->size;
+
inode_set_iversion_raw(inode, status->data_version);
/* Only update the size if the data version jumped. If the
@@ -343,16 +257,25 @@ static void afs_apply_status(struct afs_operation *op,
* idea of what the size should be that's not the same as
* what's on the server.
*/
- vnode->netfs.remote_i_size = status->size;
- if (change_size || status->size > i_size_read(inode)) {
- afs_set_i_size(vnode, status->size);
+ spin_lock(&inode->i_lock);
+
+ if (change_size || size > i_size_read(inode)) {
+ /* We can read the sizes directly as we hold i_lock. */
+ zero_point = ictx->_zero_point;
+
if (unexpected_jump)
- vnode->netfs.zero_point = status->size;
+ zero_point = size;
+ netfs_write_sizes(inode, size, size, zero_point);
+ inode_set_bytes(inode, size);
inode_set_ctime_to_ts(inode, t);
inode_set_atime_to_ts(inode, t);
+ } else {
+ netfs_write_remote_i_size(inode, size);
}
+ spin_unlock(&inode->i_lock);
+
if (op->ops == &afs_fetch_data_operation)
- op->fetch.subreq->rreq->i_size = status->size;
+ op->fetch.subreq->rreq->i_size = size;
}
}
@@ -709,7 +632,7 @@ int afs_getattr(struct mnt_idmap *idmap, const struct path *path,
* it, but we need to give userspace the server's size.
*/
if (S_ISDIR(inode->i_mode))
- stat->size = vnode->netfs.remote_i_size;
+ stat->size = netfs_read_remote_i_size(inode);
} while (read_seqretry(&vnode->cb_lock, seq));
return 0;
@@ -756,12 +679,14 @@ void afs_evict_inode(struct inode *inode)
.range_end = LLONG_MAX,
};
- afs_single_writepages(inode->i_mapping, &wbc);
+ inode->i_mapping->a_ops->writepages(inode->i_mapping, &wbc);
}
netfs_wait_for_outstanding_io(inode);
truncate_inode_pages_final(&inode->i_data);
netfs_free_folioq_buffer(vnode->directory);
+ if (vnode->symlink)
+ afs_evict_symlink(vnode);
afs_set_cache_aux(vnode, &aux);
netfs_clear_inode_writeback(inode, &aux);
@@ -889,7 +814,7 @@ int afs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
*/
if (!(attr->ia_valid & (supported & ~ATTR_SIZE & ~ATTR_MTIME)) &&
attr->ia_size < i_size &&
- attr->ia_size > vnode->netfs.remote_i_size) {
+ attr->ia_size > netfs_read_remote_i_size(inode)) {
truncate_setsize(inode, attr->ia_size);
netfs_resize_file(&vnode->netfs, size, false);
fscache_resize_cookie(afs_vnode_cache(vnode),
diff --git a/fs/afs/internal.h b/fs/afs/internal.h
index 599353c..0b72a85 100644
--- a/fs/afs/internal.h
+++ b/fs/afs/internal.h
@@ -710,6 +710,7 @@ struct afs_vnode {
#define AFS_VNODE_DIR_READ 11 /* Set if we've read a dir's contents */
struct folio_queue *directory; /* Directory contents */
+ struct afs_symlink __rcu *symlink; /* Symlink content */
struct list_head wb_keys; /* List of keys available for writeback */
struct list_head pending_locks; /* locks waiting to be granted */
struct list_head granted_locks; /* locks granted on this file */
@@ -777,6 +778,15 @@ struct afs_permits {
};
/*
+ * Copy of symlink content for normal use.
+ */
+struct afs_symlink {
+ struct rcu_head rcu;
+ refcount_t ref;
+ char content[];
+};
+
+/*
* Error prioritisation and accumulation.
*/
struct afs_error {
@@ -887,7 +897,7 @@ struct afs_operation {
struct {
int reason; /* enum afs_edit_dir_reason */
mode_t mode;
- const char *symlink;
+ struct afs_symlink *symlink;
} create;
struct {
bool need_rehash;
@@ -1098,13 +1108,10 @@ extern const struct inode_operations afs_dir_inode_operations;
extern const struct address_space_operations afs_dir_aops;
extern const struct dentry_operations afs_fs_dentry_operations;
-ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file);
ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file)
__acquires(&dvnode->validate_lock);
extern void afs_d_release(struct dentry *);
extern void afs_check_for_remote_deletion(struct afs_operation *);
-int afs_single_writepages(struct address_space *mapping,
- struct writeback_control *wbc);
/*
* dir_edit.c
@@ -1157,6 +1164,7 @@ extern int afs_open(struct inode *, struct file *);
extern int afs_release(struct inode *, struct file *);
void afs_fetch_data_async_rx(struct work_struct *work);
void afs_fetch_data_immediate_cancel(struct afs_call *call);
+void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size);
/*
* flock.c
@@ -1246,10 +1254,6 @@ extern void afs_fs_probe_cleanup(struct afs_net *);
*/
extern const struct afs_operation_ops afs_fetch_status_operation;
-void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op);
-const char *afs_get_link(struct dentry *dentry, struct inode *inode,
- struct delayed_call *callback);
-int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen);
extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *);
extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *);
extern int afs_ilookup5_test_by_fid(struct inode *, void *);
@@ -1600,6 +1604,21 @@ extern int __init afs_fs_init(void);
extern void afs_fs_exit(void);
/*
+ * symlink.c
+ */
+extern const struct inode_operations afs_symlink_inode_operations;
+extern const struct address_space_operations afs_symlink_aops;
+
+void afs_invalidate_symlink(struct afs_vnode *vnode);
+void afs_evict_symlink(struct afs_vnode *vnode);
+void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op);
+const char *afs_get_link(struct dentry *dentry, struct inode *inode,
+ struct delayed_call *callback);
+int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen);
+int afs_symlink_writepages(struct address_space *mapping,
+ struct writeback_control *wbc);
+
+/*
* validation.c
*/
bool afs_check_validity(const struct afs_vnode *vnode);
@@ -1759,16 +1778,6 @@ static inline void afs_update_dentry_version(struct afs_operation *op,
}
/*
- * Set the file size and block count. Estimate the number of 512 bytes blocks
- * used, rounded up to nearest 1K for consistency with other AFS clients.
- */
-static inline void afs_set_i_size(struct afs_vnode *vnode, u64 size)
-{
- i_size_write(&vnode->netfs.inode, size);
- vnode->netfs.inode.i_blocks = ((size + 1023) >> 10) << 1;
-}
-
-/*
* Check for a conflicting operation on a directory that we just unlinked from.
* If someone managed to sneak a link or an unlink in on the file we just
* unlinked, we won't be able to trust nlink on an AFS file (but not YFS).
diff --git a/fs/afs/symlink.c b/fs/afs/symlink.c
new file mode 100644
index 0000000..ed58683
--- /dev/null
+++ b/fs/afs/symlink.c
@@ -0,0 +1,278 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* AFS filesystem symbolic link handling
+ *
+ * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/namei.h>
+#include <linux/pagemap.h>
+#include <linux/iov_iter.h>
+#include "internal.h"
+
+static void afs_put_symlink(struct afs_symlink *symlink)
+{
+ if (refcount_dec_and_test(&symlink->ref))
+ kfree_rcu(symlink, rcu);
+}
+
+static void afs_replace_symlink(struct afs_vnode *vnode, struct afs_symlink *symlink)
+{
+ struct afs_symlink *old;
+
+ old = rcu_replace_pointer(vnode->symlink, symlink,
+ lockdep_is_held(&vnode->validate_lock));
+ if (old)
+ afs_put_symlink(old);
+}
+
+/*
+ * In the event that a third-party update of a symlink occurs, dispose of the
+ * copy of the old contents. Called under ->validate_lock.
+ */
+void afs_invalidate_symlink(struct afs_vnode *vnode)
+{
+ afs_replace_symlink(vnode, NULL);
+}
+
+/*
+ * Dispose of a symlink copy during inode deletion.
+ */
+void afs_evict_symlink(struct afs_vnode *vnode)
+{
+ struct afs_symlink *old;
+
+ old = rcu_replace_pointer(vnode->symlink, NULL, true);
+ if (old)
+ afs_put_symlink(old);
+
+}
+
+/*
+ * Set up a locally created symlink inode for immediate write to the cache.
+ */
+void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op)
+{
+ struct afs_symlink *symlink = op->create.symlink;
+ size_t dsize = 0;
+ size_t size = strlen(symlink->content) + 1;
+ char *p;
+
+ rcu_assign_pointer(vnode->symlink, symlink);
+ op->create.symlink = NULL;
+
+ if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs)))
+ return;
+
+ if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size,
+ mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0)
+ return;
+
+ vnode->directory_size = dsize;
+ p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
+ memcpy(p, symlink->content, size);
+ kunmap_local(p);
+ netfs_single_mark_inode_dirty(&vnode->netfs.inode);
+}
+
+/*
+ * Read a symlink in a single download.
+ */
+static ssize_t afs_do_read_symlink(struct afs_vnode *vnode)
+{
+ struct afs_symlink *symlink;
+ struct iov_iter iter;
+ ssize_t ret;
+ loff_t i_size;
+
+ i_size = i_size_read(&vnode->netfs.inode);
+ if (i_size > PAGE_SIZE - 1) {
+ trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big);
+ return -EFBIG;
+ }
+
+ if (!vnode->directory) {
+ size_t cur_size = 0;
+
+ ret = netfs_alloc_folioq_buffer(NULL,
+ &vnode->directory, &cur_size, PAGE_SIZE,
+ mapping_gfp_mask(vnode->netfs.inode.i_mapping));
+ vnode->directory_size = PAGE_SIZE - 1;
+ if (ret < 0)
+ return ret;
+ }
+
+ iov_iter_folio_queue(&iter, ITER_DEST, vnode->directory, 0, 0, PAGE_SIZE);
+
+ /* AFS requires us to perform the read of a symlink as a single unit to
+ * avoid issues with the content being changed between reads.
+ */
+ ret = netfs_read_single(&vnode->netfs.inode, NULL, &iter);
+ if (ret >= 0) {
+ i_size = ret;
+ if (i_size > PAGE_SIZE - 1) {
+ trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big);
+ return -EFBIG;
+ }
+ vnode->directory_size = i_size;
+
+ /* Copy the symlink. */
+ symlink = kmalloc_flex(struct afs_symlink, content, i_size + 1,
+ GFP_KERNEL);
+ if (!symlink)
+ return -ENOMEM;
+
+ refcount_set(&symlink->ref, 1);
+ symlink->content[i_size] = 0;
+
+ const char *s = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
+
+ memcpy(symlink->content, s, i_size);
+ kunmap_local(s);
+
+ afs_replace_symlink(vnode, symlink);
+ }
+
+ if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) {
+ netfs_free_folioq_buffer(vnode->directory);
+ vnode->directory = NULL;
+ vnode->directory_size = 0;
+ }
+
+ return ret;
+}
+
+static ssize_t afs_read_symlink(struct afs_vnode *vnode)
+{
+ ssize_t ret;
+
+ fscache_use_cookie(afs_vnode_cache(vnode), false);
+ ret = afs_do_read_symlink(vnode);
+ fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL);
+ return ret;
+}
+
+static void afs_put_link(void *arg)
+{
+ afs_put_symlink(arg);
+}
+
+const char *afs_get_link(struct dentry *dentry, struct inode *inode,
+ struct delayed_call *callback)
+{
+ struct afs_symlink *symlink;
+ struct afs_vnode *vnode = AFS_FS_I(inode);
+ ssize_t ret;
+
+ if (!dentry) {
+ /* RCU pathwalk. */
+ symlink = rcu_dereference(vnode->symlink);
+ if (!symlink || !afs_check_validity(vnode))
+ return ERR_PTR(-ECHILD);
+ set_delayed_call(callback, NULL, NULL);
+ return symlink->content;
+ }
+
+ if (vnode->symlink) {
+ ret = afs_validate(vnode, NULL);
+ if (ret < 0)
+ return ERR_PTR(ret);
+
+ down_read(&vnode->validate_lock);
+ if (vnode->symlink)
+ goto good;
+ up_read(&vnode->validate_lock);
+ }
+
+ if (down_write_killable(&vnode->validate_lock) < 0)
+ return ERR_PTR(-ERESTARTSYS);
+ if (!vnode->symlink) {
+ ret = afs_read_symlink(vnode);
+ if (ret < 0) {
+ up_write(&vnode->validate_lock);
+ return ERR_PTR(ret);
+ }
+ }
+
+ downgrade_write(&vnode->validate_lock);
+
+good:
+ symlink = rcu_dereference_protected(vnode->symlink,
+ lockdep_is_held(&vnode->validate_lock));
+ refcount_inc(&symlink->ref);
+ up_read(&vnode->validate_lock);
+
+ set_delayed_call(callback, afs_put_link, symlink);
+ return symlink->content;
+}
+
+int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
+{
+ DEFINE_DELAYED_CALL(done);
+ const char *content;
+ int len;
+
+ content = afs_get_link(dentry, d_inode(dentry), &done);
+ if (IS_ERR(content)) {
+ do_delayed_call(&done);
+ return PTR_ERR(content);
+ }
+
+ len = umin(strlen(content), buflen);
+ if (copy_to_user(buffer, content, len))
+ len = -EFAULT;
+ do_delayed_call(&done);
+ return len;
+}
+
+/*
+ * Write the symlink contents to the cache as a single blob. We then throw
+ * away the page we used to receive it.
+ */
+int afs_symlink_writepages(struct address_space *mapping,
+ struct writeback_control *wbc)
+{
+ struct afs_vnode *vnode = AFS_FS_I(mapping->host);
+ struct iov_iter iter;
+ int ret = 0;
+
+ if (!down_read_trylock(&vnode->validate_lock)) {
+ if (wbc->sync_mode == WB_SYNC_NONE) {
+ /* The VFS will have undirtied the inode. */
+ netfs_single_mark_inode_dirty(&vnode->netfs.inode);
+ return 0;
+ }
+ down_read(&vnode->validate_lock);
+ }
+
+ if (vnode->directory &&
+ atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) {
+ iov_iter_folio_queue(&iter, ITER_SOURCE, vnode->directory, 0, 0,
+ i_size_read(&vnode->netfs.inode));
+ ret = netfs_writeback_single(mapping, wbc, &iter);
+ }
+
+ if (ret == 0) {
+ mutex_lock(&vnode->netfs.wb_lock);
+ netfs_free_folioq_buffer(vnode->directory);
+ vnode->directory = NULL;
+ vnode->directory_size = 0;
+ mutex_unlock(&vnode->netfs.wb_lock);
+ } else if (ret == 1) {
+ ret = 0; /* Skipped write due to lock conflict. */
+ }
+
+ up_read(&vnode->validate_lock);
+ return ret;
+}
+
+const struct inode_operations afs_symlink_inode_operations = {
+ .get_link = afs_get_link,
+ .readlink = afs_readlink,
+};
+
+const struct address_space_operations afs_symlink_aops = {
+ .writepages = afs_symlink_writepages,
+};
diff --git a/fs/afs/validation.c b/fs/afs/validation.c
index 0ba8336..e997563 100644
--- a/fs/afs/validation.c
+++ b/fs/afs/validation.c
@@ -465,11 +465,17 @@ int afs_validate(struct afs_vnode *vnode, struct key *key)
vnode->cb_ro_snapshot = cb_ro_snapshot;
vnode->cb_scrub = cb_scrub;
- /* if the vnode's data version number changed then its contents are
- * different */
+ /* If the vnode's data version number changed then its contents are
+ * different. Note that afs_apply_status() doesn't set ZAP_DATA on
+ * directories.
+ */
zap |= test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags);
- if (zap)
- afs_zap_data(vnode);
+ if (zap) {
+ if (S_ISREG(vnode->netfs.inode.i_mode))
+ afs_zap_data(vnode);
+ else if (S_ISLNK(vnode->netfs.inode.i_mode))
+ afs_invalidate_symlink(vnode);
+ }
up_write(&vnode->validate_lock);
_leave(" = 0");
return 0;
diff --git a/fs/afs/write.c b/fs/afs/write.c
index fcfed9d..7f34b939 100644
--- a/fs/afs/write.c
+++ b/fs/afs/write.c
@@ -142,7 +142,7 @@ static void afs_issue_write_worker(struct work_struct *work)
afs_begin_vnode_operation(op);
op->store.write_iter = &subreq->io_iter;
- op->store.i_size = umax(pos + len, vnode->netfs.remote_i_size);
+ op->store.i_size = umax(pos + len, netfs_read_remote_i_size(&vnode->netfs.inode));
op->mtime = inode_get_mtime(&vnode->netfs.inode);
afs_wait_for_operation(op);
diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c
index 24fb562..d941179 100644
--- a/fs/afs/yfsclient.c
+++ b/fs/afs/yfsclient.c
@@ -960,7 +960,7 @@ void yfs_fs_symlink(struct afs_operation *op)
_enter("");
- contents_sz = strlen(op->create.symlink);
+ contents_sz = strlen(op->create.symlink->content);
call = afs_alloc_flat_call(op->net, &yfs_RXYFSSymlink,
sizeof(__be32) +
sizeof(struct yfs_xdr_RPCFlags) +
@@ -981,7 +981,7 @@ void yfs_fs_symlink(struct afs_operation *op)
bp = xdr_encode_u32(bp, 0); /* RPC flags */
bp = xdr_encode_YFSFid(bp, &dvp->fid);
bp = xdr_encode_name(bp, name);
- bp = xdr_encode_string(bp, op->create.symlink, contents_sz);
+ bp = xdr_encode_string(bp, op->create.symlink->content, contents_sz);
bp = xdr_encode_YFSStoreStatus(bp, &mode, &op->mtime);
yfs_check_req(call, bp);
diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c
index e6f5a17..b611c64 100644
--- a/fs/btrfs/block-group.c
+++ b/fs/btrfs/block-group.c
@@ -2412,29 +2412,25 @@ static struct btrfs_block_group *btrfs_create_block_group(
*/
static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
{
- u64 start = 0;
+ struct rb_node *node;
int ret = 0;
- while (1) {
+ /*
+ * This is called during mount from btrfs_read_block_groups(), before
+ * any background threads are started, so no concurrent writers can
+ * modify the mapping_tree. No lock is needed here.
+ */
+ for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) {
struct btrfs_chunk_map *map;
struct btrfs_block_group *bg;
- /*
- * btrfs_find_chunk_map() will return the first chunk map
- * intersecting the range, so setting @length to 1 is enough to
- * get the first chunk.
- */
- map = btrfs_find_chunk_map(fs_info, start, 1);
- if (!map)
- break;
-
+ map = rb_entry(node, struct btrfs_chunk_map, rb_node);
bg = btrfs_lookup_block_group(fs_info, map->start);
if (unlikely(!bg)) {
btrfs_err(fs_info,
"chunk start=%llu len=%llu doesn't have corresponding block group",
map->start, map->chunk_len);
ret = -EUCLEAN;
- btrfs_free_chunk_map(map);
break;
}
if (unlikely(bg->start != map->start || bg->length != map->chunk_len ||
@@ -2447,12 +2443,9 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
bg->start, bg->length,
bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
ret = -EUCLEAN;
- btrfs_free_chunk_map(map);
btrfs_put_block_group(bg);
break;
}
- start = map->start + map->chunk_len;
- btrfs_free_chunk_map(map);
btrfs_put_block_group(bg);
}
return ret;
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index b2393a4..a02b62e 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -407,22 +407,18 @@ static noinline int add_ra_bio_pages(struct inode *inode,
end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT;
- /*
- * Avoid direct reclaim when the caller does not allow it. Since
- * add_ra_bio_pages() is always speculative, suppress allocation warnings
- * in either case.
- */
+ /* Avoid direct reclaim when the caller does not allow it. */
+ constraint_gfp = ~__GFP_FS;
+ cache_gfp = GFP_NOFS | __GFP_NOWARN;
if (!direct_reclaim) {
- constraint_gfp = ~(__GFP_FS | __GFP_DIRECT_RECLAIM) | __GFP_NOWARN;
- cache_gfp = (GFP_NOFS & ~__GFP_DIRECT_RECLAIM) | __GFP_NOWARN;
- } else {
- constraint_gfp = (~__GFP_FS) | __GFP_NOWARN;
- cache_gfp = GFP_NOFS | __GFP_NOWARN;
+ constraint_gfp &= ~__GFP_DIRECT_RECLAIM;
+ cache_gfp &= ~__GFP_DIRECT_RECLAIM;
}
while (cur < compressed_end) {
pgoff_t page_end;
pgoff_t pg_index = cur >> PAGE_SHIFT;
+ gfp_t masked_constraint_gfp;
u32 add_size;
if (pg_index > end_index)
@@ -449,8 +445,14 @@ static noinline int add_ra_bio_pages(struct inode *inode,
continue;
}
- folio = filemap_alloc_folio(mapping_gfp_constraint(mapping, constraint_gfp),
- 0, NULL);
+ /*
+ * Since add_ra_bio_pages() is always speculative, suppress
+ * allocation warnings.
+ */
+ masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp);
+ masked_constraint_gfp |= __GFP_NOWARN;
+
+ folio = filemap_alloc_folio(masked_constraint_gfp, 0, NULL);
if (!folio)
break;
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 8a11be0..c0a30bb 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -4686,6 +4686,7 @@ static void btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
free_extent_buffer_stale(eb);
}
}
+ btrfs_extent_io_tree_release(dirty_pages);
}
static void btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index ca3e4b9..2275189 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -4641,7 +4641,8 @@ int try_release_extent_buffer(struct folio *folio)
* to read the block we will not block on anything.
*/
void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
- u64 bytenr, u64 owner_root, u64 gen, int level)
+ u64 bytenr, u64 owner_root, u64 gen, int level,
+ const struct btrfs_key *first_key)
{
struct btrfs_tree_parent_check check = {
.level = level,
@@ -4650,6 +4651,11 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
struct extent_buffer *eb;
int ret;
+ if (first_key) {
+ memcpy(&check.first_key, first_key, sizeof(struct btrfs_key));
+ check.has_first_key = true;
+ }
+
eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
if (IS_ERR(eb))
return;
@@ -4677,9 +4683,13 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
*/
void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
{
+ struct btrfs_key node_key;
+
+ btrfs_node_key_to_cpu(node, &node_key, slot);
btrfs_readahead_tree_block(node->fs_info,
btrfs_node_blockptr(node, slot),
btrfs_header_owner(node),
btrfs_node_ptr_generation(node, slot),
- btrfs_header_level(node) - 1);
+ btrfs_header_level(node) - 1,
+ &node_key);
}
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index fd20923..b310a51 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -287,7 +287,8 @@ static inline void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
}
void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
- u64 bytenr, u64 owner_root, u64 gen, int level);
+ u64 bytenr, u64 owner_root, u64 gen, int level,
+ const struct btrfs_key *first_key);
void btrfs_readahead_node_child(struct extent_buffer *node, int slot);
/* Note: this can be used in for loops without caching the value in a variable. */
diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c
index 9284c0a..6b79bff2 100644
--- a/fs/btrfs/extent_map.c
+++ b/fs/btrfs/extent_map.c
@@ -1246,7 +1246,9 @@ static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root,
write_unlock(&tree->lock);
next:
from = btrfs_ino(inode) + 1;
- cond_resched_lock(&root->inodes.xa_lock);
+ xa_unlock(&root->inodes);
+ cond_resched();
+ xa_lock(&root->inodes);
}
xa_unlock(&root->inodes);
diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index cf1cb5c..8c171ed 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -633,7 +633,7 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
trans->transid);
btrfs_set_file_extent_num_bytes(leaf, fi,
end - other_start);
- return 0;
+ goto mark_dirty;
}
}
@@ -661,7 +661,7 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
other_end - start);
btrfs_set_file_extent_offset(leaf, fi,
start - orig_offset);
- return 0;
+ goto mark_dirty;
}
}
@@ -788,7 +788,12 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
}
}
- return 0;
+mark_dirty:
+ ret = btrfs_inode_set_file_extent_range(inode, start, end - start);
+ if (ret)
+ btrfs_abort_transaction(trans, ret);
+
+ return ret;
}
/*
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 9efd1ec..472b306 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -259,7 +259,11 @@ int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans,
nr++;
path->slots[0]--;
} else {
- ASSERT(0);
+ btrfs_err(fs_info, "unexpected free space tree key type %u",
+ found_key.type);
+ ret = -EUCLEAN;
+ btrfs_abort_transaction(trans, ret);
+ goto out;
}
}
@@ -405,7 +409,11 @@ int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans,
nr++;
} else {
- ASSERT(0);
+ btrfs_err(fs_info, "unexpected free space tree key type %u",
+ found_key.type);
+ ret = -EUCLEAN;
+ btrfs_abort_transaction(trans, ret);
+ goto out;
}
}
@@ -1518,7 +1526,11 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans,
nr++;
path->slots[0]--;
} else {
- ASSERT(0);
+ btrfs_err(trans->fs_info, "unexpected free space tree key type %u",
+ found_key.type);
+ ret = -EUCLEAN;
+ btrfs_abort_transaction(trans, ret);
+ return ret;
}
}
diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index a4758d9..a8aa086 100644
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -155,6 +155,7 @@ enum {
BTRFS_FS_LOG_RECOVERING,
BTRFS_FS_OPEN,
BTRFS_FS_QUOTA_ENABLED,
+ BTRFS_FS_SQUOTA_ENABLING,
BTRFS_FS_UPDATE_UUID_TREE_GEN,
BTRFS_FS_CREATING_FREE_SPACE_TREE,
BTRFS_FS_BTREE_ERR,
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 4047401..1ca1cbd 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -1153,7 +1153,7 @@ static void submit_one_async_extent(struct async_chunk *async_chunk,
NULL, &cached,
EXTENT_LOCKED | EXTENT_DELALLOC |
EXTENT_DELALLOC_NEW |
- EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
+ EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV,
PAGE_UNLOCK | PAGE_START_WRITEBACK |
PAGE_END_WRITEBACK);
if (async_extent->cb)
@@ -4959,6 +4959,8 @@ static int btrfs_rmdir(struct inode *vfs_dir, struct dentry *dentry)
if (ret)
goto out;
+ btrfs_record_unlink_dir(trans, dir, inode, false);
+
/* now the directory is empty */
ret = btrfs_unlink_inode(trans, dir, inode, &fname.disk_name);
if (!ret)
@@ -9297,10 +9299,38 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
if (!(mode & FALLOC_FL_KEEP_SIZE) &&
(actual_len > inode->i_size) &&
(cur_offset > inode->i_size)) {
+ u64 range_start;
+ u64 range_end;
+
if (cur_offset > actual_len)
i_size = actual_len;
else
i_size = cur_offset;
+
+ /*
+ * Make sure the file_extent_tree covers the entire
+ * range [old_i_size, new_i_size) before we update
+ * disk_i_size. Without this, a previous KEEP_SIZE
+ * prealloc that extended past i_size (and was lost
+ * across umount/mount because file_extent_tree is
+ * only populated up to round_up(i_size) on inode
+ * load) can leave a gap inside this range. That gap
+ * would cause btrfs_inode_safe_disk_i_size_write()
+ * (via find_contiguous_extent_bit() starting at 0)
+ * to truncate disk_i_size to the start of the gap,
+ * making the persisted size smaller than i_size.
+ */
+ range_start = round_down(inode->i_size, fs_info->sectorsize);
+ range_end = round_up(i_size, fs_info->sectorsize);
+ ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode),
+ range_start, range_end - range_start);
+ if (ret) {
+ btrfs_abort_transaction(trans, ret);
+ if (own_trans)
+ btrfs_end_transaction(trans);
+ break;
+ }
+
i_size_write(inode, i_size);
btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
}
@@ -10669,7 +10699,9 @@ struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino)
break;
from = btrfs_ino(inode) + 1;
- cond_resched_lock(&root->inodes.xa_lock);
+ xa_unlock(&root->inodes);
+ cond_resched();
+ xa_lock(&root->inodes);
}
xa_unlock(&root->inodes);
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index b2e447f..a39460b 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -5102,7 +5102,6 @@ static int btrfs_ioctl_subvol_sync(struct btrfs_fs_info *fs_info, void __user *a
return 0;
}
-#ifdef CONFIG_BTRFS_EXPERIMENTAL
static int btrfs_ioctl_shutdown(struct btrfs_fs_info *fs_info, unsigned long arg)
{
int ret = 0;
@@ -5134,10 +5133,12 @@ static int btrfs_ioctl_shutdown(struct btrfs_fs_info *fs_info, unsigned long arg
case BTRFS_SHUTDOWN_FLAGS_NOLOGFLUSH:
btrfs_force_shutdown(fs_info);
break;
+ default:
+ ret = -EINVAL;
+ break;
}
return ret;
}
-#endif
long btrfs_ioctl(struct file *file, unsigned int
cmd, unsigned long arg)
@@ -5294,10 +5295,8 @@ long btrfs_ioctl(struct file *file, unsigned int
#endif
case BTRFS_IOC_SUBVOL_SYNC_WAIT:
return btrfs_ioctl_subvol_sync(fs_info, argp);
-#ifdef CONFIG_BTRFS_EXPERIMENTAL
case BTRFS_IOC_SHUTDOWN:
return btrfs_ioctl_shutdown(fs_info, arg);
-#endif
}
return -ENOTTY;
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index cdf736d..6838fac 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -1107,7 +1107,13 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
if (simple) {
fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA);
- btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid);
+ /*
+ * Set the enable generation to the next transaction, as we cannot
+ * ensure that extents written during this transaction will see any
+ * state we have set here. So we should treat all extents of the
+ * transaction as coming in before squotas was enabled.
+ */
+ btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid + 1);
} else {
fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
}
@@ -1210,7 +1216,15 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
goto out_free_path;
}
- fs_info->qgroup_enable_gen = trans->transid;
+ /*
+ * Set fs_info->qgroup_enable_gen and BTRFS_FS_SQUOTA_ENABLING
+ * under the transaction handle. We want to ensure that all extents in
+ * the next transaction definitely see them.
+ */
+ if (simple) {
+ fs_info->qgroup_enable_gen = trans->transid + 1;
+ set_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
+ }
mutex_unlock(&fs_info->qgroup_ioctl_lock);
/*
@@ -1224,9 +1238,15 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
*/
ret = btrfs_commit_transaction(trans);
trans = NULL;
+
mutex_lock(&fs_info->qgroup_ioctl_lock);
- if (ret)
+ if (ret) {
+ if (simple) {
+ clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
+ fs_info->qgroup_enable_gen = 0;
+ }
goto out_free_path;
+ }
/*
* Set quota enabled flag after committing the transaction, to avoid
@@ -1236,6 +1256,8 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
spin_lock(&fs_info->qgroup_lock);
fs_info->quota_root = quota_root;
set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
+ if (simple)
+ clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
spin_unlock(&fs_info->qgroup_lock);
/* Skip rescan for simple qgroups. */
@@ -1715,32 +1737,24 @@ int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
return ret;
}
-static bool can_delete_parent_qgroup(struct btrfs_qgroup *qgroup)
-
+static bool can_delete_parent_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup)
{
ASSERT(btrfs_qgroup_level(qgroup->qgroupid));
+ if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
+ squota_check_parent_usage(fs_info, qgroup);
return list_empty(&qgroup->members);
}
/*
- * Return true if we can delete the squota qgroup and false otherwise.
- *
- * Rules for whether we can delete:
- *
- * A subvolume qgroup can be removed iff the subvolume is fully deleted, which
- * is iff there is 0 usage in the qgroup.
- *
- * A higher level qgroup can be removed iff it has no members.
- * Note: We audit its usage to warn on inconsitencies without blocking deletion.
+ * Because a shared extent can outlive its owning subvolume, we cannot delete a
+ * subvol squota qgroup until all of the extents it owns are gone, even if the
+ * subvolume itself has been deleted.
*/
-static bool can_delete_squota_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup)
+static bool can_delete_squota_subvol_qgroup(struct btrfs_fs_info *fs_info,
+ struct btrfs_qgroup *qgroup)
{
ASSERT(btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE);
-
- if (btrfs_qgroup_level(qgroup->qgroupid) > 0) {
- squota_check_parent_usage(fs_info, qgroup);
- return can_delete_parent_qgroup(qgroup);
- }
+ ASSERT(btrfs_qgroup_level(qgroup->qgroupid) == 0);
return !(qgroup->rfer || qgroup->excl || qgroup->rfer_cmpr || qgroup->excl_cmpr);
}
@@ -1754,14 +1768,11 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup
{
struct btrfs_key key;
BTRFS_PATH_AUTO_FREE(path);
-
- /* Since squotas cannot be inconsistent, they have special rules for deletion. */
- if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
- return can_delete_squota_qgroup(fs_info, qgroup);
+ int ret;
/* For higher level qgroup, we can only delete it if it has no child. */
if (btrfs_qgroup_level(qgroup->qgroupid))
- return can_delete_parent_qgroup(qgroup);
+ return can_delete_parent_qgroup(fs_info, qgroup);
/*
* For level-0 qgroups, we can only delete it if it has no subvolume
@@ -1777,10 +1788,21 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup
return -ENOMEM;
/*
- * The @ret from btrfs_find_root() exactly matches our definition for
- * the return value, thus can be returned directly.
+ * Any subvol qgroup, regardless of mode, cannot be deleted if the
+ * subvol still exists.
*/
- return btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL);
+ ret = btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL);
+ /*
+ * btrfs_find_root returns <0 on error, 0 if found, and >0 if not,
+ * so the "found" and "error" cases match our desired return values.
+ */
+ if (ret <= 0)
+ return ret;
+
+ /* Squotas require additional checks, even if the subvol is deleted. */
+ if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
+ return can_delete_squota_subvol_qgroup(fs_info, qgroup);
+ return 1;
}
int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
@@ -4922,7 +4944,8 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
u64 num_bytes = delta->num_bytes;
const int sign = (delta->is_inc ? 1 : -1);
- if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE)
+ if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE &&
+ !test_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags))
return 0;
if (!btrfs_is_fstree(root))
@@ -4934,8 +4957,9 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
spin_lock(&fs_info->qgroup_lock);
qgroup = find_qgroup_rb(fs_info, root);
- if (!qgroup) {
- ret = -ENOENT;
+ if (WARN_ON_ONCE(!qgroup)) {
+ btrfs_warn(fs_info, "squota failed to find qgroup for root %llu", root);
+ ret = 0;
goto out;
}
@@ -4944,8 +4968,19 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
list_for_each_entry(qg, &qgroup_list, iterator) {
struct btrfs_qgroup_list *glist;
- qg->excl += num_bytes * sign;
- qg->rfer += num_bytes * sign;
+ ASSERT(qg->excl == qg->rfer);
+ if (WARN_ON_ONCE(sign < 0 && qg->excl < num_bytes)) {
+ btrfs_warn(fs_info,
+ "squota underflow qg %hu/%llu excl %llu num_bytes %llu",
+ btrfs_qgroup_level(qg->qgroupid),
+ btrfs_qgroup_subvolid(qg->qgroupid),
+ qg->excl, num_bytes);
+ qg->excl = 0;
+ qg->rfer = 0;
+ } else {
+ qg->excl += num_bytes * sign;
+ qg->rfer += num_bytes * sign;
+ }
qgroup_dirty(fs_info, qg);
list_for_each_entry(glist, &qg->groups, next_group)
diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c
index 638c4ad..4b0186c 100644
--- a/fs/btrfs/raid-stripe-tree.c
+++ b/fs/btrfs/raid-stripe-tree.c
@@ -45,8 +45,11 @@ static int btrfs_partially_delete_raid_extent(struct btrfs_trans_handle *trans,
for (int i = 0; i < btrfs_num_raid_stripes(item_size); i++) {
struct btrfs_raid_stride *stride = &extent->strides[i];
+ u64 devid;
u64 phys;
+ devid = btrfs_raid_stride_devid(leaf, stride);
+ btrfs_set_stack_raid_stride_devid(&newitem->strides[i], devid);
phys = btrfs_raid_stride_physical(leaf, stride) + frontpad;
btrfs_set_stack_raid_stride_physical(&newitem->strides[i], phys);
}
@@ -95,14 +98,26 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
while (1) {
key.objectid = start;
key.type = BTRFS_RAID_STRIPE_KEY;
- key.offset = 0;
+ key.offset = (u64)-1;
ret = btrfs_search_slot(trans, stripe_root, &key, path, -1, 1);
if (ret < 0)
break;
- if (path->slots[0] == btrfs_header_nritems(path->nodes[0]))
- path->slots[0]--;
+ /*
+ * Search with offset=(u64)-1 ensures we land on the correct
+ * leaf even when the target entry is the first item on a leaf.
+ * Since no real entry has offset=(u64)-1, ret is always 1 and
+ * slot points past the last entry with objectid==start (or
+ * past the end of the leaf if that entry is the last item).
+ * Back up one slot to find the actual entry.
+ */
+ if (path->slots[0] == 0) {
+ /* No entry with objectid <= start exists. */
+ ret = 0;
+ break;
+ }
+ path->slots[0]--;
leaf = path->nodes[0];
slot = path->slots[0];
@@ -123,7 +138,7 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
*/
if (found_start > start) {
if (slot == 0) {
- ret = btrfs_previous_item(stripe_root, path, start,
+ ret = btrfs_previous_item(stripe_root, path, 0,
BTRFS_RAID_STRIPE_KEY);
if (ret) {
if (ret > 0)
@@ -139,7 +154,10 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
btrfs_item_key_to_cpu(leaf, &key, slot);
found_start = key.objectid;
found_end = found_start + key.offset;
- ASSERT(found_start <= start);
+ if (found_start > start || found_end <= start) {
+ ret = -ENOENT;
+ break;
+ }
}
if (key.type != BTRFS_RAID_STRIPE_KEY)
@@ -176,9 +194,19 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
/* The "right" item. */
ret = btrfs_duplicate_item(trans, stripe_root, path, &newkey);
+ if (ret == -EAGAIN) {
+ btrfs_release_path(path);
+ continue;
+ }
if (ret)
break;
+ /*
+ * btrfs_duplicate_item() may have triggered a leaf
+ * split via setup_leaf_for_split(), so we must refresh
+ * our leaf pointer from the path.
+ */
+ leaf = path->nodes[0];
item_size = btrfs_item_size(leaf, path->slots[0]);
extent = btrfs_item_ptr(leaf, path->slots[0],
struct btrfs_stripe_extent);
@@ -195,8 +223,9 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
/* The "left" item. */
path->slots[0]--;
btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
- btrfs_partially_delete_raid_extent(trans, path, &key,
- diff_start, 0);
+ ret = btrfs_partially_delete_raid_extent(trans, path,
+ &key,
+ diff_start, 0);
break;
}
@@ -212,8 +241,11 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
if (found_start < start) {
u64 diff_start = start - found_start;
- btrfs_partially_delete_raid_extent(trans, path, &key,
- diff_start, 0);
+ ret = btrfs_partially_delete_raid_extent(trans, path,
+ &key,
+ diff_start, 0);
+ if (ret)
+ break;
start += (key.offset - diff_start);
length -= (key.offset - diff_start);
@@ -236,9 +268,10 @@ int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 le
if (found_end > end) {
u64 diff_end = found_end - end;
- btrfs_partially_delete_raid_extent(trans, path, &key,
- key.offset - length,
- length);
+ ret = btrfs_partially_delete_raid_extent(trans, path,
+ &key,
+ key.offset - length,
+ length);
ASSERT(key.offset - diff_end == length);
break;
}
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index 1c42c51..3ebaf58 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -2607,7 +2607,7 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans,
if (!block->key_ready)
btrfs_readahead_tree_block(fs_info, block->bytenr,
block->owner, 0,
- block->level);
+ block->level, NULL);
}
/* Get first keys */
@@ -3876,7 +3876,7 @@ static int add_remap_tree_entries(struct btrfs_trans_handle *trans, struct btrfs
ret = btrfs_insert_empty_items(trans, fs_info->remap_root, path, &batch);
btrfs_release_path(path);
- if (num_entries <= max_items)
+ if (ret || num_entries <= max_items)
break;
num_entries -= max_items;
@@ -4174,6 +4174,12 @@ static int move_existing_remap(struct btrfs_fs_info *fs_info,
return ret;
}
+ if (ins.offset < length) {
+ spin_lock(&sinfo->lock);
+ btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - length);
+ spin_unlock(&sinfo->lock);
+ }
+
dest_addr = ins.objectid;
dest_length = ins.offset;
@@ -5000,6 +5006,12 @@ static int do_remap_reloc_trans(struct btrfs_fs_info *fs_info,
return ret;
}
+ if (ins.offset < remap_length) {
+ spin_lock(&sinfo->lock);
+ btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - remap_length);
+ spin_unlock(&sinfo->lock);
+ }
+
made_reservation = true;
new_addr = ins.objectid;
@@ -5023,21 +5035,27 @@ static int do_remap_reloc_trans(struct btrfs_fs_info *fs_info,
if (bg_needs_free_space) {
ret = btrfs_add_block_group_free_space(trans, dest_bg);
- if (ret)
+ if (ret) {
+ btrfs_abort_transaction(trans, ret);
goto fail;
+ }
}
ret = copy_remapped_data(fs_info, start, new_addr, length);
- if (ret)
+ if (ret) {
+ btrfs_abort_transaction(trans, ret);
goto fail;
+ }
ret = btrfs_remove_from_free_space_tree(trans, new_addr, length);
- if (ret)
+ if (ret) {
+ btrfs_abort_transaction(trans, ret);
goto fail;
+ }
ret = add_remap_entry(trans, path, src_bg, start, new_addr, length);
if (ret) {
- btrfs_add_to_free_space_tree(trans, new_addr, length);
+ btrfs_abort_transaction(trans, ret);
goto fail;
}
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index 248adb7..194f581b 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -1293,14 +1293,13 @@ static int btrfs_write_and_wait_transaction(struct btrfs_trans_handle *trans)
blk_finish_plug(&plug);
ret2 = btrfs_wait_extents(fs_info, dirty_pages);
- btrfs_extent_io_tree_release(&trans->transaction->dirty_pages);
-
if (ret)
return ret;
- else if (ret2)
+ if (ret2)
return ret2;
- else
- return 0;
+
+ btrfs_extent_io_tree_release(&trans->transaction->dirty_pages);
+ return 0;
}
/*
diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c
index 1b83ed0..2937db6 100644
--- a/fs/cachefiles/namei.c
+++ b/fs/cachefiles/namei.c
@@ -130,6 +130,8 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
ret = cachefiles_inject_write_error();
if (ret == 0) {
subdir = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), subdir, 0700, NULL);
+ if (IS_ERR(subdir))
+ ret = PTR_ERR(subdir);
} else {
end_creating(subdir);
subdir = ERR_PTR(ret);
diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 1454760..0a86f67 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -1336,6 +1336,7 @@ void ceph_process_folio_batch(struct address_space *mapping,
ceph_wbc, folio);
if (rc == -ENODATA) {
folio_unlock(folio);
+ folio_put(folio);
ceph_wbc->fbatch.folios[i] = NULL;
continue;
} else if (rc == -E2BIG) {
@@ -1346,6 +1347,7 @@ void ceph_process_folio_batch(struct address_space *mapping,
if (!folio_clear_dirty_for_io(folio)) {
doutc(cl, "%p !folio_clear_dirty_for_io\n", folio);
folio_unlock(folio);
+ folio_put(folio);
ceph_wbc->fbatch.folios[i] = NULL;
continue;
}
diff --git a/fs/ceph/quota.c b/fs/ceph/quota.c
index 4dc9426..053d5bf 100644
--- a/fs/ceph/quota.c
+++ b/fs/ceph/quota.c
@@ -228,12 +228,19 @@ static int get_quota_realm(struct ceph_mds_client *mdsc, struct inode *inode,
restart:
realm = ceph_inode(inode)->i_snap_realm;
- if (realm)
+ if (realm) {
ceph_get_snap_realm(mdsc, realm);
- else
- pr_err_ratelimited_client(cl,
- "%p %llx.%llx null i_snap_realm\n",
- inode, ceph_vinop(inode));
+ } else {
+ /*
+ * i_snap_realm is NULL when all caps have been released, e.g.
+ * after an MDS session rejection. This is a transient state;
+ * the realm will be restored once caps are re-granted.
+ * Treat it as "no quota realm found".
+ */
+ doutc(cl, "%p %llx.%llx null i_snap_realm\n",
+ inode, ceph_vinop(inode));
+ }
+
while (realm) {
bool has_inode;
@@ -340,12 +347,19 @@ static bool check_quota_exceeded(struct inode *inode, enum quota_check_op op,
down_read(&mdsc->snap_rwsem);
restart:
realm = ceph_inode(inode)->i_snap_realm;
- if (realm)
+ if (realm) {
ceph_get_snap_realm(mdsc, realm);
- else
- pr_err_ratelimited_client(cl,
- "%p %llx.%llx null i_snap_realm\n",
- inode, ceph_vinop(inode));
+ } else {
+ /*
+ * i_snap_realm is NULL when all caps have been released, e.g.
+ * after an MDS session rejection. This is a transient state;
+ * the realm will be restored once caps are re-granted.
+ * Treat it as "quota not exceeded".
+ */
+ doutc(cl, "%p %llx.%llx null i_snap_realm\n",
+ inode, ceph_vinop(inode));
+ }
+
while (realm) {
bool has_inode;
@@ -496,6 +510,9 @@ bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf)
u64 total = 0, used, free;
bool is_updated = false;
+ if (!ceph_has_realms_with_quotas(d_inode(fsc->sb->s_root)))
+ return false;
+
down_read(&mdsc->snap_rwsem);
get_quota_realm(mdsc, d_inode(fsc->sb->s_root), QUOTA_GET_MAX_BYTES,
&realm, true);
diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c
index 5f87f62..e773be0 100644
--- a/fs/ceph/xattr.c
+++ b/fs/ceph/xattr.c
@@ -1254,6 +1254,22 @@ int __ceph_setxattr(struct inode *inode, const char *name,
ceph_vinop(inode), name, ceph_cap_string(issued));
__build_xattrs(inode);
+ /*
+ * __build_xattrs() may have released and reacquired i_ceph_lock,
+ * during which handle_cap_grant() could have replaced i_xattrs.blob
+ * with a newer MDS-provided blob and bumped i_xattrs.version. If that
+ * caused __build_xattrs() to rebuild the rb-tree from the new blob,
+ * count/names_size/vals_size may now be larger than when
+ * required_blob_size was computed above. Recompute it here so the
+ * prealloc_blob size check below reflects the current tree state.
+ */
+ required_blob_size = __get_required_blob_size(ci, name_len, val_len);
+ if (required_blob_size > mdsc->mdsmap->m_max_xattr_size) {
+ doutc(cl, "sync (size too large): %d > %llu\n",
+ required_blob_size, mdsc->mdsmap->m_max_xattr_size);
+ goto do_sync;
+ }
+
if (!ci->i_xattrs.prealloc_blob ||
required_blob_size > ci->i_xattrs.prealloc_blob->alloc_len) {
struct ceph_buffer *blob;
@@ -1294,6 +1310,7 @@ int __ceph_setxattr(struct inode *inode, const char *name,
do_sync:
spin_unlock(&ci->i_ceph_lock);
+ ceph_buffer_put(old_blob);
do_sync_unlocked:
if (lock_snap_rwsem)
up_read(&mdsc->snap_rwsem);
diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c
index 1c5224c..733c195 100644
--- a/fs/efivarfs/super.c
+++ b/fs/efivarfs/super.c
@@ -191,13 +191,10 @@ static const struct dentry_operations efivarfs_d_ops = {
static struct dentry *efivarfs_alloc_dentry(struct dentry *parent, char *name)
{
+ struct qstr q = QSTR(name);
struct dentry *d;
- struct qstr q;
int err;
- q.name = name;
- q.len = strlen(name);
-
err = efivarfs_d_hash(parent, &q);
if (err)
return ERR_PTR(err);
diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c
index 41e3110..df7ea01 100644
--- a/fs/erofs/xattr.c
+++ b/fs/erofs/xattr.c
@@ -89,13 +89,11 @@ static int erofs_init_inode_xattrs(struct inode *inode)
vi->xattr_isize - sizeof(struct erofs_xattr_ibody_header)) {
erofs_err(sb, "invalid h_shared_count %u @ nid %llu",
vi->xattr_shared_count, vi->nid);
- erofs_put_metabuf(&buf);
ret = -EFSCORRUPTED;
goto out_unlock;
}
vi->xattr_shared_xattrs = kmalloc_objs(uint, vi->xattr_shared_count);
if (!vi->xattr_shared_xattrs) {
- erofs_put_metabuf(&buf);
ret = -ENOMEM;
goto out_unlock;
}
@@ -112,12 +110,12 @@ static int erofs_init_inode_xattrs(struct inode *inode)
}
vi->xattr_shared_xattrs[i] = le32_to_cpu(*xattr_id);
}
- erofs_put_metabuf(&buf);
/* paired with smp_mb() at the beginning of the function. */
smp_mb();
set_bit(EROFS_I_EA_INITED_BIT, &vi->flags);
out_unlock:
+ erofs_put_metabuf(&buf);
clear_and_wake_up_bit(EROFS_I_BL_XATTR_BIT, &vi->flags);
return ret;
}
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 43bb5a6..27ab7bd 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1509,8 +1509,15 @@ static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page));
folio = page_folio(zbv.page);
- /* For preallocated managed folios, add them to page cache here */
+ /*
+ * Preallocated folios are added to the managed cache here rather than
+ * in z_erofs_bind_cache() in order to keep these folios locked in
+ * increasing (physical) address order.
+ * Clear folio->private before these folios become visible to others in
+ * the managed cache to avoid duplicate additions for unaligned extents.
+ */
if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) {
+ folio->private = NULL;
tocache = true;
goto out_tocache;
}
@@ -1546,14 +1553,8 @@ static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
}
return;
}
- /*
- * Already linked with another pcluster, which only appears in
- * crafted images by fuzzers for now. But handle this anyway.
- */
- tocache = false; /* use temporary short-lived pages */
} else {
DBG_BUGON(1); /* referenced managed folios can't be truncated */
- tocache = true;
}
folio_unlock(folio);
folio_put(folio);
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index c59452d..f94f3dc 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -2176,7 +2176,10 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
WARN_ON(!ap->num_folios);
- /* Reached max pages */
+ /* Reached max pages or max folio slots */
+ if (ap->num_folios >= fc->max_pages)
+ return true;
+
if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages)
return true;
diff --git a/fs/hpfs/alloc.c b/fs/hpfs/alloc.c
index 66617b1..f515037 100644
--- a/fs/hpfs/alloc.c
+++ b/fs/hpfs/alloc.c
@@ -372,8 +372,8 @@ int hpfs_check_free_dnodes(struct super_block *s, int n)
return 0;
}
}
+ hpfs_brelse4(&qbh);
}
- hpfs_brelse4(&qbh);
i = 0;
if (hpfs_sb(s)->sb_c_bitmap != -1) {
bmp = hpfs_map_bitmap(s, b, &qbh, "chkdn1");
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 8b05bec..78d61bf 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -96,15 +96,8 @@ static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
#define PGOFF_LOFFT_MAX \
(((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
-static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
+static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
{
- /* Unfortunate we have to reassign vma->vm_private_data. */
- return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
-}
-
-static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
-{
- struct file *file = desc->file;
struct inode *inode = file_inode(file);
loff_t len, vma_len;
int ret;
@@ -119,8 +112,8 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
* way when do_mmap unwinds (may be important on powerpc
* and ia64).
*/
- vma_desc_set_flags(desc, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT);
- desc->vm_ops = &hugetlb_vm_ops;
+ vma_set_flags(vma, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT);
+ vma->vm_ops = &hugetlb_vm_ops;
/*
* page based offset in vm_pgoff could be sufficiently large to
@@ -129,16 +122,16 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
* sizeof(unsigned long). So, only check in those instances.
*/
if (sizeof(unsigned long) == sizeof(loff_t)) {
- if (desc->pgoff & PGOFF_LOFFT_MAX)
+ if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
return -EINVAL;
}
/* must be huge page aligned */
- if (desc->pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
+ if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
return -EINVAL;
- vma_len = (loff_t)vma_desc_size(desc);
- len = vma_len + ((loff_t)desc->pgoff << PAGE_SHIFT);
+ vma_len = (loff_t)(vma->vm_end - vma->vm_start);
+ len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
/* check for overflow */
if (len < vma_len)
return -EINVAL;
@@ -148,7 +141,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
ret = -ENOMEM;
- vma_flags = desc->vma_flags;
+ vma_flags = vma->flags;
/*
* for SHM_HUGETLB, the pages are reserved in the shmget() call so skip
* reserving here. Note: only for SHM hugetlbfs file, the inode
@@ -158,30 +151,17 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
if (hugetlb_reserve_pages(inode,
- desc->pgoff >> huge_page_order(h),
- len >> huge_page_shift(h), desc,
- vma_flags) < 0)
+ vma->vm_pgoff >> huge_page_order(h),
+ len >> huge_page_shift(h), vma,
+ vma_flags) < 0)
goto out;
ret = 0;
- if (vma_desc_test(desc, VMA_WRITE_BIT) && inode->i_size < len)
+ if (vma_test(vma, VMA_WRITE_BIT) && inode->i_size < len)
i_size_write(inode, len);
out:
inode_unlock(inode);
- if (!ret) {
- /* Allocate the VMA lock after we set it up. */
- desc->action.success_hook = hugetlb_file_mmap_prepare_success;
- /*
- * We cannot permit the rmap finding this VMA in the time
- * between the VMA being inserted into the VMA tree and the
- * completion/success hook being invoked.
- *
- * This is because we establish a per-VMA hugetlb lock which can
- * be raced by rmap.
- */
- desc->action.hide_from_rmap_until_complete = true;
- }
return ret;
}
@@ -1227,7 +1207,7 @@ static void init_once(void *foo)
static const struct file_operations hugetlbfs_file_operations = {
.read_iter = hugetlbfs_read_iter,
- .mmap_prepare = hugetlbfs_file_mmap_prepare,
+ .mmap = hugetlbfs_file_mmap,
.fsync = noop_fsync,
.get_unmapped_area = hugetlb_get_unmapped_area,
.llseek = default_llseek,
diff --git a/fs/inode.c b/fs/inode.c
index 6a3cbc7..62c579a 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -2124,7 +2124,13 @@ static int inode_update_cmtime(struct inode *inode, unsigned int flags)
inode_iversion_need_inc(inode))
return -EAGAIN;
} else {
- if (inode_maybe_inc_iversion(inode, !!dirty))
+ /*
+ * Don't force iversion increment for pure lazytime
+ * updates (I_DIRTY_TIME only), let I_VERSION_QUERIED
+ * dictate whether the increment is needed.
+ */
+ if (inode_maybe_inc_iversion(inode,
+ dirty != I_DIRTY_TIME))
dirty |= I_DIRTY_SYNC;
}
}
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index b0a6549b..b36ee61 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -355,7 +355,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
if (dio->flags & IOMAP_DIO_BOUNCE)
ret = bio_iov_iter_bounce(bio, dio->submit.iter,
- iomap_max_bio_size(&iter->iomap));
+ iomap_max_bio_size(&iter->iomap), alignment);
else
ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
alignment - 1);
diff --git a/fs/isofs/export.c b/fs/isofs/export.c
index 421d247..78f80c1a 100644
--- a/fs/isofs/export.c
+++ b/fs/isofs/export.c
@@ -24,7 +24,7 @@ isofs_export_iget(struct super_block *sb,
{
struct inode *inode;
- if (block == 0)
+ if (block == 0 || block >= ISOFS_SB(sb)->s_nzones)
return ERR_PTR(-ESTALE);
inode = isofs_iget(sb, block, offset);
if (IS_ERR(inode))
diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c
index 8dd3911..3ace3d6 100644
--- a/fs/isofs/namei.c
+++ b/fs/isofs/namei.c
@@ -10,20 +10,13 @@
#include <linux/gfp.h>
#include "isofs.h"
-/*
- * ok, we cannot use strncmp, as the name is not in our data space.
- * Thus we'll have to use isofs_match. No big problem. Match also makes
- * some sanity tests.
- */
static int
isofs_cmp(struct dentry *dentry, const char *compare, int dlen)
{
- struct qstr qstr;
- qstr.name = compare;
- qstr.len = dlen;
if (likely(!dentry->d_op))
return dentry->d_name.len != dlen || memcmp(dentry->d_name.name, compare, dlen);
- return dentry->d_op->d_compare(NULL, dentry->d_name.len, dentry->d_name.name, &qstr);
+ return dentry->d_op->d_compare(NULL, dentry->d_name.len, dentry->d_name.name,
+ &QSTR_LEN(compare, dlen));
}
/*
diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c
index 6fe6dbd..1232fab 100644
--- a/fs/isofs/rock.c
+++ b/fs/isofs/rock.c
@@ -101,6 +101,15 @@ static int rock_continue(struct rock_state *rs)
goto out;
}
+ if ((unsigned)rs->cont_extent >= ISOFS_SB(rs->inode->i_sb)->s_nzones) {
+ printk(KERN_NOTICE "rock: corrupted directory entry. "
+ "extent=%u out of volume (nzones=%lu)\n",
+ (unsigned)rs->cont_extent,
+ ISOFS_SB(rs->inode->i_sb)->s_nzones);
+ ret = -EIO;
+ goto out;
+ }
+
if (rs->cont_extent) {
struct buffer_head *bh;
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index 60c4a0e0..442d626 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -309,7 +309,7 @@ static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip,
out1:
jfs_info("jfs_mkdir: rc:%d", rc);
- return ERR_PTR(rc);
+ return rc ? ERR_PTR(rc) : NULL;
}
/*
diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h
index a7c85ab..1db6cb3 100644
--- a/fs/lockd/lockd.h
+++ b/fs/lockd/lockd.h
@@ -332,7 +332,7 @@ int nlmsvc_dispatch(struct svc_rqst *rqstp);
* File handling for the server personality
*/
__be32 nlm_lookup_file(struct svc_rqst *, struct nlm_file **,
- struct nlm_lock *);
+ struct nlm_lock *, int);
void nlm_release_file(struct nlm_file *);
void nlmsvc_put_lockowner(struct nlm_lockowner *);
void nlmsvc_release_lockowner(struct nlm_lock *);
diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c
index 5de41e2..41cab85 100644
--- a/fs/lockd/svc4proc.c
+++ b/fs/lockd/svc4proc.c
@@ -146,8 +146,11 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host,
struct nlm_lock *lock, struct nlm_file **filp,
struct nlm4_lock *xdr_lock, unsigned char type)
{
+ bool is_test = (rqstp->rq_proc == NLMPROC4_TEST ||
+ rqstp->rq_proc == NLMPROC4_TEST_MSG);
struct file_lock *fl = &lock->fl;
struct nlm_file *file = NULL;
+ int mode;
__be32 error;
if (xdr_lock->fh.len > NFS_MAXFHSIZE)
@@ -170,7 +173,8 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host,
fl->c.flc_type = type;
lockd_set_file_lock_range4(fl, lock->lock_start, lock->lock_len);
- error = nlm_lookup_file(rqstp, &file, lock);
+ mode = is_test ? O_RDWR : lock_to_openmode(fl);
+ error = nlm_lookup_file(rqstp, &file, lock, mode);
switch (error) {
case nlm_granted:
break;
@@ -184,7 +188,8 @@ nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host,
*filp = file;
fl->c.flc_flags = FL_POSIX;
- fl->c.flc_file = file->f_file[lock_to_openmode(fl)];
+ fl->c.flc_file = is_test ? nlmsvc_file_file(file)
+ : file->f_file[mode];
fl->c.flc_pid = current->tgid;
fl->fl_lmops = &nlmsvc_lock_operations;
nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid);
diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c
index b98b1d0..f452014 100644
--- a/fs/lockd/svclock.c
+++ b/fs/lockd/svclock.c
@@ -613,7 +613,6 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file,
struct nlm_lock *conflock)
{
int error;
- int mode;
__be32 ret;
dprintk("lockd: nlmsvc_testlock(%s/%llu, ty=%d, %Ld-%Ld)\n",
@@ -631,14 +630,13 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file,
goto out;
}
- mode = lock_to_openmode(&lock->fl);
locks_init_lock(&conflock->fl);
/* vfs_test_lock only uses start, end, and owner, but tests flc_file */
conflock->fl.c.flc_file = lock->fl.c.flc_file;
conflock->fl.fl_start = lock->fl.fl_start;
conflock->fl.fl_end = lock->fl.fl_end;
conflock->fl.c.flc_owner = lock->fl.c.flc_owner;
- error = vfs_test_lock(file->f_file[mode], &conflock->fl);
+ error = vfs_test_lock(lock->fl.c.flc_file, &conflock->fl);
if (error) {
ret = nlm_lck_denied_nolocks;
goto out;
diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c
index 749abf8..c0a3487 100644
--- a/fs/lockd/svcproc.c
+++ b/fs/lockd/svcproc.c
@@ -68,6 +68,8 @@ nlmsvc_retrieve_args(struct svc_rqst *rqstp, struct nlm_args *argp,
struct nlm_host *host = NULL;
struct nlm_file *file = NULL;
struct nlm_lock *lock = &argp->lock;
+ bool is_test = (rqstp->rq_proc == NLMPROC_TEST ||
+ rqstp->rq_proc == NLMPROC_TEST_MSG);
int mode;
__be32 error = 0;
@@ -83,15 +85,22 @@ nlmsvc_retrieve_args(struct svc_rqst *rqstp, struct nlm_args *argp,
/* Obtain file pointer. Not used by FREE_ALL call. */
if (filp != NULL) {
- error = cast_status(nlm_lookup_file(rqstp, &file, lock));
+ mode = lock_to_openmode(&lock->fl);
+
+ if (is_test)
+ mode = O_RDWR;
+
+ error = cast_status(nlm_lookup_file(rqstp, &file, lock, mode));
if (error != 0)
goto no_locks;
*filp = file;
/* Set up the missing parts of the file_lock structure */
- mode = lock_to_openmode(&lock->fl);
lock->fl.c.flc_flags = FL_POSIX;
- lock->fl.c.flc_file = file->f_file[mode];
+ if (is_test)
+ lock->fl.c.flc_file = nlmsvc_file_file(file);
+ else
+ lock->fl.c.flc_file = file->f_file[mode];
lock->fl.c.flc_pid = current->tgid;
lock->fl.fl_lmops = &nlmsvc_lock_operations;
nlmsvc_locks_init_private(&lock->fl, host, (pid_t)lock->svid);
diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c
index 344e6c1..9da9d6e 100644
--- a/fs/lockd/svcsubs.c
+++ b/fs/lockd/svcsubs.c
@@ -83,23 +83,36 @@ int lock_to_openmode(struct file_lock *lock)
*
* We have to make sure we have the right credential to open
* the file.
+ *
+ * @mode is O_RDONLY, O_WRONLY, or O_RDWR. O_RDWR means success
+ * is achieved with EITHER O_RDONLY or O_WRONLY; it does not
+ * require both.
*/
static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
struct nlm_file *file, int mode)
{
- struct file **fp = &file->f_file[mode];
- __be32 nlmerr = nlm_granted;
+ __be32 nlmerr = nlm__int__failed;
+ __be32 deferred = 0;
int error;
+ int m;
- if (*fp)
- return nlmerr;
+ for (m = O_RDONLY; m <= O_WRONLY; m++) {
+ struct file **fp = &file->f_file[m];
- error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, mode);
- if (error) {
+ if (mode != O_RDWR && mode != m)
+ continue;
+ if (*fp)
+ return nlm_granted;
+
+ error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, m);
+ if (!error)
+ return nlm_granted;
+
dprintk("lockd: open failed (errno %d)\n", error);
switch (error) {
case -EWOULDBLOCK:
nlmerr = nlm__int__drop_reply;
+ deferred = nlmerr;
break;
case -ESTALE:
nlmerr = nlm__int__stale_fh;
@@ -110,7 +123,7 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
}
}
- return nlmerr;
+ return deferred ? deferred : nlmerr;
}
/*
@@ -119,17 +132,15 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
*/
__be32
nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result,
- struct nlm_lock *lock)
+ struct nlm_lock *lock, int mode)
{
struct nlm_file *file;
unsigned int hash;
__be32 nfserr;
- int mode;
nlm_debug_print_fh("nlm_lookup_file", &lock->fh);
hash = file_hash(&lock->fh);
- mode = lock_to_openmode(&lock->fl);
/* Lock file table */
mutex_lock(&nlm_file_mutex);
diff --git a/fs/mnt_idmapping.c b/fs/mnt_idmapping.c
index 6472c4e..cb61fbd 100644
--- a/fs/mnt_idmapping.c
+++ b/fs/mnt_idmapping.c
@@ -375,6 +375,8 @@ int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_
continue;
seq_printf(seq, "%u %u %u", extent->first, lower, extent->count);
+ if (seq_has_overflowed(seq))
+ return -EAGAIN;
seq->count++; /* mappings are separated by \0 */
if (seq_has_overflowed(seq))
diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c
index a8c0d86..76d0f6a 100644
--- a/fs/netfs/buffered_read.c
+++ b/fs/netfs/buffered_read.c
@@ -156,9 +156,8 @@ static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
netfs_cache_read_terminated, subreq);
}
-static void netfs_queue_read(struct netfs_io_request *rreq,
- struct netfs_io_subrequest *subreq,
- bool last_subreq)
+void netfs_queue_read(struct netfs_io_request *rreq,
+ struct netfs_io_subrequest *subreq)
{
struct netfs_io_stream *stream = &rreq->io_streams[0];
@@ -169,7 +168,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq,
* remove entries off of the front.
*/
spin_lock(&rreq->lock);
- list_add_tail(&subreq->rreq_link, &stream->subrequests);
+ /* Write IN_PROGRESS before pointer to new subreq */
+ list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
if (!stream->active) {
stream->collected_to = subreq->start;
@@ -178,11 +178,6 @@ static void netfs_queue_read(struct netfs_io_request *rreq,
}
}
- if (last_subreq) {
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
- }
-
spin_unlock(&rreq->lock);
}
@@ -214,7 +209,6 @@ static void netfs_issue_read(struct netfs_io_request *rreq,
static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
struct readahead_control *ractl)
{
- struct netfs_inode *ictx = netfs_inode(rreq->inode);
unsigned long long start = rreq->start;
ssize_t size = rreq->len;
int ret = 0;
@@ -233,10 +227,13 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
subreq->start = start;
subreq->len = size;
+ netfs_queue_read(rreq, subreq);
+
source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size);
subreq->source = source;
if (source == NETFS_DOWNLOAD_FROM_SERVER) {
- unsigned long long zp = umin(ictx->zero_point, rreq->i_size);
+ unsigned long long zero_point = netfs_read_zero_point(rreq->inode);
+ unsigned long long zp = umin(zero_point, rreq->i_size);
size_t len = subreq->len;
if (unlikely(rreq->origin == NETFS_READ_SINGLE))
@@ -252,7 +249,8 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
rreq->debug_id, subreq->debug_index,
subreq->len, size,
- subreq->start, ictx->zero_point, rreq->i_size);
+ subreq->start, zero_point, rreq->i_size);
+ netfs_cancel_read(subreq, ret);
break;
}
subreq->len = len;
@@ -261,12 +259,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
if (rreq->netfs_ops->prepare_read) {
ret = rreq->netfs_ops->prepare_read(subreq);
if (ret < 0) {
- subreq->error = ret;
- /* Not queued - release both refs. */
- netfs_put_subrequest(subreq,
- netfs_sreq_trace_put_cancel);
- netfs_put_subrequest(subreq,
- netfs_sreq_trace_put_cancel);
+ netfs_cancel_read(subreq, ret);
break;
}
trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
@@ -289,24 +282,29 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
pr_err("Unexpected read source %u\n", source);
WARN_ON_ONCE(1);
+ netfs_cancel_read(subreq, ret);
break;
issue:
slice = netfs_prepare_read_iterator(subreq, ractl);
if (slice < 0) {
ret = slice;
- subreq->error = ret;
- trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
- /* Not queued - release both refs. */
- netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
- netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+ netfs_cancel_read(subreq, ret);
break;
}
- size -= slice;
start += slice;
+ size -= slice;
+ if (size <= 0) {
+ smp_wmb(); /* Write lists before ALL_QUEUED. */
+ set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ }
- netfs_queue_read(rreq, subreq, size <= 0);
netfs_issue_read(rreq, subreq);
+
+ if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
+ netfs_wait_for_paused_read(rreq);
+ if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
+ break;
cond_resched();
} while (size > 0);
@@ -397,6 +395,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
{
struct netfs_io_request *rreq;
struct address_space *mapping = folio->mapping;
+ struct netfs_group *group = netfs_folio_group(folio);
struct netfs_folio *finfo = netfs_folio_info(folio);
struct netfs_inode *ctx = netfs_inode(mapping->host);
struct folio *sink = NULL;
@@ -458,14 +457,20 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
netfs_read_to_pagecache(rreq, NULL);
- if (sink)
- folio_put(sink);
-
ret = netfs_wait_for_read(rreq);
if (ret >= 0) {
+ if (group)
+ folio_change_private(folio, group);
+ else
+ folio_detach_private(folio);
+ kfree(finfo);
+ trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
}
+
+ if (sink)
+ folio_put(sink);
folio_unlock(folio);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
return ret < 0 ? ret : 0;
@@ -498,10 +503,10 @@ int netfs_read_folio(struct file *file, struct folio *folio)
struct netfs_inode *ctx = netfs_inode(mapping->host);
int ret;
- if (folio_test_dirty(folio)) {
- trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
+ folio_wait_writeback(folio);
+
+ if (folio_test_dirty(folio))
return netfs_read_gaps(file, folio);
- }
_enter("%lx", folio->index);
@@ -667,7 +672,7 @@ int netfs_write_begin(struct netfs_inode *ctx,
ret = PTR_ERR(rreq);
goto error;
}
- rreq->no_unlock_folio = folio->index;
+ rreq->no_unlock_folio = folio;
__set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
ret = netfs_begin_cache_read(rreq, ctx);
@@ -684,9 +689,9 @@ int netfs_write_begin(struct netfs_inode *ctx,
netfs_read_to_pagecache(rreq, NULL);
ret = netfs_wait_for_read(rreq);
+ netfs_put_request(rreq, netfs_rreq_trace_put_return);
if (ret < 0)
goto error;
- netfs_put_request(rreq, netfs_rreq_trace_put_return);
have_folio:
ret = folio_wait_private_2_killable(folio);
@@ -733,7 +738,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
goto error;
}
- rreq->no_unlock_folio = folio->index;
+ rreq->no_unlock_folio = folio;
__set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
ret = netfs_begin_cache_read(rreq, ctx);
if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 05ea5b0..6bde332 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -12,24 +12,6 @@
#include <linux/slab.h>
#include "internal.h"
-static void __netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
-{
- if (netfs_group)
- folio_attach_private(folio, netfs_get_group(netfs_group));
-}
-
-static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
-{
- void *priv = folio_get_private(folio);
-
- if (unlikely(priv != netfs_group)) {
- if (netfs_group && (!priv || priv == NETFS_FOLIO_COPY_TO_CACHE))
- folio_attach_private(folio, netfs_get_group(netfs_group));
- else if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
- folio_detach_private(folio);
- }
-}
-
/*
* Grab a folio for writing and lock it. Attempt to allocate as large a folio
* as possible to hold as much of the remaining length as possible in one go.
@@ -149,6 +131,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
}
do {
+ enum netfs_folio_trace trace;
struct netfs_folio *finfo;
struct netfs_group *group;
unsigned long long fpos;
@@ -156,6 +139,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
size_t offset; /* Offset into pagecache folio */
size_t part; /* Bytes to write to folio */
size_t copied; /* Bytes copied from user */
+ void *priv;
offset = pos & (max_chunk - 1);
part = min(max_chunk - offset, iov_iter_count(iter));
@@ -201,73 +185,99 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
goto error_folio_unlock;
}
- /* Decide how we should modify a folio. We might be attempting
- * to do write-streaming, in which case we don't want to a
- * local RMW cycle if we can avoid it. If we're doing local
- * caching or content crypto, we award that priority over
- * avoiding RMW. If the file is open readably, then we also
- * assume that we may want to read what we wrote.
- */
finfo = netfs_folio_info(folio);
group = netfs_folio_group(folio);
+ /* If the requested group differs from the group set on the
+ * page, then we need to flush out the folio if it has a group
+ * set (ie. is non-NULL). Note that COPY_TO_CACHE is a special
+ * case, being a netfs annotation rather than an actual group.
+ *
+ * The filesystem isn't permitted to mix writes with groups and
+ * writes without groups as the NULL group is used to indicate
+ * that no group is set.
+ */
if (unlikely(group != netfs_group) &&
- group != NETFS_FOLIO_COPY_TO_CACHE)
+ group != NETFS_FOLIO_COPY_TO_CACHE &&
+ group) {
+ WARN_ON_ONCE(!netfs_group);
goto flush_content;
+ }
+ /* Decide how we should modify a folio. We might be attempting
+ * to do write-streaming, as we don't want to a local RMW cycle
+ * if we can avoid it. If we're doing local caching or content
+ * crypto, we award that priority over avoiding RMW. If the
+ * file is open readably, then we let ->read_folio() fill in
+ * the gaps.
+ */
if (folio_test_uptodate(folio)) {
if (mapping_writably_mapped(mapping))
flush_dcache_folio(folio);
copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
if (unlikely(copied == 0))
goto copy_failed;
- netfs_set_group(folio, netfs_group);
- trace_netfs_folio(folio, netfs_folio_is_uptodate);
- goto copied;
+ trace = netfs_folio_is_uptodate;
+ goto copied_uptodate;
}
/* If the page is above the zero-point then we assume that the
* server would just return a block of zeros or a short read if
* we try to read it.
*/
- if (fpos >= ctx->zero_point) {
+ if (fpos >= netfs_read_zero_point(inode)) {
folio_zero_segment(folio, 0, offset);
copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
if (unlikely(copied == 0))
goto copy_failed;
folio_zero_segment(folio, offset + copied, flen);
- __netfs_set_group(folio, netfs_group);
- folio_mark_uptodate(folio);
- trace_netfs_folio(folio, netfs_modify_and_clear);
- goto copied;
+ if (finfo)
+ trace = netfs_modify_and_clear_rm_finfo;
+ else
+ trace = netfs_modify_and_clear;
+ goto mark_uptodate;
}
/* See if we can write a whole folio in one go. */
if (!maybe_trouble && offset == 0 && part >= flen) {
copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
- if (unlikely(copied == 0))
+ if (likely(copied == part)) {
+ if (finfo)
+ trace = netfs_whole_folio_modify_filled;
+ else
+ trace = netfs_whole_folio_modify;
+ goto mark_uptodate;
+ }
+ if (copied == 0)
goto copy_failed;
- if (unlikely(copied < part)) {
+ if (!finfo || copied <= finfo->dirty_offset) {
maybe_trouble = true;
iov_iter_revert(iter, copied);
copied = 0;
folio_unlock(folio);
goto retry;
}
- __netfs_set_group(folio, netfs_group);
- folio_mark_uptodate(folio);
- trace_netfs_folio(folio, netfs_whole_folio_modify);
+
+ /* We overwrote some existing dirty data, so we have to
+ * accept the partial write.
+ */
+ finfo->dirty_len += finfo->dirty_offset;
+ if (finfo->dirty_len == flen) {
+ trace = netfs_whole_folio_modify_filled_efault;
+ goto mark_uptodate;
+ }
+ if (copied > finfo->dirty_len)
+ finfo->dirty_len = copied;
+ finfo->dirty_offset = 0;
+ trace = netfs_whole_folio_modify_efault;
goto copied;
}
/* We don't want to do a streaming write on a file that loses
* caching service temporarily because the backing store got
- * culled and we don't really want to get a streaming write on
- * a file that's open for reading as ->read_folio() then has to
- * be able to flush it.
+ * culled.
*/
- if ((file->f_mode & FMODE_READ) ||
- netfs_is_cache_enabled(ctx)) {
+ if (netfs_is_cache_enabled(ctx)) {
if (finfo) {
netfs_stat(&netfs_n_wh_wstream_conflict);
goto flush_content;
@@ -282,11 +292,11 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
if (unlikely(copied == 0))
goto copy_failed;
- netfs_set_group(folio, netfs_group);
- trace_netfs_folio(folio, netfs_just_prefetch);
- goto copied;
+ trace = netfs_just_prefetch;
+ goto copied_uptodate;
}
+ /* Do a streaming write on a folio that has nothing in it yet. */
if (!finfo) {
ret = -EIO;
if (WARN_ON(folio_get_private(folio)))
@@ -295,10 +305,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
if (unlikely(copied == 0))
goto copy_failed;
if (offset == 0 && copied == flen) {
- __netfs_set_group(folio, netfs_group);
- folio_mark_uptodate(folio);
- trace_netfs_folio(folio, netfs_streaming_filled_page);
- goto copied;
+ trace = netfs_streaming_filled_page;
+ goto mark_uptodate;
}
finfo = kzalloc_obj(*finfo);
@@ -312,7 +320,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
finfo->dirty_len = copied;
folio_attach_private(folio, (void *)((unsigned long)finfo |
NETFS_FOLIO_INFO));
- trace_netfs_folio(folio, netfs_streaming_write);
+ trace = netfs_streaming_write;
goto copied;
}
@@ -326,16 +334,10 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
goto copy_failed;
finfo->dirty_len += copied;
if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
- if (finfo->netfs_group)
- folio_change_private(folio, finfo->netfs_group);
- else
- folio_detach_private(folio);
- folio_mark_uptodate(folio);
- kfree(finfo);
- trace_netfs_folio(folio, netfs_streaming_cont_filled_page);
- } else {
- trace_netfs_folio(folio, netfs_streaming_write_cont);
+ trace = netfs_streaming_cont_filled_page;
+ goto mark_uptodate;
}
+ trace = netfs_streaming_write_cont;
goto copied;
}
@@ -349,7 +351,38 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
goto out;
continue;
+ /* Mark a folio as being up to data when we've filled it
+ * completely. If the folio has a group attached, then it must
+ * be the same group, otherwise we should have flushed it out
+ * above. We have to get rid of the netfs_folio struct if
+ * there was one.
+ */
+ mark_uptodate:
+ folio_mark_uptodate(folio);
+
+ copied_uptodate:
+ priv = folio_get_private(folio);
+ if (likely(priv == netfs_group)) {
+ /* Already set correctly; no change required. */
+ } else if (priv == NETFS_FOLIO_COPY_TO_CACHE) {
+ if (!netfs_group)
+ folio_detach_private(folio);
+ else
+ folio_change_private(folio, netfs_get_group(netfs_group));
+ } else if (!priv) {
+ folio_attach_private(folio, netfs_get_group(netfs_group));
+ } else {
+ WARN_ON_ONCE(!finfo);
+ if (netfs_group)
+ /* finfo->netfs_group has a ref */
+ folio_change_private(folio, netfs_group);
+ else
+ folio_detach_private(folio);
+ kfree(finfo);
+ }
+
copied:
+ trace_netfs_folio(folio, trace);
flush_dcache_folio(folio);
/* Update the inode size if we moved the EOF marker */
@@ -510,6 +543,7 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
struct inode *inode = file_inode(file);
struct netfs_inode *ictx = netfs_inode(inode);
vm_fault_t ret = VM_FAULT_NOPAGE;
+ void *priv;
int err;
_enter("%lx", folio->index);
@@ -530,7 +564,9 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
}
group = netfs_folio_group(folio);
- if (group != netfs_group && group != NETFS_FOLIO_COPY_TO_CACHE) {
+ if (group &&
+ group != netfs_group &&
+ group != NETFS_FOLIO_COPY_TO_CACHE) {
folio_unlock(folio);
err = filemap_fdatawrite_range(mapping,
folio_pos(folio),
@@ -552,7 +588,19 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
else
trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
- netfs_set_group(folio, netfs_group);
+
+ priv = folio_get_private(folio);
+ if (priv != netfs_group) {
+ if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
+ folio_detach_private(folio);
+ else if (netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
+ folio_change_private(folio, netfs_get_group(netfs_group));
+ else if (netfs_group && !priv)
+ folio_attach_private(folio, netfs_get_group(netfs_group));
+ else
+ WARN_ON_ONCE(1);
+ }
+
file_update_time(file);
set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags);
if (ictx->ops->post_modify)
diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c
index f72e6da..6a8fb0d 100644
--- a/fs/netfs/direct_read.c
+++ b/fs/netfs/direct_read.c
@@ -45,12 +45,11 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
* Perform a read to a buffer from the server, slicing up the region to be read
* according to the network rsize.
*/
-static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
+static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
{
- struct netfs_io_stream *stream = &rreq->io_streams[0];
unsigned long long start = rreq->start;
ssize_t size = rreq->len;
- int ret = 0;
+ int ret;
do {
struct netfs_io_subrequest *subreq;
@@ -58,7 +57,10 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
subreq = netfs_alloc_subrequest(rreq);
if (!subreq) {
- ret = -ENOMEM;
+ /* Stash the error in the request if there's not
+ * already an error set.
+ */
+ cmpxchg(&rreq->error, 0, -ENOMEM);
break;
}
@@ -66,25 +68,13 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
subreq->start = start;
subreq->len = size;
- __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
-
- spin_lock(&rreq->lock);
- list_add_tail(&subreq->rreq_link, &stream->subrequests);
- if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
- if (!stream->active) {
- stream->collected_to = subreq->start;
- /* Store list pointers before active flag */
- smp_store_release(&stream->active, true);
- }
- }
- trace_netfs_sreq(subreq, netfs_sreq_trace_added);
- spin_unlock(&rreq->lock);
+ netfs_queue_read(rreq, subreq);
netfs_stat(&netfs_n_rh_download);
if (rreq->netfs_ops->prepare_read) {
ret = rreq->netfs_ops->prepare_read(subreq);
if (ret < 0) {
- netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+ netfs_cancel_read(subreq, ret);
break;
}
}
@@ -113,8 +103,6 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
netfs_wake_collector(rreq);
}
-
- return ret;
}
/*
@@ -137,21 +125,17 @@ static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
// TODO: Use bounce buffer if requested
inode_dio_begin(rreq->inode);
+ netfs_dispatch_unbuffered_reads(rreq);
- ret = netfs_dispatch_unbuffered_reads(rreq);
-
- if (!rreq->submitted) {
- netfs_put_request(rreq, netfs_rreq_trace_put_no_submit);
- inode_dio_end(rreq->inode);
- ret = 0;
- goto out;
- }
+ /* The collector will get run, even if we don't manage to submit any
+ * subreqs, so we shouldn't call inode_dio_end() here.
+ */
if (sync)
ret = netfs_wait_for_read(rreq);
else
ret = -EIOCBQUEUED;
-out:
+
_leave(" = %zd", ret);
return ret;
}
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index f9ab69d..25f8ceb 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -376,8 +376,10 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from)
if (ret < 0)
goto out;
end = iocb->ki_pos + iov_iter_count(from);
- if (end > ictx->zero_point)
- ictx->zero_point = end;
+ spin_lock(&inode->i_lock);
+ if (end > ictx->_zero_point)
+ netfs_write_zero_point(inode, end);
+ spin_unlock(&inode->i_lock);
fscache_invalidate(netfs_i_cookie(ictx), NULL, i_size_read(inode),
FSCACHE_INVAL_DIO_WRITE);
diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h
index d436e20..645996e 100644
--- a/fs/netfs/internal.h
+++ b/fs/netfs/internal.h
@@ -23,6 +23,8 @@
/*
* buffered_read.c
*/
+void netfs_queue_read(struct netfs_io_request *rreq,
+ struct netfs_io_subrequest *subreq);
void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
int netfs_prefetch_for_write(struct file *file, struct folio *folio,
size_t offset, size_t len);
@@ -108,6 +110,7 @@ static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
*/
bool netfs_read_collection(struct netfs_io_request *rreq);
void netfs_read_collection_worker(struct work_struct *work);
+void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
/*
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 154a14bb..b375567 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -22,7 +22,7 @@
*
* Extract the page fragments from the given amount of the source iterator and
* build up a second iterator that refers to all of those bits. This allows
- * the original iterator to disposed of.
+ * the original iterator to be disposed of.
*
* @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
* allowed on the pages extracted.
@@ -43,7 +43,7 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
unsigned int max_pages;
unsigned int npages = 0;
unsigned int i;
- ssize_t ret;
+ ssize_t ret = 0;
size_t count = orig_len, offset, len;
size_t bv_size, pg_size;
@@ -67,25 +67,28 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
ret = iov_iter_extract_pages(orig, &pages, count,
max_pages - npages, extraction_flags,
&offset);
- if (ret < 0) {
- pr_err("Couldn't get user pages (rc=%zd)\n", ret);
+ if (unlikely(ret <= 0)) {
+ ret = ret ?: -EIO;
break;
}
- if (ret > count) {
- pr_err("get_pages rc=%zd more than %zu\n", ret, count);
+ if (WARN(ret > count,
+ "%s: extract_pages overrun %zd > %zu bytes\n",
+ __func__, ret, count)) {
+ ret = -EIO;
+ break;
+ }
+
+ cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
+ if (WARN(cur_npages > max_pages - npages,
+ "%s: extract_pages overrun %u > %u pages\n",
+ __func__, npages + cur_npages, max_pages)) {
+ ret = -EIO;
break;
}
count -= ret;
ret += offset;
- cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE);
-
- if (npages + cur_npages > max_pages) {
- pr_err("Out of bvec array capacity (%u vs %u)\n",
- npages + cur_npages, max_pages);
- break;
- }
for (i = 0; i < cur_npages; i++) {
len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
@@ -97,6 +100,18 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
npages += cur_npages;
}
+ /* Note: Don't try to clean up after EIO. Either we got no pages, so
+ * nothing to clean up, or we got a buffer overrun, memory corruption
+ * and can't trust the stuff in the buffer (a WARN was emitted).
+ */
+
+ if (ret < 0 && (ret == -ENOMEM || npages == 0)) {
+ for (i = 0; i < npages; i++)
+ unpin_user_page(bv[i].bv_page);
+ kvfree(bv);
+ return ret;
+ }
+
iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
return npages;
}
diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c
index 6df89c9..5d55451 100644
--- a/fs/netfs/misc.c
+++ b/fs/netfs/misc.c
@@ -211,18 +211,25 @@ EXPORT_SYMBOL(netfs_clear_inode_writeback);
void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
{
struct netfs_folio *finfo;
- struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
+ struct inode *inode = folio_inode(folio);
+ struct netfs_inode *ctx = netfs_inode(inode);
size_t flen = folio_size(folio);
_enter("{%lx},%zx,%zx", folio->index, offset, length);
if (offset == 0 && length == flen) {
- unsigned long long i_size = i_size_read(&ctx->inode);
+ unsigned long long i_size, remote_i_size, zero_point;
unsigned long long fpos = folio_pos(folio), end;
+ netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
end = umin(fpos + flen, i_size);
- if (fpos < i_size && end > ctx->zero_point)
- ctx->zero_point = end;
+ if (fpos < i_size && end > zero_point) {
+ spin_lock(&inode->i_lock);
+ end = umin(fpos + flen, inode->i_size);
+ if (fpos < i_size && end > ctx->_zero_point)
+ netfs_write_zero_point(inode, end);
+ spin_unlock(&inode->i_lock);
+ }
}
folio_wait_private_2(folio); /* [DEPRECATED] */
@@ -255,7 +262,8 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
goto erase_completely;
/* Move the start of the data. */
finfo->dirty_len = fend - iend;
- finfo->dirty_offset = offset;
+ finfo->dirty_offset = iend;
+ trace_netfs_folio(folio, netfs_folio_trace_invalidate_front);
return;
}
@@ -264,12 +272,14 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
*/
if (iend >= fend) {
finfo->dirty_len = offset - fstart;
+ trace_netfs_folio(folio, netfs_folio_trace_invalidate_tail);
return;
}
/* A partial write was split. The caller has already zeroed
* it, so just absorb the hole.
*/
+ trace_netfs_folio(folio, netfs_folio_trace_invalidate_middle);
}
return;
@@ -277,8 +287,9 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
netfs_put_group(netfs_folio_group(folio));
folio_detach_private(folio);
folio_clear_uptodate(folio);
+ folio_cancel_dirty(folio);
kfree(finfo);
- return;
+ trace_netfs_folio(folio, netfs_folio_trace_invalidate_all);
}
EXPORT_SYMBOL(netfs_invalidate_folio);
@@ -292,15 +303,22 @@ EXPORT_SYMBOL(netfs_invalidate_folio);
*/
bool netfs_release_folio(struct folio *folio, gfp_t gfp)
{
- struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
- unsigned long long end;
+ struct inode *inode = folio_inode(folio);
+ struct netfs_inode *ctx = netfs_inode(inode);
+ unsigned long long i_size, remote_i_size, zero_point, end;
if (folio_test_dirty(folio))
return false;
- end = umin(folio_next_pos(folio), i_size_read(&ctx->inode));
- if (end > ctx->zero_point)
- ctx->zero_point = end;
+ netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+ end = folio_next_pos(folio);
+ if (end > zero_point) {
+ spin_lock(&inode->i_lock);
+ end = umin(end, ctx->_remote_i_size);
+ if (end > ctx->_zero_point)
+ netfs_write_zero_point(inode, end);
+ spin_unlock(&inode->i_lock);
+ }
if (folio_test_private(folio))
return false;
@@ -356,6 +374,7 @@ void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
DEFINE_WAIT(myself);
list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
+ smp_rmb(); /* Read ->next before IN_PROGRESS. */
if (!netfs_check_subreq_in_progress(subreq))
continue;
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index e5f6665..23660a5 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -83,7 +83,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
}
just_unlock:
- if (folio->index == rreq->no_unlock_folio &&
+ if (folio == rreq->no_unlock_folio &&
test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
_debug("no unlock");
} else {
@@ -205,8 +205,10 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq)
* in progress. The issuer thread may be adding stuff to the tail
* whilst we're doing this.
*/
- front = list_first_entry_or_null(&stream->subrequests,
- struct netfs_io_subrequest, rreq_link);
+ front = list_first_entry_or_null_acquire(&stream->subrequests,
+ struct netfs_io_subrequest, rreq_link);
+ /* Read first subreq pointer before IN_PROGRESS flag. */
+
while (front) {
size_t transferred;
@@ -576,6 +578,17 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
EXPORT_SYMBOL(netfs_read_subreq_terminated);
/*
+ * Cancel a read subrequest due to preparation failure.
+ */
+void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error)
+{
+ trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
+ subreq->error = error;
+ __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
+ netfs_read_subreq_terminated(subreq);
+}
+
+/*
* Handle termination of a read from the cache.
*/
void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error)
diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c
index cca9ac4..f59a70f 100644
--- a/fs/netfs/read_retry.c
+++ b/fs/netfs/read_retry.c
@@ -175,7 +175,9 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
list_for_each_entry_safe_from(subreq, tmp,
&stream->subrequests, rreq_link) {
trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous);
+ spin_lock(&rreq->lock);
list_del(&subreq->rreq_link);
+ spin_unlock(&rreq->lock);
netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
if (subreq == to)
break;
@@ -203,8 +205,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
refcount_read(&subreq->ref),
netfs_sreq_trace_new);
+ spin_lock(&rreq->lock);
list_add(&subreq->rreq_link, &to->rreq_link);
- to = list_next_entry(to, rreq_link);
+ spin_unlock(&rreq->lock);
+ to = subreq;
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
stream->sreq_max_len = umin(len, rreq->rsize);
@@ -288,8 +292,15 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
struct folio *folio = folioq_folio(p, slot);
if (folio && !folioq_is_marked2(p, slot)) {
- trace_netfs_folio(folio, netfs_folio_trace_abandon);
- folio_unlock(folio);
+ if (folio == rreq->no_unlock_folio &&
+ test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO,
+ &rreq->flags)) {
+ _debug("no unlock");
+ } else {
+ trace_netfs_folio(folio,
+ netfs_folio_trace_abandon);
+ folio_unlock(folio);
+ }
}
}
}
diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c
index d0e23bc..8833550 100644
--- a/fs/netfs/read_single.c
+++ b/fs/netfs/read_single.c
@@ -89,7 +89,6 @@ static void netfs_single_read_cache(struct netfs_io_request *rreq,
*/
static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
{
- struct netfs_io_stream *stream = &rreq->io_streams[0];
struct netfs_io_subrequest *subreq;
int ret = 0;
@@ -102,14 +101,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
subreq->len = rreq->len;
subreq->io_iter = rreq->buffer.iter;
- __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
-
- spin_lock(&rreq->lock);
- list_add_tail(&subreq->rreq_link, &stream->subrequests);
- trace_netfs_sreq(subreq, netfs_sreq_trace_added);
- /* Store list pointers before active flag */
- smp_store_release(&stream->active, true);
- spin_unlock(&rreq->lock);
+ netfs_queue_read(rreq, subreq);
netfs_single_cache_prepare_read(rreq, subreq);
switch (subreq->source) {
@@ -121,10 +113,14 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
goto cancel;
}
+ smp_wmb(); /* Write lists before ALL_QUEUED. */
+ set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
rreq->netfs_ops->issue_read(subreq);
rreq->submitted += subreq->len;
break;
case NETFS_READ_FROM_CACHE:
+ smp_wmb(); /* Write lists before ALL_QUEUED. */
+ set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
netfs_single_read_cache(rreq, subreq);
rreq->submitted += subreq->len;
@@ -134,14 +130,15 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
pr_warn("Unexpected single-read source %u\n", subreq->source);
WARN_ON_ONCE(true);
ret = -EIO;
- break;
+ goto cancel;
}
- smp_wmb(); /* Write lists before ALL_QUEUED. */
- set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
return ret;
cancel:
- netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+ netfs_cancel_read(subreq, ret);
+ smp_wmb(); /* Write lists before ALL_QUEUED. */
+ set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+ netfs_wake_collector(rreq);
return ret;
}
diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c
index b194447..24fc2bb 100644
--- a/fs/netfs/write_collect.c
+++ b/fs/netfs/write_collect.c
@@ -57,7 +57,8 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
int netfs_folio_written_back(struct folio *folio)
{
enum netfs_folio_trace why = netfs_folio_trace_clear;
- struct netfs_inode *ictx = netfs_inode(folio->mapping->host);
+ struct inode *inode = folio_inode(folio);
+ struct netfs_inode *ictx = netfs_inode(inode);
struct netfs_folio *finfo;
struct netfs_group *group = NULL;
int gcount = 0;
@@ -69,8 +70,10 @@ int netfs_folio_written_back(struct folio *folio)
unsigned long long fend;
fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len;
- if (fend > ictx->zero_point)
- ictx->zero_point = fend;
+ spin_lock(&ictx->inode.i_lock);
+ if (fend > ictx->_zero_point)
+ netfs_write_zero_point(inode, fend);
+ spin_unlock(&ictx->inode.i_lock);
folio_detach_private(folio);
group = finfo->netfs_group;
@@ -228,8 +231,10 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
if (!smp_load_acquire(&stream->active))
continue;
- front = list_first_entry_or_null(&stream->subrequests,
- struct netfs_io_subrequest, rreq_link);
+ front = list_first_entry_or_null_acquire(&stream->subrequests,
+ struct netfs_io_subrequest, rreq_link);
+ /* Read first subreq pointer before IN_PROGRESS flag. */
+
while (front) {
trace_netfs_collect_sreq(wreq, front);
//_debug("sreq [%x] %llx %zx/%zx",
diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c
index 2db688f..c03c7cc 100644
--- a/fs/netfs/write_issue.c
+++ b/fs/netfs/write_issue.c
@@ -204,7 +204,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
* remove entries off of the front.
*/
spin_lock(&wreq->lock);
- list_add_tail(&subreq->rreq_link, &stream->subrequests);
+ /* Write IN_PROGRESS before pointer to new subreq */
+ list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
if (!stream->active) {
stream->collected_to = subreq->start;
@@ -413,12 +414,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
if (streamw)
netfs_issue_write(wreq, cache);
- /* Flip the page to the writeback state and unlock. If we're called
- * from write-through, then the page has already been put into the wb
- * state.
- */
- if (wreq->origin == NETFS_WRITEBACK)
- folio_start_writeback(folio);
+ folio_start_writeback(folio);
folio_unlock(folio);
if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
@@ -646,29 +642,41 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
struct folio *folio, size_t copied, bool to_page_end,
struct folio **writethrough_cache)
{
+ int ret;
+
_enter("R=%x ic=%zu ws=%u cp=%zu tp=%u",
wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end);
- if (!*writethrough_cache) {
- if (folio_test_dirty(folio))
- /* Sigh. mmap. */
- folio_clear_dirty_for_io(folio);
+ /* The folio is locked. */
+ if (*writethrough_cache != folio) {
+ if (*writethrough_cache) {
+ /* Did the folio get moved? */
+ folio_put(*writethrough_cache);
+ *writethrough_cache = NULL;
+ }
/* We can make multiple writes to the folio... */
- folio_start_writeback(folio);
if (wreq->len == 0)
trace_netfs_folio(folio, netfs_folio_trace_wthru);
else
trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
*writethrough_cache = folio;
+ folio_get(folio);
}
wreq->len += copied;
- if (!to_page_end)
- return 0;
+ if (!to_page_end) {
+ folio_mark_dirty(folio);
+ folio_unlock(folio);
+ return 0;
+ }
+
+ ret = netfs_write_folio(wreq, wbc, folio);
+ folio_put(*writethrough_cache);
*writethrough_cache = NULL;
- return netfs_write_folio(wreq, wbc, folio);
+ wreq->submitted = wreq->len;
+ return ret;
}
/*
@@ -682,8 +690,12 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c
_enter("R=%x", wreq->debug_id);
- if (writethrough_cache)
+ if (writethrough_cache) {
+ folio_lock(writethrough_cache);
netfs_write_folio(wreq, wbc, writethrough_cache);
+ folio_put(writethrough_cache);
+ wreq->submitted = wreq->len;
+ }
netfs_end_issue_write(wreq);
@@ -818,6 +830,9 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq,
*
* Write a monolithic, non-pagecache object back to the server and/or
* the cache.
+ *
+ * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE;
+ * or a negative error code.
*/
int netfs_writeback_single(struct address_space *mapping,
struct writeback_control *wbc,
@@ -834,8 +849,10 @@ int netfs_writeback_single(struct address_space *mapping,
if (!mutex_trylock(&ictx->wb_lock)) {
if (wbc->sync_mode == WB_SYNC_NONE) {
+ /* The VFS will have undirtied the inode. */
+ netfs_single_mark_inode_dirty(&ictx->inode);
netfs_stat(&netfs_n_wb_lock_skip);
- return 0;
+ return 1;
}
netfs_stat(&netfs_n_wb_lock_wait);
mutex_lock(&ictx->wb_lock);
diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c
index 29489a2..32735ab 100644
--- a/fs/netfs/write_retry.c
+++ b/fs/netfs/write_retry.c
@@ -130,7 +130,9 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
list_for_each_entry_safe_from(subreq, tmp,
&stream->subrequests, rreq_link) {
trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
+ spin_lock(&wreq->lock);
list_del(&subreq->rreq_link);
+ spin_unlock(&wreq->lock);
netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
if (subreq == to)
break;
@@ -153,8 +155,10 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
netfs_sreq_trace_new);
trace_netfs_sreq(subreq, netfs_sreq_trace_split);
+ spin_lock(&wreq->lock);
list_add(&subreq->rreq_link, &to->rreq_link);
- to = list_next_entry(to, rreq_link);
+ spin_unlock(&wreq->lock);
+ to = subreq;
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
stream->sreq_max_len = len;
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 85e94c3..ab39ec8 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1413,6 +1413,9 @@ nfsd4_clone(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
dst, clone->cl_dst_pos, clone->cl_count,
EX_ISSYNC(cstate->current_fh.fh_export));
+ if (!status && (READ_ONCE(dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0)
+ nfsd_update_cmtime_attr(dst->nf_file, 0);
+
nfsd_file_put(dst);
nfsd_file_put(src);
out:
@@ -2118,8 +2121,10 @@ static int nfsd4_do_async_copy(void *data)
set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags);
trace_nfsd_copy_async_done(copy);
- nfsd4_send_cb_offload(copy);
atomic_dec(©->cp_nn->pending_async_copies);
+ if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update)
+ nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0);
+ nfsd4_send_cb_offload(copy);
return 0;
}
@@ -2179,6 +2184,9 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid,
sizeof(result->cb_stateid));
dup_copy_fields(copy, async_copy);
+ if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) &
+ FMODE_NOCMTIME) != 0)
+ async_copy->attr_update = true;
memcpy(async_copy->cp_cb_offload.co_referring_sessionid.data,
cstate->session->se_sessionid.data,
NFS4_MAX_SESSIONID_LEN);
@@ -2197,6 +2205,10 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
} else {
status = nfsd4_do_copy(copy, copy->nf_src->nf_file,
copy->nf_dst->nf_file, true);
+ if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) &
+ FMODE_NOCMTIME) != 0 &&
+ copy->cp_res.wr_bytes_written > 0)
+ nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0);
}
out:
trace_nfsd_copy_done(copy, status);
@@ -2535,10 +2547,6 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp,
dd = nfsd_get_dir_deleg(cstate, gdd, nf);
nfsd_file_put(nf);
if (IS_ERR(dd)) {
- int err = PTR_ERR(dd);
-
- if (err != -EAGAIN)
- return nfserrno(err);
gdd->gddrnf_status = GDD4_UNAVAIL;
return nfs_ok;
}
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index c2d13b2..6837b63 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1221,10 +1221,6 @@ static void put_deleg_file(struct nfs4_file *fp)
static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct file *f)
{
- struct iattr ia = { .ia_valid = ATTR_ATIME | ATTR_CTIME | ATTR_MTIME | ATTR_DELEG };
- struct inode *inode = file_inode(f);
- int ret;
-
/* don't do anything if FMODE_NOCMTIME isn't set */
if ((READ_ONCE(f->f_mode) & FMODE_NOCMTIME) == 0)
return;
@@ -1242,17 +1238,7 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f
return;
/* Stamp everything to "now" */
- inode_lock(inode);
- ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &ia, NULL);
- inode_unlock(inode);
- if (ret) {
- struct inode *inode = file_inode(f);
-
- pr_notice_ratelimited("nfsd: Unable to update timestamps on inode %02x:%02x:%llu: %d\n",
- MAJOR(inode->i_sb->s_dev),
- MINOR(inode->i_sb->s_dev),
- inode->i_ino, ret);
- }
+ nfsd_update_cmtime_attr(f, ATTR_ATIME);
}
static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp)
@@ -1865,6 +1851,13 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb)
break;
case SC_TYPE_LAYOUT:
ls = layoutstateid(stid);
+ spin_lock(&clp->cl_lock);
+ if (stid->sc_status == 0) {
+ stid->sc_status |=
+ SC_STATUS_ADMIN_REVOKED;
+ atomic_inc(&clp->cl_admin_revoked);
+ }
+ spin_unlock(&clp->cl_lock);
nfsd4_close_layout(ls);
break;
}
@@ -6378,7 +6371,6 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open,
}
open->op_delegate_type = deleg_ts ? OPEN_DELEGATE_WRITE_ATTRS_DELEG :
OPEN_DELEGATE_WRITE;
- dp->dl_cb_fattr.ncf_cur_fsize = stat.size;
dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat);
dp->dl_atime = stat.atime;
dp->dl_ctime = stat.ctime;
@@ -9429,11 +9421,15 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
if (status != nfserr_jukebox ||
!nfsd_wait_for_delegreturn(rqstp, inode))
goto out_status;
+ status = nfs_ok;
+ goto out_status;
}
- if (!ncf->ncf_file_modified &&
- (ncf->ncf_initial_cinfo != ncf->ncf_cb_change ||
- ncf->ncf_cur_fsize != ncf->ncf_cb_fsize))
- ncf->ncf_file_modified = true;
+ if (!ncf->ncf_file_modified) {
+ if (ncf->ncf_initial_cinfo != ncf->ncf_cb_change)
+ ncf->ncf_file_modified = true;
+ else if (i_size_read(inode) != ncf->ncf_cb_fsize)
+ ncf->ncf_file_modified = true;
+ }
if (ncf->ncf_file_modified) {
int err;
@@ -9560,3 +9556,31 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
put_nfs4_file(fp);
return ERR_PTR(status);
}
+
+/**
+ * nfsd_update_cmtime_attr - update file's delegated ctime/mtime,
+ * and optionally other attributes (ie ATTR_ATIME).
+ * @f: pointer to an opened file
+ * @flags: any additional flags that should be updated
+ *
+ * Given upon opening a file delegated attributes were issues, update
+ * @f attributes to current times.
+ */
+void nfsd_update_cmtime_attr(struct file *f, unsigned int flags)
+{
+ int ret;
+ struct inode *inode = file_inode(f);
+ struct iattr attr = {
+ .ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_DELEG | flags,
+ };
+
+ inode_lock(inode);
+ ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &attr, NULL);
+ inode_unlock(inode);
+ if (ret)
+ pr_notice_ratelimited("nfsd: Unable to update timestamps on "
+ "inode %02x:%02x:%llu: %d\n",
+ MAJOR(inode->i_sb->s_dev),
+ MINOR(inode->i_sb->s_dev),
+ inode->i_ino, ret);
+}
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 39e7012..04e3954 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1594,16 +1594,27 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
static int nfsd_nl_fh_key_set(const struct nlattr *attr, struct nfsd_net *nn)
{
siphash_key_t *fh_key = nn->fh_key;
+ u64 k0, k1;
+ bool changed;
+
+ k0 = get_unaligned_le64(nla_data(attr));
+ k1 = get_unaligned_le64(nla_data(attr) + 8);
if (!fh_key) {
fh_key = kmalloc(sizeof(siphash_key_t), GFP_KERNEL);
- if (!fh_key)
+ if (!fh_key) {
+ trace_nfsd_ctl_fh_key_set(false, -ENOMEM);
return -ENOMEM;
+ }
nn->fh_key = fh_key;
+ changed = true;
+ } else {
+ changed = fh_key->key[0] != k0 || fh_key->key[1] != k1;
}
- fh_key->key[0] = get_unaligned_le64(nla_data(attr));
- fh_key->key[1] = get_unaligned_le64(nla_data(attr) + 8);
+ fh_key->key[0] = k0;
+ fh_key->key[1] = k1;
+ trace_nfsd_ctl_fh_key_set(changed, 0);
return 0;
}
@@ -1682,7 +1693,6 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info)
attr = info->attrs[NFSD_A_SERVER_FH_KEY];
if (attr) {
ret = nfsd_nl_fh_key_set(attr, nn);
- trace_nfsd_ctl_fh_key_set((const char *)nn->fh_key, ret);
if (ret)
goto out_unlock;
}
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 953675e..c5ccea6 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -843,6 +843,7 @@ extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
void nfsd4_put_client(struct nfs4_client *clp);
void nfsd4_async_copy_reaper(struct nfsd_net *nn);
bool nfsd4_has_active_async_copies(struct nfs4_client *clp);
+void nfsd_update_cmtime_attr(struct file *f, unsigned int flags);
extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name,
struct xdr_netobj princhash, struct nfsd_net *nn);
extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn);
diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
index 5ad38f5..b631a47 100644
--- a/fs/nfsd/trace.h
+++ b/fs/nfsd/trace.h
@@ -2243,23 +2243,21 @@ TRACE_EVENT(nfsd_end_grace,
TRACE_EVENT(nfsd_ctl_fh_key_set,
TP_PROTO(
- const char *key,
+ bool changed,
int result
),
- TP_ARGS(key, result),
+ TP_ARGS(changed, result),
TP_STRUCT__entry(
- __field(u32, key_hash)
+ __field(bool, changed)
__field(int, result)
),
TP_fast_assign(
- if (key)
- __entry->key_hash = ~crc32_le(0xFFFFFFFF, key, 16);
- else
- __entry->key_hash = 0;
+ __entry->changed = changed;
__entry->result = result;
),
- TP_printk("key=0x%08x result=%d",
- __entry->key_hash, __entry->result
+ TP_printk("key %s, result=%d",
+ __entry->changed ? "updated" : "unmodified",
+ __entry->result
)
);
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 417e9ad..9a4124c 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -752,6 +752,7 @@ struct nfsd4_copy {
struct nfsd_file *nf_src;
struct nfsd_file *nf_dst;
+ bool attr_update;
copy_stateid_t cp_stateid;
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c
index bfe884d..38290b9c 100644
--- a/fs/notify/fanotify/fanotify.c
+++ b/fs/notify/fanotify/fanotify.c
@@ -457,7 +457,7 @@ static int fanotify_encode_fh(struct fanotify_fh *fh, struct inode *inode,
/*
* Unlike file_handle, type and len of struct fanotify_fh are u8.
* Traditionally, filesystem return handle_type < 0xff, but there
- * is no enforecement for that in vfs.
+ * is no enforcement for that in vfs.
*/
BUILD_BUG_ON(MAX_HANDLE_SZ > 0xff || FILEID_INVALID > 0xff);
if (type <= 0 || type >= FILEID_INVALID || fh_len != dwords << 2)
diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index b7198c4..2dac70b 100644
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -388,7 +388,7 @@ static struct fsnotify_mark *fsnotify_first_mark(struct fsnotify_mark_connector
return hlist_entry_safe(node, struct fsnotify_mark, obj_list);
}
-static struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark)
+struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark)
{
struct hlist_node *node = NULL;
diff --git a/fs/notify/mark.c b/fs/notify/mark.c
index c2ed5b1..e256b42 100644
--- a/fs/notify/mark.c
+++ b/fs/notify/mark.c
@@ -238,7 +238,12 @@ static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
return inode;
}
-static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
+/*
+ * Calculate mask of events for a list of marks.
+ *
+ * Return true if any of the attached marks want to hold an inode reference.
+ */
+static bool __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
u32 new_mask = 0;
bool want_iref = false;
@@ -262,6 +267,34 @@ static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
*/
WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask);
+ return want_iref;
+}
+
+/*
+ * Calculate mask of events for a list of marks after attach/modify mark
+ * and get an inode reference for the connector if needed.
+ *
+ * A concurrent add of evictable mark and detach of non-evictable mark can
+ * lead to __fsnotify_recalc_mask() returning false want_iref, but in this
+ * case we defer clearing iref to fsnotify_recalc_mask_clear_iref() called
+ * from fsnotify_put_mark().
+ */
+static void fsnotify_recalc_mask_set_iref(struct fsnotify_mark_connector *conn)
+{
+ bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
+ bool want_iref = __fsnotify_recalc_mask(conn) || has_iref;
+
+ (void) fsnotify_update_iref(conn, want_iref);
+}
+
+/*
+ * Calculate mask of events for a list of marks after detach mark
+ * and return the inode object if its reference is no longer needed.
+ */
+static void *fsnotify_recalc_mask_clear_iref(struct fsnotify_mark_connector *conn)
+{
+ bool want_iref = __fsnotify_recalc_mask(conn);
+
return fsnotify_update_iref(conn, want_iref);
}
@@ -298,7 +331,7 @@ void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
spin_lock(&conn->lock);
update_children = !fsnotify_conn_watches_children(conn);
- __fsnotify_recalc_mask(conn);
+ fsnotify_recalc_mask_set_iref(conn);
update_children &= fsnotify_conn_watches_children(conn);
spin_unlock(&conn->lock);
/*
@@ -419,7 +452,7 @@ void fsnotify_put_mark(struct fsnotify_mark *mark)
/* Update watched objects after detaching mark */
if (sb)
fsnotify_update_sb_watchers(sb, conn);
- objp = __fsnotify_recalc_mask(conn);
+ objp = fsnotify_recalc_mask_clear_iref(conn);
type = conn->type;
}
WRITE_ONCE(mark->connector, NULL);
@@ -457,9 +490,6 @@ EXPORT_SYMBOL_GPL(fsnotify_put_mark);
*/
static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
{
- if (!mark)
- return true;
-
if (refcount_inc_not_zero(&mark->refcnt)) {
spin_lock(&mark->lock);
if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
@@ -500,15 +530,22 @@ bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
int type;
fsnotify_foreach_iter_type(type) {
+ struct fsnotify_mark *mark = iter_info->marks[type];
+
/* This can fail if mark is being removed */
- if (!fsnotify_get_mark_safe(iter_info->marks[type])) {
- __release(&fsnotify_mark_srcu);
- goto fail;
+ while (mark && !fsnotify_get_mark_safe(mark)) {
+ if (mark->group == iter_info->current_group) {
+ __release(&fsnotify_mark_srcu);
+ goto fail;
+ }
+ /* This is a mark in an unrelated group, skip */
+ mark = fsnotify_next_mark(mark);
+ iter_info->marks[type] = mark;
}
}
/*
- * Now that both marks are pinned by refcount in the inode / vfsmount
+ * Now that all marks are pinned by refcount in the inode / vfsmount / etc
* lists, we can drop SRCU lock, and safely resume the list iteration
* once userspace returns.
*/
diff --git a/fs/nsfs.c b/fs/nsfs.c
index 51e8c94..160018c 100644
--- a/fs/nsfs.c
+++ b/fs/nsfs.c
@@ -266,7 +266,7 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl,
else
tsk = find_task_by_pid_ns(arg, pid_ns);
if (!tsk)
- break;
+ return ret;
switch (ioctl) {
case NS_GET_PID_FROM_PIDNS:
diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c
index 97b660e..421c6cd 100644
--- a/fs/ntfs/attrib.c
+++ b/fs/ntfs/attrib.c
@@ -583,24 +583,13 @@ static u32 ntfs_resident_attr_min_value_length(const __le32 type)
case AT_STANDARD_INFORMATION:
return offsetof(struct standard_information, ver) +
sizeof(((struct standard_information *)0)->ver.v1.reserved12);
- case AT_ATTRIBUTE_LIST:
- return offsetof(struct attr_list_entry, name);
case AT_FILE_NAME:
- return offsetof(struct file_name_attr, file_name);
- case AT_OBJECT_ID:
- return sizeof(struct guid);
- case AT_SECURITY_DESCRIPTOR:
- return sizeof(struct security_descriptor_relative);
+ return offsetof(struct file_name_attr, file_name) +
+ sizeof(__le16) * 1;
case AT_VOLUME_INFORMATION:
return sizeof(struct volume_information);
- case AT_INDEX_ROOT:
- return sizeof(struct index_root);
- case AT_REPARSE_POINT:
- return offsetof(struct reparse_point, reparse_data);
case AT_EA_INFORMATION:
return sizeof(struct ea_information);
- case AT_EA:
- return offsetof(struct ea_attr, ea_name) + 1;
default:
return 0;
}
@@ -672,6 +661,9 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
__le16 *upcase = vol->upcase;
u32 upcase_len = vol->upcase_len;
unsigned int space;
+ u16 name_offset;
+ u32 attr_len;
+ u32 name_size;
/*
* Iterate over attributes in mft record starting at @ctx->attr, or the
@@ -699,6 +691,20 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
return -ENOENT;
if (unlikely(!a->length))
break;
+ if (a->name_length) {
+ name_offset = le16_to_cpu(a->name_offset);
+ attr_len = le32_to_cpu(a->length);
+ name_size = a->name_length * sizeof(__le16);
+
+ if (name_offset > attr_len ||
+ attr_len - name_offset < name_size) {
+ ntfs_error(vol->sb,
+ "Corrupt attribute name in MFT record %llu\n",
+ ctx->ntfs_ino->mft_no);
+ break;
+ }
+ }
+
if (type == AT_UNUSED)
return 0;
if (a->type != type)
@@ -712,14 +718,6 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
if (a->name_length)
return -ENOENT;
} else {
- if (a->name_length && ((le16_to_cpu(a->name_offset) +
- a->name_length * sizeof(__le16)) >
- le32_to_cpu(a->length))) {
- ntfs_error(vol->sb, "Corrupt attribute name in MFT record %llu\n",
- ctx->ntfs_ino->mft_no);
- break;
- }
-
if (!ntfs_are_names_equal(name, name_len,
(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
a->name_length, ic, upcase, upcase_len)) {
@@ -2924,12 +2922,12 @@ int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type,
struct ntfs_inode *base_ni;
int err;
- ntfs_debug("Entering for inode %lld, attr 0x%x.\n",
- (unsigned long long)ni->mft_no, type);
-
if (!ni || !ni->vol)
return -EINVAL;
+ ntfs_debug("Entering for inode %lld, attr 0x%x.\n",
+ ni->mft_no, type);
+
if (NInoAttr(ni))
base_ni = ni->ext.base_ntfs_ino;
else
diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c
index bd501e8..c2594d4 100644
--- a/fs/ntfs/attrlist.c
+++ b/fs/ntfs/attrlist.c
@@ -119,15 +119,14 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr)
struct mft_record *ni_mrec;
u8 *old_al;
- ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
- (long long) ni->mft_no,
- (unsigned int) le32_to_cpu(attr->type));
-
if (!ni || !attr) {
ntfs_debug("Invalid arguments.\n");
return -EINVAL;
}
+ ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+ ni->mft_no, (unsigned int) le32_to_cpu(attr->type));
+
ni_mrec = map_mft_record(ni);
if (IS_ERR(ni_mrec)) {
ntfs_debug("Invalid arguments.\n");
diff --git a/fs/ntfs/bdev-io.c b/fs/ntfs/bdev-io.c
index 67e65c8..27d7c27 100644
--- a/fs/ntfs/bdev-io.c
+++ b/fs/ntfs/bdev-io.c
@@ -97,6 +97,8 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size
idx_end++;
for (; idx < idx_end; idx++, from = 0) {
+ u32 len;
+
folio = read_mapping_folio(sb->s_bdev->bd_mapping, idx, NULL);
if (IS_ERR(folio)) {
ntfs_error(sb, "Unable to read %ld page", idx);
@@ -105,9 +107,10 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size
offset = (loff_t)idx << PAGE_SHIFT;
to = min_t(u32, end - offset, PAGE_SIZE);
+ len = to - from;
- memcpy_to_folio(folio, from, buf + buf_off, to);
- buf_off += to;
+ memcpy_to_folio(folio, from, buf + buf_off, len);
+ buf_off += len;
folio_mark_uptodate(folio);
folio_mark_dirty(folio);
folio_put(folio);
diff --git a/fs/ntfs/bitmap.c b/fs/ntfs/bitmap.c
index 656d802..b1436b3 100644
--- a/fs/ntfs/bitmap.c
+++ b/fs/ntfs/bitmap.c
@@ -125,7 +125,7 @@ int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
struct address_space *mapping;
struct folio *folio;
u8 *kaddr;
- int pos, len;
+ int pos, len, err;
u8 bit;
struct ntfs_inode *ni = NTFS_I(vi);
struct ntfs_volume *vol = ni->vol;
@@ -201,8 +201,10 @@ int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
/* If we are not in the last page, deal with all subsequent pages. */
while (index < end_index) {
- if (cnt <= 0)
+ if (cnt <= 0) {
+ err = -EIO;
goto rollback;
+ }
/* Update @index and get the next folio. */
folio_mark_dirty(folio);
@@ -214,6 +216,7 @@ int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
ntfs_error(vi->i_sb,
"Failed to map subsequent page (error %li), aborting.",
PTR_ERR(folio));
+ err = PTR_ERR(folio);
goto rollback;
}
@@ -265,7 +268,7 @@ int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
* - @count - @cnt is the number of bits that have been modified
*/
if (is_rollback)
- return PTR_ERR(folio);
+ return err;
if (count != cnt)
pos = __ntfs_bitmap_set_bits_in_run(vi, start_bit, count - cnt,
value ? 0 : 1, true);
@@ -274,14 +277,14 @@ int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit,
if (!pos) {
/* Rollback was successful. */
ntfs_error(vi->i_sb,
- "Failed to map subsequent page (error %li), aborting.",
- PTR_ERR(folio));
+ "Failed to map subsequent page (error %i), aborting.",
+ err);
} else {
/* Rollback failed. */
ntfs_error(vi->i_sb,
- "Failed to map subsequent page (error %li) and rollback failed (error %i). Aborting and leaving inconsistent metadata. Unmount and run chkdsk.",
- PTR_ERR(folio), pos);
+ "Failed to map subsequent page (error %i) and rollback failed (error %i). Aborting and leaving inconsistent metadata. Unmount and run chkdsk.",
+ err, pos);
NVolSetErrors(NTFS_SB(vi->i_sb));
}
- return PTR_ERR(folio);
+ return err;
}
diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c
index bfa904d..20f5c70 100644
--- a/fs/ntfs/dir.c
+++ b/fs/ntfs/dir.c
@@ -911,8 +911,8 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor)
if (next->flags & INDEX_ENTRY_NODE) {
next = ntfs_index_walk_down(next, ictx);
- if (!next) {
- err = -EIO;
+ if (IS_ERR(next)) {
+ err = PTR_ERR(next);
goto out;
}
}
@@ -920,7 +920,14 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor)
if (next && !(next->flags & INDEX_ENTRY_END))
goto nextdir;
- while ((next = ntfs_index_next(next, ictx)) != NULL) {
+ while (1) {
+ next = ntfs_index_next(next, ictx);
+ if (IS_ERR(next)) {
+ err = PTR_ERR(next);
+ goto out;
+ }
+ if (!next)
+ break;
nextdir:
/* Check the consistency of an index entry */
if (ntfs_index_entry_inconsistent(ictx, vol, next, COLLATION_FILE_NAME,
diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c
index 2080f39..146e011 100644
--- a/fs/ntfs/index.c
+++ b/fs/ntfs/index.c
@@ -677,11 +677,11 @@ static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_bl
static int ntfs_icx_parent_inc(struct ntfs_index_context *icx)
{
- icx->pindex++;
- if (icx->pindex >= MAX_PARENT_VCN) {
+ if (icx->pindex >= MAX_PARENT_VCN - 1) {
ntfs_error(icx->idx_ni->vol->sb, "Index is over %d level deep", MAX_PARENT_VCN);
return -EOPNOTSUPP;
}
+ icx->pindex++;
return 0;
}
@@ -1969,20 +1969,31 @@ int ntfs_index_remove(struct ntfs_inode *dir_ni, const void *key, const u32 keyl
struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_index_context *ictx)
{
struct index_entry *entry;
+ struct index_block *ib;
+ int err;
s64 vcn;
entry = ie;
do {
vcn = ntfs_ie_get_vcn(entry);
if (ictx->is_in_root) {
- /* down from level zero */
- ictx->ir = NULL;
- ictx->ib = kvzalloc(ictx->block_size, GFP_NOFS);
+ ib = kvzalloc(ictx->block_size, GFP_NOFS);
+ if (!ib)
+ return ERR_PTR(-ENOMEM);
+ /*
+ * Descending from root index (level 0) to the first
+ * child level. is_in_root == true implies pindex == 0,
+ * so advance to level 1.
+ */
ictx->pindex = 1;
+ ictx->ir = NULL;
+ ictx->ib = ib;
ictx->is_in_root = false;
} else {
/* down from non-zero level */
- ictx->pindex++;
+ err = ntfs_icx_parent_inc(ictx);
+ if (err)
+ return ERR_PTR(err);
}
ictx->parent_pos[ictx->pindex] = 0;
@@ -1991,8 +2002,8 @@ struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_ind
ictx->entry = ntfs_ie_get_first(&ictx->ib->index);
entry = ictx->entry;
} else
- entry = NULL;
- } while (entry && (entry->flags & INDEX_ENTRY_NODE));
+ entry = ERR_PTR(-EIO);
+ } while (!IS_ERR(entry) && (entry->flags & INDEX_ENTRY_NODE));
return entry;
}
@@ -2097,10 +2108,15 @@ struct index_entry *ntfs_index_next(struct index_entry *ie, struct ntfs_index_co
/* walk down if it has a subnode */
if (flags & INDEX_ENTRY_NODE) {
- if (!ictx->ia_ni)
+ if (!ictx->ia_ni) {
ictx->ia_ni = ntfs_ia_open(ictx, ictx->idx_ni);
+ if (!ictx->ia_ni)
+ return ERR_PTR(-EIO);
+ }
next = ntfs_index_walk_down(next, ictx);
+ if (IS_ERR(next))
+ return next;
} else {
/* walk up it has no subnode, nor data */
diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c
index 16890d4..360bebd 100644
--- a/fs/ntfs/inode.c
+++ b/fs/ntfs/inode.c
@@ -2582,8 +2582,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
mutex_lock_nested(&index_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
if (NInoBeingDeleted(ni)) {
- iput(index_vi);
mutex_unlock(&index_ni->mrec_lock);
+ iput(index_vi);
continue;
}
@@ -2591,8 +2591,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
if (!ictx) {
ntfs_error(sb, "Failed to get index ctx, inode %llu",
index_ni->mft_no);
- iput(index_vi);
mutex_unlock(&index_ni->mrec_lock);
+ iput(index_vi);
continue;
}
@@ -2601,8 +2601,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
ntfs_debug("Index lookup failed, inode %llu",
index_ni->mft_no);
ntfs_index_ctx_put(ictx);
- iput(index_vi);
mutex_unlock(&index_ni->mrec_lock);
+ iput(index_vi);
continue;
}
/* Update flags and file size. */
diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c
index 74a4d3e..dc7d8c8 100644
--- a/fs/ntfs/iomap.c
+++ b/fs/ntfs/iomap.c
@@ -788,8 +788,7 @@ static int ntfs_write_iomap_end_resident(struct inode *inode, loff_t pos,
ctx = ntfs_attr_get_search_ctx(ni, NULL);
if (!ctx) {
written = -ENOMEM;
- mutex_unlock(&ni->mrec_lock);
- return written;
+ goto err_out;
}
err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len,
@@ -810,7 +809,8 @@ static int ntfs_write_iomap_end_resident(struct inode *inode, loff_t pos,
memcpy(kattr + pos, iomap_inline_data(iomap, pos), written);
mark_mft_record_dirty(ctx->ntfs_ino);
err_out:
- ntfs_attr_put_search_ctx(ctx);
+ if (ctx)
+ ntfs_attr_put_search_ctx(ctx);
put_page(ipage);
mutex_unlock(&ni->mrec_lock);
return written;
diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c
index 3f8d164..d3f25d8 100644
--- a/fs/ntfs/logfile.c
+++ b/fs/ntfs/logfile.c
@@ -710,6 +710,9 @@ bool ntfs_empty_logfile(struct inode *log_vi)
if (unlikely(lcn == LCN_RL_NOT_MAPPED)) {
vcn = rl->vcn;
kvfree(empty_buf);
+ empty_buf = NULL;
+ kfree(ra);
+ ra = NULL;
goto map_vcn;
}
/* If this run is not valid abort with an error. */
@@ -753,7 +756,7 @@ bool ntfs_empty_logfile(struct inode *log_vi)
} while (start < end);
} while ((++rl)->vcn < end_vcn);
up_write(&log_ni->runlist.lock);
- kfree(empty_buf);
+ kvfree(empty_buf);
kfree(ra);
truncate_inode_pages(log_vi->i_mapping, 0);
/* Set the flag so we do not have to do it again on remount. */
diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c
index 7d98926..a7d10ee 100644
--- a/fs/ntfs/mft.c
+++ b/fs/ntfs/mft.c
@@ -30,6 +30,8 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
{
struct attr_record *a;
struct super_block *sb = vol->sb;
+ u16 attrs_offset;
+ u32 bytes_in_use;
if (!ntfs_is_file_record(m->magic)) {
ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n",
@@ -65,7 +67,16 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
goto err_out;
}
- a = (struct attr_record *)((char *)m + le16_to_cpu(m->attrs_offset));
+ attrs_offset = le16_to_cpu(m->attrs_offset);
+ bytes_in_use = le32_to_cpu(m->bytes_in_use);
+
+ if (attrs_offset > bytes_in_use ||
+ bytes_in_use - attrs_offset < sizeof_field(struct attr_record, type)) {
+ ntfs_error(sb, "Record %llu has corrupt attribute offset\n", mft_no);
+ goto err_out;
+ }
+
+ a = (struct attr_record *)((char *)m + attrs_offset);
if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) {
ntfs_error(sb, "Record %llu is corrupt\n", mft_no);
goto err_out;
@@ -449,7 +460,7 @@ static void ntfs_bio_end_io(struct bio *bio)
int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
struct mft_record *m)
{
- u8 *kmirr = NULL;
+ u8 *kmirr;
struct folio *folio;
unsigned int folio_ofs, lcn_folio_off = 0;
int err = 0;
@@ -479,6 +490,7 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
kmirr = kmap_local_folio(folio, 0) + folio_ofs;
/* Copy the mst protected mft record to the mirror. */
memcpy(kmirr, m, vol->mft_record_size);
+ kunmap_local(kmirr);
if (vol->cluster_size_bits > PAGE_SHIFT) {
lcn_folio_off = folio->index << PAGE_SHIFT;
@@ -490,20 +502,22 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) +
lcn_folio_off + folio_ofs);
- if (!bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) {
+ if (bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs))
+ err = submit_bio_wait(bio);
+ else
err = -EIO;
- bio_put(bio);
- goto unlock_folio;
- }
+ bio_put(bio);
- bio->bi_end_io = ntfs_bio_end_io;
- submit_bio(bio);
- /* Current state: all buffers are clean, unlocked, and uptodate. */
+ /*
+ * The in-memory mirror is now valid because we just memcpy()'d the
+ * mst-protected mft record into it. Mark the folio uptodate even on
+ * write error so a subsequent read_mapping_folio() does not refetch
+ * the stale on-disk mirror and overwrite this copy. The error is
+ * propagated to the caller via @err.
+ */
folio_mark_uptodate(folio);
-unlock_folio:
folio_unlock(folio);
- kunmap_local(kmirr);
folio_put(folio);
if (likely(!err)) {
ntfs_debug("Done.");
@@ -588,20 +602,36 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn
}
/* Synchronize the mft mirror now if not @sync. */
- if (!sync && ni->mft_no < vol->mftmirr_size)
- ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+ if (!sync && ni->mft_no < vol->mftmirr_size) {
+ int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no,
+ fixup_m);
+ if (unlikely(sub_err) && !err)
+ err = sub_err;
+ }
- folio_get(folio);
- bio->bi_private = folio;
- bio->bi_end_io = ntfs_bio_end_io;
- submit_bio(bio);
+ if (sync) {
+ int sub_err = submit_bio_wait(bio);
+
+ bio_put(bio);
+ if (unlikely(sub_err) && !err)
+ err = sub_err;
+ } else {
+ folio_get(folio);
+ bio->bi_private = folio;
+ bio->bi_end_io = ntfs_bio_end_io;
+ submit_bio(bio);
+ }
offset += vol->cluster_size;
i++;
}
/* If @sync, now synchronize the mft mirror. */
- if (sync && ni->mft_no < vol->mftmirr_size)
- ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+ if (sync && ni->mft_no < vol->mftmirr_size) {
+ int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+
+ if (unlikely(sub_err) && !err)
+ err = sub_err;
+ }
kunmap_local(kaddr);
if (unlikely(err)) {
/* I/O error during writing. This is really bad! */
@@ -617,10 +647,10 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn
bio_put(bio);
err_out:
/*
- * Current state: all buffers are clean, unlocked, and uptodate.
- * The caller should mark the base inode as bad so that no more i/o
- * happens. ->drop_inode() will still be invoked so all extent inodes
- * and other allocated memory will be freed.
+ * The caller should mark the base inode as bad so no more I/O
+ * happens. ->drop_inode() will still be invoked so all extent inodes
+ * and other allocated memory will be freed. ENOMEM is retried by
+ * redirtying the mft record below.
*/
if (err == -ENOMEM) {
ntfs_error(vol->sb,
@@ -833,7 +863,7 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no,
vi = igrab(mft_vi);
WARN_ON(vi != mft_vi);
} else {
- vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na);
+ vi = find_inode_nowait(sb, na.mft_no, ntfs_test_inode_wb, &na);
if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated)
return false;
}
@@ -1034,7 +1064,7 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo
b = ffz((unsigned long)*byte);
if (b < 8 && b >= (bit & 7)) {
ll = data_pos + (bit & ~7ull) + b;
- if (unlikely(ll > (1ll << 32))) {
+ if (unlikely(ll >= (1ll << 32))) {
folio_unlock(folio);
kunmap_local(buf);
folio_put(folio);
@@ -2721,8 +2751,11 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w
ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.",
ni->mft_no, ni->type, folio->index);
- if (!locked_nis || !ref_inos)
+ if (!locked_nis || !ref_inos) {
+ folio_redirty_for_writepage(wbc, folio);
+ folio_unlock(folio);
return -ENOMEM;
+ }
/* We have to zero every time due to mmap-at-end-of-file. */
if (folio->index >= (i_size >> folio_shift(folio)))
@@ -2840,9 +2873,13 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w
}
prev_mft_ofs = mft_ofs;
- if (mft_no < vol->mftmirr_size)
- ntfs_sync_mft_mirror(vol, mft_no,
+ if (mft_no < vol->mftmirr_size) {
+ int sub_err = ntfs_sync_mft_mirror(vol, mft_no,
(struct mft_record *)(kaddr + mft_ofs));
+
+ if (unlikely(sub_err) && !err)
+ err = sub_err;
+ }
} else if (ref_inos[nr_ref_inos])
nr_ref_inos++;
}
diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c
index 10894de..c4f8284 100644
--- a/fs/ntfs/namei.c
+++ b/fs/ntfs/namei.c
@@ -344,9 +344,9 @@ static int ntfs_sd_add_everyone(struct ntfs_inode *ni)
sd_len = sizeof(struct security_descriptor_relative) + 2 *
(sizeof(struct ntfs_sid) + 8) + sizeof(struct ntfs_acl) +
sizeof(struct ntfs_ace) + 4;
- sd = kmalloc(sd_len, GFP_NOFS);
+ sd = kzalloc(sd_len, GFP_NOFS);
if (!sd)
- return -1;
+ return -ENOMEM;
sd->revision = 1;
sd->control = SE_DACL_PRESENT | SE_SELF_RELATIVE;
@@ -945,7 +945,8 @@ static int ntfs_delete(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
ni_mrec = actx->base_mrec ? actx->base_mrec : actx->mrec;
ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) - 1);
- drop_nlink(VFS_I(ni));
+ if (!S_ISDIR(VFS_I(ni)->i_mode))
+ drop_nlink(VFS_I(ni));
mark_mft_record_dirty(ni);
if (looking_for_dos_name) {
@@ -956,6 +957,13 @@ static int ntfs_delete(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
}
/*
+ * For directories, Drop VFS nlink only when mft record link count
+ * becomes zero. Because we fixes VFS nlink to 1 for directories.
+ */
+ if (S_ISDIR(VFS_I(ni)->i_mode) && !le16_to_cpu(ni_mrec->link_count))
+ drop_nlink(VFS_I(ni));
+
+ /*
* If hard link count is not equal to zero then we are done. In other
* case there are no reference to this inode left, so we should free all
* non-resident attributes and mark all MFT record as not in use.
@@ -1221,7 +1229,8 @@ static int __ntfs_link(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
}
/* Increment hard links count. */
ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) + 1);
- inc_nlink(VFS_I(ni));
+ if (!S_ISDIR(vi->i_mode))
+ inc_nlink(VFS_I(ni));
/* Done! */
mark_mft_record_dirty(ni);
diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c
index 8f60ec6..7471371 100644
--- a/fs/ntfs/reparse.c
+++ b/fs/ntfs/reparse.c
@@ -505,7 +505,6 @@ int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni,
struct reparse_point *reparse;
struct wsl_link_reparse_data *data;
- utarget = (char *)NULL;
len = ntfs_ucstonls(ni->vol, target, target_len, &utarget, 0);
if (len <= 0)
return -EINVAL;
@@ -514,7 +513,7 @@ int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni,
reparse = kvzalloc(reparse_len, GFP_NOFS);
if (!reparse) {
err = -ENOMEM;
- kvfree(utarget);
+ kfree(utarget);
} else {
data = (struct wsl_link_reparse_data *)reparse->reparse_data;
reparse->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK;
@@ -528,6 +527,8 @@ int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni,
kvfree(reparse);
if (!err)
ni->target = utarget;
+ else
+ kfree(utarget);
}
return err;
}
diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c
index b213b49..e7de3d0 100644
--- a/fs/ntfs/runlist.c
+++ b/fs/ntfs/runlist.c
@@ -15,6 +15,8 @@
* Copyright (c) 2007-2022 Jean-Pierre Andre
*/
+#include <linux/overflow.h>
+
#include "ntfs.h"
#include "attrib.h"
@@ -739,6 +741,7 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *
int rlsize; /* Size of runlist buffer. */
u16 rlpos; /* Current runlist position in units of struct runlist_elements. */
u8 b; /* Current byte offset in buf. */
+ u64 lowest_vcn; /* Raw on-disk lowest_vcn. */
#ifdef DEBUG
/* Make sure attr exists and is non-resident. */
@@ -747,8 +750,14 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *
return ERR_PTR(-EINVAL);
}
#endif
+ lowest_vcn = le64_to_cpu(attr->data.non_resident.lowest_vcn);
+ /* Validate lowest_vcn from on-disk metadata to ensure it is sane. */
+ if (overflows_type(lowest_vcn, vcn)) {
+ ntfs_error(vol->sb, "Invalid lowest_vcn in mapping pairs.");
+ return ERR_PTR(-EIO);
+ }
/* Start at vcn = lowest_vcn and lcn 0. */
- vcn = le64_to_cpu(attr->data.non_resident.lowest_vcn);
+ vcn = lowest_vcn;
lcn = 0;
/* Get start of the mapping pairs array. */
buf = (u8 *)attr +
@@ -823,8 +832,17 @@ struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *
* element.
*/
rl[rlpos].length = deltaxcn;
- /* Increment the current vcn by the current run length. */
- vcn += deltaxcn;
+ /*
+ * Increment the current vcn by the current run length.
+ * Guard against s64 overflow from a crafted mapping
+ * pairs array to preserve the monotonically-increasing
+ * vcn invariant.
+ */
+ if (unlikely(check_add_overflow(vcn, deltaxcn, &vcn))) {
+ ntfs_error(vol->sb, "VCN overflow in mapping pairs array.");
+ goto err_out;
+ }
+
/*
* There might be no lcn change at all, as is the case for
* sparse clusters on NTFS 3.0+, in which case we set the lcn
@@ -2038,10 +2056,11 @@ struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, i
* consists of holes.
*/
merge_cnt = 0;
- i = new_1st_cnt == 0 ? 1 : new_1st_cnt;
- if (ntfs_rle_lcn_contiguous(&new_rl[i - 1], &new_rl[i])) {
- /* Merge right and left */
- s_rl = &new_rl[new_1st_cnt - 1];
+ if (new_1st_cnt > 0 &&
+ ntfs_rle_lcn_contiguous(&new_rl[new_1st_cnt - 1],
+ &new_rl[new_1st_cnt])) {
+ /* Merge right and left. */
+ s_rl = &new_rl[new_1st_cnt - 1];
s_rl->length += s_rl[1].length;
merge_cnt = 1;
}
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
index 22dc786..9e321cc 100644
--- a/fs/ntfs/super.c
+++ b/fs/ntfs/super.c
@@ -413,6 +413,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
{
struct ntfs_inode *vol_ni = NTFS_I(vol->vol_ino);
struct ntfs_attr_search_ctx *ctx;
+ char *new_label;
__le16 *uname;
int uname_len, ret;
@@ -425,7 +426,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
return uname_len;
}
- if (uname_len > NTFS_MAX_LABEL_LEN) {
+ if (uname_len > NTFS_MAX_LABEL_LEN) {
ntfs_error(vol->sb,
"Volume label is too long (max %d characters).",
NTFS_MAX_LABEL_LEN);
@@ -433,11 +434,22 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
return -EINVAL;
}
+ /*
+ * Allocate the in-memory label copy up front. If kstrdup() fails we
+ * bail out before touching on-disk metadata, so the in-memory label
+ * and the on-disk label stay in sync.
+ */
+ new_label = kstrdup(label, GFP_KERNEL);
+ if (!new_label) {
+ kvfree(uname);
+ return -ENOMEM;
+ }
+
mutex_lock(&vol_ni->mrec_lock);
ctx = ntfs_attr_get_search_ctx(vol_ni, NULL);
if (!ctx) {
ret = -ENOMEM;
- goto out;
+ goto out;
}
if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0,
@@ -450,12 +462,14 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
out:
mutex_unlock(&vol_ni->mrec_lock);
kvfree(uname);
- mark_inode_dirty_sync(vol->vol_ino);
if (ret >= 0) {
kfree(vol->volume_label);
- vol->volume_label = kstrdup(label, GFP_KERNEL);
+ vol->volume_label = new_label;
+ mark_inode_dirty_sync(vol->vol_ino);
ret = 0;
+ } else {
+ kfree(new_label);
}
return ret;
}
@@ -979,6 +993,13 @@ static bool check_mft_mirror(struct ntfs_volume *vol)
ntfs_is_baad_recordp((__le32 *)kmirr))
bytes = vol->mft_record_size;
}
+ /* Compare the two records. */
+ if (memcmp(kmft, kmirr, bytes)) {
+ ntfs_error(sb,
+ "$MFT and $MFTMirr record %i do not match. Run chkdsk.",
+ i);
+ goto mm_unmap_out;
+ }
kmft += vol->mft_record_size;
kmirr += vol->mft_record_size;
} while (++i < vol->mftmirr_size);
@@ -1671,7 +1692,7 @@ static bool load_system_files(struct ntfs_volume *vol)
iput_upcase_err_out:
vol->upcase_len = 0;
mutex_lock(&ntfs_lock);
- if (vol->upcase == default_upcase) {
+ if (vol->upcase && vol->upcase == default_upcase) {
ntfs_nr_upcase_users--;
vol->upcase = NULL;
}
@@ -1701,7 +1722,7 @@ static void ntfs_volume_free(struct ntfs_volume *vol)
* the number of upcase users if we are a user.
*/
mutex_lock(&ntfs_lock);
- if (vol->upcase == default_upcase) {
+ if (vol->upcase && vol->upcase == default_upcase) {
ntfs_nr_upcase_users--;
vol->upcase = NULL;
}
@@ -2494,7 +2515,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
}
vol->upcase_len = 0;
mutex_lock(&ntfs_lock);
- if (vol->upcase == default_upcase) {
+ if (vol->upcase && vol->upcase == default_upcase) {
ntfs_nr_upcase_users--;
vol->upcase = NULL;
}
diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c
index bec5475..75e65e7 100644
--- a/fs/orangefs/namei.c
+++ b/fs/orangefs/namei.c
@@ -362,7 +362,7 @@ static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
__orangefs_setattr(dir, &iattr);
out:
op_release(new_op);
- return ERR_PTR(ret);
+ return ret ? ERR_PTR(ret) : NULL;
}
static int orangefs_rename(struct mnt_idmap *idmap,
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index 7b86a6ba..b41f478 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -1354,7 +1354,7 @@ int ovl_ensure_verity_loaded(const struct path *datapath)
struct inode *inode = d_inode(datapath->dentry);
struct file *filp;
- if (!fsverity_active(inode) && IS_VERITY(inode)) {
+ if (IS_VERITY(inode) && fsverity_get_info(inode) == NULL) {
/*
* If this inode was not yet opened, the verity info hasn't been
* loaded yet, so we need to do that here to force it into memory.
diff --git a/fs/proc/Makefile b/fs/proc/Makefile
index 7b4db9c..8bc615f 100644
--- a/fs/proc/Makefile
+++ b/fs/proc/Makefile
@@ -16,7 +16,9 @@
proc-y += consoles.o
proc-y += cpuinfo.o
proc-y += devices.o
-proc-y += interrupts.o
+ifneq ($(CONFIG_GENERIC_IRQ_SHOW),y)
+proc-y += interrupts.o
+endif
proc-y += loadavg.o
proc-y += meminfo.o
proc-y += stat.o
diff --git a/fs/proc/stat.c b/fs/proc/stat.c
index 8b444e8..79b42f2 100644
--- a/fs/proc/stat.c
+++ b/fs/proc/stat.c
@@ -18,41 +18,6 @@
#ifndef arch_irq_stat_cpu
#define arch_irq_stat_cpu(cpu) 0
#endif
-#ifndef arch_irq_stat
-#define arch_irq_stat() 0
-#endif
-
-u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
-{
- u64 idle, idle_usecs = -1ULL;
-
- if (cpu_online(cpu))
- idle_usecs = get_cpu_idle_time_us(cpu, NULL);
-
- if (idle_usecs == -1ULL)
- /* !NO_HZ or cpu offline so we can rely on cpustat.idle */
- idle = kcs->cpustat[CPUTIME_IDLE];
- else
- idle = idle_usecs * NSEC_PER_USEC;
-
- return idle;
-}
-
-static u64 get_iowait_time(struct kernel_cpustat *kcs, int cpu)
-{
- u64 iowait, iowait_usecs = -1ULL;
-
- if (cpu_online(cpu))
- iowait_usecs = get_cpu_iowait_time_us(cpu, NULL);
-
- if (iowait_usecs == -1ULL)
- /* !NO_HZ or cpu offline so we can rely on cpustat.iowait */
- iowait = kcs->cpustat[CPUTIME_IOWAIT];
- else
- iowait = iowait_usecs * NSEC_PER_USEC;
-
- return iowait;
-}
static void show_irq_gap(struct seq_file *p, unsigned int gap)
{
@@ -105,8 +70,8 @@ static int show_stat(struct seq_file *p, void *v)
user += cpustat[CPUTIME_USER];
nice += cpustat[CPUTIME_NICE];
system += cpustat[CPUTIME_SYSTEM];
- idle += get_idle_time(&kcpustat, i);
- iowait += get_iowait_time(&kcpustat, i);
+ idle += cpustat[CPUTIME_IDLE];
+ iowait += cpustat[CPUTIME_IOWAIT];
irq += cpustat[CPUTIME_IRQ];
softirq += cpustat[CPUTIME_SOFTIRQ];
steal += cpustat[CPUTIME_STEAL];
@@ -122,7 +87,6 @@ static int show_stat(struct seq_file *p, void *v)
sum_softirq += softirq_stat;
}
}
- sum += arch_irq_stat();
seq_put_decimal_ull(p, "cpu ", nsec_to_clock_t(user));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice));
@@ -146,8 +110,8 @@ static int show_stat(struct seq_file *p, void *v)
user = cpustat[CPUTIME_USER];
nice = cpustat[CPUTIME_NICE];
system = cpustat[CPUTIME_SYSTEM];
- idle = get_idle_time(&kcpustat, i);
- iowait = get_iowait_time(&kcpustat, i);
+ idle = cpustat[CPUTIME_IDLE];
+ iowait = cpustat[CPUTIME_IOWAIT];
irq = cpustat[CPUTIME_IRQ];
softirq = cpustat[CPUTIME_SOFTIRQ];
steal = cpustat[CPUTIME_STEAL];
diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c
index b5343d2..433aa94 100644
--- a/fs/proc/uptime.c
+++ b/fs/proc/uptime.c
@@ -18,12 +18,8 @@ static int uptime_proc_show(struct seq_file *m, void *v)
int i;
idle_nsec = 0;
- for_each_possible_cpu(i) {
- struct kernel_cpustat kcs;
-
- kcpustat_cpu_fetch(&kcs, i);
- idle_nsec += get_idle_time(&kcs, i);
- }
+ for_each_possible_cpu(i)
+ idle_nsec += kcpustat_field(CPUTIME_IDLE, i);
ktime_get_boottime_ts64(&uptime);
timens_add_boottime(&uptime);
diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h
index 1a9b2911..48af75b 100644
--- a/fs/resctrl/internal.h
+++ b/fs/resctrl/internal.h
@@ -408,6 +408,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free);
void resctrl_file_fflags_init(const char *config, unsigned long fflags);
+void resctrl_file_mode_init(const char *config, umode_t mode);
+
void rdt_staged_configs_clear(void);
bool closid_allocated(unsigned int closid);
diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
index 9fd901c..0e6a389 100644
--- a/fs/resctrl/monitor.c
+++ b/fs/resctrl/monitor.c
@@ -1211,9 +1211,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_
* NULL; otherwise, assign the counter to the specified domain @d.
*
* If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr()
- * will fail. The assignment process will abort at the first failure encountered
- * during domain traversal, which may result in the event being only partially
- * assigned.
+ * will fail. When attempting to assign counters to all domains, carry on trying
+ * to assign counters after a failure since only some domains may have counters
+ * and the goal is to assign counters where possible. If any counter assignment
+ * fails, return the error from the last failing assignment.
*
* Return:
* 0 on success, < 0 on failure.
@@ -1226,9 +1227,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro
if (!d) {
list_for_each_entry(d, &r->mon_domains, hdr.list) {
- ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
- if (ret)
- return ret;
+ int err;
+
+ err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
+ if (err)
+ ret = err;
}
} else {
ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
@@ -1422,6 +1425,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes
ret = -EINVAL;
goto out_unlock;
}
+ if (!r->mon.mbm_cntr_configurable) {
+ rdt_last_cmd_puts("event_filter is not configurable\n");
+ ret = -EPERM;
+ goto out_unlock;
+ }
ret = resctrl_parse_mem_transactions(buf, &evt_cfg);
if (!ret && mevt->evt_cfg != evt_cfg) {
@@ -1451,7 +1459,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of,
else
seq_puts(s, "[default]\n");
- if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) {
+ if (!r->mon.mbm_cntr_assign_fixed) {
if (enabled)
seq_puts(s, "default\n");
else
@@ -1502,6 +1510,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf,
}
if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) {
+ if (r->mon.mbm_cntr_assign_fixed) {
+ ret = -EINVAL;
+ rdt_last_cmd_puts("Counter assignment mode is not configurable\n");
+ goto out_unlock;
+ }
+
ret = resctrl_arch_mbm_cntr_assign_set(r, enable);
if (ret)
goto out_unlock;
@@ -1886,6 +1900,8 @@ int resctrl_l3_mon_resource_init(void)
resctrl_file_fflags_init("available_mbm_cntrs",
RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG);
+ if (r->mon.mbm_cntr_configurable)
+ resctrl_file_mode_init("event_filter", 0644);
resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO |
RFTYPE_RES_CACHE);
resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE);
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index 5dfdaa6..af2cbab 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -2022,7 +2022,7 @@ static struct rftype res_common_files[] = {
},
{
.name = "event_filter",
- .mode = 0644,
+ .mode = 0444,
.kf_ops = &rdtgroup_kf_single_ops,
.seq_show = event_filter_show,
.write = event_filter_write,
@@ -2215,6 +2215,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags)
rft->fflags = fflags;
}
+void resctrl_file_mode_init(const char *config, umode_t mode)
+{
+ struct rftype *rft;
+
+ rft = rdtgroup_get_rftype_by_name(config);
+ if (rft)
+ rft->mode = mode;
+}
+
/**
* rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file
* @r: The resource group with which the file is associated.
@@ -2331,22 +2340,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod
continue;
kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt);
- if (IS_ERR(kn_subdir2)) {
- ret = PTR_ERR(kn_subdir2);
- goto out;
- }
+ if (IS_ERR(kn_subdir2))
+ return PTR_ERR(kn_subdir2);
ret = rdtgroup_kn_set_ugid(kn_subdir2);
if (ret)
- goto out;
+ return ret;
ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG);
if (ret)
- break;
+ return ret;
}
-out:
- return ret;
+ return 0;
}
static int rdtgroup_mkdir_info_resdir(void *priv, char *name,
@@ -2510,10 +2516,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r,
}
/*
- * MBA software controller is supported only if
- * MBM is supported and MBA is in linear scale,
- * and the MBM monitor scope is the same as MBA
- * control scope.
+ * The MBA software controller is supported only if MBM is supported and MBA is
+ * in linear scale, and the MBM monitor scope is the same as MBA control scope.
+ *
+ * The software controller cannot be supported when the MBM counters are
+ * assignable. There is no guarantee that MBM counters are assigned to the
+ * event backing the software controller in all monitoring domains of all
+ * monitoring groups.
*/
static bool supports_mba_mbps(void)
{
@@ -2522,7 +2531,8 @@ static bool supports_mba_mbps(void)
return (resctrl_is_mbm_enabled() &&
r->alloc_capable && is_mba_linear() &&
- r->ctrl_scope == rmbm->mon_scope);
+ r->ctrl_scope == rmbm->mon_scope &&
+ !rmbm->mon.mbm_cntr_assignable);
}
/*
@@ -2937,7 +2947,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param)
ctx->enable_cdpl2 = true;
return 0;
case Opt_mba_mbps:
- msg = "mba_MBps requires MBM and linear scale MBA at L3 scope";
+ msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope";
if (!supports_mba_mbps())
return invalfc(fc, msg);
ctx->enable_mba_mbps = true;
diff --git a/fs/select.c b/fs/select.c
index 75978b1..bf71c98 100644
--- a/fs/select.c
+++ b/fs/select.c
@@ -708,6 +708,17 @@ static int kern_select(int n, fd_set __user *inp, fd_set __user *outp,
if (copy_from_user(&tv, tvp, sizeof(tv)))
return -EFAULT;
+ /*
+ * Reject negative components before normalisation. The seconds
+ * sum below is performed in signed long and a crafted negative
+ * timeval can wrap to a positive value that passes
+ * timespec64_valid() and turns into an effectively-infinite
+ * deadline via timespec64_add_safe()'s saturation, instead of
+ * the -EINVAL POSIX requires for negative timeouts.
+ */
+ if (tv.tv_sec < 0 || tv.tv_usec < 0)
+ return -EINVAL;
+
to = &end_time;
if (poll_select_set_timeout(to,
tv.tv_sec + (tv.tv_usec / USEC_PER_SEC),
diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c
index 02791ec..88d5e9a 100644
--- a/fs/smb/client/cached_dir.c
+++ b/fs/smb/client/cached_dir.c
@@ -286,6 +286,14 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon,
&rqst[0], &oplock, &oparms, utf16_path);
if (rc)
goto oshr_free;
+
+ if (oplock != SMB2_OPLOCK_LEVEL_II) {
+ rc = -EINVAL;
+ cifs_dbg(FYI, "%s: Oplock level %d not suitable for cached directory\n",
+ __func__, oplock);
+ goto oshr_free;
+ }
+
smb2_set_next_command(tcon, &rqst[0]);
memset(&qi_iov, 0, sizeof(qi_iov));
diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c
index 3a41bba..44c40727 100644
--- a/fs/smb/client/cifs_spnego.c
+++ b/fs/smb/client/cifs_spnego.c
@@ -8,6 +8,7 @@
*/
#include <linux/list.h>
+#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <keys/user-type.h>
@@ -40,12 +41,27 @@ cifs_spnego_key_destroy(struct key *key)
kfree(key->payload.data[0]);
}
+static int
+cifs_spnego_key_vet_description(const char *description)
+{
+ /*
+ * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall.
+ * They are only valid when produced by CIFS while using the private
+ * spnego_cred installed below. Do not let userspace create this type
+ * of key through request_key(2)/add_key(2), since the helper treats
+ * pid/uid/creduid/upcall_target as kernel-originating fields.
+ */
+ if (current_cred() != spnego_cred)
+ return -EPERM;
+ return 0;
+}
/*
* keytype for CIFS spnego keys
*/
struct key_type cifs_spnego_key_type = {
.name = "cifs.spnego",
+ .vet_description = cifs_spnego_key_vet_description,
.instantiate = cifs_spnego_key_instantiate,
.destroy = cifs_spnego_key_destroy,
.describe = user_describe,
diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c
index ec5d477..786dbbc 100644
--- a/fs/smb/client/cifsacl.c
+++ b/fs/smb/client/cifsacl.c
@@ -1264,6 +1264,17 @@ static int parse_sid(struct smb_sid *psid, char *end_of_acl)
return 0;
}
+static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset)
+{
+ if (acl_len < sizeof(struct smb_acl))
+ return false;
+
+ if (dacloffset < sizeof(struct smb_ntsd))
+ return false;
+
+ return dacloffset <= acl_len - sizeof(struct smb_acl);
+}
+
/* Convert CIFS ACL to POSIX form */
static int parse_sec_desc(struct cifs_sb_info *cifs_sb,
@@ -1284,7 +1295,6 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb,
group_sid_ptr = (struct smb_sid *)((char *)pntsd +
le32_to_cpu(pntsd->gsidoffset));
dacloffset = le32_to_cpu(pntsd->dacloffset);
- dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset);
cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n",
pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset),
le32_to_cpu(pntsd->gsidoffset),
@@ -1315,11 +1325,18 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb,
return rc;
}
- if (dacloffset)
+ if (dacloffset) {
+ if (!dacl_offset_valid(acl_len, dacloffset)) {
+ cifs_dbg(VFS, "Server returned illegal DACL offset\n");
+ return -EINVAL;
+ }
+
+ dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset);
parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr,
group_sid_ptr, fattr, get_mode_from_special_sid);
- else
+ } else {
cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */
+ }
return rc;
}
@@ -1342,6 +1359,11 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
dacloffset = le32_to_cpu(pntsd->dacloffset);
if (dacloffset) {
+ if (!dacl_offset_valid(secdesclen, dacloffset)) {
+ cifs_dbg(VFS, "Server returned illegal DACL offset\n");
+ return -EINVAL;
+ }
+
dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset);
rc = validate_dacl(dacl_ptr, end_of_acl);
if (rc)
@@ -1710,6 +1732,12 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode,
nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2);
dacloffset = le32_to_cpu(pntsd->dacloffset);
if (dacloffset) {
+ if (!dacl_offset_valid(secdesclen, dacloffset)) {
+ cifs_dbg(VFS, "Server returned illegal DACL offset\n");
+ rc = -EINVAL;
+ goto id_mode_to_cifs_acl_exit;
+ }
+
dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset);
rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen);
if (rc) {
@@ -1732,7 +1760,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode,
* descriptor parameters, and security descriptor itself
*/
nsecdesclen = max_t(u32, nsecdesclen, DEFAULT_SEC_DESC_LEN);
- pnntsd = kmalloc(nsecdesclen, GFP_KERNEL);
+ pnntsd = kzalloc(nsecdesclen, GFP_KERNEL);
if (!pnntsd) {
kfree(pntsd);
cifs_put_tlink(tlink);
@@ -1752,6 +1780,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode,
rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag);
cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc);
}
+id_mode_to_cifs_acl_exit:
cifs_put_tlink(tlink);
kfree(pnntsd);
diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c
index 9f76b03..ce23924 100644
--- a/fs/smb/client/cifsfs.c
+++ b/fs/smb/client/cifsfs.c
@@ -306,6 +306,8 @@ static void cifs_kill_sb(struct super_block *sb)
/* Wait for all pending oplock breaks to complete */
flush_workqueue(cifsoplockd_wq);
+ /* Wait for all opened files to release */
+ flush_workqueue(deferredclose_wq);
/* finally release root dentry */
dput(cifs_sb->root);
@@ -434,7 +436,8 @@ cifs_alloc_inode(struct super_block *sb)
spin_lock_init(&cifs_inode->writers_lock);
cifs_inode->writers = 0;
cifs_inode->netfs.inode.i_blkbits = 14; /* 2**14 = CIFS_MAX_MSGSIZE */
- cifs_inode->netfs.remote_i_size = 0;
+ cifs_inode->netfs._remote_i_size = 0;
+ cifs_inode->netfs._zero_point = 0;
cifs_inode->uniqueid = 0;
cifs_inode->createtime = 0;
cifs_inode->epoch = 0;
@@ -1303,7 +1306,8 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
struct cifsFileInfo *smb_file_src = src_file->private_data;
struct cifsFileInfo *smb_file_target = dst_file->private_data;
struct cifs_tcon *target_tcon, *src_tcon;
- unsigned long long destend, fstart, fend, old_size, new_size;
+ unsigned long long i_size, new_size;
+ unsigned long long destend, fstart, fend;
unsigned int xid;
int rc;
@@ -1347,7 +1351,7 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
* Advance the EOF marker after the flush above to the end of the range
* if it's short of that.
*/
- if (src_cifsi->netfs.remote_i_size < off + len) {
+ if (netfs_read_remote_i_size(src_inode) < off + len) {
rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len);
if (rc < 0)
goto unlock;
@@ -1368,22 +1372,24 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
rc = cifs_flush_folio(target_inode, destend, &fstart, &fend, false);
if (rc)
goto unlock;
- if (fend > target_cifsi->netfs.zero_point)
- target_cifsi->netfs.zero_point = fend + 1;
- old_size = target_cifsi->netfs.remote_i_size;
+
+ spin_lock(&target_inode->i_lock);
+ if (fend > target_cifsi->netfs._zero_point)
+ netfs_write_zero_point(target_inode, fend + 1);
+ i_size = target_inode->i_size;
+ spin_unlock(&target_inode->i_lock);
/* Discard all the folios that overlap the destination region. */
cifs_dbg(FYI, "about to discard pages %llx-%llx\n", fstart, fend);
truncate_inode_pages_range(&target_inode->i_data, fstart, fend);
- fscache_invalidate(cifs_inode_cookie(target_inode), NULL,
- i_size_read(target_inode), 0);
+ fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size, 0);
rc = -EOPNOTSUPP;
if (target_tcon->ses->server->ops->duplicate_extents) {
rc = target_tcon->ses->server->ops->duplicate_extents(xid,
smb_file_src, smb_file_target, off, len, destoff);
- if (rc == 0 && new_size > old_size) {
+ if (rc == 0 && new_size > i_size) {
truncate_setsize(target_inode, new_size);
fscache_resize_cookie(cifs_inode_cookie(target_inode),
new_size);
@@ -1402,8 +1408,12 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
rc = -EINVAL;
}
}
- if (rc == 0 && new_size > target_cifsi->netfs.zero_point)
- target_cifsi->netfs.zero_point = new_size;
+ if (rc == 0) {
+ spin_lock(&target_inode->i_lock);
+ if (new_size > target_cifsi->netfs._zero_point)
+ netfs_write_zero_point(target_inode, new_size);
+ spin_unlock(&target_inode->i_lock);
+ }
}
/* force revalidate of size and timestamps of target file now
@@ -1474,7 +1484,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid,
* Advance the EOF marker after the flush above to the end of the range
* if it's short of that.
*/
- if (src_cifsi->netfs.remote_i_size < off + len) {
+ if (netfs_read_remote_i_size(src_inode) < off + len) {
rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len);
if (rc < 0)
goto unlock;
@@ -1502,8 +1512,12 @@ ssize_t cifs_file_copychunk_range(unsigned int xid,
fscache_resize_cookie(cifs_inode_cookie(target_inode),
i_size_read(target_inode));
}
- if (rc > 0 && destoff + rc > target_cifsi->netfs.zero_point)
- target_cifsi->netfs.zero_point = destoff + rc;
+ if (rc > 0) {
+ spin_lock(&target_inode->i_lock);
+ if (destoff + rc > target_cifsi->netfs._zero_point)
+ netfs_write_zero_point(target_inode, destoff + rc);
+ spin_unlock(&target_inode->i_lock);
+ }
}
file_accessed(src_file);
diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h
index 4a25afd..79d891f 100644
--- a/fs/smb/client/cifsproto.h
+++ b/fs/smb/client/cifsproto.h
@@ -89,7 +89,6 @@ int cifs_handle_standard(struct TCP_Server_Info *server,
struct mid_q_entry *mid);
char *smb3_fs_context_fullpath(const struct smb3_fs_context *ctx, char dirsep);
int smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx);
-int smb3_parse_opt(const char *options, const char *key, char **val);
int cifs_ipaddr_cmp(struct sockaddr *srcaddr, struct sockaddr *rhs);
bool cifs_match_ipaddr(struct sockaddr *srcaddr, struct sockaddr *rhs);
int cifs_discard_remaining_data(struct TCP_Server_Info *server);
diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c
index 3990a90..9e27bfa 100644
--- a/fs/smb/client/cifssmb.c
+++ b/fs/smb/client/cifssmb.c
@@ -1465,6 +1465,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
struct cifs_io_subrequest *rdata = mid->callback_data;
struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode);
struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink);
+ struct inode *inode = &ictx->inode;
struct smb_rqst rqst = { .rq_iov = rdata->iov,
.rq_nvec = 1,
.rq_iter = rdata->subreq.io_iter };
@@ -1538,7 +1539,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
} else {
size_t trans = rdata->subreq.transferred + rdata->got_bytes;
if (trans < rdata->subreq.len &&
- rdata->subreq.start + trans >= ictx->remote_i_size) {
+ rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) {
rdata->result = 0;
__set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
} else if (rdata->got_bytes > 0) {
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 664a2c2..b603441 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -2517,18 +2517,23 @@ int cifs_lock(struct file *file, int cmd, struct file_lock *flock)
void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, ssize_t result)
{
struct netfs_io_request *wreq = wdata->rreq;
- struct netfs_inode *ictx = netfs_inode(wreq->inode);
+ struct inode *inode = wreq->inode;
+ struct netfs_inode *ictx = netfs_inode(inode);
loff_t wrend;
if (result > 0) {
+ spin_lock(&inode->i_lock);
+
wrend = wdata->subreq.start + wdata->subreq.transferred + result;
- if (wrend > ictx->zero_point &&
+ if (wrend > ictx->_zero_point &&
(wdata->rreq->origin == NETFS_UNBUFFERED_WRITE ||
wdata->rreq->origin == NETFS_DIO_WRITE))
- ictx->zero_point = wrend;
- if (wrend > ictx->remote_i_size)
+ netfs_write_zero_point(inode, wrend);
+ if (wrend > ictx->_remote_i_size)
netfs_resize_file(ictx, wrend, true);
+
+ spin_unlock(&inode->i_lock);
}
netfs_write_subrequest_terminated(&wdata->subreq, result);
diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c
index b9544eb..2f86158 100644
--- a/fs/smb/client/fs_context.c
+++ b/fs/smb/client/fs_context.c
@@ -420,7 +420,7 @@ static int parse_symlink_flavor(struct fs_context *fc, char *value,
#define DUP_CTX_STR(field) \
do { \
if (ctx->field) { \
- new_ctx->field = kstrdup(ctx->field, GFP_ATOMIC); \
+ new_ctx->field = kstrdup(ctx->field, GFP_KERNEL); \
if (new_ctx->field == NULL) { \
smb3_cleanup_fs_context_contents(new_ctx); \
return -ENOMEM; \
@@ -536,37 +536,6 @@ cifs_parse_smb_version(struct fs_context *fc, char *value, struct smb3_fs_contex
return 0;
}
-int smb3_parse_opt(const char *options, const char *key, char **val)
-{
- int rc = -ENOENT;
- char *opts, *orig, *p;
-
- orig = opts = kstrdup(options, GFP_KERNEL);
- if (!opts)
- return -ENOMEM;
-
- while ((p = strsep(&opts, ","))) {
- char *nval;
-
- if (!*p)
- continue;
- if (strncasecmp(p, key, strlen(key)))
- continue;
- nval = strchr(p, '=');
- if (nval) {
- if (nval == p)
- continue;
- *nval++ = 0;
- *val = kstrdup(nval, GFP_KERNEL);
- rc = !*val ? -ENOMEM : 0;
- goto out;
- }
- }
-out:
- kfree(orig);
- return rc;
-}
-
/*
* Remove duplicate path delimiters. Windows is supposed to do that
* but there are some bugs that prevent rename from working if there are
@@ -767,7 +736,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc,
static int smb3_fs_context_parse_monolithic(struct fs_context *fc,
void *data);
static int smb3_get_tree(struct fs_context *fc);
-static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channels);
+static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels);
static int smb3_reconfigure(struct fs_context *fc);
static const struct fs_context_operations smb3_fs_context_ops = {
@@ -1041,25 +1010,34 @@ do { \
int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses)
{
+ char *password = NULL, *password2 = NULL;
+
if (ses->password &&
cifs_sb->ctx->password &&
strcmp(ses->password, cifs_sb->ctx->password)) {
- kfree_sensitive(cifs_sb->ctx->password);
- cifs_sb->ctx->password = kstrdup(ses->password, GFP_KERNEL);
- if (!cifs_sb->ctx->password)
+ password = kstrdup(ses->password, GFP_KERNEL);
+ if (!password)
return -ENOMEM;
}
if (ses->password2 &&
cifs_sb->ctx->password2 &&
strcmp(ses->password2, cifs_sb->ctx->password2)) {
- kfree_sensitive(cifs_sb->ctx->password2);
- cifs_sb->ctx->password2 = kstrdup(ses->password2, GFP_KERNEL);
- if (!cifs_sb->ctx->password2) {
- kfree_sensitive(cifs_sb->ctx->password);
- cifs_sb->ctx->password = NULL;
+ password2 = kstrdup(ses->password2, GFP_KERNEL);
+ if (!password2) {
+ kfree_sensitive(password);
return -ENOMEM;
}
}
+
+ if (password) {
+ kfree_sensitive(cifs_sb->ctx->password);
+ cifs_sb->ctx->password = password;
+ }
+ if (password2) {
+ kfree_sensitive(cifs_sb->ctx->password2);
+ cifs_sb->ctx->password2 = password2;
+ }
+
return 0;
}
@@ -1072,7 +1050,7 @@ int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_se
* with the session's channel lock. This should be called whenever the maximum
* allowed channels for a session changes (e.g., after a remount or reconfigure).
*/
-static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channels)
+static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels)
{
spin_lock(&ses->chan_lock);
ses->chan_max = max_channels;
@@ -1082,12 +1060,15 @@ static void smb3_sync_ses_chan_max(struct cifs_ses *ses, unsigned int max_channe
static int smb3_reconfigure(struct fs_context *fc)
{
struct smb3_fs_context *ctx = smb3_fc2context(fc);
+ struct smb3_fs_context *new_ctx = NULL;
+ struct smb3_fs_context *old_ctx = NULL;
struct dentry *root = fc->root;
struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb);
struct cifs_ses *ses = cifs_sb_master_tcon(cifs_sb)->ses;
unsigned int rsize = ctx->rsize, wsize = ctx->wsize;
char *new_password = NULL, *new_password2 = NULL;
bool need_recon = false;
+ bool need_mchan_update;
int rc;
if (ses->expired_pwd)
@@ -1097,6 +1078,16 @@ static int smb3_reconfigure(struct fs_context *fc)
if (rc)
return rc;
+ old_ctx = kzalloc_obj(*old_ctx);
+ if (!old_ctx)
+ return -ENOMEM;
+
+ rc = smb3_fs_context_dup(old_ctx, cifs_sb->ctx);
+ if (rc) {
+ kfree(old_ctx);
+ return rc;
+ }
+
/*
* We can not change UNC/username/password/domainname/
* workstation_name/nodename/iocharset
@@ -1106,16 +1097,22 @@ static int smb3_reconfigure(struct fs_context *fc)
STEAL_STRING(cifs_sb, ctx, UNC);
STEAL_STRING(cifs_sb, ctx, source);
STEAL_STRING(cifs_sb, ctx, username);
+ STEAL_STRING(cifs_sb, ctx, domainname);
+ STEAL_STRING(cifs_sb, ctx, nodename);
+ STEAL_STRING(cifs_sb, ctx, iocharset);
- if (need_recon == false)
+ if (!need_recon) {
STEAL_STRING_SENSITIVE(cifs_sb, ctx, password);
- else {
+ } else {
if (ctx->password) {
new_password = kstrdup(ctx->password, GFP_KERNEL);
- if (!new_password)
- return -ENOMEM;
- } else
+ if (!new_password) {
+ rc = -ENOMEM;
+ goto restore_ctx;
+ }
+ } else {
STEAL_STRING_SENSITIVE(cifs_sb, ctx, password);
+ }
}
/*
@@ -1125,11 +1122,29 @@ static int smb3_reconfigure(struct fs_context *fc)
if (ctx->password2) {
new_password2 = kstrdup(ctx->password2, GFP_KERNEL);
if (!new_password2) {
- kfree_sensitive(new_password);
- return -ENOMEM;
+ rc = -ENOMEM;
+ goto restore_ctx;
}
- } else
+ } else {
STEAL_STRING_SENSITIVE(cifs_sb, ctx, password2);
+ }
+
+ /* if rsize or wsize not passed in on remount, use previous values */
+ ctx->rsize = rsize ? CIFS_ALIGN_RSIZE(fc, rsize) : cifs_sb->ctx->rsize;
+ ctx->wsize = wsize ? CIFS_ALIGN_WSIZE(fc, wsize) : cifs_sb->ctx->wsize;
+
+ new_ctx = kzalloc_obj(*new_ctx);
+ if (!new_ctx) {
+ rc = -ENOMEM;
+ goto restore_ctx;
+ }
+
+ rc = smb3_fs_context_dup(new_ctx, ctx);
+ if (rc)
+ goto restore_ctx;
+
+ need_mchan_update = ctx->multichannel != cifs_sb->ctx->multichannel ||
+ ctx->max_channels != cifs_sb->ctx->max_channels;
/*
* we may update the passwords in the ses struct below. Make sure we do
@@ -1140,54 +1155,55 @@ static int smb3_reconfigure(struct fs_context *fc)
/*
* smb2_reconnect may swap password and password2 in case session setup
* failed. First get ctx passwords in sync with ses passwords. It should
- * be okay to do this even if this function were to return an error at a
- * later stage
+ * be done before committing new passwords.
*/
rc = smb3_sync_session_ctx_passwords(cifs_sb, ses);
if (rc) {
mutex_unlock(&ses->session_mutex);
- kfree_sensitive(new_password);
- kfree_sensitive(new_password2);
- return rc;
- }
-
- /*
- * now that allocations for passwords are done, commit them
- */
- if (new_password) {
- kfree_sensitive(ses->password);
- ses->password = new_password;
- }
- if (new_password2) {
- kfree_sensitive(ses->password2);
- ses->password2 = new_password2;
+ goto cleanup_new_ctx;
}
/*
* If multichannel or max_channels has changed, update the session's channels accordingly.
* This may add or remove channels to match the new configuration.
*/
- if ((ctx->multichannel != cifs_sb->ctx->multichannel) ||
- (ctx->max_channels != cifs_sb->ctx->max_channels)) {
-
- /* Synchronize ses->chan_max with the new mount context */
- smb3_sync_ses_chan_max(ses, ctx->max_channels);
- /* Now update the session's channels to match the new configuration */
+ if (need_mchan_update) {
/* Prevent concurrent scaling operations */
spin_lock(&ses->ses_lock);
if (ses->flags & CIFS_SES_FLAG_SCALE_CHANNELS) {
spin_unlock(&ses->ses_lock);
mutex_unlock(&ses->session_mutex);
- return -EINVAL;
+ rc = -EINVAL;
+ goto cleanup_new_ctx;
}
ses->flags |= CIFS_SES_FLAG_SCALE_CHANNELS;
spin_unlock(&ses->ses_lock);
+ }
+
+ /*
+ * Commit session passwords before any channel work so newly added
+ * channels authenticate with the new credentials.
+ */
+ if (new_password) {
+ kfree_sensitive(ses->password);
+ ses->password = new_password;
+ new_password = NULL;
+ }
+ if (new_password2) {
+ kfree_sensitive(ses->password2);
+ ses->password2 = new_password2;
+ new_password2 = NULL;
+ }
+
+ if (need_mchan_update) {
+ /* Synchronize ses->chan_max with the new mount context */
+ smb3_sync_ses_chan_max(ses, ctx->max_channels);
mutex_unlock(&ses->session_mutex);
- rc = smb3_update_ses_channels(ses, ses->server,
- false /* from_reconnect */,
- false /* disable_mchan */);
+ smb3_update_ses_channels(ses, ses->server,
+ false /* from_reconnect */,
+ false /* disable_mchan */);
/* Clear scaling flag after operation */
spin_lock(&ses->ses_lock);
@@ -1197,16 +1213,12 @@ static int smb3_reconfigure(struct fs_context *fc)
mutex_unlock(&ses->session_mutex);
}
- STEAL_STRING(cifs_sb, ctx, domainname);
- STEAL_STRING(cifs_sb, ctx, nodename);
- STEAL_STRING(cifs_sb, ctx, iocharset);
-
- /* if rsize or wsize not passed in on remount, use previous values */
- ctx->rsize = rsize ? CIFS_ALIGN_RSIZE(fc, rsize) : cifs_sb->ctx->rsize;
- ctx->wsize = wsize ? CIFS_ALIGN_WSIZE(fc, wsize) : cifs_sb->ctx->wsize;
-
smb3_cleanup_fs_context_contents(cifs_sb->ctx);
- rc = smb3_fs_context_dup(cifs_sb->ctx, ctx);
+ memcpy(cifs_sb->ctx, new_ctx, sizeof(*new_ctx));
+ kfree(new_ctx);
+ new_ctx = NULL;
+ smb3_cleanup_fs_context(old_ctx);
+ old_ctx = NULL;
smb3_update_mnt_flags(cifs_sb);
#ifdef CONFIG_CIFS_DFS_UPCALL
if (!rc)
@@ -1214,6 +1226,18 @@ static int smb3_reconfigure(struct fs_context *fc)
#endif
return rc;
+
+cleanup_new_ctx:
+ smb3_cleanup_fs_context_contents(new_ctx);
+restore_ctx:
+ kfree(new_ctx);
+ kfree_sensitive(new_password);
+ kfree_sensitive(new_password2);
+ smb3_cleanup_fs_context_contents(cifs_sb->ctx);
+ memcpy(cifs_sb->ctx, old_ctx, sizeof(*old_ctx));
+ kfree(old_ctx);
+
+ return rc;
}
static int smb3_fs_context_parse_param(struct fs_context *fc,
diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c
index 16a5310..9472c0a 100644
--- a/fs/smb/client/inode.c
+++ b/fs/smb/client/inode.c
@@ -119,7 +119,7 @@ cifs_revalidate_cache(struct inode *inode, struct cifs_fattr *fattr)
fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode);
mtime = inode_get_mtime(inode);
if (timespec64_equal(&mtime, &fattr->cf_mtime) &&
- cifs_i->netfs.remote_i_size == fattr->cf_eof) {
+ netfs_read_remote_i_size(inode) == fattr->cf_eof) {
cifs_dbg(FYI, "%s: inode %llu is unchanged\n",
__func__, cifs_i->uniqueid);
return;
@@ -173,12 +173,12 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr,
CIFS_I(inode)->time = 0; /* force reval */
return -ESTALE;
}
- if (inode_state_read_once(inode) & I_NEW)
- CIFS_I(inode)->netfs.zero_point = fattr->cf_eof;
-
cifs_revalidate_cache(inode, fattr);
spin_lock(&inode->i_lock);
+ if (inode_state_read_once(inode) & I_NEW)
+ netfs_write_zero_point(inode, fattr->cf_eof);
+
fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode);
fattr->cf_atime = timestamp_truncate(fattr->cf_atime, inode);
fattr->cf_ctime = timestamp_truncate(fattr->cf_ctime, inode);
@@ -212,7 +212,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr,
else
clear_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags);
- cifs_i->netfs.remote_i_size = fattr->cf_eof;
+ netfs_write_remote_i_size(inode, fattr->cf_eof);
/*
* Can't safely change the file size here if the client is writing to
* it due to potential races.
@@ -2772,7 +2772,9 @@ cifs_revalidate_mapping(struct inode *inode)
if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE)
goto skip_invalidate;
- cifs_inode->netfs.zero_point = cifs_inode->netfs.remote_i_size;
+ spin_lock(&inode->i_lock);
+ netfs_write_zero_point(inode, netfs_inode(inode)->_remote_i_size);
+ spin_unlock(&inode->i_lock);
rc = filemap_invalidate_inode(inode, true, 0, LLONG_MAX);
if (rc) {
cifs_dbg(VFS, "%s: invalidate inode %p failed with rc %d\n",
diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c
index 9afab32..17408bb 100644
--- a/fs/smb/client/ioctl.c
+++ b/fs/smb/client/ioctl.c
@@ -296,7 +296,7 @@ static int cifs_dump_full_key(struct cifs_tcon *tcon, struct smb3_full_key_debug
break;
case SMB2_ENCRYPTION_AES256_CCM:
case SMB2_ENCRYPTION_AES256_GCM:
- out.session_key_length = CIFS_SESS_KEY_SIZE;
+ out.session_key_length = ses->auth_key.len;
out.server_in_key_length = out.server_out_key_length = SMB3_GCM256_CRYPTKEY_SIZE;
break;
default:
diff --git a/fs/smb/client/netlink.c b/fs/smb/client/netlink.c
index 147d940..0dd1091 100644
--- a/fs/smb/client/netlink.c
+++ b/fs/smb/client/netlink.c
@@ -33,13 +33,17 @@ static const struct nla_policy cifs_genl_policy[CIFS_GENL_ATTR_MAX + 1] = {
static const struct genl_ops cifs_genl_ops[] = {
{
.cmd = CIFS_GENL_CMD_SWN_NOTIFY,
+ .flags = GENL_ADMIN_PERM,
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
.doit = cifs_swn_notify,
},
};
static const struct genl_multicast_group cifs_genl_mcgrps[] = {
- [CIFS_GENL_MCGRP_SWN] = { .name = CIFS_GENL_MCGRP_SWN_NAME },
+ [CIFS_GENL_MCGRP_SWN] = {
+ .name = CIFS_GENL_MCGRP_SWN_NAME,
+ .flags = GENL_MCAST_CAP_NET_ADMIN,
+ },
};
struct genl_family cifs_genl_family = {
diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c
index be22bbc..e860fa0 100644
--- a/fs/smb/client/readdir.c
+++ b/fs/smb/client/readdir.c
@@ -143,7 +143,8 @@ cifs_prime_dcache(struct dentry *parent, struct qstr *name,
fattr->cf_rdev = inode->i_rdev;
fattr->cf_uid = inode->i_uid;
fattr->cf_gid = inode->i_gid;
- fattr->cf_eof = CIFS_I(inode)->netfs.remote_i_size;
+ fattr->cf_eof =
+ netfs_read_remote_i_size(inode);
fattr->cf_symlink_target = NULL;
} else {
CIFS_I(inode)->time = 0;
diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c
index b292aa9..6860eff 100644
--- a/fs/smb/client/smb2file.c
+++ b/fs/smb/client/smb2file.c
@@ -49,6 +49,9 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov)
__func__, le32_to_cpu(p->ErrorId));
len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8);
+ if (len > end - ((u8 *)p + sizeof(*p)))
+ return ERR_PTR(-EINVAL);
+
p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len);
}
} else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) &&
diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c
index c6dd282..6c9c229 100644
--- a/fs/smb/client/smb2inode.c
+++ b/fs/smb/client/smb2inode.c
@@ -111,7 +111,7 @@ static int check_wsl_eas(struct kvec *rsp_iov)
u32 outlen, next;
u16 vlen;
u8 nlen;
- u8 *end;
+ u8 *ea_end, *iov_end;
outlen = le32_to_cpu(rsp->OutputBufferLength);
if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE ||
@@ -120,15 +120,19 @@ static int check_wsl_eas(struct kvec *rsp_iov)
ea = (void *)((u8 *)rsp_iov->iov_base +
le16_to_cpu(rsp->OutputBufferOffset));
- end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len;
+ ea_end = (u8 *)ea + outlen;
+ iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len;
+ if (ea_end > iov_end)
+ return -EINVAL;
+
for (;;) {
- if ((u8 *)ea > end - sizeof(*ea))
+ if ((u8 *)ea > ea_end - sizeof(*ea))
return -EINVAL;
nlen = ea->ea_name_length;
vlen = le16_to_cpu(ea->ea_value_length);
if (nlen != SMB2_WSL_XATTR_NAME_LEN ||
- (u8 *)ea->ea_data + nlen + 1 + vlen > end)
+ (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end)
return -EINVAL;
switch (vlen) {
@@ -230,7 +234,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon,
num_rqst = 0;
server = cifs_pick_channel(ses);
- vars = kzalloc_obj(*vars, GFP_ATOMIC);
+ vars = kzalloc_obj(*vars, GFP_KERNEL);
if (vars == NULL) {
rc = -ENOMEM;
goto out;
diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c
index 973fce3..2a7355c 100644
--- a/fs/smb/client/smb2misc.c
+++ b/fs/smb/client/smb2misc.c
@@ -241,7 +241,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len,
if (len != calc_len) {
/* create failed on symlink */
if (command == SMB2_CREATE_HE &&
- shdr->Status == STATUS_STOPPED_ON_SYMLINK)
+ shdr->Status == STATUS_STOPPED_ON_SYMLINK &&
+ len > calc_len)
return 0;
/* Windows 7 server returns 24 bytes more */
if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE)
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 7f346ee..d4875f9 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -111,10 +111,21 @@ smb2_add_credits(struct TCP_Server_Info *server,
cifs_trace_rw_credits_zero_in_flight);
}
server->in_flight--;
+
+ /*
+ * Rebalance credits when an op drains in_flight. For session setup,
+ * do this only when the total accumulated credits are high enough (>2)
+ * so that a newly established secondary channel can reserve credits for
+ * echoes and oplocks. We expect this to happen at the end of the final
+ * session setup response.
+ */
if (server->in_flight == 0 &&
((optype & CIFS_OP_MASK) != CIFS_NEG_OP) &&
((optype & CIFS_OP_MASK) != CIFS_SESS_OP))
rc = change_conf(server);
+ else if (server->in_flight == 0 &&
+ ((optype & CIFS_OP_MASK) == CIFS_SESS_OP) && *val > 2)
+ rc = change_conf(server);
/*
* Sometimes server returns 0 credits on oplock break ack - we need to
* rebalance credits in this case.
@@ -3391,8 +3402,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
struct inode *inode = file_inode(file);
struct cifsInodeInfo *cifsi = CIFS_I(inode);
struct cifsFileInfo *cfile = file->private_data;
- struct netfs_inode *ictx = netfs_inode(inode);
- unsigned long long i_size, new_size, remote_size;
+ unsigned long long i_size, new_size, remote_i_size, zero_point;
long rc;
unsigned int xid;
@@ -3403,9 +3413,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
filemap_invalidate_lock(inode->i_mapping);
- i_size = i_size_read(inode);
- remote_size = ictx->remote_i_size;
- if (offset + len >= remote_size && offset < i_size) {
+ netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+ if (offset + len >= remote_i_size && offset < i_size) {
unsigned long long top = umin(offset + len, i_size);
rc = filemap_write_and_wait_range(inode->i_mapping, offset, top - 1);
@@ -3438,9 +3447,11 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
cfile->fid.volatile_fid, cfile->pid, new_size);
if (rc >= 0) {
truncate_setsize(inode, new_size);
+ spin_lock(&inode->i_lock);
netfs_resize_file(&cifsi->netfs, new_size, true);
- if (offset < cifsi->netfs.zero_point)
- cifsi->netfs.zero_point = offset;
+ if (offset < cifsi->netfs._zero_point)
+ netfs_write_zero_point(inode, offset);
+ spin_unlock(&inode->i_lock);
fscache_resize_cookie(cifs_inode_cookie(inode), new_size);
}
}
@@ -3463,7 +3474,7 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon,
struct inode *inode = file_inode(file);
struct cifsFileInfo *cfile = file->private_data;
struct file_zero_data_information fsctl_buf;
- unsigned long long end = offset + len, i_size, remote_i_size;
+ unsigned long long end = offset + len, i_size, remote_i_size, zero_point;
long rc;
unsigned int xid;
__u8 set_sparse = 1;
@@ -3505,14 +3516,17 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon,
* that we locally hole-punch the tail of the dirty data, the proposed
* EOF update will end up in the wrong place.
*/
- i_size = i_size_read(inode);
- remote_i_size = netfs_inode(inode)->remote_i_size;
+ netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+
if (end > remote_i_size && i_size > remote_i_size) {
unsigned long long extend_to = umin(end, i_size);
rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
cfile->fid.volatile_fid, cfile->pid, extend_to);
- if (rc >= 0)
- netfs_inode(inode)->remote_i_size = extend_to;
+ if (rc >= 0) {
+ spin_lock(&inode->i_lock);
+ netfs_write_remote_i_size(inode, extend_to);
+ spin_unlock(&inode->i_lock);
+ }
}
unlock:
@@ -3776,7 +3790,6 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
struct inode *inode = file_inode(file);
struct cifsInodeInfo *cifsi = CIFS_I(inode);
struct cifsFileInfo *cfile = file->private_data;
- struct netfs_inode *ictx = &cifsi->netfs;
loff_t old_eof, new_eof;
xid = get_xid();
@@ -3794,7 +3807,9 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
goto out_2;
truncate_pagecache_range(inode, off, old_eof);
- ictx->zero_point = old_eof;
+ spin_lock(&inode->i_lock);
+ netfs_write_zero_point(inode, old_eof);
+ spin_unlock(&inode->i_lock);
netfs_wait_for_outstanding_io(inode);
rc = smb2_copychunk_range(xid, cfile, cfile, off + len,
@@ -3811,8 +3826,10 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
rc = 0;
truncate_setsize(inode, new_eof);
+ spin_lock(&inode->i_lock);
netfs_resize_file(&cifsi->netfs, new_eof, true);
- ictx->zero_point = new_eof;
+ netfs_write_zero_point(inode, new_eof);
+ spin_unlock(&inode->i_lock);
fscache_resize_cookie(cifs_inode_cookie(inode), new_eof);
out_2:
filemap_invalidate_unlock(inode->i_mapping);
@@ -3855,13 +3872,17 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon,
goto out_2;
truncate_setsize(inode, new_eof);
+ spin_lock(&inode->i_lock);
netfs_resize_file(&cifsi->netfs, i_size_read(inode), true);
+ spin_unlock(&inode->i_lock);
fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode));
rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len);
if (rc < 0)
goto out_2;
- cifsi->netfs.zero_point = new_eof;
+ spin_lock(&inode->i_lock);
+ netfs_write_zero_point(inode, new_eof);
+ spin_unlock(&inode->i_lock);
rc = smb3_zero_data(file, tcon, off, len, xid);
if (rc < 0)
@@ -4685,9 +4706,15 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size,
{
for (; folioq; folioq = folioq->next) {
for (int s = 0; s < folioq_count(folioq); s++) {
- struct folio *folio = folioq_folio(folioq, s);
- size_t fsize = folio_size(folio);
- size_t n, len = umin(fsize - skip, data_size);
+ struct folio *folio;
+ size_t fsize, n, len;
+
+ if (data_size == 0)
+ return 0;
+
+ folio = folioq_folio(folioq, s);
+ fsize = folio_size(folio);
+ len = umin(fsize - skip, data_size);
n = copy_folio_to_iter(folio, skip, len, iter);
if (n != len) {
@@ -4700,6 +4727,12 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size,
}
}
+ if (data_size != 0) {
+ cifs_dbg(VFS, "%s: short copy, %zu bytes missing\n",
+ __func__, data_size);
+ return smb_EIO2(smb_eio_trace_rx_copy_to_iter, 0, data_size);
+ }
+
return 0;
}
@@ -4710,6 +4743,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
{
unsigned int data_offset;
unsigned int data_len;
+ unsigned int end_off;
unsigned int cur_off;
unsigned int cur_page_idx;
unsigned int pad_len;
@@ -4814,7 +4848,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
}
/* Copy the data to the output I/O iterator. */
- rdata->result = cifs_copy_folioq_to_iter(buffer, buffer_len,
+ rdata->result = cifs_copy_folioq_to_iter(buffer, data_len,
cur_off, &rdata->subreq.io_iter);
if (rdata->result != 0) {
if (is_offloaded)
@@ -4823,9 +4857,10 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
dequeue_mid(server, mid, rdata->result);
return 0;
}
- rdata->got_bytes = buffer_len;
+ rdata->got_bytes = data_len;
- } else if (buf_len >= data_offset + data_len) {
+ } else if (!check_add_overflow(data_offset, data_len, &end_off) &&
+ buf_len >= end_off) {
/* read response payload is in buf */
WARN_ONCE(buffer, "read data can be either in buf or in buffer");
copied = copy_to_iter(buf + data_offset, data_len, &rdata->subreq.io_iter);
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index cb61051..fbeb215 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -1713,17 +1713,30 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data)
is_binding = (ses->ses_status == SES_GOOD);
spin_unlock(&ses->ses_lock);
+ /*
+ * Per MS-SMB2 3.2.5.3, Session.SessionKey is the first 16 bytes of the
+ * GSS cryptographic key, right-padded with zero bytes if shorter.
+ * Allocate at least SMB2_NTLMV2_SESSKEY_SIZE bytes (zeroed) so the KDF
+ * input buffer is always valid for HMAC-SHA256 even with deprecated
+ * Kerberos enctypes that return a short session key.
+ */
+ if (unlikely(msg->sesskey_len < SMB2_NTLMV2_SESSKEY_SIZE))
+ cifs_dbg(VFS,
+ "short GSS session key (%u bytes); zero-padding per MS-SMB2 3.2.5.3\n",
+ msg->sesskey_len);
+
kfree_sensitive(ses->auth_key.response);
- ses->auth_key.response = kmemdup(msg->data,
- msg->sesskey_len,
- GFP_KERNEL);
+ ses->auth_key.len = max_t(unsigned int, msg->sesskey_len,
+ SMB2_NTLMV2_SESSKEY_SIZE);
+ ses->auth_key.response = kzalloc(ses->auth_key.len, GFP_KERNEL);
if (!ses->auth_key.response) {
cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n",
- __func__, msg->sesskey_len);
+ __func__, ses->auth_key.len);
+ ses->auth_key.len = 0;
rc = -ENOMEM;
goto out_put_spnego_key;
}
- ses->auth_key.len = msg->sesskey_len;
+ memcpy(ses->auth_key.response, msg->data, msg->sesskey_len);
sess_data->iov[1].iov_base = msg->data + msg->sesskey_len;
sess_data->iov[1].iov_len = msg->secblob_len;
@@ -4595,6 +4608,7 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode);
struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink);
struct smb2_hdr *shdr = (struct smb2_hdr *)rdata->iov[0].iov_base;
+ struct inode *inode = &ictx->inode;
struct cifs_credits credits = {
.value = 0,
.instance = 0,
@@ -4708,7 +4722,7 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
} else {
size_t trans = rdata->subreq.transferred + rdata->got_bytes;
if (trans < rdata->subreq.len &&
- rdata->subreq.start + trans >= ictx->remote_i_size) {
+ rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) {
__set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
rdata->result = 0;
}
@@ -4941,7 +4955,7 @@ smb2_writev_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
unsigned int rreq_debug_id = wdata->rreq->debug_id;
unsigned int subreq_debug_index = wdata->subreq.debug_index;
ssize_t result = 0;
- size_t written;
+ size_t written = 0;
WARN_ONCE(wdata->server != server,
"wdata server %p != mid server %p",
diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c
index 4100903..1143ee5 100644
--- a/fs/smb/client/smb2transport.c
+++ b/fs/smb/client/smb2transport.c
@@ -169,7 +169,9 @@ smb2_find_smb_sess_tcon_unlocked(struct cifs_ses *ses, __u32 tid)
list_for_each_entry(tcon, &ses->tcon_list, tcon_list) {
if (tcon->tid != tid)
continue;
+ spin_lock(&tcon->tc_lock);
++tcon->tc_count;
+ spin_unlock(&tcon->tc_lock);
trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count,
netfs_trace_tcon_ref_get_find_sess_tcon);
return tcon;
@@ -251,7 +253,8 @@ smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server)
}
static void generate_key(struct cifs_ses *ses, struct kvec label,
- struct kvec context, __u8 *key, unsigned int key_size)
+ struct kvec context, __u8 *key, unsigned int key_size,
+ unsigned int full_key_size)
{
unsigned char zero = 0x0;
__u8 i[4] = {0, 0, 0, 1};
@@ -265,7 +268,7 @@ static void generate_key(struct cifs_ses *ses, struct kvec label,
memset(key, 0x0, key_size);
hmac_sha256_init_usingrawkey(&hmac_ctx, ses->auth_key.response,
- SMB2_NTLMV2_SESSKEY_SIZE);
+ full_key_size);
hmac_sha256_update(&hmac_ctx, i, 4);
hmac_sha256_update(&hmac_ctx, label.iov_base, label.iov_len);
hmac_sha256_update(&hmac_ctx, &zero, 1);
@@ -298,6 +301,7 @@ generate_smb3signingkey(struct cifs_ses *ses,
struct TCP_Server_Info *server,
const struct derivation_triplet *ptriplet)
{
+ unsigned int full_key_size = SMB2_NTLMV2_SESSKEY_SIZE;
bool is_binding = false;
int chan_index = 0;
@@ -330,12 +334,24 @@ generate_smb3signingkey(struct cifs_ses *ses,
if (is_binding) {
generate_key(ses, ptriplet->signing.label,
ptriplet->signing.context,
- ses->chans[chan_index].signkey,
- SMB3_SIGN_KEY_SIZE);
+ ses->chans[chan_index].signkey, SMB3_SIGN_KEY_SIZE,
+ SMB2_NTLMV2_SESSKEY_SIZE);
} else {
generate_key(ses, ptriplet->signing.label,
- ptriplet->signing.context,
- ses->smb3signingkey, SMB3_SIGN_KEY_SIZE);
+ ptriplet->signing.context, ses->smb3signingkey,
+ SMB3_SIGN_KEY_SIZE, SMB2_NTLMV2_SESSKEY_SIZE);
+
+ /*
+ * Per MS-SMB2 3.2.5.3.1, signing key always uses Session.SessionKey
+ * (first 16 bytes). Encryption/decryption keys use
+ * Session.FullSessionKey when dialect is 3.1.1 and cipher is
+ * AES-256-CCM or AES-256-GCM, otherwise Session.SessionKey.
+ */
+
+ if (server->dialect == SMB311_PROT_ID &&
+ (server->cipher_type == SMB2_ENCRYPTION_AES256_CCM ||
+ server->cipher_type == SMB2_ENCRYPTION_AES256_GCM))
+ full_key_size = ses->auth_key.len;
/* safe to access primary channel, since it will never go away */
spin_lock(&ses->chan_lock);
@@ -345,10 +361,13 @@ generate_smb3signingkey(struct cifs_ses *ses,
generate_key(ses, ptriplet->encryption.label,
ptriplet->encryption.context,
- ses->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE);
+ ses->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE,
+ full_key_size);
+
generate_key(ses, ptriplet->decryption.label,
ptriplet->decryption.context,
- ses->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE);
+ ses->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE,
+ full_key_size);
}
#ifdef CONFIG_CIFS_DEBUG_DUMP_KEYS
@@ -361,7 +380,7 @@ generate_smb3signingkey(struct cifs_ses *ses,
&ses->Suid);
cifs_dbg(VFS, "Cipher type %d\n", server->cipher_type);
cifs_dbg(VFS, "Session Key %*ph\n",
- SMB2_NTLMV2_SESSKEY_SIZE, ses->auth_key.response);
+ (int)ses->auth_key.len, ses->auth_key.response);
cifs_dbg(VFS, "Signing Key %*ph\n",
SMB3_SIGN_KEY_SIZE, ses->smb3signingkey);
if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) ||
diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c
index 75f9f91..563ef488 100644
--- a/fs/smb/client/smbdirect.c
+++ b/fs/smb/client/smbdirect.c
@@ -9,7 +9,6 @@
#include "cifs_debug.h"
#include "cifsproto.h"
#include "smb2proto.h"
-#include "../smbdirect/public.h"
/* Port numbers for SMBD transport */
#define SMB_PORT 445
@@ -558,3 +557,5 @@ void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m)
server->rdma_readwrite_threshold,
m);
}
+
+MODULE_IMPORT_NS("SMBDIRECT");
diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h
index 287ac84..be205ec 100644
--- a/fs/smb/client/smbdirect.h
+++ b/fs/smb/client/smbdirect.h
@@ -12,7 +12,7 @@
#include "cifsglob.h"
-#include "../smbdirect/smbdirect.h"
+#include <linux/smbdirect.h>
extern int rdma_readwrite_threshold;
extern int smbd_max_frmr_depth;
diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c
index 05f8099..fdf4e50 100644
--- a/fs/smb/client/transport.c
+++ b/fs/smb/client/transport.c
@@ -1158,7 +1158,7 @@ int
cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
{
int length, len;
- unsigned int data_offset, data_len;
+ unsigned int data_offset, data_len, end_off;
struct cifs_io_subrequest *rdata = mid->callback_data;
char *buf = server->smallbuf;
unsigned int buflen = server->pdu_size;
@@ -1256,11 +1256,14 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid)
use_rdma_mr = rdata->mr;
#endif
data_len = server->ops->read_data_length(buf, use_rdma_mr);
- if (!use_rdma_mr && (data_offset + data_len > buflen)) {
- /* data_len is corrupt -- discard frame */
- rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed,
- data_offset + data_len, buflen);
- return cifs_readv_discard(server, mid);
+ if (!use_rdma_mr) {
+ if (check_add_overflow(data_offset, data_len, &end_off) ||
+ end_off > buflen) {
+ /* data_len is corrupt -- discard frame */
+ rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed,
+ end_off, buflen);
+ return cifs_readv_discard(server, mid);
+ }
}
#ifdef CONFIG_CIFS_SMB_DIRECT
diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h
index b4ccddc..bc3012c 100644
--- a/fs/smb/common/fscc.h
+++ b/fs/smb/common/fscc.h
@@ -260,12 +260,12 @@ typedef struct {
char FileName[];
} __packed FILE_DIRECTORY_INFO; /* level 0x101 FF resp data */
-/* See MS-FSCC 2.4.13 */
+/* See MS-FSCC 2.4.14 */
struct smb2_file_eof_info { /* encoding of request for level 10 */
__le64 EndOfFile; /* new end of file value */
} __packed; /* level 20 Set */
-/* See MS-FSCC 2.4.14 */
+/* See MS-FSCC 2.4.15 */
typedef struct {
__le32 NextEntryOffset;
__u32 FileIndex;
diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h
index a4b12eb..aeb0a24 100644
--- a/fs/smb/common/smb2pdu.h
+++ b/fs/smb/common/smb2pdu.h
@@ -1566,6 +1566,10 @@ struct validate_negotiate_info_rsp {
#define FILE_STANDARD_LINK_INFORMATION 54
#define FILE_ID_INFORMATION 59
#define FILE_ID_EXTD_DIRECTORY_INFORMATION 60 /* also for QUERY_DIR */
+#define FileId64ExtdDirectoryInformation 78 /* also for QUERY_DIR */
+#define FileId64ExtdBothDirectoryInformation 79 /* also for QUERY_DIR */
+#define FileIdAllExtdDirectoryInformation 80 /* also for QUERY_DIR */
+#define FileIdAllExtdBothDirectoryInformation 81 /* also for QUERY_DIR */
/* Used for Query Info and Find File POSIX Info for SMB3.1.1 and SMB1 */
#define SMB_FIND_FILE_POSIX_INFO 0x064
diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c
index fbbc052..8347495 100644
--- a/fs/smb/server/connection.c
+++ b/fs/smb/server/connection.c
@@ -79,6 +79,85 @@ static int create_proc_clients(void) { return 0; }
static void delete_proc_clients(void) {}
#endif
+static struct workqueue_struct *ksmbd_conn_wq;
+
+int ksmbd_conn_wq_init(void)
+{
+ ksmbd_conn_wq = alloc_workqueue("ksmbd-conn-release",
+ WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
+ if (!ksmbd_conn_wq)
+ return -ENOMEM;
+ return 0;
+}
+
+void ksmbd_conn_wq_destroy(void)
+{
+ if (ksmbd_conn_wq) {
+ destroy_workqueue(ksmbd_conn_wq);
+ ksmbd_conn_wq = NULL;
+ }
+}
+
+/*
+ * __ksmbd_conn_release_work() - perform the final, once-per-struct cleanup
+ * of a ksmbd_conn whose refcount has just dropped to zero.
+ *
+ * This is the common release path used by ksmbd_conn_put() for the embedded
+ * state that outlives the connection thread: async_ida and the attached
+ * transport (which owns the socket and iov for TCP). Called from a workqueue
+ * so that sleep-allowed teardown (sock_release -> tcp_close ->
+ * lock_sock_nested) never runs from an RCU softirq callback (free_opinfo_rcu)
+ * or any other non-sleeping putter context.
+ */
+static void __ksmbd_conn_release_work(struct work_struct *work)
+{
+ struct ksmbd_conn *conn =
+ container_of(work, struct ksmbd_conn, release_work);
+
+ ida_destroy(&conn->async_ida);
+ conn->transport->ops->free_transport(conn->transport);
+ kfree(conn);
+}
+
+/**
+ * ksmbd_conn_get() - take a reference on @conn and return it.
+ *
+ * @conn: connection instance to get a reference to
+ *
+ * Returns @conn unchanged so callers can write
+ * "fp->conn = ksmbd_conn_get(work->conn);" in one expression. Returns NULL
+ * if @conn is NULL.
+ */
+struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn)
+{
+ if (!conn)
+ return NULL;
+
+ atomic_inc(&conn->refcnt);
+ return conn;
+}
+
+/**
+ * ksmbd_conn_put() - drop a reference and, if it was the last, queue the
+ * release onto ksmbd_conn_wq so it runs from process context.
+ *
+ * @conn: connection instance to put a reference to
+ *
+ * Callable from any context including RCU softirq callbacks and non-sleeping
+ * locks; the actual release is deferred to the workqueue. ksmbd_conn_wq is
+ * created in ksmbd_server_init() before any conn can be allocated and is
+ * destroyed in ksmbd_server_exit() after rcu_barrier(), so it is always
+ * non-NULL while a conn reference is held.
+ */
+void ksmbd_conn_put(struct ksmbd_conn *conn)
+{
+ if (!conn)
+ return;
+
+ if (atomic_dec_and_test(&conn->refcnt))
+ queue_work(ksmbd_conn_wq, &conn->release_work);
+}
+
/**
* ksmbd_conn_free() - free resources of the connection instance
*
@@ -93,23 +172,19 @@ void ksmbd_conn_free(struct ksmbd_conn *conn)
hash_del(&conn->hlist);
up_write(&conn_list_lock);
+ /*
+ * request_buf / preauth_info / mechToken are only ever accessed by the
+ * connection handler thread that owns @conn. ksmbd_conn_free() is
+ * called from the transport free_transport() path when that thread is
+ * exiting, so it is safe to release them unconditionally even when
+ * ksmbd_conn_put() below is not the final putter (oplock / ksmbd_file
+ * holders only retain the conn pointer, not these per-thread buffers).
+ */
xa_destroy(&conn->sessions);
kvfree(conn->request_buf);
kfree(conn->preauth_info);
kfree(conn->mechToken);
- if (atomic_dec_and_test(&conn->refcnt)) {
- /*
- * async_ida is embedded in struct ksmbd_conn, so pair
- * ida_destroy() with the final kfree() rather than with
- * the unconditional field teardown above. This keeps
- * the IDA valid for the entire lifetime of the struct,
- * even while other refcount holders (oplock / vfs
- * durable handles) still reference the connection.
- */
- ida_destroy(&conn->async_ida);
- conn->transport->ops->free_transport(conn->transport);
- kfree(conn);
- }
+ ksmbd_conn_put(conn);
}
/**
@@ -136,6 +211,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void)
conn->um = ERR_PTR(-EOPNOTSUPP);
if (IS_ERR(conn->um))
conn->um = NULL;
+ INIT_WORK(&conn->release_work, __ksmbd_conn_release_work);
atomic_set(&conn->req_running, 0);
atomic_set(&conn->r_count, 0);
atomic_set(&conn->refcnt, 1);
@@ -512,8 +588,7 @@ void ksmbd_conn_r_count_dec(struct ksmbd_conn *conn)
if (!atomic_dec_return(&conn->r_count) && waitqueue_active(&conn->r_count_q))
wake_up(&conn->r_count_q);
- if (atomic_dec_and_test(&conn->refcnt))
- kfree(conn);
+ ksmbd_conn_put(conn);
}
int ksmbd_conn_transport_init(void)
@@ -540,24 +615,54 @@ int ksmbd_conn_transport_init(void)
static void stop_sessions(void)
{
- struct ksmbd_conn *conn;
+ struct ksmbd_conn *conn, *target;
struct ksmbd_transport *t;
+ bool any;
int bkt;
+ /*
+ * Serialised via init_lock; no concurrent stop_sessions() can
+ * touch conn->stop_called, so writing it under the read lock is
+ * safe.
+ */
again:
+ target = NULL;
+ any = false;
down_read(&conn_list_lock);
hash_for_each(conn_list, bkt, conn, hlist) {
- t = conn->transport;
- ksmbd_conn_set_exiting(conn);
- if (t->ops->shutdown) {
- up_read(&conn_list_lock);
- t->ops->shutdown(t);
- down_read(&conn_list_lock);
- }
+ any = true;
+ if (conn->stop_called)
+ continue;
+ atomic_inc(&conn->refcnt);
+ conn->stop_called = true;
+ /*
+ * Mark the connection EXITING while still holding the
+ * read lock so the selection and the status transition
+ * happen together. Do not regress a connection that has
+ * already advanced to RELEASING on its own (e.g. the
+ * handler exited its receive loop for an unrelated
+ * reason).
+ */
+ if (READ_ONCE(conn->status) != KSMBD_SESS_RELEASING)
+ ksmbd_conn_set_exiting(conn);
+ target = conn;
+ break;
}
up_read(&conn_list_lock);
- if (!hash_empty(conn_list)) {
+ if (target) {
+ t = target->transport;
+ if (t->ops->shutdown)
+ t->ops->shutdown(t);
+ if (atomic_dec_and_test(&target->refcnt)) {
+ ida_destroy(&target->async_ida);
+ t->ops->free_transport(t);
+ kfree(target);
+ }
+ goto again;
+ }
+
+ if (any) {
msleep(100);
goto again;
}
diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h
index ae21a1b..e074be9 100644
--- a/fs/smb/server/connection.h
+++ b/fs/smb/server/connection.h
@@ -16,6 +16,7 @@
#include <linux/kthread.h>
#include <linux/nls.h>
#include <linux/unicode.h>
+#include <linux/workqueue.h>
#include "smb_common.h"
#include "ksmbd_work.h"
@@ -49,6 +50,7 @@ struct ksmbd_conn {
struct mutex srv_mutex;
int status;
unsigned int cli_cap;
+ bool stop_called;
union {
__be32 inet_addr;
#if IS_ENABLED(CONFIG_IPV6)
@@ -119,6 +121,7 @@ struct ksmbd_conn {
bool binding;
atomic_t refcnt;
bool is_aapl;
+ struct work_struct release_work;
};
struct ksmbd_conn_ops {
@@ -163,6 +166,10 @@ void ksmbd_conn_wait_idle(struct ksmbd_conn *conn);
int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id);
struct ksmbd_conn *ksmbd_conn_alloc(void);
void ksmbd_conn_free(struct ksmbd_conn *conn);
+struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn);
+void ksmbd_conn_put(struct ksmbd_conn *conn);
+int ksmbd_conn_wq_init(void);
+void ksmbd_conn_wq_destroy(void);
bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c);
int ksmbd_conn_write(struct ksmbd_work *work);
int ksmbd_conn_rdma_read(struct ksmbd_conn *conn,
diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c
index 53f44ff..6f97f8d 100644
--- a/fs/smb/server/mgmt/share_config.c
+++ b/fs/smb/server/mgmt/share_config.c
@@ -167,7 +167,10 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work,
share->path = kstrndup(ksmbd_share_config_path(resp), path_len,
KSMBD_DEFAULT_GFP);
- if (share->path) {
+ if (!share->path) {
+ ret = -ENOMEM;
+ } else {
+ ret = 0;
share->path_sz = strlen(share->path);
while (share->path_sz > 1 &&
share->path[share->path_sz - 1] == '/')
@@ -179,9 +182,10 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work,
share->force_directory_mode = resp->force_directory_mode;
share->force_uid = resp->force_uid;
share->force_gid = resp->force_gid;
- ret = parse_veto_list(share,
- KSMBD_SHARE_CONFIG_VETO_LIST(resp),
- resp->veto_list_sz);
+ if (!ret)
+ ret = parse_veto_list(share,
+ KSMBD_SHARE_CONFIG_VETO_LIST(resp),
+ resp->veto_list_sz);
if (!ret && share->path) {
if (__ksmbd_override_fsids(work, share)) {
kill_share(share);
diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c
index cd3f28b..0f5c185 100644
--- a/fs/smb/server/oplock.c
+++ b/fs/smb/server/oplock.c
@@ -30,7 +30,6 @@ static DEFINE_RWLOCK(lease_list_lock);
static struct oplock_info *alloc_opinfo(struct ksmbd_work *work,
u64 id, __u16 Tid)
{
- struct ksmbd_conn *conn = work->conn;
struct ksmbd_session *sess = work->sess;
struct oplock_info *opinfo;
@@ -39,7 +38,7 @@ static struct oplock_info *alloc_opinfo(struct ksmbd_work *work,
return NULL;
opinfo->sess = sess;
- opinfo->conn = conn;
+ opinfo->conn = ksmbd_conn_get(work->conn);
opinfo->level = SMB2_OPLOCK_LEVEL_NONE;
opinfo->op_state = OPLOCK_STATE_NONE;
opinfo->pending_break = 0;
@@ -50,7 +49,6 @@ static struct oplock_info *alloc_opinfo(struct ksmbd_work *work,
init_waitqueue_head(&opinfo->oplock_brk);
atomic_set(&opinfo->refcount, 1);
atomic_set(&opinfo->breaking_cnt, 0);
- atomic_inc(&opinfo->conn->refcnt);
return opinfo;
}
@@ -132,8 +130,7 @@ static void __free_opinfo(struct oplock_info *opinfo)
{
if (opinfo->is_lease)
free_lease(opinfo);
- if (opinfo->conn && atomic_dec_and_test(&opinfo->conn->refcnt))
- kfree(opinfo->conn);
+ ksmbd_conn_put(opinfo->conn);
kfree(opinfo);
}
@@ -484,8 +481,12 @@ static inline int compare_guid_key(struct oplock_info *opinfo,
const char *guid1, const char *key1)
{
const char *guid2, *key2;
+ struct ksmbd_conn *conn;
- guid2 = opinfo->conn->ClientGUID;
+ conn = READ_ONCE(opinfo->conn);
+ if (!conn)
+ return 0;
+ guid2 = conn->ClientGUID;
key2 = opinfo->o_lease->lease_key;
if (!memcmp(guid1, guid2, SMB2_CLIENT_GUID_SIZE) &&
!memcmp(key1, key2, SMB2_LEASE_KEY_SIZE))
diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c
index 58ef02c..5d799b2 100644
--- a/fs/smb/server/server.c
+++ b/fs/smb/server/server.c
@@ -596,8 +596,14 @@ static int __init ksmbd_server_init(void)
if (ret)
goto err_crypto_destroy;
+ ret = ksmbd_conn_wq_init();
+ if (ret)
+ goto err_workqueue_destroy;
+
return 0;
+err_workqueue_destroy:
+ ksmbd_workqueue_destroy();
err_crypto_destroy:
ksmbd_crypto_destroy();
err_release_inode_hash:
@@ -623,6 +629,12 @@ static void __exit ksmbd_server_exit(void)
{
ksmbd_server_shutdown();
rcu_barrier();
+ /*
+ * ksmbd_conn_put() defers the final release onto ksmbd_conn_wq,
+ * so drain it after rcu_barrier() has fired any pending RCU
+ * callbacks that may have queued a release.
+ */
+ ksmbd_conn_wq_destroy();
ksmbd_release_inode_hash();
}
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 21825a6..620bcfb 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -3767,8 +3767,10 @@ int smb2_open(struct ksmbd_work *work)
err_out2:
if (!rc) {
- ksmbd_update_fstate(&work->sess->file_table, fp, FP_INITED);
- rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len);
+ rc = ksmbd_update_fstate(&work->sess->file_table, fp,
+ FP_INITED);
+ if (!rc)
+ rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len);
}
if (rc) {
if (rc == -EINVAL)
@@ -3802,8 +3804,19 @@ int smb2_open(struct ksmbd_work *work)
ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status);
}
- if (dh_info.reconnected)
- ksmbd_put_durable_fd(dh_info.fp);
+ if (dh_info.reconnected) {
+ /*
+ * If reconnect succeeded, fp was republished in the
+ * session file table. On a later error, ksmbd_fd_put()
+ * above drops the session reference; drop the durable
+ * lookup reference through the same session-aware path so
+ * final close removes the volatile id before freeing fp.
+ */
+ if (rc && fp == dh_info.fp)
+ ksmbd_fd_put(work, dh_info.fp);
+ else
+ ksmbd_put_durable_fd(dh_info.fp);
+ }
kfree(name);
kfree(lc);
@@ -3946,7 +3959,13 @@ static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level,
goto free_conv_name;
}
- struct_sz = readdir_info_level_struct_sz(info_level) + conv_len;
+ struct_sz = readdir_info_level_struct_sz(info_level);
+ if (struct_sz == -EOPNOTSUPP) {
+ rc = -EINVAL;
+ goto free_conv_name;
+ }
+
+ struct_sz += conv_len;
next_entry_offset = ALIGN(struct_sz, KSMBD_DIR_INFO_ALIGNMENT);
d_info->last_entry_off_align = next_entry_offset - struct_sz;
@@ -8183,9 +8202,20 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id,
int ret = 0;
__le32 old_fattr;
+ if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) {
+ ksmbd_debug(SMB, "User does not have write permission\n");
+ return -EACCES;
+ }
+
fp = ksmbd_lookup_fd_fast(work, id);
if (!fp)
return -ENOENT;
+
+ if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_WRITE_ATTRIBUTES_LE))) {
+ ret = -EACCES;
+ goto out;
+ }
+
idmap = file_mnt_idmap(fp->filp);
old_fattr = fp->f_ci->m_fattr;
diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c
index 4bbc2c2..664b1b4 100644
--- a/fs/smb/server/smbacl.c
+++ b/fs/smb/server/smbacl.c
@@ -643,8 +643,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
ntace = (struct smb_ace *)((char *)pndace + *size);
ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags,
pace->e_perm, 0777);
- if (check_add_overflow(*size, ace_sz, size))
+ if (check_add_overflow(*size, ace_sz, size)) {
+ kfree(sid);
break;
+ }
(*num_aces)++;
if (pace->e_tag == ACL_USER)
ntace->access_req |=
@@ -655,8 +657,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
ntace = (struct smb_ace *)((char *)pndace + *size);
ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED,
0x03, pace->e_perm, 0777);
- if (check_add_overflow(*size, ace_sz, size))
+ if (check_add_overflow(*size, ace_sz, size)) {
+ kfree(sid);
break;
+ }
(*num_aces)++;
if (pace->e_tag == ACL_USER)
ntace->access_req |=
@@ -698,8 +702,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
ntace = (struct smb_ace *)((char *)pndace + *size);
ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b,
pace->e_perm, 0777);
- if (check_add_overflow(*size, ace_sz, size))
+ if (check_add_overflow(*size, ace_sz, size)) {
+ kfree(sid);
break;
+ }
(*num_aces)++;
if (pace->e_tag == ACL_USER)
ntace->access_req |=
@@ -1068,7 +1074,60 @@ static void smb_set_ace(struct smb_ace *ace, const struct smb_sid *sid, u8 type,
ace->flags = flags;
ace->access_req = access_req;
smb_copy_sid(&ace->sid, sid);
- ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + (sid->num_subauth * 4));
+ ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 +
+ (ace->sid.num_subauth * 4));
+}
+
+static int smb_append_inherited_ace(struct smb_ace **ace, int *nt_size,
+ u16 *ace_cnt, const struct smb_sid *sid,
+ u8 type, u8 flags, __le32 access_req)
+{
+ int ace_size;
+
+ smb_set_ace(*ace, sid, type, flags, access_req);
+ ace_size = le16_to_cpu((*ace)->size);
+ /* pdacl->size is __le16 and includes struct smb_acl. */
+ if (check_add_overflow(*nt_size, ace_size, nt_size) ||
+ *nt_size > U16_MAX - (int)sizeof(struct smb_acl))
+ return -EINVAL;
+
+ (*ace_cnt)++;
+ *ace = (struct smb_ace *)((char *)*ace + ace_size);
+ return 0;
+}
+
+static int smb_validate_ntsd_sid(struct smb_ntsd *pntsd, size_t pntsd_size,
+ unsigned int sid_offset, struct smb_sid **sid,
+ size_t *sid_size)
+{
+ size_t sid_end;
+
+ *sid = NULL;
+ *sid_size = 0;
+
+ if (!sid_offset)
+ return 0;
+
+ if (sid_offset < sizeof(struct smb_ntsd) ||
+ check_add_overflow(sid_offset, (size_t)CIFS_SID_BASE_SIZE,
+ &sid_end) ||
+ sid_end > pntsd_size)
+ return -EINVAL;
+
+ *sid = (struct smb_sid *)((char *)pntsd + sid_offset);
+ if ((*sid)->num_subauth > SID_MAX_SUB_AUTHORITIES)
+ return -EINVAL;
+
+ if (check_add_overflow((size_t)CIFS_SID_BASE_SIZE,
+ sizeof(__le32) * (size_t)(*sid)->num_subauth,
+ &sid_end))
+ return -EINVAL;
+
+ if (sid_offset > pntsd_size || sid_end > pntsd_size - sid_offset)
+ return -EINVAL;
+
+ *sid_size = sid_end;
+ return 0;
}
int smb_inherit_dacl(struct ksmbd_conn *conn,
@@ -1083,28 +1142,28 @@ int smb_inherit_dacl(struct ksmbd_conn *conn,
struct dentry *parent = path->dentry->d_parent;
struct mnt_idmap *idmap = mnt_idmap(path->mnt);
int inherited_flags = 0, flags = 0, i, nt_size = 0, pdacl_size;
- int rc = 0, pntsd_type, pntsd_size, acl_len, aces_size;
+ int rc = 0, pntsd_type, ppntsd_size, acl_len, aces_size;
unsigned int dacloffset;
size_t dacl_struct_end;
u16 num_aces, ace_cnt = 0;
char *aces_base;
bool is_dir = S_ISDIR(d_inode(path->dentry)->i_mode);
- pntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap,
+ ppntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap,
parent, &parent_pntsd);
- if (pntsd_size <= 0)
+ if (ppntsd_size <= 0)
return -ENOENT;
dacloffset = le32_to_cpu(parent_pntsd->dacloffset);
if (!dacloffset ||
check_add_overflow(dacloffset, sizeof(struct smb_acl), &dacl_struct_end) ||
- dacl_struct_end > (size_t)pntsd_size) {
+ dacl_struct_end > (size_t)ppntsd_size) {
rc = -EINVAL;
goto free_parent_pntsd;
}
parent_pdacl = (struct smb_acl *)((char *)parent_pntsd + dacloffset);
- acl_len = pntsd_size - dacloffset;
+ acl_len = ppntsd_size - dacloffset;
num_aces = le16_to_cpu(parent_pdacl->num_aces);
pntsd_type = le16_to_cpu(parent_pntsd->type);
pdacl_size = le16_to_cpu(parent_pdacl->size);
@@ -1157,6 +1216,12 @@ int smb_inherit_dacl(struct ksmbd_conn *conn,
CIFS_SID_BASE_SIZE)
break;
+ if (parent_aces->sid.num_subauth > SID_MAX_SUB_AUTHORITIES ||
+ pace_size < offsetof(struct smb_ace, sid) +
+ CIFS_SID_BASE_SIZE +
+ sizeof(__le32) * parent_aces->sid.num_subauth)
+ break;
+
aces_size -= pace_size;
flags = parent_aces->flags;
@@ -1186,22 +1251,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn,
}
if (is_dir && creator && flags & CONTAINER_INHERIT_ACE) {
- smb_set_ace(aces, psid, parent_aces->type, inherited_flags,
- parent_aces->access_req);
- nt_size += le16_to_cpu(aces->size);
- ace_cnt++;
- aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size));
+ rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt,
+ psid, parent_aces->type,
+ inherited_flags,
+ parent_aces->access_req);
+ if (rc)
+ goto free_aces_base;
flags |= INHERIT_ONLY_ACE;
psid = creator;
} else if (is_dir && !(parent_aces->flags & NO_PROPAGATE_INHERIT_ACE)) {
psid = &parent_aces->sid;
}
- smb_set_ace(aces, psid, parent_aces->type, flags | inherited_flags,
- parent_aces->access_req);
- nt_size += le16_to_cpu(aces->size);
- aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size));
- ace_cnt++;
+ rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, psid,
+ parent_aces->type,
+ flags | inherited_flags,
+ parent_aces->access_req);
+ if (rc)
+ goto free_aces_base;
pass:
parent_aces = (struct smb_ace *)((char *)parent_aces + pace_size);
}
@@ -1210,22 +1277,33 @@ int smb_inherit_dacl(struct ksmbd_conn *conn,
struct smb_ntsd *pntsd;
struct smb_acl *pdacl;
struct smb_sid *powner_sid = NULL, *pgroup_sid = NULL;
- int powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size;
- int pntsd_alloc_size;
+ size_t powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size;
+ size_t pntsd_alloc_size;
- if (parent_pntsd->osidoffset) {
- powner_sid = (struct smb_sid *)((char *)parent_pntsd +
- le32_to_cpu(parent_pntsd->osidoffset));
- powner_sid_size = 1 + 1 + 6 + (powner_sid->num_subauth * 4);
- }
- if (parent_pntsd->gsidoffset) {
- pgroup_sid = (struct smb_sid *)((char *)parent_pntsd +
- le32_to_cpu(parent_pntsd->gsidoffset));
- pgroup_sid_size = 1 + 1 + 6 + (pgroup_sid->num_subauth * 4);
- }
+ rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size,
+ le32_to_cpu(parent_pntsd->osidoffset),
+ &powner_sid, &powner_sid_size);
+ if (rc)
+ goto free_aces_base;
+ rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size,
+ le32_to_cpu(parent_pntsd->gsidoffset),
+ &pgroup_sid, &pgroup_sid_size);
+ if (rc)
+ goto free_aces_base;
- pntsd_alloc_size = sizeof(struct smb_ntsd) + powner_sid_size +
- pgroup_sid_size + sizeof(struct smb_acl) + nt_size;
+ if (check_add_overflow(sizeof(struct smb_ntsd),
+ (size_t)powner_sid_size,
+ &pntsd_alloc_size) ||
+ check_add_overflow(pntsd_alloc_size,
+ (size_t)pgroup_sid_size,
+ &pntsd_alloc_size) ||
+ check_add_overflow(pntsd_alloc_size, sizeof(struct smb_acl),
+ &pntsd_alloc_size) ||
+ check_add_overflow(pntsd_alloc_size, (size_t)nt_size,
+ &pntsd_alloc_size)) {
+ rc = -EINVAL;
+ goto free_aces_base;
+ }
pntsd = kzalloc(pntsd_alloc_size, KSMBD_DEFAULT_GFP);
if (!pntsd) {
@@ -1368,8 +1446,8 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
aces_size = acl_size - sizeof(struct smb_acl);
for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) {
- if (offsetof(struct smb_ace, sid) +
- aces_size < CIFS_SID_BASE_SIZE)
+ if (aces_size < offsetof(struct smb_ace, sid) +
+ CIFS_SID_BASE_SIZE)
break;
ace_size = le16_to_cpu(ace->size);
if (ace_size > aces_size ||
@@ -1389,8 +1467,8 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
aces_size = acl_size - sizeof(struct smb_acl);
for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) {
- if (offsetof(struct smb_ace, sid) +
- aces_size < CIFS_SID_BASE_SIZE)
+ if (aces_size < offsetof(struct smb_ace, sid) +
+ CIFS_SID_BASE_SIZE)
break;
ace_size = le16_to_cpu(ace->size);
if (ace_size > aces_size ||
diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c
index a8242c0..b6d63ff 100644
--- a/fs/smb/server/transport_rdma.c
+++ b/fs/smb/server/transport_rdma.c
@@ -18,7 +18,6 @@
#include "smb_common.h"
#include "../common/smb2status.h"
#include "transport_rdma.h"
-#include "../smbdirect/public.h"
#define SMB_DIRECT_PORT_IWARP 5445
@@ -540,3 +539,5 @@ static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
.rdma_write = smb_direct_rdma_write,
.free_transport = smb_direct_free_transport,
};
+
+MODULE_IMPORT_NS("SMBDIRECT");
diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h
index bde3d88..8b78917 100644
--- a/fs/smb/server/transport_rdma.h
+++ b/fs/smb/server/transport_rdma.h
@@ -25,6 +25,6 @@ static inline void init_smbd_max_io_size(unsigned int sz) { }
static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; }
#endif
-#include "../smbdirect/smbdirect.h"
+#include <linux/smbdirect.h>
#endif /* __KSMBD_TRANSPORT_RDMA_H__ */
diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c
index 3551f01..4d2d33d 100644
--- a/fs/smb/server/vfs_cache.c
+++ b/fs/smb/server/vfs_cache.c
@@ -81,7 +81,7 @@ static int proc_show_files(struct seq_file *m, void *v)
read_lock(&global_ft.lock);
idr_for_each_entry(global_ft.idr, fp, id) {
seq_printf(m, "%#-10x %#-10llx %#-10llx %#-10x",
- fp->tcon->id,
+ fp->tcon ? fp->tcon->id : 0,
fp->persistent_id,
fp->volatile_id,
atomic_read(&fp->refcount));
@@ -211,13 +211,13 @@ int ksmbd_query_inode_status(struct dentry *dentry)
return ret;
down_read(&ci->m_lock);
- if (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS))
+ if (ci->m_flags & S_DEL_PENDING)
ret = KSMBD_INODE_STATUS_PENDING_DELETE;
else
ret = KSMBD_INODE_STATUS_OK;
up_read(&ci->m_lock);
- atomic_dec(&ci->m_count);
+ ksmbd_inode_put(ci);
return ret;
}
@@ -227,7 +227,7 @@ bool ksmbd_inode_pending_delete(struct ksmbd_file *fp)
int ret;
down_read(&ci->m_lock);
- ret = (ci->m_flags & (S_DEL_PENDING | S_DEL_ON_CLS));
+ ret = (ci->m_flags & S_DEL_PENDING);
up_read(&ci->m_lock);
return ret;
@@ -395,12 +395,20 @@ static void __ksmbd_inode_close(struct ksmbd_file *fp)
}
}
+ down_write(&ci->m_lock);
+ /* Promote S_DEL_ON_CLS to S_DEL_PENDING when close */
+ if (ci->m_flags & S_DEL_ON_CLS) {
+ ci->m_flags &= ~S_DEL_ON_CLS;
+ ci->m_flags |= S_DEL_PENDING;
+ }
+ up_write(&ci->m_lock);
+
if (atomic_dec_and_test(&ci->m_count)) {
bool do_unlink = false;
down_write(&ci->m_lock);
- if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_PENDING)) {
- ci->m_flags &= ~(S_DEL_ON_CLS | S_DEL_PENDING);
+ if (ci->m_flags & S_DEL_PENDING) {
+ ci->m_flags &= ~S_DEL_PENDING;
do_unlink = true;
}
up_write(&ci->m_lock);
@@ -418,6 +426,14 @@ static void __ksmbd_remove_durable_fd(struct ksmbd_file *fp)
return;
idr_remove(global_ft.idr, fp->persistent_id);
+ /*
+ * Clear persistent_id so a later __ksmbd_close_fd() that runs from a
+ * delayed putter (e.g. when a concurrent ksmbd_lookup_fd_inode()
+ * walker held the final reference) does not re-issue idr_remove() on
+ * an id that idr_alloc_cyclic() may have already handed out to a new
+ * durable handle.
+ */
+ fp->persistent_id = KSMBD_NO_FID;
}
static void ksmbd_remove_durable_fd(struct ksmbd_file *fp)
@@ -431,13 +447,13 @@ static void ksmbd_remove_durable_fd(struct ksmbd_file *fp)
static void __ksmbd_remove_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp)
{
- if (!has_file_id(fp->volatile_id))
- return;
-
down_write(&fp->f_ci->m_lock);
list_del_init(&fp->node);
up_write(&fp->f_ci->m_lock);
+ if (!has_file_id(fp->volatile_id))
+ return;
+
write_lock(&ft->lock);
idr_remove(ft->idr, fp->volatile_id);
write_unlock(&ft->lock);
@@ -475,6 +491,17 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp)
kfree(smb_lock);
}
+ /*
+ * Drop fp's strong reference on conn (taken in ksmbd_open_fd() /
+ * ksmbd_reopen_durable_fd()). Durable fps that reached the
+ * scavenger have already had fp->conn cleared by session_fd_check(),
+ * in which case there is nothing to drop here.
+ */
+ if (fp->conn) {
+ ksmbd_conn_put(fp->conn);
+ fp->conn = NULL;
+ }
+
if (ksmbd_stream_fd(fp))
kfree(fp->stream.name);
kfree(fp->owner.name);
@@ -510,6 +537,20 @@ static struct ksmbd_file *__ksmbd_lookup_fd(struct ksmbd_file_table *ft,
static void __put_fd_final(struct ksmbd_work *work, struct ksmbd_file *fp)
{
+ /*
+ * Detached durable fp -- session_fd_check() cleared fp->conn at
+ * preserve, so this fp is no longer tracked by any conn's
+ * stats.open_files_count. This happens when
+ * ksmbd_scavenger_dispose_dh() hands the final close off to an
+ * m_fp_list walker (e.g. ksmbd_lookup_fd_inode()) whose work->conn
+ * is unrelated to the conn that originally opened the handle; close
+ * via the NULL-ft path so we do not underflow that unrelated
+ * counter.
+ */
+ if (!fp->conn) {
+ __ksmbd_close_fd(NULL, fp);
+ return;
+ }
__ksmbd_close_fd(&work->sess->file_table, fp);
atomic_dec(&work->conn->stats.open_files_count);
}
@@ -678,14 +719,14 @@ struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry)
down_read(&ci->m_lock);
list_for_each_entry(lfp, &ci->m_fp_list, node) {
if (inode == file_inode(lfp->filp)) {
- atomic_dec(&ci->m_count);
lfp = ksmbd_fp_get(lfp);
up_read(&ci->m_lock);
+ ksmbd_inode_put(ci);
return lfp;
}
}
- atomic_dec(&ci->m_count);
up_read(&ci->m_lock);
+ ksmbd_inode_put(ci);
return NULL;
}
@@ -752,7 +793,14 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp)
atomic_set(&fp->refcount, 1);
fp->filp = filp;
- fp->conn = work->conn;
+ /*
+ * fp owns a strong reference on fp->conn for as long as fp->conn is
+ * non-NULL, so session_fd_check() and __ksmbd_close_fd() never
+ * dereference a dangling pointer. Paired with ksmbd_conn_put() in
+ * session_fd_check() (durable preserve), in __ksmbd_close_fd()
+ * (final close), and on the error paths below.
+ */
+ fp->conn = ksmbd_conn_get(work->conn);
fp->tcon = work->tcon;
fp->volatile_id = KSMBD_NO_FID;
fp->persistent_id = KSMBD_NO_FID;
@@ -774,19 +822,64 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp)
return fp;
err_out:
+ /* fp->conn was set and refcounted before every branch here. */
+ ksmbd_conn_put(fp->conn);
kmem_cache_free(filp_cache, fp);
return ERR_PTR(ret);
}
-void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp,
- unsigned int state)
+/**
+ * ksmbd_update_fstate() - update an fp state under the file-table lock
+ * @ft: file table that publishes @fp's volatile id
+ * @fp: file pointer to update
+ * @state: new state
+ *
+ * Return: 0 on success. The FP_NEW -> FP_INITED transition is special:
+ * -ENOENT if teardown already unpublished @fp by advancing the state or
+ * clearing the volatile id. Other state updates preserve the historical
+ * fire-and-forget behavior.
+ */
+int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp,
+ unsigned int state)
{
+ int ret;
+
if (!fp)
- return;
+ return -ENOENT;
write_lock(&ft->lock);
- fp->f_state = state;
+ if (state == FP_INITED &&
+ (fp->f_state != FP_NEW || !has_file_id(fp->volatile_id))) {
+ ret = -ENOENT;
+ } else {
+ fp->f_state = state;
+ ret = 0;
+ }
write_unlock(&ft->lock);
+
+ return ret;
+}
+
+/*
+ * ksmbd_mark_fp_closed() - mark fp closed under ft->lock and return how many
+ * refs the teardown path owns.
+ *
+ * FP_INITED has a normal idr-owned reference, so teardown owns both that
+ * reference and the transient lookup reference. FP_NEW is still owned by the
+ * in-flight opener/reopener, which will drop the original reference after
+ * ksmbd_update_fstate(..., FP_INITED) observes the cleared volatile id.
+ * FP_CLOSED on entry means an earlier ksmbd_close_fd() already consumed the
+ * idr-owned ref.
+ */
+static int ksmbd_mark_fp_closed(struct ksmbd_file *fp)
+{
+ if (fp->f_state == FP_INITED) {
+ set_close_state_blocked_works(fp);
+ fp->f_state = FP_CLOSED;
+ return 2;
+ }
+
+ return 1;
}
static int
@@ -794,7 +887,8 @@ __close_file_table_ids(struct ksmbd_session *sess,
struct ksmbd_tree_connect *tcon,
bool (*skip)(struct ksmbd_tree_connect *tcon,
struct ksmbd_file *fp,
- struct ksmbd_user *user))
+ struct ksmbd_user *user),
+ bool skip_preserves_fp)
{
struct ksmbd_file_table *ft = &sess->file_table;
struct ksmbd_file *fp;
@@ -802,32 +896,120 @@ __close_file_table_ids(struct ksmbd_session *sess,
int num = 0;
while (1) {
+ int n_to_drop;
+
write_lock(&ft->lock);
fp = idr_get_next(ft->idr, &id);
if (!fp) {
write_unlock(&ft->lock);
break;
}
-
- if (skip(tcon, fp, sess->user) ||
- !atomic_dec_and_test(&fp->refcount)) {
+ if (!atomic_inc_not_zero(&fp->refcount)) {
id++;
write_unlock(&ft->lock);
continue;
}
- set_close_state_blocked_works(fp);
- idr_remove(ft->idr, fp->volatile_id);
- fp->volatile_id = KSMBD_NO_FID;
- write_unlock(&ft->lock);
+ if (skip_preserves_fp) {
+ /*
+ * Session teardown: skip() is session_fd_check(),
+ * which may sleep and mutates fp->conn / fp->tcon /
+ * fp->volatile_id when it chooses to preserve fp
+ * for durable reconnect. Unpublish fp from the
+ * session idr here, under ft->lock, so that
+ * __ksmbd_lookup_fd() through this session cannot
+ * grant a new ksmbd_fp_get() reference to an fp
+ * whose fields are about to be rewritten outside
+ * the lock. Durable reconnect still reaches fp via
+ * global_ft.
+ */
+ idr_remove(ft->idr, id);
+ fp->volatile_id = KSMBD_NO_FID;
+ write_unlock(&ft->lock);
+ if (skip(tcon, fp, sess->user)) {
+ /*
+ * session_fd_check() has converted fp to
+ * durable-preserve state and cleared its
+ * per-conn fields. fp is already unpublished
+ * above; the original idr-owned ref keeps it
+ * alive for the durable scavenger. Drop only
+ * the transient ref. atomic_dec() is safe --
+ * atomic_inc_not_zero() succeeded on a
+ * positive value and we added one more, so
+ * refcount cannot be zero here.
+ */
+ atomic_dec(&fp->refcount);
+ id++;
+ continue;
+ }
+
+ /*
+ * Keep the close-state decision under the same lock
+ * observed by ksmbd_update_fstate(), which is how an
+ * in-flight FP_NEW opener learns that teardown has
+ * cleared its volatile id.
+ */
+ write_lock(&ft->lock);
+ n_to_drop = ksmbd_mark_fp_closed(fp);
+ write_unlock(&ft->lock);
+ } else {
+ /*
+ * Tree teardown: skip() is tree_conn_fd_check(), a
+ * cheap pointer compare that doesn't sleep and has
+ * no side effects, so keep the skip decision plus
+ * the unpublish-and-mark-closed sequence atomic
+ * under ft->lock. fps belonging to other tree
+ * connects (skip() == true) stay fully published in
+ * the session idr with no lock window.
+ */
+ if (skip(tcon, fp, sess->user)) {
+ atomic_dec(&fp->refcount);
+ write_unlock(&ft->lock);
+ id++;
+ continue;
+ }
+ idr_remove(ft->idr, id);
+ fp->volatile_id = KSMBD_NO_FID;
+ n_to_drop = ksmbd_mark_fp_closed(fp);
+ write_unlock(&ft->lock);
+ }
+
+ /*
+ * fp->volatile_id is already cleared to prevent stale idr
+ * removal from a deferred final close. Remove fp from
+ * m_fp_list here because __ksmbd_remove_fd() will skip the
+ * list unlink when volatile_id is KSMBD_NO_FID.
+ */
down_write(&fp->f_ci->m_lock);
list_del_init(&fp->node);
up_write(&fp->f_ci->m_lock);
- __ksmbd_close_fd(ft, fp);
-
- num++;
+ /*
+ * Drop the references this iteration owns:
+ *
+ * n_to_drop == 2: we observed FP_INITED and committed
+ * the FP_CLOSED transition ourselves, so we own the
+ * transient (+1) and the still-intact idr-owned ref.
+ *
+ * n_to_drop == 1: either a prior ksmbd_close_fd()
+ * already consumed the idr-owned ref, or fp was still
+ * FP_NEW and the in-flight opener/reopener must keep
+ * the original reference until ksmbd_update_fstate()
+ * observes the cleared volatile id.
+ *
+ * If we end up as the final putter, finalize fp and
+ * account the open_files_count decrement via the caller's
+ * atomic_sub(num, ...). Otherwise the remaining user's
+ * ksmbd_fd_put() reaches __put_fd_final(), which does its
+ * own atomic_dec(&open_files_count), so we must not count
+ * this fp here -- doing so would double-decrement the
+ * connection-wide counter.
+ */
+ if (atomic_sub_and_test(n_to_drop, &fp->refcount)) {
+ __ksmbd_close_fd(NULL, fp);
+ num++;
+ }
id++;
}
@@ -881,24 +1063,37 @@ static bool ksmbd_durable_scavenger_alive(void)
return true;
}
-static void ksmbd_scavenger_dispose_dh(struct list_head *head)
+static void ksmbd_scavenger_dispose_dh(struct ksmbd_file *fp)
{
- while (!list_empty(head)) {
- struct ksmbd_file *fp;
+ /*
+ * Durable-preserved fp can remain linked on f_ci->m_fp_list for
+ * share-mode checks. Unlink it before final close; fp->node is not
+ * available as a scavenger-private list node because re-adding it to
+ * another list corrupts m_fp_list.
+ */
+ down_write(&fp->f_ci->m_lock);
+ list_del_init(&fp->node);
+ up_write(&fp->f_ci->m_lock);
- fp = list_first_entry(head, struct ksmbd_file, node);
- list_del_init(&fp->node);
+ /*
+ * Drop both the durable lifetime reference and the transient reference
+ * taken by the scavenger under global_ft.lock. If a concurrent
+ * ksmbd_lookup_fd_inode() (or any other m_fp_list walker) snatched fp
+ * before the unlink above, that holder owns the final close via
+ * ksmbd_fd_put() -> __ksmbd_close_fd(). Otherwise the scavenger is
+ * the last putter and finalises fp here.
+ */
+ if (atomic_sub_and_test(2, &fp->refcount))
__ksmbd_close_fd(NULL, fp);
- }
}
static int ksmbd_durable_scavenger(void *dummy)
{
struct ksmbd_file *fp = NULL;
+ struct ksmbd_file *expired_fp;
unsigned int id;
unsigned int min_timeout = 1;
bool found_fp_timeout;
- LIST_HEAD(scavenger_list);
unsigned long remaining_jiffies;
__module_get(THIS_MODULE);
@@ -908,8 +1103,6 @@ static int ksmbd_durable_scavenger(void *dummy)
if (try_to_freeze())
continue;
- found_fp_timeout = false;
-
remaining_jiffies = wait_event_timeout(dh_wq,
ksmbd_durable_scavenger_alive() == false,
__msecs_to_jiffies(min_timeout));
@@ -918,23 +1111,39 @@ static int ksmbd_durable_scavenger(void *dummy)
else
min_timeout = DURABLE_HANDLE_MAX_TIMEOUT;
- write_lock(&global_ft.lock);
- idr_for_each_entry(global_ft.idr, fp, id) {
- if (!fp->durable_timeout)
- continue;
+ do {
+ expired_fp = NULL;
+ found_fp_timeout = false;
- if (atomic_read(&fp->refcount) > 1 ||
- fp->conn)
- continue;
-
- found_fp_timeout = true;
- if (fp->durable_scavenger_timeout <=
- jiffies_to_msecs(jiffies)) {
- __ksmbd_remove_durable_fd(fp);
- list_add(&fp->node, &scavenger_list);
- } else {
+ write_lock(&global_ft.lock);
+ idr_for_each_entry(global_ft.idr, fp, id) {
unsigned long durable_timeout;
+ if (!fp->durable_timeout)
+ continue;
+
+ if (atomic_read(&fp->refcount) > 1 ||
+ fp->conn)
+ continue;
+
+ found_fp_timeout = true;
+ if (fp->durable_scavenger_timeout <=
+ jiffies_to_msecs(jiffies)) {
+ __ksmbd_remove_durable_fd(fp);
+ /*
+ * Take a transient reference so fp
+ * cannot be freed by an in-flight
+ * ksmbd_lookup_fd_inode() that found
+ * it through f_ci->m_fp_list while we
+ * drop global_ft.lock and reach the
+ * m_fp_list unlink in
+ * ksmbd_scavenger_dispose_dh().
+ */
+ atomic_inc(&fp->refcount);
+ expired_fp = fp;
+ break;
+ }
+
durable_timeout =
fp->durable_scavenger_timeout -
jiffies_to_msecs(jiffies);
@@ -942,10 +1151,11 @@ static int ksmbd_durable_scavenger(void *dummy)
if (min_timeout > durable_timeout)
min_timeout = durable_timeout;
}
- }
- write_unlock(&global_ft.lock);
+ write_unlock(&global_ft.lock);
- ksmbd_scavenger_dispose_dh(&scavenger_list);
+ if (expired_fp)
+ ksmbd_scavenger_dispose_dh(expired_fp);
+ } while (expired_fp);
if (found_fp_timeout == false)
break;
@@ -1062,25 +1272,35 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon,
if (!is_reconnectable(fp))
return false;
+ if (fp->f_state != FP_INITED)
+ return false;
+
+ if (WARN_ON_ONCE(!fp->conn))
+ return false;
+
if (ksmbd_vfs_copy_durable_owner(fp, user))
return false;
+ /*
+ * fp owns a strong reference on fp->conn (taken in ksmbd_open_fd()
+ * / ksmbd_reopen_durable_fd()), so conn stays valid for the whole
+ * body of this function regardless of any op->conn puts below.
+ */
conn = fp->conn;
ci = fp->f_ci;
down_write(&ci->m_lock);
list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) {
if (op->conn != conn)
continue;
- if (op->conn && atomic_dec_and_test(&op->conn->refcnt))
- kfree(op->conn);
+ ksmbd_conn_put(op->conn);
op->conn = NULL;
}
up_write(&ci->m_lock);
list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) {
- spin_lock(&fp->conn->llist_lock);
+ spin_lock(&conn->llist_lock);
list_del_init(&smb_lock->clist);
- spin_unlock(&fp->conn->llist_lock);
+ spin_unlock(&conn->llist_lock);
}
fp->conn = NULL;
@@ -1091,6 +1311,8 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon,
fp->durable_scavenger_timeout =
jiffies_to_msecs(jiffies) + fp->durable_timeout;
+ /* Drop fp's own reference on conn. */
+ ksmbd_conn_put(conn);
return true;
}
@@ -1098,7 +1320,8 @@ void ksmbd_close_tree_conn_fds(struct ksmbd_work *work)
{
int num = __close_file_table_ids(work->sess,
work->tcon,
- tree_conn_fd_check);
+ tree_conn_fd_check,
+ false);
atomic_sub(num, &work->conn->stats.open_files_count);
}
@@ -1107,7 +1330,8 @@ void ksmbd_close_session_fds(struct ksmbd_work *work)
{
int num = __close_file_table_ids(work->sess,
work->tcon,
- session_fd_check);
+ session_fd_check,
+ true);
atomic_sub(num, &work->conn->stats.open_files_count);
}
@@ -1178,15 +1402,27 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp)
old_f_state = fp->f_state;
fp->f_state = FP_NEW;
+
+ /*
+ * Initialize fp's connection binding before publishing fp into the
+ * session's file table. If __open_id() is ordered first, a
+ * concurrent teardown that iterates the table can observe a valid
+ * volatile_id with fp->conn == NULL and preserve a
+ * partially-initialized fp. fp owns a strong reference on the new
+ * conn (see ksmbd_open_fd()); undo it on __open_id() failure.
+ */
+ fp->conn = ksmbd_conn_get(conn);
+ fp->tcon = work->tcon;
+
__open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID);
if (!has_file_id(fp->volatile_id)) {
+ fp->conn = NULL;
+ fp->tcon = NULL;
+ ksmbd_conn_put(conn);
fp->f_state = old_f_state;
return -EBADF;
}
- fp->conn = conn;
- fp->tcon = work->tcon;
-
list_for_each_entry(smb_lock, &fp->lock_list, flist) {
spin_lock(&conn->llist_lock);
list_add_tail(&smb_lock->clist, &conn->lock_list);
@@ -1198,8 +1434,7 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp)
list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) {
if (op->conn)
continue;
- op->conn = fp->conn;
- atomic_inc(&op->conn->refcnt);
+ op->conn = ksmbd_conn_get(fp->conn);
}
up_write(&ci->m_lock);
@@ -1228,7 +1463,7 @@ void ksmbd_destroy_file_table(struct ksmbd_session *sess)
if (!ft->idr)
return;
- __close_file_table_ids(sess, NULL, session_fd_check);
+ __close_file_table_ids(sess, NULL, session_fd_check, true);
idr_destroy(ft->idr);
kfree(ft->idr);
ft->idr = NULL;
diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h
index 866f32c1..e687126 100644
--- a/fs/smb/server/vfs_cache.h
+++ b/fs/smb/server/vfs_cache.h
@@ -172,8 +172,8 @@ int ksmbd_close_inode_fds(struct ksmbd_work *work, struct inode *inode);
int ksmbd_init_global_file_table(void);
void ksmbd_free_global_file_table(void);
void ksmbd_set_fd_limit(unsigned long limit);
-void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp,
- unsigned int state);
+int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp,
+ unsigned int state);
bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp,
struct ksmbd_user *user);
diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c
index 704b271..5297400 100644
--- a/fs/smb/smbdirect/accept.c
+++ b/fs/smb/smbdirect/accept.c
@@ -854,4 +854,4 @@ struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc,
return nsc;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_accept);
+EXPORT_SYMBOL_GPL(smbdirect_socket_accept);
diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c
index 8addee4..cd726b3 100644
--- a/fs/smb/smbdirect/connect.c
+++ b/fs/smb/smbdirect/connect.c
@@ -60,7 +60,7 @@ int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst)
*/
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect);
+EXPORT_SYMBOL_GPL(smbdirect_connect);
static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc)
{
@@ -922,4 +922,4 @@ int smbdirect_connect_sync(struct smbdirect_socket *sc,
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect_sync);
+EXPORT_SYMBOL_GPL(smbdirect_connect_sync);
diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c
index 8223667..8adf580 100644
--- a/fs/smb/smbdirect/connection.c
+++ b/fs/smb/smbdirect/connection.c
@@ -706,7 +706,7 @@ bool smbdirect_connection_is_connected(struct smbdirect_socket *sc)
return false;
return true;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_is_connected);
+EXPORT_SYMBOL_GPL(smbdirect_connection_is_connected);
int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc)
{
@@ -779,7 +779,7 @@ int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc)
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_wait_for_connected);
+EXPORT_SYMBOL_GPL(smbdirect_connection_wait_for_connected);
void smbdirect_connection_idle_timer_work(struct work_struct *work)
{
@@ -958,7 +958,7 @@ int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc,
return ret;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_batch_flush);
+EXPORT_SYMBOL_GPL(smbdirect_connection_send_batch_flush);
struct smbdirect_send_batch *
smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage,
@@ -976,7 +976,7 @@ smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage,
return batch;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_init_send_batch_storage);
+EXPORT_SYMBOL_GPL(smbdirect_init_send_batch_storage);
static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc,
struct smbdirect_send_batch *batch)
@@ -1263,7 +1263,7 @@ int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc,
bcredit_failed:
return ret;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_single_iter);
+EXPORT_SYMBOL_GPL(smbdirect_connection_send_single_iter);
int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc)
{
@@ -1288,7 +1288,7 @@ int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc)
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_wait_zero_pending);
+EXPORT_SYMBOL_GPL(smbdirect_connection_send_wait_zero_pending);
int smbdirect_connection_send_iter(struct smbdirect_socket *sc,
struct iov_iter *iter,
@@ -1373,7 +1373,7 @@ int smbdirect_connection_send_iter(struct smbdirect_socket *sc,
return total_count;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_iter);
+EXPORT_SYMBOL_GPL(smbdirect_connection_send_iter);
static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc)
{
@@ -1937,7 +1937,7 @@ int smbdirect_connection_recvmsg(struct smbdirect_socket *sc,
goto again;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_recvmsg);
+EXPORT_SYMBOL_GPL(smbdirect_connection_recvmsg);
static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state,
struct page *page, size_t off, size_t len)
@@ -2168,7 +2168,7 @@ static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len,
if (ret < 0) {
while (state->num_sge > before) {
- struct ib_sge *sge = &state->sge[state->num_sge--];
+ struct ib_sge *sge = &state->sge[--state->num_sge];
ib_dma_unmap_page(state->device,
sge->addr,
diff --git a/fs/smb/smbdirect/debug.c b/fs/smb/smbdirect/debug.c
index a66a19d..3445843 100644
--- a/fs/smb/smbdirect/debug.c
+++ b/fs/smb/smbdirect/debug.c
@@ -40,7 +40,7 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
seq_puts(m, "\n");
seq_printf(m, "Conn keep_alive_interval: %u ",
- sp->keepalive_interval_msec * 1000);
+ sp->keepalive_interval_msec / 1000);
seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u",
sp->max_read_write_size,
rdma_readwrite_threshold);
@@ -85,4 +85,4 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
atomic_read(&sc->mr_io.ready.count),
atomic_read(&sc->mr_io.used.count));
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_legacy_debug_proc_show);
+EXPORT_SYMBOL_GPL(smbdirect_connection_legacy_debug_proc_show);
diff --git a/fs/smb/smbdirect/devices.c b/fs/smb/smbdirect/devices.c
index 44962f2..7adacbd 100644
--- a/fs/smb/smbdirect/devices.c
+++ b/fs/smb/smbdirect/devices.c
@@ -238,7 +238,7 @@ u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev)
return RDMA_NODE_UNSPECIFIED;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_netdev_rdma_capable_node_type);
+EXPORT_SYMBOL_GPL(smbdirect_netdev_rdma_capable_node_type);
__init int smbdirect_devices_init(void)
{
diff --git a/fs/smb/smbdirect/internal.h b/fs/smb/smbdirect/internal.h
index 2d5acf2..e9959e6 100644
--- a/fs/smb/smbdirect/internal.h
+++ b/fs/smb/smbdirect/internal.h
@@ -6,11 +6,11 @@
#ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__
#define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__
+#define DEFAULT_SYMBOL_NAMESPACE "SMBDIRECT"
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include "smbdirect.h"
+#include <linux/smbdirect.h>
#include "pdu.h"
-#include "public.h"
#include <linux/mutex.h>
diff --git a/fs/smb/smbdirect/listen.c b/fs/smb/smbdirect/listen.c
index 143a761..2f78bca 100644
--- a/fs/smb/smbdirect/listen.c
+++ b/fs/smb/smbdirect/listen.c
@@ -90,7 +90,7 @@ int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog)
*/
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_listen);
+EXPORT_SYMBOL_GPL(smbdirect_socket_listen);
static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id,
struct rdma_cm_event *event)
diff --git a/fs/smb/smbdirect/mr.c b/fs/smb/smbdirect/mr.c
index 5228e69..15c6363 100644
--- a/fs/smb/smbdirect/mr.c
+++ b/fs/smb/smbdirect/mr.c
@@ -269,7 +269,7 @@ smbdirect_connection_register_mr_io(struct smbdirect_socket *sc,
{
const struct smbdirect_socket_parameters *sp = &sc->parameters;
struct smbdirect_mr_io *mr;
- int ret, num_pages;
+ int ret, num_pages, num_mapped;
struct ib_reg_wr *reg_wr;
num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1);
@@ -300,19 +300,22 @@ smbdirect_connection_register_mr_io(struct smbdirect_socket *sc,
num_pages, iov_iter_count(iter), sp->max_frmr_depth);
smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth);
- ret = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
- if (!ret) {
+ num_mapped = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir);
+ if (!num_mapped) {
smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
- "ib_dma_map_sg num_pages=%u dir=%x ret=%d (%1pe)\n",
- num_pages, mr->dir, ret, SMBDIRECT_DEBUG_ERR_PTR(ret));
+ "ib_dma_map_sg num_pages=%u dir=%x num_mapped=%d\n",
+ num_pages, mr->dir, num_mapped);
+ ret = -EIO;
goto dma_map_error;
}
- ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE);
- if (ret != mr->sgt.nents) {
+ ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, num_mapped, NULL, PAGE_SIZE);
+ if (ret != num_mapped) {
smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR,
- "ib_map_mr_sg failed ret = %d nents = %u\n",
- ret, mr->sgt.nents);
+ "ib_map_mr_sg failed ret = %d num_mapped = %u\n",
+ ret, num_mapped);
+ if (ret >= 0)
+ ret = -EIO;
goto map_mr_error;
}
@@ -380,7 +383,7 @@ smbdirect_connection_register_mr_io(struct smbdirect_socket *sc,
mutex_unlock(&mr->mutex);
return NULL;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_register_mr_io);
+EXPORT_SYMBOL_GPL(smbdirect_connection_register_mr_io);
void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
struct smbdirect_buffer_descriptor_v1 *v1)
@@ -397,7 +400,7 @@ void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr,
}
mutex_unlock(&mr->mutex);
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_mr_io_fill_buffer_descriptor);
+EXPORT_SYMBOL_GPL(smbdirect_mr_io_fill_buffer_descriptor);
/*
* Deregister a MR after I/O is done
@@ -490,4 +493,4 @@ void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr)
if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked))
mutex_unlock(&mr->mutex);
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_deregister_mr_io);
+EXPORT_SYMBOL_GPL(smbdirect_connection_deregister_mr_io);
diff --git a/fs/smb/smbdirect/rw.c b/fs/smb/smbdirect/rw.c
index c2f46b1..6fe3804 100644
--- a/fs/smb/smbdirect/rw.c
+++ b/fs/smb/smbdirect/rw.c
@@ -252,4 +252,4 @@ int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc,
kfree(msg);
goto out;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_rdma_xmit);
+EXPORT_SYMBOL_GPL(smbdirect_connection_rdma_xmit);
diff --git a/fs/smb/smbdirect/smbdirect.h b/fs/smb/smbdirect/smbdirect.h
deleted file mode 100644
index bbab5f7..0000000
--- a/fs/smb/smbdirect/smbdirect.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2025 Stefan Metzmacher
- */
-
-#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__
-#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__
-
-#include <linux/types.h>
-
-/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */
-struct smbdirect_buffer_descriptor_v1 {
- __le64 offset;
- __le32 token;
- __le32 length;
-} __packed;
-
-/*
- * Connection parameters mostly from [MS-SMBD] 3.1.1.1
- *
- * These are setup and negotiated at the beginning of a
- * connection and remain constant unless explicitly changed.
- *
- * Some values are important for the upper layer.
- */
-struct smbdirect_socket_parameters {
- __u64 flags;
-#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1)
-#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2)
- __u32 resolve_addr_timeout_msec;
- __u32 resolve_route_timeout_msec;
- __u32 rdma_connect_timeout_msec;
- __u32 negotiate_timeout_msec;
- __u16 initiator_depth; /* limited to U8_MAX */
- __u16 responder_resources; /* limited to U8_MAX */
- __u16 recv_credit_max;
- __u16 send_credit_target;
- __u32 max_send_size;
- __u32 max_fragmented_send_size;
- __u32 max_recv_size;
- __u32 max_fragmented_recv_size;
- __u32 max_read_write_size;
- __u32 max_frmr_depth;
- __u32 keepalive_interval_msec;
- __u32 keepalive_timeout_msec;
-} __packed;
-
-#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \
- SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \
- SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW)
-
-#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ */
diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c
index 1b4ab01..39cca72 100644
--- a/fs/smb/smbdirect/socket.c
+++ b/fs/smb/smbdirect/socket.c
@@ -20,7 +20,7 @@ bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs)
return false;
return true;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_frwr_is_supported);
+EXPORT_SYMBOL_GPL(smbdirect_frwr_is_supported);
static void smbdirect_socket_cleanup_work(struct work_struct *work);
@@ -107,7 +107,7 @@ int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc)
alloc_failed:
return ret;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_kern);
+EXPORT_SYMBOL_GPL(smbdirect_socket_create_kern);
int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc)
{
@@ -148,7 +148,7 @@ int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_so
alloc_failed:
return ret;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_accepting);
+EXPORT_SYMBOL_GPL(smbdirect_socket_create_accepting);
int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc,
const struct smbdirect_socket_parameters *sp)
@@ -189,14 +189,14 @@ int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc,
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_initial_parameters);
+EXPORT_SYMBOL_GPL(smbdirect_socket_set_initial_parameters);
const struct smbdirect_socket_parameters *
smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc)
{
return &sc->parameters;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_get_current_parameters);
+EXPORT_SYMBOL_GPL(smbdirect_socket_get_current_parameters);
int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc,
enum ib_poll_context poll_ctx,
@@ -220,7 +220,7 @@ int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc,
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_kernel_settings);
+EXPORT_SYMBOL_GPL(smbdirect_socket_set_kernel_settings);
void smbdirect_socket_set_logging(struct smbdirect_socket *sc,
void *private_ptr,
@@ -240,7 +240,7 @@ void smbdirect_socket_set_logging(struct smbdirect_socket *sc,
sc->logging.needed = needed;
sc->logging.vaprintf = vaprintf;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_logging);
+EXPORT_SYMBOL_GPL(smbdirect_socket_set_logging);
static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc)
{
@@ -663,13 +663,13 @@ int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr)
return 0;
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_bind);
+EXPORT_SYMBOL_GPL(smbdirect_socket_bind);
void smbdirect_socket_shutdown(struct smbdirect_socket *sc)
{
smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN);
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_shutdown);
+EXPORT_SYMBOL_GPL(smbdirect_socket_shutdown);
static void smbdirect_socket_release_disconnect(struct kref *kref)
{
@@ -712,7 +712,7 @@ void smbdirect_socket_release(struct smbdirect_socket *sc)
*/
kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy);
}
-__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_release);
+EXPORT_SYMBOL_GPL(smbdirect_socket_release);
int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc,
enum smbdirect_socket_status expected_status,
diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c
index 182e54e..4e1e4f1 100644
--- a/fs/sysfs/group.c
+++ b/fs/sysfs/group.c
@@ -188,7 +188,7 @@ static int internal_create_group(struct kobject *kobj, int update,
kernfs_get(kn);
error = create_files(kn, kobj, uid, gid, grp, update);
if (error) {
- if (grp->name)
+ if (grp->name && !update)
kernfs_remove(kn);
}
kernfs_put(kn);
diff --git a/fs/timerfd.c b/fs/timerfd.c
index 73104f3..fe845af 100644
--- a/fs/timerfd.c
+++ b/fs/timerfd.c
@@ -55,6 +55,15 @@ static inline bool isalarm(struct timerfd_ctx *ctx)
ctx->clockid == CLOCK_BOOTTIME_ALARM;
}
+static void __timerfd_triggered(struct timerfd_ctx *ctx)
+{
+ lockdep_assert_held(&ctx->wqh.lock);
+
+ ctx->expired = 1;
+ ctx->ticks++;
+ wake_up_locked_poll(&ctx->wqh, EPOLLIN);
+}
+
/*
* This gets called when the timer event triggers. We set the "expired"
* flag, but we do not re-arm the timer (in case it's necessary,
@@ -62,13 +71,8 @@ static inline bool isalarm(struct timerfd_ctx *ctx)
*/
static void timerfd_triggered(struct timerfd_ctx *ctx)
{
- unsigned long flags;
-
- spin_lock_irqsave(&ctx->wqh.lock, flags);
- ctx->expired = 1;
- ctx->ticks++;
- wake_up_locked_poll(&ctx->wqh, EPOLLIN);
- spin_unlock_irqrestore(&ctx->wqh.lock, flags);
+ guard(spinlock_irqsave)(&ctx->wqh.lock);
+ __timerfd_triggered(ctx);
}
static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr)
@@ -184,15 +188,54 @@ static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx)
return remaining < 0 ? 0: remaining;
}
+static void timerfd_alarm_start(struct timerfd_ctx *ctx, ktime_t exp, bool relative)
+{
+ /* Start the timer. If it's expired already, handle the callback. */
+ if (!alarm_start_timer(&ctx->t.alarm, exp, relative))
+ __timerfd_triggered(ctx);
+}
+
+static u64 timerfd_alarm_restart(struct timerfd_ctx *ctx)
+{
+ /* -1 to account for ctx->ticks++ in __timerfd_triggered() */
+ u64 ticks = alarm_forward_now(&ctx->t.alarm, ctx->tintv) - 1;
+
+ timerfd_alarm_start(ctx, alarm_get_expires(&ctx->t.alarm), false);
+ return ticks;
+}
+
+static void timerfd_hrtimer_start(struct timerfd_ctx *ctx, ktime_t exp,
+ const enum hrtimer_mode mode)
+{
+ /* Start the timer. If it's expired already, handle the callback. */
+ if (!hrtimer_start_range_ns_user(&ctx->t.tmr, exp, 0, mode))
+ __timerfd_triggered(ctx);
+}
+
+static u64 timerfd_hrtimer_restart(struct timerfd_ctx *ctx)
+{
+ /* -1 to account for ctx->ticks++ in __timerfd_triggered() */
+ u64 ticks = hrtimer_forward_now(&ctx->t.tmr, ctx->tintv) - 1;
+
+ timerfd_hrtimer_start(ctx, hrtimer_get_expires(&ctx->t.tmr), HRTIMER_MODE_ABS);
+ return ticks;
+}
+
+static u64 timerfd_restart(struct timerfd_ctx *ctx)
+{
+ if (isalarm(ctx))
+ return timerfd_alarm_restart(ctx);
+ return timerfd_hrtimer_restart(ctx);
+}
+
static int timerfd_setup(struct timerfd_ctx *ctx, int flags,
const struct itimerspec64 *ktmr)
{
+ int clockid = ctx->clockid;
enum hrtimer_mode htmode;
ktime_t texp;
- int clockid = ctx->clockid;
- htmode = (flags & TFD_TIMER_ABSTIME) ?
- HRTIMER_MODE_ABS: HRTIMER_MODE_REL;
+ htmode = (flags & TFD_TIMER_ABSTIME) ? HRTIMER_MODE_ABS: HRTIMER_MODE_REL;
texp = timespec64_to_ktime(ktmr->it_value);
ctx->expired = 0;
@@ -206,20 +249,15 @@ static int timerfd_setup(struct timerfd_ctx *ctx, int flags,
timerfd_alarmproc);
} else {
hrtimer_setup(&ctx->t.tmr, timerfd_tmrproc, clockid, htmode);
- hrtimer_set_expires(&ctx->t.tmr, texp);
}
if (texp != 0) {
if (flags & TFD_TIMER_ABSTIME)
texp = timens_ktime_to_host(clockid, texp);
- if (isalarm(ctx)) {
- if (flags & TFD_TIMER_ABSTIME)
- alarm_start(&ctx->t.alarm, texp);
- else
- alarm_start_relative(&ctx->t.alarm, texp);
- } else {
- hrtimer_start(&ctx->t.tmr, texp, htmode);
- }
+ if (isalarm(ctx))
+ timerfd_alarm_start(ctx, texp, !(flags & TFD_TIMER_ABSTIME));
+ else
+ timerfd_hrtimer_start(ctx, texp, htmode);
if (timerfd_canceled(ctx))
return -ECANCELED;
@@ -287,27 +325,19 @@ static ssize_t timerfd_read_iter(struct kiocb *iocb, struct iov_iter *to)
}
if (ctx->ticks) {
- ticks = ctx->ticks;
+ unsigned int expired = ctx->expired;
- if (ctx->expired && ctx->tintv) {
- /*
- * If tintv != 0, this is a periodic timer that
- * needs to be re-armed. We avoid doing it in the timer
- * callback to avoid DoS attacks specifying a very
- * short timer period.
- */
- if (isalarm(ctx)) {
- ticks += alarm_forward_now(
- &ctx->t.alarm, ctx->tintv) - 1;
- alarm_restart(&ctx->t.alarm);
- } else {
- ticks += hrtimer_forward_now(&ctx->t.tmr,
- ctx->tintv) - 1;
- hrtimer_restart(&ctx->t.tmr);
- }
- }
+ ticks = ctx->ticks;
ctx->expired = 0;
ctx->ticks = 0;
+
+ /*
+ * If tintv != 0, this is a periodic timer that needs to be
+ * re-armed. We avoid doing it in the timer callback to avoid
+ * DoS attacks specifying a very short timer period.
+ */
+ if (expired && ctx->tintv)
+ ticks += timerfd_restart(ctx);
}
spin_unlock_irq(&ctx->wqh.lock);
if (ticks) {
@@ -526,18 +556,7 @@ static int do_timerfd_gettime(int ufd, struct itimerspec64 *t)
spin_lock_irq(&ctx->wqh.lock);
if (ctx->expired && ctx->tintv) {
ctx->expired = 0;
-
- if (isalarm(ctx)) {
- ctx->ticks +=
- alarm_forward_now(
- &ctx->t.alarm, ctx->tintv) - 1;
- alarm_restart(&ctx->t.alarm);
- } else {
- ctx->ticks +=
- hrtimer_forward_now(&ctx->t.tmr, ctx->tintv)
- - 1;
- hrtimer_restart(&ctx->t.tmr);
- }
+ ctx->ticks += timerfd_restart(ctx);
}
t->it_value = ktime_to_timespec64(timerfd_get_remaining(ctx));
t->it_interval = ktime_to_timespec64(ctx->tintv);
diff --git a/fs/udf/misc.c b/fs/udf/misc.c
index 0788593b..6928e37 100644
--- a/fs/udf/misc.c
+++ b/fs/udf/misc.c
@@ -230,8 +230,12 @@ struct buffer_head *udf_read_tagged(struct super_block *sb, uint32_t block,
}
/* Verify the descriptor CRC */
- if (le16_to_cpu(tag_p->descCRCLength) + sizeof(struct tag) > sb->s_blocksize ||
- le16_to_cpu(tag_p->descCRC) == crc_itu_t(0,
+ if (le16_to_cpu(tag_p->descCRCLength) + sizeof(struct tag) > sb->s_blocksize) {
+ udf_err(sb, "block %u: CRC length %u exceeds block size\n",
+ block, le16_to_cpu(tag_p->descCRCLength));
+ goto error_out;
+ }
+ if (le16_to_cpu(tag_p->descCRC) == crc_itu_t(0,
bh->b_data + sizeof(struct tag),
le16_to_cpu(tag_p->descCRCLength)))
return bh;
diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c
index 80ba94f..aecbab6 100644
--- a/fs/xfs/libxfs/xfs_dir2_data.c
+++ b/fs/xfs/libxfs/xfs_dir2_data.c
@@ -382,6 +382,7 @@ xfs_dir3_data_write_verify(
struct xfs_mount *mp = bp->b_mount;
struct xfs_buf_log_item *bip = bp->b_log_item;
struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr;
+ struct xfs_dir3_data_hdr *datahdr3 = bp->b_addr;
xfs_failaddr_t fa;
fa = xfs_dir3_data_verify(bp);
@@ -396,6 +397,11 @@ xfs_dir3_data_write_verify(
if (bip)
hdr3->lsn = cpu_to_be64(bip->bli_item.li_lsn);
+ /*
+ * Zero padding that may be stale from old kernels.
+ */
+ datahdr3->pad = 0;
+
xfs_buf_update_cksum(bp, XFS_DIR3_DATA_CRC_OFF);
}
@@ -728,7 +734,6 @@ xfs_dir3_data_init(
struct xfs_dir2_data_unused *dup;
struct xfs_dir2_data_free *bf;
int error;
- int i;
/*
* Get the buffer set up for the block.
@@ -741,13 +746,16 @@ xfs_dir3_data_init(
xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_DATA_BUF);
/*
- * Initialize the header.
+ * Initialize the whole directory header region to zero
+ * so that all padding, bestfree entries, and any
+ * future header fields are clean.
*/
hdr = bp->b_addr;
+ memset(hdr, 0, geo->data_entry_offset);
+
if (xfs_has_crc(mp)) {
struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr;
- memset(hdr3, 0, sizeof(*hdr3));
hdr3->magic = cpu_to_be32(XFS_DIR3_DATA_MAGIC);
hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp));
hdr3->owner = cpu_to_be64(args->owner);
@@ -759,10 +767,6 @@ xfs_dir3_data_init(
bf = xfs_dir2_data_bestfree_p(mp, hdr);
bf[0].offset = cpu_to_be16(geo->data_entry_offset);
bf[0].length = cpu_to_be16(geo->blksize - geo->data_entry_offset);
- for (i = 1; i < XFS_DIR2_DATA_FD_COUNT; i++) {
- bf[i].length = 0;
- bf[i].offset = 0;
- }
/*
* Set up an unused entry for the block's body.
diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c
index 40c7f0f..0ec6ccd 100644
--- a/fs/xfs/libxfs/xfs_refcount.c
+++ b/fs/xfs/libxfs/xfs_refcount.c
@@ -1414,8 +1414,7 @@ xfs_refcount_finish_one(
if (rcur == NULL) {
struct xfs_perag *pag = to_perag(ri->ri_group);
- error = xfs_alloc_read_agf(pag, tp,
- XFS_ALLOC_FLAG_FREEING, &agbp);
+ error = xfs_alloc_read_agf(pag, tp, 0, &agbp);
if (error)
return error;
diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c
index 20e6306..3d40cb0 100644
--- a/fs/xfs/scrub/common.c
+++ b/fs/xfs/scrub/common.c
@@ -251,6 +251,17 @@ xchk_ino_set_preen(
trace_xchk_ino_preen(sc, ino, __return_address);
}
+/* Record a block indexed by a file fork that could be optimized. */
+void
+xchk_fblock_set_preen(
+ struct xfs_scrub *sc,
+ int whichfork,
+ xfs_fileoff_t offset)
+{
+ sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN;
+ trace_xchk_fblock_preen(sc, whichfork, offset, __return_address);
+}
+
/* Record something being wrong with the filesystem primary superblock. */
void
xchk_set_corrupt(
diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h
index f2ecc68..b494d74 100644
--- a/fs/xfs/scrub/common.h
+++ b/fs/xfs/scrub/common.h
@@ -25,6 +25,8 @@ bool xchk_fblock_xref_process_error(struct xfs_scrub *sc,
void xchk_block_set_preen(struct xfs_scrub *sc,
struct xfs_buf *bp);
void xchk_ino_set_preen(struct xfs_scrub *sc, xfs_ino_t ino);
+void xchk_fblock_set_preen(struct xfs_scrub *sc,
+ int whichfork, xfs_fileoff_t offset);
void xchk_set_corrupt(struct xfs_scrub *sc);
void xchk_block_set_corrupt(struct xfs_scrub *sc,
diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c
index 1a71d36..c2d6ad5 100644
--- a/fs/xfs/scrub/dabtree.c
+++ b/fs/xfs/scrub/dabtree.c
@@ -454,7 +454,12 @@ xchk_da_btree_block(
}
}
- /* XXX: Check hdr3.pad32 once we know how to fix it. */
+ if (xfs_has_crc(ip->i_mount)) {
+ struct xfs_da3_node_hdr *nodehdr3 = blk->bp->b_addr;
+
+ if (nodehdr3->__pad32)
+ xchk_da_set_preen(ds, level);
+ }
break;
default:
xchk_da_set_corrupt(ds, level);
diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c
index e09724c..09715a4 100644
--- a/fs/xfs/scrub/dir.c
+++ b/fs/xfs/scrub/dir.c
@@ -492,7 +492,12 @@ xchk_directory_data_bestfree(
goto out;
xchk_buffer_recheck(sc, bp);
- /* XXX: Check xfs_dir3_data_hdr.pad is zero once we start setting it. */
+ if (xfs_has_crc(sc->mp)) {
+ struct xfs_dir3_data_hdr *hdr3 = bp->b_addr;
+
+ if (hdr3->pad)
+ xchk_fblock_set_preen(sc, XFS_DATA_FORK, lblk);
+ }
if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)
goto out_buf;
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 580d40a..0cea458 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -472,6 +472,7 @@ xfs_buf_find_insert(
/* The new buffer keeps the perag reference until it is freed. */
new_bp->b_pag = pag;
+retry:
rcu_read_lock();
bp = rhashtable_lookup_get_insert_fast(&btp->bt_hash,
&new_bp->b_rhash_head, xfs_buf_hash_params);
@@ -480,8 +481,16 @@ xfs_buf_find_insert(
error = PTR_ERR(bp);
goto out_free_buf;
}
- if (bp && lockref_get_not_dead(&bp->b_lockref)) {
- /* found an existing buffer */
+ if (bp) {
+ /*
+ * If there is an existing buffer with a dead lockref, retry
+ * until the new buffer is added, or a usable buffer is found.
+ */
+ if (!lockref_get_not_dead(&bp->b_lockref)) {
+ rcu_read_unlock();
+ cpu_relax();
+ goto retry;
+ }
rcu_read_unlock();
error = xfs_buf_find_lock(bp, flags);
if (error)
@@ -820,15 +829,20 @@ xfs_buf_destroy(
ASSERT(__lockref_is_dead(&bp->b_lockref));
ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
+ if (bp->b_pag)
+ xfs_perag_put(bp->b_pag);
+ xfs_buf_free(bp);
+}
+
+static inline void
+xfs_buf_kill(
+ struct xfs_buf *bp)
+{
+ lockref_mark_dead(&bp->b_lockref);
if (!xfs_buf_is_uncached(bp)) {
rhashtable_remove_fast(&bp->b_target->bt_hash,
&bp->b_rhash_head, xfs_buf_hash_params);
-
- if (bp->b_pag)
- xfs_perag_put(bp->b_pag);
}
-
- xfs_buf_free(bp);
}
/*
@@ -851,7 +865,7 @@ xfs_buf_rele(
return;
kill:
- lockref_mark_dead(&bp->b_lockref);
+ xfs_buf_kill(bp);
list_lru_del_obj(&bp->b_target->bt_lru, &bp->b_lru);
spin_unlock(&bp->b_lockref.lock);
@@ -1433,7 +1447,7 @@ xfs_buftarg_drain_rele(
return LRU_SKIP;
}
- lockref_mark_dead(&bp->b_lockref);
+ xfs_buf_kill(bp);
list_lru_isolate_move(lru, item, dispose);
spin_unlock(&bp->b_lockref.lock);
return LRU_REMOVED;
@@ -1545,7 +1559,7 @@ xfs_buftarg_isolate(
return LRU_ROTATE;
}
- lockref_mark_dead(&bp->b_lockref);
+ xfs_buf_kill(bp);
list_lru_isolate_move(lru, item, dispose);
spin_unlock(&bp->b_lockref.lock);
return LRU_REMOVED;
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index beaa26e..9978ac1 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -699,12 +699,6 @@ xfs_create(
*/
error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks,
&tp);
- if (error == -ENOSPC) {
- /* flush outstanding delalloc blocks and retry */
- xfs_flush_inodes(mp);
- error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp,
- resblks, &tp);
- }
if (error)
goto out_parent;
diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c
index 64c8afb..b994ff1 100644
--- a/fs/xfs/xfs_notify_failure.c
+++ b/fs/xfs/xfs_notify_failure.c
@@ -350,7 +350,7 @@ xfs_dax_notify_dev_failure(
/*
* Shutdown fs from a force umount in pre-remove case which won't fail,
* so errors can be ignored. Otherwise, shutdown the filesystem with
- * CORRUPT flag if error occured or notify.want_shutdown was set during
+ * CORRUPT flag if error occurred or notify.want_shutdown was set during
* RMAP querying.
*/
if (mf_flags & MF_MEM_PRE_REMOVE)
diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c
index bcc470f..148cc32 100644
--- a/fs/xfs/xfs_trans.c
+++ b/fs/xfs/xfs_trans.c
@@ -1199,10 +1199,21 @@ xfs_trans_alloc_icreate(
{
struct xfs_trans *tp;
bool retried = false;
+ bool flushed = false;
int error;
retry:
error = xfs_trans_alloc(mp, resv, dblocks, 0, 0, &tp);
+ if (error == -ENOSPC && !flushed) {
+ /*
+ * Flush all delalloc blocks to reclaim space from speculative
+ * preallocation. This is similar to the quota retry below
+ * but targets FS-wide ENOSPC.
+ */
+ xfs_flush_inodes(mp);
+ flushed = true;
+ goto retry;
+ }
if (error)
return error;
diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c
index a851b98..5e297b7 100644
--- a/fs/xfs/xfs_zone_alloc.c
+++ b/fs/xfs/xfs_zone_alloc.c
@@ -1170,7 +1170,7 @@ xfs_calc_open_zones(
if (bdev_open_zones && bdev_open_zones < mp->m_max_open_zones) {
mp->m_max_open_zones = bdev_open_zones;
- xfs_info(mp, "limiting open zones to %u due to hardware limit.\n",
+ xfs_info(mp, "limiting open zones to %u due to hardware limit.",
bdev_open_zones);
}
@@ -1217,7 +1217,7 @@ xfs_alloc_zone_info(
return zi;
out_free_bitmaps:
- while (--i > 0)
+ while (--i >= 0)
kvfree(zi->zi_used_bucket_bitmap[i]);
kfree(zi);
return NULL;
diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c
index fedcc47..c8a1d5c 100644
--- a/fs/xfs/xfs_zone_gc.c
+++ b/fs/xfs/xfs_zone_gc.c
@@ -1221,7 +1221,7 @@ xfs_zone_gc_mount(
if (data->oz)
xfs_open_zone_put(data->oz);
out_free_gc_data:
- kfree(data);
+ xfs_zone_gc_data_free(data);
return error;
}
diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
index 9b646cb..ff43d6d 100644
--- a/fs/zonefs/super.c
+++ b/fs/zonefs/super.c
@@ -610,10 +610,14 @@ static long zonefs_fname_to_fno(const struct qstr *fname)
return c - '0';
for (i = 0, rname = name + len - 1; i < len; i++, rname--) {
+ long digit;
+
c = *rname;
if (!isdigit(c))
return -ENOENT;
- fno += (c - '0') * shift;
+ digit = (c - '0') * shift;
+ if (check_add_overflow(fno, digit, &fno))
+ return -ENOENT;
shift *= 10;
}
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index b701b5f..c41d9a75 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -17,6 +17,8 @@
#include <linux/property.h>
#include <linux/types.h>
+struct notifier_block;
+
struct acpi_handle_list {
u32 count;
acpi_handle *handles;
diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h
index 49d1749..a4b5627 100644
--- a/include/acpi/acpixf.h
+++ b/include/acpi/acpixf.h
@@ -726,6 +726,11 @@ ACPI_HW_DEPENDENT_RETURN_STATUS(acpi_status
ACPI_HW_DEPENDENT_RETURN_STATUS(acpi_status acpi_update_all_gpes(void))
ACPI_HW_DEPENDENT_RETURN_STATUS(acpi_status
+ acpi_enable_gpe_cond(acpi_handle gpe_device,
+ u32 gpe_number,
+ u8 dispatch_type))
+
+ACPI_HW_DEPENDENT_RETURN_STATUS(acpi_status
acpi_enable_gpe(acpi_handle gpe_device,
u32 gpe_number))
diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h
index 4e15583..f72e005 100644
--- a/include/acpi/actbl1.h
+++ b/include/acpi/actbl1.h
@@ -1386,6 +1386,12 @@ enum acpi_einj_command_status {
#define ACPI_EINJ_CXL_MEM_FATAL (1<<17)
#define ACPI_EINJ_VENDOR_DEFINED (1<<31)
+/* EINJV2 error types from EINJV2_GET_ERROR_TYPE (ACPI 6.6) */
+
+#define ACPI_EINJV2_PROCESSOR (1)
+#define ACPI_EINJV2_MEMORY (1<<1)
+#define ACPI_EINJV2_PCIE (1<<2)
+
/*******************************************************************************
*
* ERST - Error Record Serialization Table (ACPI 4.0)
diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild
index 2c53a1e..15df9dc 100644
--- a/include/asm-generic/Kbuild
+++ b/include/asm-generic/Kbuild
@@ -44,6 +44,7 @@
mandatory-y += msi.h
mandatory-y += pci.h
mandatory-y += percpu.h
+mandatory-y += percpu_types.h
mandatory-y += pgalloc.h
mandatory-y += preempt.h
mandatory-y += rqspinlock.h
diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
index d4f581c..b99cb57 100644
--- a/include/asm-generic/barrier.h
+++ b/include/asm-generic/barrier.h
@@ -235,7 +235,7 @@ do { \
/**
* smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
* @ptr: pointer to the variable to wait on
- * @cond: boolean expression to wait for
+ * @cond_expr: boolean expression to wait for
*
* Equivalent to using READ_ONCE() on the condition variable.
*
@@ -259,7 +259,7 @@ do { \
/**
* smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
* @ptr: pointer to the variable to wait on
- * @cond: boolean expression to wait for
+ * @cond_expr: boolean expression to wait for
*
* Equivalent to using smp_load_acquire() on the condition variable but employs
* the control dependency of the wait to reduce the barrier on many platforms.
diff --git a/include/asm-generic/kprobes.h b/include/asm-generic/kprobes.h
index 060eab0..5290a2b 100644
--- a/include/asm-generic/kprobes.h
+++ b/include/asm-generic/kprobes.h
@@ -14,7 +14,7 @@ static unsigned long __used \
_kbl_addr_##fname = (unsigned long)fname;
# define NOKPROBE_SYMBOL(fname) __NOKPROBE_SYMBOL(fname)
/* Use this to forbid a kprobes attach on very low level functions */
-# define __kprobes __section(".kprobes.text")
+# define __kprobes notrace __section(".kprobes.text")
# define nokprobe_inline __always_inline
#else
# define NOKPROBE_SYMBOL(fname)
diff --git a/include/asm-generic/percpu_types.h b/include/asm-generic/percpu_types.h
new file mode 100644
index 0000000..a095cea
--- /dev/null
+++ b/include/asm-generic/percpu_types.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_GENERIC_PERCPU_TYPES_H_
+#define _ASM_GENERIC_PERCPU_TYPES_H_
+
+#ifndef __ASSEMBLER__
+/*
+ * __percpu_qual is the qualifier for the percpu named address space.
+ *
+ * Most architectures use generic named address space for percpu variables but
+ * some architectures define percpu variables in different named address space.
+ * E.g. on x86, percpu variable may be declared as being relative to the %fs or
+ * %gs segments using __seg_fs or __seg_gs named address space qualifier.
+ */
+#ifndef __percpu_qual
+# define __percpu_qual
+#endif
+
+#endif /* __ASSEMBLER__ */
+#endif /* _ASM_GENERIC_PERCPU_TYPES_H_ */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index 2fd1fb8..7d17b9b 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -53,8 +53,7 @@ typedef struct qspinlock {
*
* When NR_CPUS < 16K
* 0- 7: locked byte
- * 8: pending
- * 9-15: not used
+ * 8-15: pending byte
* 16-17: tail index
* 18-31: tail cpu (+1)
*
diff --git a/include/asm-generic/ring_buffer.h b/include/asm-generic/ring_buffer.h
new file mode 100644
index 0000000..201d2ae
--- /dev/null
+++ b/include/asm-generic/ring_buffer.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Generic arch dependent ring_buffer macros.
+ */
+#ifndef __ASM_GENERIC_RING_BUFFER_H__
+#define __ASM_GENERIC_RING_BUFFER_H__
+
+#include <linux/cacheflush.h>
+
+/* Flush cache on ring buffer range if needed. Do nothing by default. */
+#define arch_ring_buffer_flush_range(start, end) do { } while (0)
+
+#endif /* __ASM_GENERIC_RING_BUFFER_H__ */
diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h
index 71dd38f..aac3ecf 100644
--- a/include/crypto/krb5.h
+++ b/include/crypto/krb5.h
@@ -121,9 +121,12 @@ size_t crypto_krb5_how_much_buffer(const struct krb5_enctype *krb5,
size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5,
enum krb5_crypto_mode mode,
size_t *_buffer_size, size_t *_offset);
-void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
- enum krb5_crypto_mode mode,
- size_t *_offset, size_t *_len);
+int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t *_offset, size_t *_len);
+int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+ enum krb5_crypto_mode mode,
+ size_t len, size_t min_content);
struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *krb5,
const struct krb5_buffer *TK,
u32 usage, gfp_t gfp);
diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h
index bc78fb7..768a8da 100644
--- a/include/drm/drm_device.h
+++ b/include/drm/drm_device.h
@@ -375,6 +375,13 @@ struct drm_device {
* Root directory for debugfs files.
*/
struct dentry *debugfs_root;
+
+ /**
+ * @gem_lru_mutex:
+ *
+ * Lock protecting movement of GEM objects between LRUs.
+ */
+ struct mutex gem_lru_mutex;
};
void drm_dev_set_dma_dev(struct drm_device *dev, struct device *dma_dev);
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index bf39190..0c5e5ed 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -273,6 +273,12 @@ int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper);
int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper);
bool drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
const struct drm_gem_object *obj);
+#else
+static inline bool drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
+ const struct drm_gem_object *obj)
+{
+ return false;
+}
#endif
#endif
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 86f58461..8a704f6 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -245,18 +245,12 @@ struct drm_gem_object_funcs {
* for lockless &shrinker.count_objects, and provides
* &drm_gem_lru_scan for driver's &shrinker.scan_objects
* implementation.
+ *
+ * Any access to this kind of object must be done with
+ * drm_device::gem_lru_mutex held.
*/
struct drm_gem_lru {
/**
- * @lock:
- *
- * Lock protecting movement of GEM objects between LRUs. All
- * LRUs that the object can move between should be protected
- * by the same lock.
- */
- struct mutex *lock;
-
- /**
* @count:
*
* The total number of backing pages of the GEM objects in
@@ -453,6 +447,9 @@ struct drm_gem_object {
* @lru:
*
* The current LRU list that the GEM object is on.
+ *
+ * Access to this field must be done with drm_device::gem_lru_mutex
+ * held.
*/
struct drm_gem_lru *lru;
};
@@ -610,12 +607,13 @@ void drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
u32 handle, u64 *offset);
-void drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock);
+void drm_gem_lru_init(struct drm_gem_lru *lru);
void drm_gem_lru_remove(struct drm_gem_object *obj);
void drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj);
void drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj);
unsigned long
-drm_gem_lru_scan(struct drm_gem_lru *lru,
+drm_gem_lru_scan(struct drm_device *dev,
+ struct drm_gem_lru *lru,
unsigned int nr_to_scan,
unsigned long *remaining,
bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 33e80f3..a5d3865 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -448,6 +448,8 @@ void ttm_resource_add_bulk_move(struct ttm_resource *res,
struct ttm_buffer_object *bo);
void ttm_resource_del_bulk_move(struct ttm_resource *res,
struct ttm_buffer_object *bo);
+void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res,
+ struct ttm_buffer_object *bo);
void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
void ttm_resource_init(struct ttm_buffer_object *bo,
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 9cd1594..ce0573e 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -613,6 +613,7 @@ unsigned long kunit_vm_mmap(struct kunit *test, struct file *file,
unsigned long offset);
void kunit_cleanup(struct kunit *test);
+void kunit_free_boot_suites(void);
void __printf(2, 3) kunit_log_append(struct string_stream *log, const char *fmt, ...);
diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h
index 3ffa534..2014288 100644
--- a/include/linux/alarmtimer.h
+++ b/include/linux/alarmtimer.h
@@ -42,11 +42,14 @@ struct alarm {
void *data;
};
+static __always_inline ktime_t alarm_get_expires(struct alarm *alarm)
+{
+ return alarm->node.expires;
+}
+
void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
void (*function)(struct alarm *, ktime_t));
-void alarm_start(struct alarm *alarm, ktime_t start);
-void alarm_start_relative(struct alarm *alarm, ktime_t start);
-void alarm_restart(struct alarm *alarm);
+bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative);
int alarm_try_to_cancel(struct alarm *alarm);
int alarm_cancel(struct alarm *alarm);
diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
index 50b47eb..e719575 100644
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -105,6 +105,12 @@
ARM_SMCCC_SMC_32, \
0, 0x3fff)
+/* C1-Pro erratum 4193714: SME DVMSync early acknowledgement */
+#define ARM_SMCCC_CPU_WORKAROUND_4193714 \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+ ARM_SMCCC_OWNER_CPU, 0x10)
+
#define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
ARM_SMCCC_SMC_32, \
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 97d7473..dc17780 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -475,7 +475,8 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty);
extern void bio_set_pages_dirty(struct bio *bio);
extern void bio_check_pages_dirty(struct bio *bio);
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen);
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen,
+ size_t minsize);
void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index b4b703c..cd191c5 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2917,7 +2917,13 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+ return 0;
+}
#endif
struct btf *bpf_get_btf_vmlinux(void);
@@ -3725,6 +3731,7 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
+extern const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
extern const struct bpf_func_proto bpf_find_vma_proto;
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b148f81..185b2aa 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -729,6 +729,7 @@ struct bpf_subprog_info {
*/
s16 fastcall_stack_off;
bool has_tail_call: 1;
+ bool might_throw: 1;
bool tail_call_reachable: 1;
bool has_ld_abs: 1;
bool is_cb: 1;
@@ -1308,6 +1309,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
+bool bpf_is_throw_kfunc(struct bpf_insn *insn);
int bpf_compute_const_regs(struct bpf_verifier_env *env);
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
int bpf_check_cfg(struct bpf_verifier_env *env);
diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
index c8f4f0a..fc879ac 100644
--- a/include/linux/cacheinfo.h
+++ b/include/linux/cacheinfo.h
@@ -89,6 +89,7 @@ int populate_cache_leaves(unsigned int cpu);
int cache_setup_acpi(unsigned int cpu);
bool last_level_cache_is_valid(unsigned int cpu);
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y);
+struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu);
int fetch_cache_info(unsigned int cpu);
int detect_cache_attributes(unsigned int cpu);
#ifndef CONFIG_ACPI_PPTT
diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h
index b907e6c..260d796 100644
--- a/include/linux/cdrom.h
+++ b/include/linux/cdrom.h
@@ -108,6 +108,7 @@ int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev,
extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi,
unsigned int clearing);
+extern void cdrom_probe_write_features(struct cdrom_device_info *cdi);
extern int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi);
extern void unregister_cdrom(struct cdrom_device_info *cdi);
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index f425637..50a784d 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -611,8 +611,8 @@ struct cgroup {
/* used to wait for offlining of csses */
wait_queue_head_t offline_waitq;
- /* used by cgroup_rmdir() to wait for dying tasks to leave */
- wait_queue_head_t dying_populated_waitq;
+ /* defers killing csses after removal until cgroup is depopulated */
+ struct work_struct finish_destroy_work;
/* used to schedule release agent */
struct work_struct release_agent_work;
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index e52160e..c5648fc 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -53,6 +53,7 @@ struct kernel_clone_args;
enum css_task_iter_flags {
CSS_TASK_ITER_PROCS = (1U << 0), /* walk only threadgroup leaders */
CSS_TASK_ITER_THREADED = (1U << 1), /* walk all threaded css_sets in the domain */
+ CSS_TASK_ITER_WITH_DEAD = (1U << 2), /* include exiting tasks */
CSS_TASK_ITER_SKIPPED = (1U << 16), /* internal flags */
};
@@ -776,6 +777,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen)
/*
* cgroup scalable recursive statistics.
*/
+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
void css_rstat_flush(struct cgroup_subsys_state *css);
diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
index ea95ca4..b1b5698 100644
--- a/include/linux/cleanup.h
+++ b/include/linux/cleanup.h
@@ -397,7 +397,8 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond
__DEFINE_GUARD_LOCK_PTR(_name, _T)
#define DEFINE_GUARD(_name, _type, _lock, _unlock) \
- DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
+ static __always_inline __nonnull_args(1) _type class_##_name##_constructor(_type _T); \
+ DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \
DEFINE_CLASS_IS_GUARD(_name)
#define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \
@@ -491,13 +492,14 @@ typedef struct { \
static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \
__no_context_analysis \
{ \
- if (_T->lock) { _unlock; } \
+ _unlock; \
} \
\
__DEFINE_GUARD_LOCK_PTR(_name, &_T->lock)
#define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \
-static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \
+static __always_inline __nonnull_args(1) \
+class_##_name##_t class_##_name##_constructor(_type *l) \
__no_context_analysis \
{ \
class_##_name##_t _t = { .lock = l }, *_T = &_t; \
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index 7c38190..283d729 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -32,6 +32,21 @@ struct module;
#include <vdso/clocksource.h>
/**
+ * struct clocksource_hw_snapshot - Snapshot for the underlying hardware counter of derived
+ * clocksources like kvmclock or Hyper-V scaled TSC
+ * @hw_cycles: The hardware counter value
+ * @hw_csid: Clocksource ID of the hardware counter
+ *
+ * Such clocksources must implement the read_snapshot() callback and fill in the
+ * hardware counter value, the clocksource ID of the hardware counter and derive
+ * the actual clocksource cycles from @hw_cycles to provide an atomic snapshot
+ */
+struct clocksource_hw_snapshot {
+ u64 hw_cycles;
+ enum clocksource_ids hw_csid;
+};
+
+/**
* struct clocksource - hardware abstraction for a free running counter
* Provides mostly state-free accessors to the underlying hardware.
* This is the structure used for system time.
@@ -72,6 +87,14 @@ struct module;
* @flags: Flags describing special properties
* @base: Hardware abstraction for clock on which a clocksource
* is based
+ * @read_snapshot: Extended @read() function for clocksources such as
+ * kvmclock or the Hyper-V scaled TSC where the actual
+ * clocksource value for timekeeping is calculated from an
+ * underlying hardware counter. Returns the timekeeping
+ * relevant cycle value and stores the raw value of the
+ * underlying counter from which it was calculated
+ * including the clocksource ID of that counter in the
+ * clocksource hardware snapshot.
* @enable: Optional function to enable the clocksource
* @disable: Optional function to disable the clocksource
* @suspend: Optional suspend function for the clocksource
@@ -113,6 +136,7 @@ struct clocksource {
unsigned long flags;
struct clocksource_base *base;
+ u64 (*read_snapshot)(struct clocksource *cs, struct clocksource_hw_snapshot *chs);
int (*enable)(struct clocksource *cs);
void (*disable)(struct clocksource *cs);
void (*suspend)(struct clocksource *cs);
@@ -236,8 +260,9 @@ clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec);
*/
extern int
__clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq);
-extern void
-__clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq);
+extern int
+__devm_clocksource_register_scale(struct device *dev, struct clocksource *cs,
+ u32 scale, u32 freq);
/*
* Don't call this unless you are a default clocksource
@@ -258,14 +283,16 @@ static inline int clocksource_register_khz(struct clocksource *cs, u32 khz)
return __clocksource_register_scale(cs, 1000, khz);
}
-static inline void __clocksource_update_freq_hz(struct clocksource *cs, u32 hz)
+static inline int devm_clocksource_register_hz(struct device *dev,
+ struct clocksource *cs, u32 hz)
{
- __clocksource_update_freq_scale(cs, 1, hz);
+ return __devm_clocksource_register_scale(dev, cs, 1, hz);
}
-static inline void __clocksource_update_freq_khz(struct clocksource *cs, u32 khz)
+static inline int devm_clocksource_register_khz(struct device *dev,
+ struct clocksource *cs, u32 khz)
{
- __clocksource_update_freq_scale(cs, 1000, khz);
+ return __devm_clocksource_register_scale(dev, cs, 1000, khz);
}
#ifdef CONFIG_ARCH_CLOCKSOURCE_INIT
diff --git a/include/linux/compat.h b/include/linux/compat.h
index 56cebaf..8da0a15 100644
--- a/include/linux/compat.h
+++ b/include/linux/compat.h
@@ -72,6 +72,10 @@
__diag_push(); \
__diag_ignore(GCC, 8, "-Wattribute-alias", \
"Type aliasing is used to sanitize syscall arguments");\
+ __diag_ignore(clang, 23, "-Wunknown-warning-option", \
+ "Avoid breaking versions without -Wattribute-alias"); \
+ __diag_ignore(clang, 23, "-Wattribute-alias", \
+ "Type aliasing is used to sanitize syscall arguments"); \
asmlinkage long compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \
__attribute__((alias(__stringify(__se_compat_sys##name)))); \
ALLOW_ERROR_INJECTION(compat_sys##name, ERRNO); \
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index e1123dd..527e4e1 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -131,6 +131,12 @@
#define __diag_str(s) __diag_str1(s)
#define __diag(s) _Pragma(__diag_str(clang diagnostic s))
+#if CONFIG_CLANG_VERSION >= 230000
+#define __diag_clang_23(s) __diag(s)
+#else
+#define __diag_clang_23(s)
+#endif
+
#define __diag_clang_13(s) __diag(s)
#define __diag_ignore_all(option, comment) \
diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h
index a931757..8302ebc 100644
--- a/include/linux/compiler-context-analysis.h
+++ b/include/linux/compiler-context-analysis.h
@@ -39,12 +39,14 @@
# define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__)))
/**
- * __guarded_by - struct member and globals attribute, declares variable
- * only accessible within active context
+ * __guarded_by() - struct member and globals attribute, declares variable
+ * only accessible within active context
+ * @...: context lock instance pointer(s)
*
* Declares that the struct member or global variable is only accessible within
- * the context entered by the given context lock. Read operations on the data
- * require shared access, while write operations require exclusive access.
+ * the context entered by the given context lock(s). Read operations on the data
+ * require shared access to at least one of the context locks, while write
+ * operations require exclusive access to all listed context locks.
*
* .. code-block:: c
*
@@ -52,17 +54,24 @@
* spinlock_t lock;
* long counter __guarded_by(&lock);
* };
+ *
+ * struct some_state {
+ * spinlock_t lock1, lock2;
+ * long counter __guarded_by(&lock1, &lock2);
+ * };
*/
# define __guarded_by(...) __attribute__((guarded_by(__VA_ARGS__)))
/**
- * __pt_guarded_by - struct member and globals attribute, declares pointed-to
- * data only accessible within active context
+ * __pt_guarded_by() - struct member and globals attribute, declares pointed-to
+ * data only accessible within active context
+ * @...: context lock instance pointer(s)
*
* Declares that the data pointed to by the struct member pointer or global
* pointer is only accessible within the context entered by the given context
- * lock. Read operations on the data require shared access, while write
- * operations require exclusive access.
+ * lock(s). Read operations on the data require shared access to at least one
+ * of the context locks, while write operations require exclusive access to all
+ * listed context locks.
*
* .. code-block:: c
*
@@ -70,6 +79,11 @@
* spinlock_t lock;
* long *counter __pt_guarded_by(&lock);
* };
+ *
+ * struct some_state {
+ * spinlock_t lock1, lock2;
+ * long *counter __pt_guarded_by(&lock1, &lock2);
+ * };
*/
# define __pt_guarded_by(...) __attribute__((pt_guarded_by(__VA_ARGS__)))
diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h
index c16d419..476c4c5 100644
--- a/include/linux/compiler_attributes.h
+++ b/include/linux/compiler_attributes.h
@@ -231,6 +231,15 @@
#define noinline __attribute__((__noinline__))
/*
+ * Note: deliberately not named '__nonnull', to avoid clashing with glibc's
+ * __nonnull() when kernel and userspace headers are combined.
+ *
+ * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-nonnull
+ * clang: https://clang.llvm.org/docs/AttributeReference.html#nonnull
+ */
+#define __nonnull_args(x...) __attribute__((__nonnull__(x)))
+
+/*
* Optional: only supported since gcc >= 8
* Optional: not supported by clang
*
@@ -397,6 +406,17 @@
#endif
/*
+ * Optional: not supported by clang
+ *
+ * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-noipa
+ */
+#if __has_attribute(noipa)
+# define __noipa __attribute__((noipa))
+#else
+# define __noipa
+#endif
+
+/*
* gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-weak-function-attribute
* gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-weak-variable-attribute
*/
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index e8fd775..c5921f1 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -634,6 +634,9 @@ struct ftrace_likely_data {
#else
#define __unqual_scalar_typeof(x) __typeof_unqual__(x)
#endif
+
+#include <asm/percpu_types.h>
+
#endif /* !__ASSEMBLY__ */
/*
@@ -711,6 +714,10 @@ struct ftrace_likely_data {
#define __diag_GCC(version, severity, string)
#endif
+#ifndef __diag_clang
+#define __diag_clang(version, severity, string)
+#endif
+
#define __diag_push() __diag(push)
#define __diag_pop() __diag(pop)
diff --git a/include/linux/cpuhplock.h b/include/linux/cpuhplock.h
index 286b3ab..42f6a09 100644
--- a/include/linux/cpuhplock.h
+++ b/include/linux/cpuhplock.h
@@ -12,9 +12,6 @@
struct device;
-extern int lockdep_is_cpus_held(void);
-extern int lockdep_is_cpus_write_held(void);
-
#ifdef CONFIG_HOTPLUG_CPU
void cpus_write_lock(void);
void cpus_write_unlock(void);
@@ -22,6 +19,8 @@ void cpus_read_lock(void);
void cpus_read_unlock(void);
int cpus_read_trylock(void);
void lockdep_assert_cpus_held(void);
+int lockdep_is_cpus_held(void);
+int lockdep_is_cpus_write_held(void);
void cpu_hotplug_disable_offlining(void);
void cpu_hotplug_disable(void);
void cpu_hotplug_enable(void);
@@ -38,6 +37,8 @@ static inline void cpus_read_lock(void) { }
static inline void cpus_read_unlock(void) { }
static inline int cpus_read_trylock(void) { return true; }
static inline void lockdep_assert_cpus_held(void) { }
+static inline int lockdep_is_cpus_held(void) { return 1; }
+static inline int lockdep_is_cpus_write_held(void) { return 1; }
static inline void cpu_hotplug_disable_offlining(void) { }
static inline void cpu_hotplug_disable(void) { }
static inline void cpu_hotplug_enable(void) { }
diff --git a/include/linux/delay.h b/include/linux/delay.h
index 46412c0..68b2a69dd 100644
--- a/include/linux/delay.h
+++ b/include/linux/delay.h
@@ -110,7 +110,7 @@ static const unsigned int max_slack_shift = 2;
* fsleep - flexible sleep which autoselects the best mechanism
* @usecs: requested sleep duration in microseconds
*
- * flseep() selects the best mechanism that will provide maximum 25% slack
+ * fsleep() selects the best mechanism that will provide maximum 25% slack
* to the requested sleep duration. Therefore it uses:
*
* * udelay() loop for sleep durations <= 10 microseconds to avoid hrtimer
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 166933b..d1203da 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -322,13 +322,13 @@ struct dma_buf {
* @vmapping_counter:
*
* Used internally to refcnt the vmaps returned by dma_buf_vmap().
- * Protected by @lock.
+ * Protected by @resv.
*/
unsigned vmapping_counter;
/**
* @vmap_ptr:
- * The current vmap ptr if @vmapping_counter > 0. Protected by @lock.
+ * The current vmap ptr if @vmapping_counter > 0. Protected by @resv.
*/
struct iosys_map vmap_ptr;
diff --git a/include/linux/dpll.h b/include/linux/dpll.h
index b7277a8..2dbe856 100644
--- a/include/linux/dpll.h
+++ b/include/linux/dpll.h
@@ -284,8 +284,10 @@ void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin,
int dpll_pin_ref_sync_pair_add(struct dpll_pin *pin,
struct dpll_pin *ref_sync_pin);
+int __dpll_device_change_ntf(struct dpll_device *dpll);
int dpll_device_change_ntf(struct dpll_device *dpll);
+int __dpll_pin_change_ntf(struct dpll_pin *pin);
int dpll_pin_change_ntf(struct dpll_pin *pin);
int register_dpll_notifier(struct notifier_block *nb);
diff --git a/include/linux/efi.h b/include/linux/efi.h
index 72e76ec..ccbc354 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -61,7 +61,7 @@ typedef void *efi_handle_t;
/*
* The UEFI spec and EDK2 reference implementation both define EFI_GUID as
- * struct { u32 a; u16; b; u16 c; u8 d[8]; }; and so the implied alignment
+ * struct { u32 a; u16 b; u16 c; u8 d[8]; }; and so the implied alignment
* is 32 bits not 8 bits like our guid_t. In some cases (i.e., on 32-bit ARM),
* this means that firmware services invoked by the kernel may assume that
* efi_guid_t* arguments are 32-bit aligned, and use memory accessors that
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 1ec6d5b..88a241a 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1151,9 +1151,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-#define __bpf_call_base_args \
- ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
- (void *)__bpf_call_base)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
void bpf_jit_compile(struct bpf_prog *prog);
diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h
index 0a3bcd1..be1b38c 100644
--- a/include/linux/fprobe.h
+++ b/include/linux/fprobe.h
@@ -94,6 +94,7 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter
int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num);
int register_fprobe_syms(struct fprobe *fp, const char **syms, int num);
int unregister_fprobe(struct fprobe *fp);
+int unregister_fprobe_async(struct fprobe *fp);
bool fprobe_is_registered(struct fprobe *fp);
int fprobe_count_ips_from_filter(const char *filter, const char *notfilter);
#else
@@ -113,6 +114,10 @@ static inline int unregister_fprobe(struct fprobe *fp)
{
return -EOPNOTSUPP;
}
+static inline int unregister_fprobe_async(struct fprobe *fp)
+{
+ return -EOPNOTSUPP;
+}
static inline bool fprobe_is_registered(struct fprobe *fp)
{
return false;
diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index 9598540..e5cde39 100644
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -915,6 +915,7 @@ extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
unsigned int obj_type);
extern void fsnotify_get_mark(struct fsnotify_mark *mark);
extern void fsnotify_put_mark(struct fsnotify_mark *mark);
+struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark);
extern void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info);
extern bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info);
diff --git a/include/linux/futex.h b/include/linux/futex.h
index 9e9750f..51f4ccd 100644
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -64,14 +64,10 @@ enum {
static inline void futex_init_task(struct task_struct *tsk)
{
- tsk->robust_list = NULL;
-#ifdef CONFIG_COMPAT
- tsk->compat_robust_list = NULL;
-#endif
- INIT_LIST_HEAD(&tsk->pi_state_list);
- tsk->pi_state_cache = NULL;
- tsk->futex_state = FUTEX_STATE_OK;
- mutex_init(&tsk->futex_exit_mutex);
+ memset(&tsk->futex, 0, sizeof(tsk->futex));
+ INIT_LIST_HEAD(&tsk->futex.pi_state_list);
+ tsk->futex.state = FUTEX_STATE_OK;
+ mutex_init(&tsk->futex.exit_mutex);
}
void futex_exit_recursive(struct task_struct *tsk);
@@ -85,22 +81,18 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
#ifdef CONFIG_FUTEX_PRIVATE_HASH
int futex_hash_allocate_default(void);
void futex_hash_free(struct mm_struct *mm);
-int futex_mm_init(struct mm_struct *mm);
-
-#else /* !CONFIG_FUTEX_PRIVATE_HASH */
+#else /* CONFIG_FUTEX_PRIVATE_HASH */
static inline int futex_hash_allocate_default(void) { return 0; }
static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
-static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
-#endif /* CONFIG_FUTEX_PRIVATE_HASH */
+#endif /* !CONFIG_FUTEX_PRIVATE_HASH */
-#else /* !CONFIG_FUTEX */
+#else /* CONFIG_FUTEX */
static inline void futex_init_task(struct task_struct *tsk) { }
static inline void futex_exit_recursive(struct task_struct *tsk) { }
static inline void futex_exit_release(struct task_struct *tsk) { }
static inline void futex_exec_release(struct task_struct *tsk) { }
-static inline long do_futex(u32 __user *uaddr, int op, u32 val,
- ktime_t *timeout, u32 __user *uaddr2,
- u32 val2, u32 val3)
+static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
+ u32 __user *uaddr2, u32 val2, u32 val3)
{
return -EINVAL;
}
@@ -108,13 +100,63 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig
{
return -EINVAL;
}
-static inline int futex_hash_allocate_default(void)
-{
- return 0;
-}
+static inline int futex_hash_allocate_default(void) { return 0; }
static inline int futex_hash_free(struct mm_struct *mm) { return 0; }
-static inline int futex_mm_init(struct mm_struct *mm) { return 0; }
+#endif /* !CONFIG_FUTEX */
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+#include <asm/futex_robust.h>
+
+void futex_reset_cs_ranges(struct futex_mm_data *fd);
+void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr);
+
+static inline bool futex_within_robust_unlock(struct pt_regs *regs,
+ struct futex_unlock_cs_range *csr)
+{
+ unsigned long ip = instruction_pointer(regs);
+
+ return ip >= csr->start_ip && ip < csr->start_ip + csr->len;
+}
+
+static inline void futex_fixup_robust_unlock(struct pt_regs *regs)
+{
+ struct futex_unlock_cs_range *csr;
+
+ /*
+ * Avoid dereferencing current->mm if not returning from interrupt.
+ * current->rseq.event is going to be used subsequently, so bringing the
+ * cache line in is not a big deal.
+ */
+ if (!current->rseq.event.user_irq)
+ return;
+
+ csr = current->mm->futex.unlock.cs_ranges;
+
+ /* The loop is optimized out for !COMPAT */
+ for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) {
+ if (unlikely(futex_within_robust_unlock(regs, csr))) {
+ __futex_fixup_robust_unlock(regs, csr);
+ return;
+ }
+ }
+}
+
+static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx,
+ unsigned long start, unsigned long end, bool sz32)
+{
+ fd->unlock.cs_ranges[idx].start_ip = start;
+ fd->unlock.cs_ranges[idx].len = end - start;
+ fd->unlock.cs_ranges[idx].pop_size32 = sz32;
+}
+#else /* CONFIG_FUTEX_ROBUST_UNLOCK */
+static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { }
+#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
+
+
+#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK)
+void futex_mm_init(struct mm_struct *mm);
+#else
+static inline void futex_mm_init(struct mm_struct *mm) { }
#endif
-#endif
+#endif /* _LINUX_FUTEX_H */
diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h
new file mode 100644
index 0000000..d320c05
--- /dev/null
+++ b/include/linux/futex_types.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_FUTEX_TYPES_H
+#define _LINUX_FUTEX_TYPES_H
+
+#ifdef CONFIG_FUTEX
+#include <linux/compiler_types.h>
+#include <linux/mutex_types.h>
+#include <linux/types.h>
+
+struct compat_robust_list_head;
+struct futex_pi_state;
+struct robust_list_head;
+
+/**
+ * struct futex_sched_data - Futex related per task data
+ * @robust_list: User space registered robust list pointer
+ * @compat_robust_list: User space registered robust list pointer for compat tasks
+ * @pi_state_list: List head for Priority Inheritance (PI) state management
+ * @pi_state_cache: Pointer to cache one PI state object per task
+ * @exit_mutex: Mutex for serializing exit
+ * @state: Futex handling state to handle exit races correctly
+ */
+struct futex_sched_data {
+ struct robust_list_head __user *robust_list;
+#ifdef CONFIG_COMPAT
+ struct compat_robust_list_head __user *compat_robust_list;
+#endif
+ struct list_head pi_state_list;
+ struct futex_pi_state *pi_state_cache;
+ struct mutex exit_mutex;
+ unsigned int state;
+};
+
+#ifdef CONFIG_FUTEX_PRIVATE_HASH
+/**
+ * struct futex_mm_phash - Futex private hash related per MM data
+ * @lock: Mutex to protect the private hash operations
+ * @hash: RCU managed pointer to the private hash
+ * @hash_new: Pointer to a newly allocated private hash
+ * @batches: Batch state for RCU synchronization
+ * @rcu: RCU head for call_rcu()
+ * @atomic: Aggregate value for @hash_ref
+ * @ref: Per CPU reference counter for a private hash
+ */
+struct futex_mm_phash {
+ struct mutex lock;
+ struct futex_private_hash __rcu *hash;
+ struct futex_private_hash *hash_new;
+ unsigned long batches;
+ struct rcu_head rcu;
+ atomic_long_t atomic;
+ unsigned int __percpu *ref;
+};
+#else /* CONFIG_FUTEX_ROBUST_UNLOCK */
+struct futex_mm_phash { };
+#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+/**
+ * struct futex_unlock_cs_range - Range for the VDSO unlock critical section
+ * @start_ip: The start IP of the robust futex unlock critical section (inclusive)
+ * @len: The length of the robust futex unlock critical section
+ * @pop_size32: Pending OP pointer size indicator. 0 == 64-bit, 1 == 32-bit
+ */
+struct futex_unlock_cs_range {
+ unsigned long start_ip;
+ unsigned int len;
+ unsigned int pop_size32;
+};
+
+#define FUTEX_ROBUST_MAX_CS_RANGES (1 + IS_ENABLED(CONFIG_COMPAT))
+
+/**
+ * struct futex_unlock_cs_ranges - Futex unlock VSDO critical sections
+ * @cs_ranges: Array of critical section ranges
+ */
+struct futex_unlock_cs_ranges {
+ struct futex_unlock_cs_range cs_ranges[FUTEX_ROBUST_MAX_CS_RANGES];
+};
+#else /* CONFIG_FUTEX_ROBUST_UNLOCK */
+struct futex_unlock_cs_ranges { };
+#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
+
+/**
+ * struct futex_mm_data - Futex related per MM data
+ * @phash: Futex private hash related data
+ * @unlock: Futex unlock VDSO critical sections
+ */
+struct futex_mm_data {
+ struct futex_mm_phash phash;
+ struct futex_unlock_cs_ranges unlock;
+};
+#else /* CONFIG_FUTEX */
+struct futex_sched_data { };
+struct futex_mm_data { };
+#endif /* !CONFIG_FUTEX */
+
+#endif /* _LINUX_FUTEX_TYPES_H */
diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h
index 80b38fb..31df760 100644
--- a/include/linux/fwnode.h
+++ b/include/linux/fwnode.h
@@ -208,6 +208,7 @@ struct fwnode_operations {
static inline void fwnode_init(struct fwnode_handle *fwnode,
const struct fwnode_operations *ops)
{
+ fwnode->secondary = NULL;
fwnode->ops = ops;
INIT_LIST_HEAD(&fwnode->consumers);
INIT_LIST_HEAD(&fwnode->suppliers);
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 6c75df3..cd4972a 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -273,11 +273,11 @@ enum {
*
* %__GFP_ZERO returns a zeroed page on success.
*
- * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself
- * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
- * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
- * memory tags at the same time as zeroing memory has minimal additional
- * performance impact.
+ * %__GFP_ZEROTAGS zeroes memory tags at allocation time. Setting memory tags at
+ * the same time as zeroing memory (e.g., with __GFP_ZERO) has minimal
+ * additional performance impact. However, __GFP_ZEROTAGS also zeroes the tags
+ * even if memory is not getting zeroed at allocation time (e.g.,
+ * with init_on_free).
*
* %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation.
* Used for userspace and vmalloc pages; the latter are unpoisoned by
diff --git a/include/linux/hid.h b/include/linux/hid.h
index 442a80d..47dc0bc 100644
--- a/include/linux/hid.h
+++ b/include/linux/hid.h
@@ -1030,6 +1030,8 @@ struct hid_field *hid_find_field(struct hid_device *hdev, unsigned int report_ty
int hid_set_field(struct hid_field *, unsigned, __s32);
int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
int interrupt);
+int hid_safe_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data,
+ size_t bufsize, u32 size, int interrupt);
struct hid_field *hidinput_get_led_field(struct hid_device *hid);
unsigned int hidinput_count_leds(struct hid_device *hid);
__s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code);
@@ -1298,8 +1300,8 @@ static inline u32 hid_report_len(struct hid_report *report)
return DIV_ROUND_UP(report->size, 8) + (report->id > 0);
}
-int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size,
- int interrupt);
+int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data,
+ size_t bufsize, u32 size, int interrupt);
/* HID quirks API */
unsigned long hid_lookup_quirk(const struct hid_device *hdev);
@@ -1314,8 +1316,6 @@ void hid_quirks_exit(__u16 bus);
dev_notice(&(hid)->dev, fmt, ##__VA_ARGS__)
#define hid_warn(hid, fmt, ...) \
dev_warn(&(hid)->dev, fmt, ##__VA_ARGS__)
-#define hid_warn_ratelimited(hid, fmt, ...) \
- dev_warn_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__)
#define hid_info(hid, fmt, ...) \
dev_info(&(hid)->dev, fmt, ##__VA_ARGS__)
#define hid_dbg(hid, fmt, ...) \
diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h
index a2e47db..19fffa4 100644
--- a/include/linux/hid_bpf.h
+++ b/include/linux/hid_bpf.h
@@ -72,8 +72,8 @@ struct hid_ops {
int (*hid_hw_output_report)(struct hid_device *hdev, __u8 *buf, size_t len,
u64 source, bool from_bpf);
int (*hid_input_report)(struct hid_device *hid, enum hid_report_type type,
- u8 *data, u32 size, int interrupt, u64 source, bool from_bpf,
- bool lock_already_taken);
+ u8 *data, size_t bufsize, u32 size, int interrupt, u64 source,
+ bool from_bpf, bool lock_already_taken);
struct module *owner;
const struct bus_type *bus_type;
};
@@ -200,7 +200,8 @@ struct hid_bpf {
#ifdef CONFIG_HID_BPF
u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, u8 *data,
- u32 *size, int interrupt, u64 source, bool from_bpf);
+ size_t *buf_size, u32 *size, int interrupt, u64 source,
+ bool from_bpf);
int dispatch_hid_bpf_raw_requests(struct hid_device *hdev,
unsigned char reportnum, __u8 *buf,
u32 size, enum hid_report_type rtype,
@@ -215,8 +216,11 @@ int hid_bpf_device_init(struct hid_device *hid);
const u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, const u8 *rdesc, unsigned int *size);
#else /* CONFIG_HID_BPF */
static inline u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type,
- u8 *data, u32 *size, int interrupt,
- u64 source, bool from_bpf) { return data; }
+ u8 *data, size_t *buf_size, u32 *size,
+ int interrupt, u64 source, bool from_bpf)
+{
+ return data;
+}
static inline int dispatch_hid_bpf_raw_requests(struct hid_device *hdev,
unsigned char reportnum, u8 *buf,
u32 size, enum hid_report_type rtype,
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index af03db8..d7aac9de1 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -347,10 +347,11 @@ static inline void clear_highpage_kasan_tagged(struct page *page)
#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGES
-/* Return false to let people know we did not initialize the pages */
-static inline bool tag_clear_highpages(struct page *page, int numpages)
+/* Returns true if the caller has to initialize the pages */
+static inline bool tag_clear_highpages(struct page *page, int numpages,
+ bool clear_pages)
{
- return false;
+ return clear_pages;
}
#endif
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 9ced498..6862dea 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -206,6 +206,9 @@ static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
u64 range_ns, const enum hrtimer_mode mode);
+extern bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim,
+ u64 range_ns, const enum hrtimer_mode mode);
+
/**
* hrtimer_start - (re)start an hrtimer
* @timer: the timer to be added
@@ -223,17 +226,28 @@ static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
extern int hrtimer_cancel(struct hrtimer *timer);
extern int hrtimer_try_to_cancel(struct hrtimer *timer);
-static inline void hrtimer_start_expires(struct hrtimer *timer,
- enum hrtimer_mode mode)
+static inline void hrtimer_start_expires(struct hrtimer *timer, enum hrtimer_mode mode)
{
- u64 delta;
ktime_t soft, hard;
+ u64 delta;
+
soft = hrtimer_get_softexpires(timer);
hard = hrtimer_get_expires(timer);
delta = ktime_to_ns(ktime_sub(hard, soft));
hrtimer_start_range_ns(timer, soft, delta, mode);
}
+static inline bool hrtimer_start_expires_user(struct hrtimer *timer, enum hrtimer_mode mode)
+{
+ ktime_t soft, hard;
+ u64 delta;
+
+ soft = hrtimer_get_softexpires(timer);
+ hard = hrtimer_get_expires(timer);
+ delta = ktime_to_ns(ktime_sub(hard, soft));
+ return hrtimer_start_range_ns_user(timer, soft, delta, mode);
+}
+
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
enum hrtimer_mode mode);
@@ -254,8 +268,8 @@ static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
return __hrtimer_get_remaining(timer, false);
}
-extern u64 hrtimer_get_next_event(void);
-extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
+extern ktime_t hrtimer_get_next_event(void);
+extern ktime_t hrtimer_next_event_without(const struct hrtimer *exclude);
extern bool hrtimer_active(const struct hrtimer *timer);
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 9341862..2abaf99 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -148,13 +148,11 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
struct folio **foliop);
#endif /* CONFIG_USERFAULTFD */
long hugetlb_reserve_pages(struct inode *inode, long from, long to,
- struct vm_area_desc *desc, vma_flags_t vma_flags);
+ struct vm_area_struct *vma, vma_flags_t vma_flags);
long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
long freed);
bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
-int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
- bool *migratable_cleared);
void folio_putback_hugetlb(struct folio *folio);
void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
void hugetlb_fix_reserve_counts(struct inode *inode);
@@ -276,7 +274,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
void fixup_hugetlb_reservations(struct vm_area_struct *vma);
void hugetlb_split(struct vm_area_struct *vma, unsigned long addr);
-int hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
unsigned int arch_hugetlb_cma_order(void);
@@ -422,12 +419,6 @@ static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb,
return 0;
}
-static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
- bool *migratable_cleared)
-{
- return 0;
-}
-
static inline void folio_putback_hugetlb(struct folio *folio)
{
}
@@ -469,11 +460,6 @@ static inline void fixup_hugetlb_reservations(struct vm_area_struct *vma)
static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {}
-static inline int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
-{
- return 0;
-}
-
#endif /* !CONFIG_HUGETLB_PAGE */
#ifndef pgd_write
diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h
index 565b473..5c29cd32 100644
--- a/include/linux/hugetlb_inline.h
+++ b/include/linux/hugetlb_inline.h
@@ -6,23 +6,13 @@
#ifdef CONFIG_HUGETLB_PAGE
-static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
-{
- return !!(vm_flags & VM_HUGETLB);
-}
-
static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
{
- return vma_flags_test_any(flags, VMA_HUGETLB_BIT);
+ return vma_flags_test(flags, VMA_HUGETLB_BIT);
}
#else
-static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
-{
- return false;
-}
-
static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
{
return false;
@@ -32,7 +22,7 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma)
{
- return is_vm_hugetlb_flags(vma->vm_flags);
+ return is_vma_hugetlb_flags(&vma->flags);
}
#endif
diff --git a/include/linux/intel_tpmi.h b/include/linux/intel_tpmi.h
index 94c06bf..15f0242 100644
--- a/include/linux/intel_tpmi.h
+++ b/include/linux/intel_tpmi.h
@@ -28,6 +28,12 @@ enum intel_tpmi_id {
TPMI_INFO_ID = 0x81, /* Special ID for PCI BDF and Package ID information */
};
+#define TPMI_CORE_INIT 0
+#define TPMI_CORE_EXIT 1
+
+int tpmi_register_notifier(struct notifier_block *nb);
+int tpmi_unregister_notifier(struct notifier_block *nb);
+
struct oobmsm_plat_info *tpmi_get_platform_data(struct auxiliary_device *auxdev);
struct resource *tpmi_get_resource_at_index(struct auxiliary_device *auxdev, int index);
int tpmi_get_resource_count(struct auxiliary_device *auxdev);
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 6cd26ff..3bf969a 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -864,6 +864,7 @@ static inline void init_irq_proc(void)
struct seq_file;
int show_interrupts(struct seq_file *p, void *v);
int arch_show_interrupts(struct seq_file *p, int prec);
+void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts);
extern int early_irq_init(void);
extern int arch_probe_nr_irqs(void);
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index 167fba7..1fabf0f 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -218,14 +218,6 @@ static __always_inline void __exit_to_user_mode_validate(void)
lockdep_sys_exit();
}
-/* Temporary workaround to keep ARM64 alive */
-static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *regs)
-{
- __exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK);
- rseq_exit_to_user_mode_legacy();
- __exit_to_user_mode_validate();
-}
-
/**
* syscall_exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required
* @regs: Pointer to pt_regs on entry stack
diff --git a/include/linux/irq.h b/include/linux/irq.h
index efa514e..f485369 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -103,6 +103,7 @@ enum {
IRQ_DISABLE_UNLAZY = (1 << 19),
IRQ_HIDDEN = (1 << 20),
IRQ_NO_DEBUG = (1 << 21),
+ IRQ_RESERVED = (1 << 22),
};
#define IRQF_MODIFY_MASK \
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index 0225121..ea5fd23 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -604,7 +604,7 @@
#include <asm/arch_gicv3.h>
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
/*
* We need a value to serve as a irq-type for LPIs. Choose one that will
diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h
index 40d2fce..f78787e 100644
--- a/include/linux/irqchip/arm-gic-v5.h
+++ b/include/linux/irqchip/arm-gic-v5.h
@@ -425,9 +425,6 @@ struct gicv5_its_itt_cfg {
void gicv5_init_lpis(u32 max);
void gicv5_deinit_lpis(void);
-int gicv5_alloc_lpi(void);
-void gicv5_free_lpi(u32 lpi);
-
void __init gicv5_its_of_probe(struct device_node *parent);
void __init gicv5_its_acpi_probe(void);
#endif
diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h
index d45fa19..849386d 100644
--- a/include/linux/irqchip/arm-gic.h
+++ b/include/linux/irqchip/arm-gic.h
@@ -131,7 +131,7 @@
#define GICV_PMR_PRIORITY_SHIFT 3
#define GICV_PMR_PRIORITY_MASK (0x1f << GICV_PMR_PRIORITY_SHIFT)
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
#include <linux/irqdomain.h>
@@ -162,5 +162,5 @@ int gic_get_cpu_id(unsigned int cpu);
void gic_migrate_target(unsigned int new_cpu_id);
unsigned long gic_get_sgir_physaddr(void);
-#endif /* __ASSEMBLY */
+#endif /* __ASSEMBLER__ */
#endif
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index dae9a9b..8080db1 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -52,8 +52,8 @@ struct irq_redirect {
* @depth: disable-depth, for nested irq_disable() calls
* @wake_depth: enable depth, for multiple irq_set_irq_wake() callers
* @tot_count: stats field for non-percpu irqs
- * @irq_count: stats field to detect stalled irqs
* @last_unhandled: aging timer for unhandled count
+ * @irq_count: stats field to detect stalled irqs
* @irqs_unhandled: stats field for spurious unhandled interrupts
* @threads_handled: stats field for deferred spurious detection of threaded handlers
* @threads_handled_last: comparator field for deferred spurious detection of threaded handlers
@@ -70,6 +70,7 @@ struct irq_redirect {
* IRQF_NO_SUSPEND set
* @force_resume_depth: number of irqactions on a irq descriptor with
* IRQF_FORCE_RESUME set
+ * @refcnt: Reference count mainly for /proc/interrupts
* @rcu: rcu head for delayed free
* @kobj: kobject used to represent this struct in sysfs
* @request_mutex: mutex to protect request/free before locking desc->lock
@@ -87,9 +88,9 @@ struct irq_desc {
unsigned int core_internal_state__do_not_mess_with_it;
unsigned int depth; /* nested irq disables */
unsigned int wake_depth; /* nested wake enables */
- unsigned int tot_count;
- unsigned int irq_count; /* For detecting broken IRQs */
+ unsigned long tot_count;
unsigned long last_unhandled; /* Aging timer for unhandled count */
+ unsigned int irq_count; /* For detecting broken IRQs */
unsigned int irqs_unhandled;
atomic_t threads_handled;
int threads_handled_last;
@@ -119,6 +120,7 @@ struct irq_desc {
struct dentry *debugfs_file;
const char *dev_name;
#endif
+ rcuref_t refcnt;
#ifdef CONFIG_SPARSE_IRQ
struct rcu_head rcu;
struct kobject kobj;
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index b97ce2d..fce1392 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -34,7 +34,14 @@ enum cpu_usage_stat {
};
struct kernel_cpustat {
- u64 cpustat[NR_STATS];
+#ifdef CONFIG_NO_HZ_COMMON
+ bool idle_dyntick;
+ bool idle_elapse;
+ seqcount_t idle_sleeptime_seq;
+ u64 idle_entrytime;
+ u64 idle_stealtime[2];
+#endif
+ u64 cpustat[NR_STATS];
};
struct kernel_stat {
@@ -99,23 +106,68 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu)
return kstat_cpu(cpu).irqs_sum;
}
+#ifdef CONFIG_NO_HZ_COMMON
+extern void kcpustat_dyntick_start(u64 now);
+extern void kcpustat_dyntick_stop(u64 now);
+extern void kcpustat_irq_enter(u64 now);
+extern void kcpustat_irq_exit(u64 now);
+extern u64 kcpustat_field_idle(int cpu);
+extern u64 kcpustat_field_iowait(int cpu);
+
+static inline bool kcpustat_idle_dyntick(void)
+{
+ return __this_cpu_read(kernel_cpustat.idle_dyntick);
+}
+#else
+static inline u64 kcpustat_field_idle(int cpu)
+{
+ return kcpustat_cpu(cpu).cpustat[CPUTIME_IDLE];
+}
+static inline u64 kcpustat_field_iowait(int cpu)
+{
+ return kcpustat_cpu(cpu).cpustat[CPUTIME_IOWAIT];
+}
+
+static inline bool kcpustat_idle_dyntick(void)
+{
+ return false;
+}
+#endif /* CONFIG_NO_HZ_COMMON */
+
+extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
+extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
+
+/* Fetch cputime values when vtime is disabled on a CPU */
+static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cpu)
+{
+ if (usage == CPUTIME_IDLE)
+ return kcpustat_field_idle(cpu);
+ if (usage == CPUTIME_IOWAIT)
+ return kcpustat_field_iowait(cpu);
+ return kcpustat_cpu(cpu).cpustat[usage];
+}
+
+static inline void kcpustat_cpu_fetch_default(struct kernel_cpustat *dst, int cpu)
+{
+ *dst = kcpustat_cpu(cpu);
+ dst->cpustat[CPUTIME_IDLE] = kcpustat_field_idle(cpu);
+ dst->cpustat[CPUTIME_IOWAIT] = kcpustat_field_iowait(cpu);
+}
+
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
-extern u64 kcpustat_field(struct kernel_cpustat *kcpustat,
- enum cpu_usage_stat usage, int cpu);
+extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu);
extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu);
#else
-static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat,
- enum cpu_usage_stat usage, int cpu)
+static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu)
{
- return kcpustat->cpustat[usage];
+ return kcpustat_field_default(usage, cpu);
}
static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
{
- *dst = kcpustat_cpu(cpu);
+ kcpustat_cpu_fetch_default(dst, cpu);
}
-
-#endif
+#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */
extern void account_user_time(struct task_struct *, u64);
extern void account_guest_time(struct task_struct *, u64);
@@ -124,19 +176,17 @@ extern void account_system_index_time(struct task_struct *, u64,
enum cpu_usage_stat);
extern void account_steal_time(u64);
extern void account_idle_time(u64);
-extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu);
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
static inline void account_process_tick(struct task_struct *tsk, int user)
{
- vtime_flush(tsk);
+ if (!kcpustat_idle_dyntick())
+ vtime_flush(tsk);
}
#else
extern void account_process_tick(struct task_struct *, int user);
#endif
-extern void account_idle_ticks(unsigned long ticks);
-
#ifdef CONFIG_SCHED_CORE
extern void __account_forceidle_time(struct task_struct *tsk, u64 delta);
#endif
diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h
index 7e847a2..db9bda6 100644
--- a/include/linux/kho/abi/kexec_handover.h
+++ b/include/linux/kho/abi/kexec_handover.h
@@ -274,7 +274,7 @@ enum kho_radix_consts {
* and 1 bitmap level.
*/
KHO_TREE_MAX_DEPTH =
- DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2,
+ DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2 + 1,
KHO_TABLE_SIZE_LOG2) + 1,
};
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 5c085ef..127229f 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -371,6 +371,7 @@ enum {
/* return values for ->qc_defer */
ATA_DEFER_LINK = 1,
ATA_DEFER_PORT = 2,
+ ATA_DEFER_LINK_EXCL = 3,
/* desc_len for ata_eh_info and context */
ATA_EH_DESC_LEN = 80,
@@ -854,6 +855,9 @@ struct ata_link {
unsigned int sata_spd; /* current SATA PHY speed */
enum ata_lpm_policy lpm_policy;
+ struct work_struct deferred_qc_work;
+ struct ata_queued_cmd *deferred_qc;
+
/* record runtime error info, protected by host_set lock */
struct ata_eh_info eh_info;
/* EH context */
@@ -899,9 +903,6 @@ struct ata_port {
u64 qc_active;
int nr_active_links; /* #links with active qcs */
- struct work_struct deferred_qc_work;
- struct ata_queued_cmd *deferred_qc;
-
struct ata_link link; /* host default link */
struct ata_link *slave_link; /* see ata_slave_link_init() */
diff --git a/include/linux/list.h b/include/linux/list.h
index 00ea8e5..09d9799 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -191,6 +191,29 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head)
__list_add(new, head->prev, head);
}
+/**
+ * list_add_tail_release - add a new entry with release barrier
+ * @new: new entry to be added
+ * @head: list head to add it before
+ *
+ * Insert a new entry before the specified head, using a release barrier to set
+ * the ->next pointer that points to it. This is useful for implementing
+ * queues, in particular one that the elements will be walked through forwards
+ * locklessly.
+ */
+static inline void list_add_tail_release(struct list_head *new,
+ struct list_head *head)
+{
+ struct list_head *prev = head->prev;
+
+ if (__list_add_valid(new, prev, head)) {
+ new->next = head;
+ new->prev = prev;
+ head->prev = new;
+ smp_store_release(&prev->next, new);
+ }
+}
+
/*
* Delete a list entry by making the prev/next entries
* point to each other.
@@ -645,6 +668,20 @@ static inline void list_splice_tail_init(struct list_head *list,
})
/**
+ * list_first_entry_or_null_acquire - get the first element from a list with barrier
+ * @ptr: the list head to take the element from.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_head within the struct.
+ *
+ * Note that if the list is empty, it returns NULL.
+ */
+#define list_first_entry_or_null_acquire(ptr, type, member) ({ \
+ struct list_head *head__ = (ptr); \
+ struct list_head *pos__ = smp_load_acquire(&head__->next); \
+ pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
+})
+
+/**
* list_last_entry_or_null - get the last element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
diff --git a/include/linux/mailbox_client.h b/include/linux/mailbox_client.h
index c6eea9a..e599712 100644
--- a/include/linux/mailbox_client.h
+++ b/include/linux/mailbox_client.h
@@ -45,6 +45,7 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg);
int mbox_flush(struct mbox_chan *chan, unsigned long timeout);
void mbox_client_txdone(struct mbox_chan *chan, int r); /* atomic */
bool mbox_client_peek_data(struct mbox_chan *chan); /* atomic */
+unsigned int mbox_chan_tx_slots_available(struct mbox_chan *chan); /* atomic */
void mbox_free_channel(struct mbox_chan *chan); /* may sleep */
#endif /* __MAILBOX_CLIENT_H */
diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h
index 80a427c..dc93287 100644
--- a/include/linux/mailbox_controller.h
+++ b/include/linux/mailbox_controller.h
@@ -3,6 +3,7 @@
#ifndef __MAILBOX_CONTROLLER_H
#define __MAILBOX_CONTROLLER_H
+#include <linux/bits.h>
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/hrtimer.h>
@@ -11,6 +12,13 @@
struct mbox_chan;
+/* Sentinel value distinguishing "no active request" from "NULL message data" */
+#define MBOX_NO_MSG ((void *)-1)
+
+#define MBOX_TXDONE_BY_IRQ BIT(0) /* controller has remote RTR irq */
+#define MBOX_TXDONE_BY_POLL BIT(1) /* controller can read status of last TX */
+#define MBOX_TXDONE_BY_ACK BIT(2) /* S/W ACK received by Client ticks the TX */
+
/**
* struct mbox_chan_ops - methods to control mailbox channels
* @send_data: The API asks the MBOX controller driver, in atomic
@@ -54,10 +62,10 @@ struct mbox_chan_ops {
/**
* struct mbox_controller - Controller of a class of communication channels
- * @dev: Device backing this controller
- * @ops: Operators that work on each communication chan
- * @chans: Array of channels
- * @num_chans: Number of channels in the 'chans' array.
+ * @dev: Device backing this controller. Required.
+ * @ops: Operators that work on each communication chan. Required.
+ * @chans: Array of channels. Required.
+ * @num_chans: Number of channels in the 'chans' array. Required.
* @txdone_irq: Indicates if the controller can report to API when
* the last transmitted data was read by the remote.
* Eg, if it has some TX ACK irq.
@@ -70,6 +78,7 @@ struct mbox_chan_ops {
* @of_xlate: Controller driver specific mapping of channel via DT
* @poll_hrt: API private. hrtimer used to poll for TXDONE on all
* channels.
+ * @poll_hrt_lock: API private. Lock protecting access to poll_hrt.
* @node: API private. To hook into list of controllers.
*/
struct mbox_controller {
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 0c464ea..4a56319 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -4,7 +4,7 @@
/*
* Maple Tree - An RCU-safe adaptive tree for storing ranges
* Copyright (c) 2018-2022 Oracle
- * Authors: Liam R. Howlett <Liam.Howlett@Oracle.com>
+ * Authors: Liam R. Howlett <liam@infradead.org>
* Matthew Wilcox <willy@infradead.org>
*/
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0b77690..fc2aced 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4391,7 +4391,7 @@ static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc,
int mmap_action_prepare(struct vm_area_desc *desc);
int mmap_action_complete(struct vm_area_struct *vma,
- struct mmap_action *action);
+ struct mmap_action *action, bool is_compat);
/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
@@ -4975,8 +4975,6 @@ extern int soft_offline_page(unsigned long pfn, int flags);
*/
extern const struct attribute_group memory_failure_attr_group;
extern void memory_failure_queue(unsigned long pfn, int flags);
-extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
- bool *migratable_cleared);
void num_poisoned_pages_inc(unsigned long pfn);
void num_poisoned_pages_sub(unsigned long pfn, long i);
#else
@@ -4984,12 +4982,6 @@ static inline void memory_failure_queue(unsigned long pfn, int flags)
{
}
-static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
- bool *migratable_cleared)
-{
- return 0;
-}
-
static inline void num_poisoned_pages_inc(unsigned long pfn)
{
}
@@ -5174,9 +5166,10 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
* DMA mapping IDs for page_pool
*
* When DMA-mapping a page, page_pool allocates an ID (from an xarray) and
- * stashes it in the upper bits of page->pp_magic. Non-PP pages can have
- * arbitrary kernel pointers stored in the same field as pp_magic (since
- * it overlaps with page->lru.next), so we must ensure that we cannot
+ * stashes it in the upper bits of page->pp_magic. We always want to be able to
+ * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP
+ * pages can have arbitrary kernel pointers stored in the same field as pp_magic
+ * (since it overlaps with page->lru.next), so we must ensure that we cannot
* mistake a valid kernel pointer with any of the values we write into this
* field.
*
@@ -5211,6 +5204,26 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
#define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \
PP_DMA_INDEX_SHIFT)
+/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is
+ * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for
+ * the head page of compound page and bit 1 for pfmemalloc page, as well as the
+ * bits used for the DMA index. page_is_pfmemalloc() is checked in
+ * __page_pool_put_page() to avoid recycling the pfmemalloc page.
+ */
+#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
+
+#ifdef CONFIG_PAGE_POOL
+static inline bool page_pool_page_is_pp(const struct page *page)
+{
+ return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;
+}
+#else
+static inline bool page_pool_page_is_pp(const struct page *page)
+{
+ return false;
+}
+#endif
+
#define PAGE_SNAPSHOT_FAITHFUL (1 << 0)
#define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)
#define PAGE_SNAPSHOT_PG_IDLE (1 << 2)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a308e2c..a8ed62c 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -20,6 +20,7 @@
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>
+#include <linux/futex_types.h>
#include <linux/rseq_types.h>
#include <linux/bitmap.h>
@@ -1222,6 +1223,8 @@ struct mm_struct {
/* MM CID related storage */
struct mm_mm_cid mm_cid;
+ /* sched_cache related statistics */
+ struct sched_cache_stat sc_stat;
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1270,16 +1273,7 @@ struct mm_struct {
*/
seqcount_t mm_lock_seq;
#endif
-#ifdef CONFIG_FUTEX_PRIVATE_HASH
- struct mutex futex_hash_lock;
- struct futex_private_hash __rcu *futex_phash;
- struct futex_private_hash *futex_phash_new;
- /* futex-ref */
- unsigned long futex_batches;
- struct rcu_head futex_rcu;
- atomic_long_t futex_atomic;
- unsigned int __percpu *futex_ref;
-#endif
+ struct futex_mm_data futex;
unsigned long hiwater_rss; /* High-watermark of RSS usage */
unsigned long hiwater_vm; /* High-water virtual memory usage */
@@ -1628,6 +1622,36 @@ static inline unsigned int mm_cid_size(void)
# define MM_CID_STATIC_SIZE 0
#endif /* CONFIG_SCHED_MM_CID */
+#ifdef CONFIG_SCHED_CACHE
+void mm_init_sched(struct mm_struct *mm,
+ struct sched_cache_time __percpu *pcpu_sched);
+
+static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
+{
+ struct sched_cache_time __percpu *pcpu_sched =
+ alloc_percpu_noprof(struct sched_cache_time);
+
+ if (!pcpu_sched)
+ return -ENOMEM;
+
+ mm_init_sched(mm, pcpu_sched);
+ return 0;
+}
+
+#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
+
+static inline void mm_destroy_sched(struct mm_struct *mm)
+{
+ free_percpu(mm->sc_stat.pcpu_sched);
+ mm->sc_stat.pcpu_sched = NULL;
+}
+#else /* !CONFIG_SCHED_CACHE */
+
+static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
+static inline void mm_destroy_sched(struct mm_struct *mm) { }
+
+#endif /* CONFIG_SCHED_CACHE */
+
struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h
index cf33745..5d75cc5 100644
--- a/include/linux/mroute_base.h
+++ b/include/linux/mroute_base.h
@@ -226,6 +226,7 @@ struct mr_table_ops {
/**
* struct mr_table - a multicast routing table
+ * @work: used for table destruction
* @list: entry within a list of multicast routing tables
* @net: net where this table belongs
* @ops: protocol specific operations
@@ -243,6 +244,7 @@ struct mr_table_ops {
* @mroute_reg_vif_num: PIM-device vif index
*/
struct mr_table {
+ struct rcu_work work;
struct list_head list;
possible_net_t net;
struct mr_table_ops ops;
@@ -274,6 +276,7 @@ void vif_device_init(struct vif_device *v,
unsigned short flags,
unsigned short get_iflink_mask);
+void mr_table_free(struct mr_table *mrt);
struct mr_table *
mr_table_alloc(struct net *net, u32 id,
struct mr_table_ops *ops,
diff --git a/include/linux/msi.h b/include/linux/msi.h
index fa41eed..a4613de 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -444,7 +444,7 @@ struct msi_domain_info;
*
* @domain_alloc_irqs, @domain_free_irqs can be used to override the
* default allocation/free functions (__msi_domain_alloc/free_irqs). This
- * is initially for a wrapper around XENs seperate MSI universe which can't
+ * is initially for a wrapper around XEN's separate MSI universe which can't
* be wrapped into the regular irq domains concepts by mere mortals. This
* allows to universally use msi_domain_alloc/free_irqs without having to
* special case XEN all over the place.
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index 58abd30..782984b 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -290,6 +290,12 @@
SPI_MEM_OP_NO_DUMMY, \
SPI_MEM_OP_NO_DATA)
+#define SPINAND_PAGE_READ_PACKED_8D_8D_0_OP(addr) \
+ SPI_MEM_OP(SPI_MEM_DTR_OP_PACKED_CMD(0x13, addr >> 16, 8), \
+ SPI_MEM_DTR_OP_ADDR(2, addr & 0xffff, 8), \
+ SPI_MEM_OP_NO_DUMMY, \
+ SPI_MEM_OP_NO_DATA)
+
#define SPINAND_PAGE_READ_FROM_CACHE_8D_8D_8D_OP(addr, ndummy, buf, len, freq) \
SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x9d, 8), \
SPI_MEM_DTR_OP_ADDR(2, addr, 8), \
@@ -483,6 +489,7 @@ struct spinand_ecc_info {
#define SPINAND_HAS_PROG_PLANE_SELECT_BIT BIT(2)
#define SPINAND_HAS_READ_PLANE_SELECT_BIT BIT(3)
#define SPINAND_NO_RAW_ACCESS BIT(4)
+#define SPINAND_ODTR_PACKED_PAGE_READ BIT(5)
/**
* struct spinand_ondie_ecc_conf - private SPI-NAND on-die ECC engine structure
diff --git a/include/linux/netfilter/nf_conntrack_proto_gre.h b/include/linux/netfilter/nf_conntrack_proto_gre.h
index 9ee7014..ad5563f 100644
--- a/include/linux/netfilter/nf_conntrack_proto_gre.h
+++ b/include/linux/netfilter/nf_conntrack_proto_gre.h
@@ -18,9 +18,10 @@ struct nf_ct_gre_keymap {
struct rcu_head rcu;
};
-/* add new tuple->key_reply pair to keymap */
-int nf_ct_gre_keymap_add(struct nf_conn *ct, enum ip_conntrack_dir dir,
- struct nf_conntrack_tuple *t);
+/* add tuple->key_reply pairs to keymap */
+bool nf_ct_gre_keymap_add(struct nf_conn *ct,
+ const struct nf_conntrack_tuple *orig,
+ const struct nf_conntrack_tuple *repl);
/* delete keymap entries */
void nf_ct_gre_keymap_destroy(struct nf_conn *ct);
diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h
index 77c778d..5a1c5c3 100644
--- a/include/linux/netfilter/x_tables.h
+++ b/include/linux/netfilter/x_tables.h
@@ -146,6 +146,9 @@ struct xt_match {
/* Called when user tries to insert an entry of this type. */
int (*checkentry)(const struct xt_mtchk_param *);
+ /* Called to validate hooks based on the match configuration. */
+ int (*check_hooks)(const struct xt_mtchk_param *);
+
/* Called when entry of this type deleted. */
void (*destroy)(const struct xt_mtdtor_param *);
#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
@@ -187,6 +190,9 @@ struct xt_target {
/* Should return 0 on success or an error code otherwise (-Exxxx). */
int (*checkentry)(const struct xt_tgchk_param *);
+ /* Called to validate hooks based on the target configuration. */
+ int (*check_hooks)(const struct xt_tgchk_param *);
+
/* Called when entry of this type deleted. */
void (*destroy)(const struct xt_tgdtor_param *);
#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
@@ -279,8 +285,10 @@ bool xt_find_jump_offset(const unsigned int *offsets,
int xt_check_proc_name(const char *name, unsigned int size);
+int xt_check_hooks_match(struct xt_mtchk_param *par);
int xt_check_match(struct xt_mtchk_param *, unsigned int size, u16 proto,
bool inv_proto);
+int xt_check_hooks_target(struct xt_tgchk_param *par);
int xt_check_target(struct xt_tgchk_param *, unsigned int size, u16 proto,
bool inv_proto);
@@ -297,9 +305,11 @@ struct xt_counters *xt_counters_alloc(unsigned int counters);
struct xt_table *xt_register_table(struct net *net,
const struct xt_table *table,
+ const struct nf_hook_ops *template_ops,
struct xt_table_info *bootstrap,
struct xt_table_info *newinfo);
-void *xt_unregister_table(struct xt_table *table);
+void xt_unregister_table_pre_exit(struct net *net, u8 af, const char *name);
+struct xt_table *xt_unregister_table_exit(struct net *net, u8 af, const char *name);
struct xt_table_info *xt_replace_table(struct xt_table *table,
unsigned int num_counters,
diff --git a/include/linux/netfilter_arp/arp_tables.h b/include/linux/netfilter_arp/arp_tables.h
index a40aaf6..05631a2 100644
--- a/include/linux/netfilter_arp/arp_tables.h
+++ b/include/linux/netfilter_arp/arp_tables.h
@@ -53,7 +53,6 @@ int arpt_register_table(struct net *net, const struct xt_table *table,
const struct arpt_replace *repl,
const struct nf_hook_ops *ops);
void arpt_unregister_table(struct net *net, const char *name);
-void arpt_unregister_table_pre_exit(struct net *net, const char *name);
extern unsigned int arpt_do_table(void *priv, struct sk_buff *skb,
const struct nf_hook_state *state);
diff --git a/include/linux/netfilter_ipv4/ip_tables.h b/include/linux/netfilter_ipv4/ip_tables.h
index 132b0e4..1359339 100644
--- a/include/linux/netfilter_ipv4/ip_tables.h
+++ b/include/linux/netfilter_ipv4/ip_tables.h
@@ -26,7 +26,6 @@ int ipt_register_table(struct net *net, const struct xt_table *table,
const struct ipt_replace *repl,
const struct nf_hook_ops *ops);
-void ipt_unregister_table_pre_exit(struct net *net, const char *name);
void ipt_unregister_table_exit(struct net *net, const char *name);
/* Standard entry. */
diff --git a/include/linux/netfilter_ipv6/ip6_tables.h b/include/linux/netfilter_ipv6/ip6_tables.h
index 8b8885a..c6d5b92 100644
--- a/include/linux/netfilter_ipv6/ip6_tables.h
+++ b/include/linux/netfilter_ipv6/ip6_tables.h
@@ -27,7 +27,6 @@ extern void *ip6t_alloc_initial_table(const struct xt_table *);
int ip6t_register_table(struct net *net, const struct xt_table *table,
const struct ip6t_replace *repl,
const struct nf_hook_ops *ops);
-void ip6t_unregister_table_pre_exit(struct net *net, const char *name);
void ip6t_unregister_table_exit(struct net *net, const char *name);
extern unsigned int ip6t_do_table(void *priv, struct sk_buff *skb,
const struct nf_hook_state *state);
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index ba17ac5b..243c0f7 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -62,8 +62,8 @@ struct netfs_inode {
struct fscache_cookie *cache;
#endif
struct mutex wb_lock; /* Writeback serialisation */
- loff_t remote_i_size; /* Size of the remote file */
- loff_t zero_point; /* Size after which we assume there's no data
+ loff_t _remote_i_size; /* Size of the remote file */
+ loff_t _zero_point; /* Size after which we assume there's no data
* on the server */
atomic_t io_count; /* Number of outstanding reqs */
unsigned long flags;
@@ -252,7 +252,7 @@ struct netfs_io_request {
unsigned long long collected_to; /* Point we've collected to */
unsigned long long cleaned_to; /* Position we've cleaned folios to */
unsigned long long abandon_to; /* Position to abandon folios to */
- pgoff_t no_unlock_folio; /* Don't unlock this folio after read */
+ const struct folio *no_unlock_folio; /* Don't unlock this folio after read */
unsigned int direct_bv_count; /* Number of elements in direct_bv[] */
unsigned int debug_id;
unsigned int rsize; /* Maximum read size (0 for none) */
@@ -475,6 +475,254 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode)
}
/**
+ * netfs_read_remote_i_size - Read remote_i_size safely
+ * @inode: The inode to access
+ *
+ * Read remote_i_size safely without the potential for tearing on 32-bit
+ * arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode)
+{
+ const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+ unsigned long long remote_i_size;
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ unsigned int seq;
+
+ do {
+ seq = read_seqcount_begin(&inode->i_size_seqcount);
+ remote_i_size = ictx->_remote_i_size;
+ } while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ remote_i_size = ictx->_remote_i_size;
+ preempt_enable();
+#else
+ /* Pairs with smp_store_release() in netfs_write_remote_i_size() */
+ remote_i_size = smp_load_acquire(&ictx->_remote_i_size);
+#endif
+ return remote_i_size;
+}
+
+/*
+ * netfs_write_remote_i_size - Set remote_i_size safely
+ * @inode: The inode to access
+ * @remote_i_size: The new value for the size of the file on the server
+ *
+ * Set remote_i_size safely without the potential for tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_remote_i_size(), netfs_write_remote_i_size() does
+ * need locking around it (normally i_rwsem), otherwise on 32bit/SMP an update
+ * of i_size_seqcount can be lost, resulting in subsequent i_size_read() calls
+ * spinning forever.
+ */
+static inline void netfs_write_remote_i_size(struct inode *inode,
+ unsigned long long remote_i_size)
+{
+ struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ write_seqcount_begin(&inode->i_size_seqcount);
+ ictx->_remote_i_size = remote_i_size;
+ write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ ictx->_remote_i_size = remote_i_size;
+ preempt_enable();
+#else
+ /*
+ * Pairs with smp_load_acquire() in netfs_read_remote_i_size() to
+ * ensure changes related to inode size (such as page contents) are
+ * visible before we see the changed inode size.
+ */
+ smp_store_release(&ictx->_remote_i_size, remote_i_size);
+#endif
+}
+
+/**
+ * netfs_read_zero_point - Read zero_point safely
+ * @inode: The inode to access
+ *
+ * Read zero_point safely without the potential for tearing on 32-bit
+ * arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline unsigned long long netfs_read_zero_point(const struct inode *inode)
+{
+ struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+ unsigned long long zero_point;
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ unsigned int seq;
+
+ do {
+ seq = read_seqcount_begin(&inode->i_size_seqcount);
+ zero_point = ictx->_zero_point;
+ } while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ zero_point = ictx->_zero_point;
+ preempt_enable();
+#else
+ /* Pairs with smp_store_release() in netfs_write_zero_point() */
+ zero_point = smp_load_acquire(&ictx->_zero_point);
+#endif
+ return zero_point;
+}
+
+/*
+ * netfs_write_zero_point - Set zero_point safely
+ * @inode: The inode to access
+ * @zero_point: The new value for the point beyond which the server has no data
+ *
+ * Set zero_point safely without the potential for tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need
+ * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of
+ * i_size_seqcount can be lost, resulting in subsequent read calls spinning
+ * forever.
+ */
+static inline void netfs_write_zero_point(struct inode *inode,
+ unsigned long long zero_point)
+{
+ struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ write_seqcount_begin(&inode->i_size_seqcount);
+ ictx->_zero_point = zero_point;
+ write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ ictx->_zero_point = zero_point;
+ preempt_enable();
+#else
+ /*
+ * Pairs with smp_load_acquire() in netfs_read_zero_point() to
+ * ensure changes related to inode size (such as page contents) are
+ * visible before we see the changed inode size.
+ */
+ smp_store_release(&ictx->_zero_point, zero_point);
+#endif
+}
+
+/**
+ * netfs_read_sizes - Read remote_i_size and zero_point safely
+ * @inode: The inode to access
+ * @i_size: Where to return the local file size.
+ * @remote_i_size: Where to return the size of the file on the server
+ * @zero_point: Where to return the the point beyond which the server has no data
+ *
+ * Read remote_i_size and zero_point safely without the potential for tearing
+ * on 32-bit arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline void netfs_read_sizes(const struct inode *inode,
+ unsigned long long *i_size,
+ unsigned long long *remote_i_size,
+ unsigned long long *zero_point)
+{
+ const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ unsigned int seq;
+
+ do {
+ seq = read_seqcount_begin(&inode->i_size_seqcount);
+ *i_size = inode->i_size;
+ *remote_i_size = ictx->_remote_i_size;
+ *zero_point = ictx->_zero_point;
+ } while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ *i_size = inode->i_size;
+ *remote_i_size = ictx->_remote_i_size;
+ *zero_point = ictx->_zero_point;
+ preempt_enable();
+#else
+ /* Pairs with smp_store_release() in i_size_write() */
+ *i_size = smp_load_acquire(&inode->i_size);
+ /* Pairs with smp_store_release() in netfs_write_remote_i_size() */
+ *remote_i_size = smp_load_acquire(&ictx->_remote_i_size);
+ /* Pairs with smp_store_release() in netfs_write_zero_point() */
+ *zero_point = smp_load_acquire(&ictx->_zero_point);
+#endif
+}
+
+/*
+ * netfs_write_sizes - Set i_size, remote_i_size and zero_point safely
+ * @inode: The inode to access
+ * @i_size: The new value for the local size of the file
+ * @remote_i_size: The new value for the size of the file on the server
+ * @zero_point: The new value for the point beyond which the server has no data
+ *
+ * Set both remote_i_size and zero_point safely without the potential for
+ * tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need
+ * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of
+ * i_size_seqcount can be lost, resulting in subsequent read calls spinning
+ * forever.
+ */
+static inline void netfs_write_sizes(struct inode *inode,
+ unsigned long long i_size,
+ unsigned long long remote_i_size,
+ unsigned long long zero_point)
+{
+ struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ write_seqcount_begin(&inode->i_size_seqcount);
+ inode->i_size = i_size;
+ ictx->_remote_i_size = remote_i_size;
+ ictx->_zero_point = zero_point;
+ write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ inode->i_size = i_size;
+ ictx->_remote_i_size = remote_i_size;
+ ictx->_zero_point = zero_point;
+ preempt_enable();
+#else
+ /*
+ * Pairs with smp_load_acquire() in i_size_read(),
+ * netfs_read_remote_i_size() and netfs_read_zero_point() to ensure
+ * changes related to inode size (such as page contents) are visible
+ * before we see the changed inode size.
+ */
+ smp_store_release(&inode->i_size, i_size);
+ smp_store_release(&ictx->_remote_i_size, remote_i_size);
+ smp_store_release(&ictx->_zero_point, zero_point);
+#endif
+}
+
+/**
* netfs_inode_init - Initialise a netfslib inode context
* @ctx: The netfs inode to initialise
* @ops: The netfs's operations list
@@ -488,8 +736,8 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
bool use_zero_point)
{
ctx->ops = ops;
- ctx->remote_i_size = i_size_read(&ctx->inode);
- ctx->zero_point = LLONG_MAX;
+ ctx->_remote_i_size = i_size_read(&ctx->inode);
+ ctx->_zero_point = LLONG_MAX;
ctx->flags = 0;
atomic_set(&ctx->io_count, 0);
#if IS_ENABLED(CONFIG_FSCACHE)
@@ -498,7 +746,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
mutex_init(&ctx->wb_lock);
/* ->releasepage() drives zero_point */
if (use_zero_point) {
- ctx->zero_point = ctx->remote_i_size;
+ ctx->_zero_point = ctx->_remote_i_size;
mapping_set_release_always(ctx->inode.i_mapping);
}
}
@@ -511,13 +759,40 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
*
* Inform the netfs lib that a file got resized so that it can adjust its state.
*/
-static inline void netfs_resize_file(struct netfs_inode *ctx, loff_t new_i_size,
+static inline void netfs_resize_file(struct netfs_inode *ictx,
+ unsigned long long new_i_size,
bool changed_on_server)
{
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+ struct inode *inode = &ictx->inode;
+
+ preempt_disable();
+ write_seqcount_begin(&inode->i_size_seqcount);
if (changed_on_server)
- ctx->remote_i_size = new_i_size;
- if (new_i_size < ctx->zero_point)
- ctx->zero_point = new_i_size;
+ ictx->_remote_i_size = new_i_size;
+ if (new_i_size < ictx->_zero_point)
+ ictx->_zero_point = new_i_size;
+ write_seqcount_end(&inode->i_size_seqcount);
+ preempt_enable();
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+ preempt_disable();
+ if (changed_on_server)
+ ictx->_remote_i_size = new_i_size;
+ if (new_i_size < ictx->_zero_point)
+ ictx->_zero_point = new_i_size;
+ preempt_enable();
+#else
+ /*
+ * Pairs with smp_load_acquire() in netfs_read_remote_i_size and
+ * netfs_read_zero_point() to ensure changes related to inode size
+ * (such as page contents) are visible before we see the changed inode
+ * size.
+ */
+ if (changed_on_server)
+ smp_store_release(&ictx->_remote_i_size, new_i_size);
+ if (new_i_size < ictx->_zero_point)
+ smp_store_release(&ictx->_zero_point, new_i_size);
+#endif
}
/**
diff --git a/include/linux/nvme-auth.h b/include/linux/nvme-auth.h
index 682f810..d674d8a 100644
--- a/include/linux/nvme-auth.h
+++ b/include/linux/nvme-auth.h
@@ -49,9 +49,9 @@ int nvme_auth_augmented_challenge(u8 hmac_id, const u8 *skey, size_t skey_len,
int nvme_auth_gen_privkey(struct crypto_kpp *dh_tfm, u8 dh_gid);
int nvme_auth_gen_pubkey(struct crypto_kpp *dh_tfm,
u8 *host_key, size_t host_key_len);
-int nvme_auth_gen_shared_secret(struct crypto_kpp *dh_tfm,
- const u8 *ctrl_key, size_t ctrl_key_len,
- u8 *sess_key, size_t sess_key_len);
+int nvme_auth_gen_session_key(struct crypto_kpp *dh_tfm,
+ const u8 *public_key, size_t public_key_len,
+ u8 *sess_key, size_t sess_key_len, u8 hash_id);
int nvme_auth_generate_psk(u8 hmac_id, const u8 *skey, size_t skey_len,
const u8 *c1, const u8 *c2, size_t hash_len,
u8 **ret_psk, size_t *ret_len);
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 0e03d81..7223f6f 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -923,7 +923,6 @@ enum pagetype {
PGTY_zsmalloc = 0xf6,
PGTY_unaccepted = 0xf7,
PGTY_large_kmalloc = 0xf8,
- PGTY_netpp = 0xf9,
PGTY_mapcount_underflow = 0xff
};
@@ -1056,11 +1055,6 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
-/*
- * Marks page_pool allocated pages.
- */
-PAGE_TYPE_OPS(Netpp, netpp, netpp)
-
/**
* PageHuge - Determine if the page belongs to hugetlbfs
* @page: The page to test.
diff --git a/include/linux/parport.h b/include/linux/parport.h
index 464c2ad..f64cb06 100644
--- a/include/linux/parport.h
+++ b/include/linux/parport.h
@@ -240,6 +240,7 @@ struct parport {
unsigned long devflags;
#define PARPORT_DEVPROC_REGISTERED 0
+#define PARPORT_ANNOUNCED 1
struct pardevice *proc_device; /* Currently register proc device */
struct list_head full_list;
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 85bf8dd..2f5a889 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -3,13 +3,14 @@
#define __LINUX_PERCPU_H
#include <linux/alloc_tag.h>
-#include <linux/mmdebug.h>
-#include <linux/preempt.h>
-#include <linux/smp.h>
-#include <linux/pfn.h>
-#include <linux/init.h>
#include <linux/cleanup.h>
+#include <linux/compiler_types.h>
+#include <linux/init.h>
+#include <linux/mmdebug.h>
+#include <linux/pfn.h>
+#include <linux/preempt.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <asm/percpu.h>
diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h
index aab0aeb..9f088c9 100644
--- a/include/linux/pps_kernel.h
+++ b/include/linux/pps_kernel.h
@@ -99,12 +99,14 @@ static inline void timespec_to_pps_ktime(struct pps_ktime *kt,
static inline void pps_get_ts(struct pps_event_time *ts)
{
+#ifdef CONFIG_NTP_PPS
struct system_time_snapshot snap;
- ktime_get_snapshot(&snap);
- ts->ts_real = ktime_to_timespec64(snap.real);
-#ifdef CONFIG_NTP_PPS
- ts->ts_raw = ktime_to_timespec64(snap.raw);
+ ktime_get_snapshot_id(CLOCK_REALTIME, &snap);
+ ts->ts_real = ktime_to_timespec64(snap.systime);
+ ts->ts_raw = ktime_to_timespec64(snap.monoraw);
+#else
+ ktime_get_real_ts64(&ts->ts_real);
#endif
}
diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h
index 8843645..36a27a9 100644
--- a/include/linux/ptp_clock_kernel.h
+++ b/include/linux/ptp_clock_kernel.h
@@ -12,6 +12,7 @@
#include <linux/pps_kernel.h>
#include <linux/ptp_clock.h>
#include <linux/timecounter.h>
+#include <linux/timekeeping.h>
#include <linux/skbuff.h>
#define PTP_CLOCK_NAME_LEN 32
@@ -45,13 +46,13 @@ struct system_device_crosststamp;
/**
* struct ptp_system_timestamp - system time corresponding to a PHC timestamp
- * @pre_ts: system timestamp before capturing PHC
- * @post_ts: system timestamp after capturing PHC
- * @clockid: clock-base used for capturing the system timestamps
+ * @pre_sts: system time snapshot before capturing PHC
+ * @post_sts: system time snapshot after capturing PHC
+ * @clockid: clock-base used for capturing the system timestamps
*/
struct ptp_system_timestamp {
- struct timespec64 pre_ts;
- struct timespec64 post_ts;
+ struct system_time_snapshot pre_sts;
+ struct system_time_snapshot post_sts;
clockid_t clockid;
};
@@ -510,13 +511,13 @@ static inline ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp,
static inline void ptp_read_system_prets(struct ptp_system_timestamp *sts)
{
if (sts)
- ktime_get_clock_ts64(sts->clockid, &sts->pre_ts);
+ ktime_get_snapshot_id(sts->clockid, &sts->pre_sts);
}
static inline void ptp_read_system_postts(struct ptp_system_timestamp *sts)
{
if (sts)
- ktime_get_clock_ts64(sts->clockid, &sts->post_ts);
+ ktime_get_snapshot_id(sts->clockid, &sts->post_sts);
}
#endif
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 2abba75..e3bc442 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -261,6 +261,35 @@ static inline void list_replace_rcu(struct list_head *old,
old->prev = LIST_POISON2;
}
+static inline void __list_splice_rcu(struct list_head *list,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ last->next = next;
+ first->prev = prev;
+ next->prev = last;
+ rcu_assign_pointer(list_next_rcu(prev), first);
+}
+
+/**
+ * list_splice_rcu - splice a non-RCU list into an RCU-protected list,
+ * designed for stacks.
+ * @list: the non RCU-protected list to splice
+ * @head: the place in the existing RCU-protected list to splice
+ *
+ * The list pointed to by @head can be RCU-read traversed concurrently with
+ * this function.
+ */
+static inline void list_splice_rcu(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice_rcu(list, head, head->next);
+}
+
/**
* __list_splice_init_rcu - join an RCU-protected list into an existing list.
* @list: the RCU-protected list to splice
diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index 006e57f..73ff522 100644
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -286,13 +286,15 @@ enum resctrl_schema_fmt {
/**
* struct resctrl_mon - Monitoring related data of a resctrl resource.
- * @num_rmid: Number of RMIDs available.
- * @mbm_cfg_mask: Memory transactions that can be tracked when bandwidth
- * monitoring events can be configured.
- * @num_mbm_cntrs: Number of assignable counters.
- * @mbm_cntr_assignable:Is system capable of supporting counter assignment?
- * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM
- * events of monitor groups created via mkdir.
+ * @num_rmid: Number of RMIDs available.
+ * @mbm_cfg_mask: Memory transactions that can be tracked when
+ * bandwidth monitoring events can be configured.
+ * @num_mbm_cntrs: Number of assignable counters.
+ * @mbm_cntr_assignable: Is system capable of supporting counter assignment?
+ * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM
+ * events of monitor groups created via mkdir.
+ * @mbm_cntr_configurable: True if assignable counters are configurable.
+ * @mbm_cntr_assign_fixed: True if the counter assignment mode is fixed.
*/
struct resctrl_mon {
u32 num_rmid;
@@ -300,6 +302,8 @@ struct resctrl_mon {
int num_mbm_cntrs;
bool mbm_cntr_assignable;
bool mbm_assign_on_mkdir;
+ bool mbm_cntr_configurable;
+ bool mbm_cntr_assign_fixed;
};
/**
diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h
index 015c829..fc2f596 100644
--- a/include/linux/rhashtable-types.h
+++ b/include/linux/rhashtable-types.h
@@ -12,6 +12,7 @@
#include <linux/alloc_tag.h>
#include <linux/atomic.h>
#include <linux/compiler.h>
+#include <linux/irq_work_types.h>
#include <linux/mutex.h>
#include <linux/workqueue_types.h>
@@ -49,6 +50,7 @@ typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg,
* @head_offset: Offset of rhash_head in struct to be hashed
* @max_size: Maximum size while expanding
* @min_size: Minimum size while shrinking
+ * @insecure_elasticity: Set to true to disable chain length checks
* @automatic_shrinking: Enable automatic shrinking of tables
* @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash)
* @obj_hashfn: Function to hash object
@@ -61,6 +63,7 @@ struct rhashtable_params {
u16 head_offset;
unsigned int max_size;
u16 min_size;
+ bool insecure_elasticity;
bool automatic_shrinking;
rht_hashfn_t hashfn;
rht_obj_hashfn_t obj_hashfn;
@@ -75,6 +78,7 @@ struct rhashtable_params {
* @p: Configuration parameters
* @rhlist: True if this is an rhltable
* @run_work: Deferred worker to expand/shrink asynchronously
+ * @run_irq_work: Bounces the @run_work kick through hard IRQ context.
* @mutex: Mutex to protect current/future table swapping
* @lock: Spin lock to protect walker list
* @nelems: Number of elements in table
@@ -86,6 +90,7 @@ struct rhashtable {
struct rhashtable_params p;
bool rhlist;
struct work_struct run_work;
+ struct irq_work run_irq_work;
struct mutex mutex;
spinlock_t lock;
atomic_t nelems;
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index 0480509..ef5230c 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -20,6 +20,7 @@
#include <linux/err.h>
#include <linux/errno.h>
+#include <linux/irq_work.h>
#include <linux/jhash.h>
#include <linux/list_nulls.h>
#include <linux/workqueue.h>
@@ -821,14 +822,15 @@ static __always_inline void *__rhashtable_insert_fast(
goto out;
}
- if (elasticity <= 0)
+ if (elasticity <= 0 && !params.insecure_elasticity)
goto slow_path;
data = ERR_PTR(-E2BIG);
if (unlikely(rht_grow_above_max(ht, tbl)))
goto out_unlock;
- if (unlikely(rht_grow_above_100(ht, tbl)))
+ if (unlikely(rht_grow_above_100(ht, tbl)) &&
+ !params.insecure_elasticity)
goto slow_path;
/* Inserting at head of list makes unlocking free. */
@@ -846,7 +848,7 @@ static __always_inline void *__rhashtable_insert_fast(
rht_assign_unlock(tbl, bkt, obj, flags);
if (rht_grow_above_75(ht, tbl))
- schedule_work(&ht->run_work);
+ irq_work_queue(&ht->run_irq_work);
data = NULL;
out:
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index b9d62fc..7ef79b2 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -9,6 +9,11 @@
void __rseq_handle_slowpath(struct pt_regs *regs);
+static __always_inline bool rseq_v2(struct task_struct *t)
+{
+ return IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && likely(t->rseq.event.has_rseq > 1);
+}
+
/* Invoked from resume_user_mode_work() */
static inline void rseq_handle_slowpath(struct pt_regs *regs)
{
@@ -16,8 +21,7 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs)
if (current->rseq.event.slowpath)
__rseq_handle_slowpath(regs);
} else {
- /* '&' is intentional to spare one conditional branch */
- if (current->rseq.event.sched_switch & current->rseq.event.has_rseq)
+ if (current->rseq.event.sched_switch && current->rseq.event.has_rseq)
__rseq_handle_slowpath(regs);
}
}
@@ -30,9 +34,9 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs);
*/
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs)
{
- if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) {
- /* '&' is intentional to spare one conditional branch */
- if (current->rseq.event.has_rseq & current->rseq.event.user_irq)
+ if (rseq_v2(current)) {
+ /* has_rseq is implied in rseq_v2() */
+ if (current->rseq.event.user_irq)
__rseq_signal_deliver(ksig->sig, regs);
} else {
if (current->rseq.event.has_rseq)
@@ -50,15 +54,22 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
{
struct rseq_event *ev = &t->rseq.event;
- if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) {
+ /*
+ * Only apply the user_irq optimization for RSEQ ABI V2 registrations.
+ * Legacy users like TCMalloc rely on the original ABI V1 behaviour
+ * which updates IDs on every context swtich.
+ */
+ if (rseq_v2(t)) {
/*
- * Avoid a boat load of conditionals by using simple logic
- * to determine whether NOTIFY_RESUME needs to be raised.
+ * Avoid a boat load of conditionals by using simple logic to
+ * determine whether TIF_NOTIFY_RESUME or TIF_RSEQ needs to be
+ * raised.
*
- * It's required when the CPU or MM CID has changed or
- * the entry was from user space.
+ * It's required when the CPU or MM CID has changed or the entry
+ * was via interrupt from user space. ev->has_rseq does not have
+ * to be evaluated here because rseq_v2() implies has_rseq.
*/
- bool raise = (ev->user_irq | ev->ids_changed) & ev->has_rseq;
+ bool raise = ev->user_irq | ev->ids_changed;
if (raise) {
ev->sched_switch = true;
@@ -66,6 +77,7 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
}
} else {
if (ev->has_rseq) {
+ t->rseq.event.ids_changed = true;
t->rseq.event.sched_switch = true;
rseq_raise_notify_resume(t);
}
@@ -119,6 +131,8 @@ static inline void rseq_virt_userspace_exit(void)
static inline void rseq_reset(struct task_struct *t)
{
+ /* Protect against preemption and membarrier IPI */
+ guard(irqsave)();
memset(&t->rseq, 0, sizeof(t->rseq));
t->rseq.ids.cpu_id = RSEQ_CPU_ID_UNINITIALIZED;
}
@@ -159,6 +173,7 @@ static inline unsigned int rseq_alloc_align(void)
}
#else /* CONFIG_RSEQ */
+static inline bool rseq_v2(struct task_struct *t) { return false; }
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
index f11ebd3..ed9da6e 100644
--- a/include/linux/rseq_entry.h
+++ b/include/linux/rseq_entry.h
@@ -111,6 +111,20 @@ static __always_inline void rseq_slice_clear_grant(struct task_struct *t)
t->rseq.slice.state.granted = false;
}
+/*
+ * Open coded, so it can be invoked within a user access region.
+ *
+ * This clears the user space state of the time slice extensions field only when
+ * the task has registered the optimized RSEQ_ABI V2. Some legacy registrations,
+ * e.g. TCMalloc, have conflicting non-ABI fields in struct RSEQ, which would be
+ * overwritten by an unconditional write.
+ */
+#define rseq_slice_clear_user(rseq, efault) \
+do { \
+ if (rseq_slice_extension_enabled()) \
+ unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); \
+} while (0)
+
static __always_inline bool __rseq_grant_slice_extension(bool work_pending)
{
struct task_struct *curr = current;
@@ -230,10 +244,10 @@ static __always_inline bool rseq_slice_extension_enabled(void) { return false; }
static __always_inline bool rseq_arm_slice_extension_timer(void) { return false; }
static __always_inline void rseq_slice_clear_grant(struct task_struct *t) { }
static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
+#define rseq_slice_clear_user(rseq, efault) do { } while (0)
#endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
-bool rseq_debug_validate_ids(struct task_struct *t);
static __always_inline void rseq_note_user_irq_entry(void)
{
@@ -353,43 +367,6 @@ bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs,
return false;
}
-/*
- * On debug kernels validate that user space did not mess with it if the
- * debug branch is enabled.
- */
-bool rseq_debug_validate_ids(struct task_struct *t)
-{
- struct rseq __user *rseq = t->rseq.usrptr;
- u32 cpu_id, uval, node_id;
-
- /*
- * On the first exit after registering the rseq region CPU ID is
- * RSEQ_CPU_ID_UNINITIALIZED and node_id in user space is 0!
- */
- node_id = t->rseq.ids.cpu_id != RSEQ_CPU_ID_UNINITIALIZED ?
- cpu_to_node(t->rseq.ids.cpu_id) : 0;
-
- scoped_user_read_access(rseq, efault) {
- unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault);
- if (cpu_id != t->rseq.ids.cpu_id)
- goto die;
- unsafe_get_user(uval, &rseq->cpu_id, efault);
- if (uval != cpu_id)
- goto die;
- unsafe_get_user(uval, &rseq->node_id, efault);
- if (uval != node_id)
- goto die;
- unsafe_get_user(uval, &rseq->mm_cid, efault);
- if (uval != t->rseq.ids.mm_cid)
- goto die;
- }
- return true;
-die:
- t->rseq.event.fatal = true;
-efault:
- return false;
-}
-
#endif /* RSEQ_BUILD_SLOW_PATH */
/*
@@ -499,37 +476,50 @@ rseq_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long c
* faults in task context are fatal too.
*/
static rseq_inline
-bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids,
- u32 node_id, u64 *csaddr)
+bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, u64 *csaddr)
{
struct rseq __user *rseq = t->rseq.usrptr;
- if (static_branch_unlikely(&rseq_debug_enabled)) {
- if (!rseq_debug_validate_ids(t))
- return false;
- }
-
scoped_user_rw_access(rseq, efault) {
+ /* Validate the R/O fields for debug and optimized mode */
+ if (static_branch_unlikely(&rseq_debug_enabled) || rseq_v2(t)) {
+ u32 cpu_id, uval;
+
+ unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault);
+ if (cpu_id != t->rseq.ids.cpu_id)
+ goto die;
+ unsafe_get_user(uval, &rseq->cpu_id, efault);
+ if (uval != cpu_id)
+ goto die;
+ unsafe_get_user(uval, &rseq->node_id, efault);
+ if (uval != t->rseq.ids.node_id)
+ goto die;
+ unsafe_get_user(uval, &rseq->mm_cid, efault);
+ if (uval != t->rseq.ids.mm_cid)
+ goto die;
+ }
+
unsafe_put_user(ids->cpu_id, &rseq->cpu_id_start, efault);
unsafe_put_user(ids->cpu_id, &rseq->cpu_id, efault);
- unsafe_put_user(node_id, &rseq->node_id, efault);
+ unsafe_put_user(ids->node_id, &rseq->node_id, efault);
unsafe_put_user(ids->mm_cid, &rseq->mm_cid, efault);
if (csaddr)
unsafe_get_user(*csaddr, &rseq->rseq_cs, efault);
- /* Open coded, so it's in the same user access region */
- if (rseq_slice_extension_enabled()) {
- /* Unconditionally clear it, no point in conditionals */
- unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
- }
+ /* RSEQ ABI V2 only operations */
+ if (rseq_v2(t))
+ rseq_slice_clear_user(rseq, efault);
}
rseq_slice_clear_grant(t);
/* Cache the new values */
- t->rseq.ids.cpu_cid = ids->cpu_cid;
+ t->rseq.ids = *ids;
rseq_stat_inc(rseq_stats.ids);
rseq_trace_update(t, ids);
return true;
+
+die:
+ t->rseq.event.fatal = true;
efault:
return false;
}
@@ -539,11 +529,11 @@ bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids,
* is in a critical section.
*/
static rseq_inline bool rseq_update_usr(struct task_struct *t, struct pt_regs *regs,
- struct rseq_ids *ids, u32 node_id)
+ struct rseq_ids *ids)
{
u64 csaddr;
- if (!rseq_set_ids_get_csaddr(t, ids, node_id, &csaddr))
+ if (!rseq_set_ids_get_csaddr(t, ids, &csaddr))
return false;
/*
@@ -612,6 +602,14 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
* interrupts disabled
*/
guard(pagefault)();
+ /*
+ * This optimization is only valid when the task registered for the
+ * optimized RSEQ_ABI_V2 variant. Some legacy users rely on the original
+ * RSEQ implementation behaviour which unconditionally updated the IDs.
+ * rseq_sched_switch_event() ensures that legacy registrations always
+ * have both sched_switch and ids_changed set, which is compatible with
+ * the historical TIF_NOTIFY_RESUME behaviour.
+ */
if (likely(!t->rseq.event.ids_changed)) {
struct rseq __user *rseq = t->rseq.usrptr;
/*
@@ -623,11 +621,9 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
scoped_user_rw_access(rseq, efault) {
unsafe_get_user(csaddr, &rseq->rseq_cs, efault);
- /* Open coded, so it's in the same user access region */
- if (rseq_slice_extension_enabled()) {
- /* Unconditionally clear it, no point in conditionals */
- unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
- }
+ /* RSEQ ABI V2 only operations */
+ if (rseq_v2(t))
+ rseq_slice_clear_user(rseq, efault);
}
rseq_slice_clear_grant(t);
@@ -639,13 +635,14 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
return true;
}
+ int cpu = task_cpu(t);
struct rseq_ids ids = {
- .cpu_id = task_cpu(t),
- .mm_cid = task_mm_cid(t),
+ .cpu_id = cpu,
+ .mm_cid = task_mm_cid(t),
+ .node_id = cpu_to_node(cpu),
};
- u32 node_id = cpu_to_node(ids.cpu_id);
- return rseq_update_usr(t, regs, &ids, node_id);
+ return rseq_update_usr(t, regs, &ids);
efault:
return false;
}
@@ -753,24 +750,6 @@ static __always_inline void rseq_irqentry_exit_to_user_mode(void)
ev->events = 0;
}
-/* Required to keep ARM64 working */
-static __always_inline void rseq_exit_to_user_mode_legacy(void)
-{
- struct rseq_event *ev = ¤t->rseq.event;
-
- rseq_stat_inc(rseq_stats.exit);
-
- if (static_branch_unlikely(&rseq_debug_enabled))
- WARN_ON_ONCE(ev->sched_switch);
-
- /*
- * Ensure that event (especially user_irq) is cleared when the
- * interrupt did not result in a schedule and therefore the
- * rseq processing did not clear it.
- */
- ev->events = 0;
-}
-
void __rseq_debug_syscall_return(struct pt_regs *regs);
static __always_inline void rseq_debug_syscall_return(struct pt_regs *regs)
@@ -786,7 +765,6 @@ static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs, unsigned
}
static inline void rseq_syscall_exit_to_user_mode(void) { }
static inline void rseq_irqentry_exit_to_user_mode(void) { }
-static inline void rseq_exit_to_user_mode_legacy(void) { }
static inline void rseq_debug_syscall_return(struct pt_regs *regs) { }
static inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; }
#endif /* !CONFIG_RSEQ */
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 0b42045..85739a6 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -9,6 +9,12 @@
#ifdef CONFIG_RSEQ
struct rseq;
+/*
+ * rseq_event::has_rseq contains the ABI version number so preserving it
+ * in AND operations requires a mask.
+ */
+#define RSEQ_HAS_RSEQ_VERSION_MASK 0xff
+
/**
* struct rseq_event - Storage for rseq related event management
* @all: Compound to initialize and clear the data efficiently
@@ -17,7 +23,8 @@ struct rseq;
* exit to user
* @ids_changed: Indicator that IDs need to be updated
* @user_irq: True on interrupt entry from user mode
- * @has_rseq: True if the task has a rseq pointer installed
+ * @has_rseq: Greater than 0 if the task has a rseq pointer installed.
+ * Contains the RSEQ version number
* @error: Compound error code for the slow path to analyze
* @fatal: User space data corrupted or invalid
* @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME
@@ -59,8 +66,9 @@ struct rseq_event {
* compiler emit a single compare on 64-bit
* @cpu_id: The CPU ID which was written last to user space
* @mm_cid: The MM CID which was written last to user space
+ * @node_id: The node ID which was written last to user space
*
- * @cpu_id and @mm_cid are updated when the data is written to user space.
+ * @cpu_id, @mm_cid and @node_id are updated when the data is written to user space.
*/
struct rseq_ids {
union {
@@ -70,6 +78,7 @@ struct rseq_ids {
u32 mm_cid;
};
};
+ u32 node_id;
};
/**
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index 78e7e588..9e1f012 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -56,6 +56,8 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock)
#endif
extern void rt_mutex_base_init(struct rt_mutex_base *rtb);
+context_lock_struct(rt_mutex);
+
/**
* The rt_mutex structure
*
@@ -108,8 +110,10 @@ do { \
extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock_class_key *key);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
-extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass);
-extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock);
+extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
+ __acquires(lock);
+extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
+ __acquires(lock);
#define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0)
#define rt_mutex_lock_nest_lock(lock, nest_lock) \
do { \
@@ -118,15 +122,19 @@ extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *
} while (0)
#else
-extern void rt_mutex_lock(struct rt_mutex *lock);
+extern void rt_mutex_lock(struct rt_mutex *lock) __acquires(lock);
#define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock)
#define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock)
#endif
-extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
-extern int rt_mutex_lock_killable(struct rt_mutex *lock);
-extern int rt_mutex_trylock(struct rt_mutex *lock);
+extern int rt_mutex_lock_interruptible(struct rt_mutex *lock)
+ __cond_acquires(0, lock);
+extern int rt_mutex_lock_killable(struct rt_mutex *lock)
+ __cond_acquires(0, lock);
+extern int rt_mutex_trylock(struct rt_mutex *lock)
+ __cond_acquires(true, lock);
-extern void rt_mutex_unlock(struct rt_mutex *lock);
+extern void rt_mutex_unlock(struct rt_mutex *lock)
+ __releases(lock);
#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 368c7b4..d49e52a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -16,6 +16,7 @@
#include <linux/cpumask_types.h>
#include <linux/cache.h>
+#include <linux/futex_types.h>
#include <linux/irqflags_types.h>
#include <linux/smp_types.h>
#include <linux/pid_types.h>
@@ -64,7 +65,6 @@ struct bpf_net_context;
struct capture_control;
struct cfs_rq;
struct fs_struct;
-struct futex_pi_state;
struct io_context;
struct io_uring_task;
struct mempolicy;
@@ -76,7 +76,6 @@ struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
-struct robust_list_head;
struct root_domain;
struct rq;
struct sched_attr;
@@ -161,7 +160,7 @@ struct user_event_mm;
*/
#define is_special_task_state(state) \
((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | \
- TASK_DEAD | TASK_FROZEN))
+ TASK_DEAD | TASK_WAKING | TASK_FROZEN))
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define debug_normal_state_change(state_value) \
@@ -702,6 +701,11 @@ struct sched_dl_entity {
* running, skipping the defer phase.
*
* @dl_defer_idle tracks idle state
+ *
+ * @dl_bw_attached tells if this server's bandwidth currently
+ * contributes to the root domain's total_bw. Only meaningful for server
+ * entities (@dl_server == 1). Allows toggling the reservation on/off
+ * without losing the configured @dl_runtime/@dl_period.
*/
unsigned int dl_throttled : 1;
unsigned int dl_yielded : 1;
@@ -713,6 +717,7 @@ struct sched_dl_entity {
unsigned int dl_defer_armed : 1;
unsigned int dl_defer_running : 1;
unsigned int dl_defer_idle : 1;
+ unsigned int dl_bw_attached : 1;
/*
* Bandwidth enforcement timer. Each -deadline task has its
@@ -846,7 +851,11 @@ struct task_struct {
struct alloc_tag *alloc_tag;
#endif
- int on_cpu;
+ u8 on_cpu;
+ u8 on_rq;
+ u8 is_blocked;
+ u8 __pad;
+
struct __call_single_node wake_entry;
unsigned int wakee_flips;
unsigned long wakee_flip_decay_ts;
@@ -861,7 +870,6 @@ struct task_struct {
*/
int recent_used_cpu;
int wake_cpu;
- int on_rq;
int prio;
int static_prio;
@@ -1002,6 +1010,9 @@ struct task_struct {
unsigned sched_rt_mutex:1;
#endif
+ /* Save user-dumpable when mm goes away */
+ unsigned user_dumpable:1;
+
/* Bit to tell TOMOYO we're in execve(): */
unsigned in_execve:1;
unsigned in_iowait:1;
@@ -1241,6 +1252,13 @@ struct task_struct {
struct mutex *blocked_on; /* lock we're blocked on */
raw_spinlock_t blocked_lock;
+ /*
+ * The task that is boosting this task; a back link for the current
+ * donor stack. Set in schedule() -> find_proxy_task() and only stable
+ * under preempt_disable().
+ */
+ struct task_struct *blocked_donor;
+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
/*
* Encoded lock address causing task block (lower 2 bits = type from
@@ -1331,16 +1349,9 @@ struct task_struct {
u32 closid;
u32 rmid;
#endif
-#ifdef CONFIG_FUTEX
- struct robust_list_head __user *robust_list;
-#ifdef CONFIG_COMPAT
- struct compat_robust_list_head __user *compat_robust_list;
-#endif
- struct list_head pi_state_list;
- struct futex_pi_state *pi_state_cache;
- struct mutex futex_exit_mutex;
- unsigned int futex_state;
-#endif
+
+ struct futex_sched_data futex;
+
#ifdef CONFIG_PERF_EVENTS
u8 perf_recursion[PERF_NR_CONTEXTS];
struct perf_event_context *perf_event_ctxp;
@@ -1408,6 +1419,13 @@ struct task_struct {
unsigned long numa_pages_migrated;
#endif /* CONFIG_NUMA_BALANCING */
+#ifdef CONFIG_SCHED_CACHE
+ struct callback_head cache_work;
+ int preferred_llc;
+ /* 1: task was enqueued to its preferred LLC, 0 otherwise */
+ int pref_llc_queued;
+#endif
+
struct rseq_data rseq;
struct sched_mm_cid mm_cid;
@@ -2182,19 +2200,10 @@ extern int __cond_resched_rwlock_write(rwlock_t *lock) __must_hold(lock);
#ifndef CONFIG_PREEMPT_RT
-/*
- * With proxy exec, if a task has been proxy-migrated, it may be a donor
- * on a cpu that it can't actually run on. Thus we need a special state
- * to denote that the task is being woken, but that it needs to be
- * evaluated for return-migration before it is run. So if the task is
- * blocked_on PROXY_WAKING, return migrate it before running it.
- */
-#define PROXY_WAKING ((struct mutex *)(-1L))
-
static inline struct mutex *__get_task_blocked_on(struct task_struct *p)
{
lockdep_assert_held_once(&p->blocked_lock);
- return p->blocked_on == PROXY_WAKING ? NULL : p->blocked_on;
+ return p->blocked_on;
}
static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m)
@@ -2222,7 +2231,7 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct mutex *
* blocked_on relationships, but make sure we are not
* clearing the relationship with a different lock.
*/
- WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
+ WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m);
p->blocked_on = NULL;
}
@@ -2231,35 +2240,6 @@ static inline void clear_task_blocked_on(struct task_struct *p, struct mutex *m)
guard(raw_spinlock_irqsave)(&p->blocked_lock);
__clear_task_blocked_on(p, m);
}
-
-static inline void __set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
-{
- /* Currently we serialize blocked_on under the task::blocked_lock */
- lockdep_assert_held_once(&p->blocked_lock);
-
- if (!sched_proxy_exec()) {
- __clear_task_blocked_on(p, m);
- return;
- }
-
- /* Don't set PROXY_WAKING if blocked_on was already cleared */
- if (!p->blocked_on)
- return;
- /*
- * There may be cases where we set PROXY_WAKING on tasks that were
- * already set to waking, but make sure we are not changing
- * the relationship with a different lock.
- */
- WARN_ON_ONCE(m && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
- p->blocked_on = PROXY_WAKING;
-}
-
-static inline void set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
-{
- guard(raw_spinlock_irqsave)(&p->blocked_lock);
- __set_task_blocked_on_waking(p, m);
-}
-
#else
static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
{
@@ -2268,14 +2248,6 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mute
static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
{
}
-
-static inline void __set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
-{
-}
-
-static inline void set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
-{
-}
#endif /* !CONFIG_PREEMPT_RT */
static __always_inline bool need_resched(void)
@@ -2408,6 +2380,29 @@ static __always_inline int task_mm_cid(struct task_struct *t)
}
#endif
+#ifdef CONFIG_SCHED_CACHE
+
+struct sched_cache_time {
+ u64 runtime;
+ unsigned long epoch;
+};
+
+struct sched_cache_stat {
+ struct sched_cache_time __percpu *pcpu_sched;
+ raw_spinlock_t lock;
+ unsigned long epoch;
+ u64 nr_running_avg;
+ unsigned long next_scan;
+ unsigned long footprint;
+ int cpu;
+} ____cacheline_aligned_in_smp;
+
+#else
+
+struct sched_cache_stat { };
+
+#endif
+
#ifndef MODULE
#ifndef COMPILE_OFFSETS
diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h
index 196f0ca..39f0a7f 100644
--- a/include/linux/sched/clock.h
+++ b/include/linux/sched/clock.h
@@ -33,6 +33,11 @@ extern u64 sched_clock_cpu(int cpu);
extern void sched_clock_init(void);
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
+static inline int sched_clock_stable(void)
+{
+ return 1;
+}
+
static inline void sched_clock_tick(void)
{
}
diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
index 1198138c..2735382 100644
--- a/include/linux/sched/deadline.h
+++ b/include/linux/sched/deadline.h
@@ -33,6 +33,15 @@ struct root_domain;
extern void dl_add_task_root_domain(struct task_struct *p);
extern void dl_clear_root_domain(struct root_domain *rd);
extern void dl_clear_root_domain_cpu(int cpu);
+/*
+ * Return whether moving DL task @p to @new_mask requires moving DL
+ * bandwidth accounting between root domains. This helper is specific to
+ * DL bandwidth move accounting semantics and is shared by
+ * cpuset_can_attach() and set_cpus_allowed_dl() so both paths use the
+ * same source root-domain test.
+ */
+extern bool dl_task_needs_bw_move(struct task_struct *p,
+ const struct cpumask *new_mask);
extern u64 dl_cookie;
extern bool dl_bw_visited(int cpu, u64 cookie);
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 1a3af2e..2129e18 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -103,21 +103,25 @@ enum scx_ent_flags {
SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
/*
- * Bits 8 and 9 are used to carry task state:
+ * Bits 8 to 10 are used to carry task state:
*
* NONE ops.init_task() not called yet
+ * INIT_BEGIN ops.init_task() in flight; see sched_ext_dead()
* INIT ops.init_task() succeeded, but task can be cancelled
* READY fully initialized, but not in sched_ext
* ENABLED fully initialized and in sched_ext
+ * DEAD terminal state set by sched_ext_dead()
*/
- SCX_TASK_STATE_SHIFT = 8, /* bits 8 and 9 are used to carry task state */
- SCX_TASK_STATE_BITS = 2,
+ SCX_TASK_STATE_SHIFT = 8,
+ SCX_TASK_STATE_BITS = 3,
SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT,
SCX_TASK_NONE = 0 << SCX_TASK_STATE_SHIFT,
- SCX_TASK_INIT = 1 << SCX_TASK_STATE_SHIFT,
- SCX_TASK_READY = 2 << SCX_TASK_STATE_SHIFT,
- SCX_TASK_ENABLED = 3 << SCX_TASK_STATE_SHIFT,
+ SCX_TASK_INIT_BEGIN = 1 << SCX_TASK_STATE_SHIFT,
+ SCX_TASK_INIT = 2 << SCX_TASK_STATE_SHIFT,
+ SCX_TASK_READY = 3 << SCX_TASK_STATE_SHIFT,
+ SCX_TASK_ENABLED = 4 << SCX_TASK_STATE_SHIFT,
+ SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT,
/*
* Bits 12 and 13 are used to carry reenqueue reason. In addition to
diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index dc3975f..cf0fd03 100644
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -21,6 +21,11 @@ enum hk_type {
HK_TYPE_MAX,
/*
+ * HK_TYPE_KTHREAD is now an alias of HK_TYPE_DOMAIN
+ */
+ HK_TYPE_KTHREAD = HK_TYPE_DOMAIN,
+
+ /*
* The following housekeeping types are only set by the nohz_full
* boot commandline option. So they can share the same value.
*/
@@ -29,7 +34,6 @@ enum hk_type {
HK_TYPE_RCU = HK_TYPE_KERNEL_NOISE,
HK_TYPE_MISC = HK_TYPE_KERNEL_NOISE,
HK_TYPE_WQ = HK_TYPE_KERNEL_NOISE,
- HK_TYPE_KTHREAD = HK_TYPE_KERNEL_NOISE
};
#ifdef CONFIG_CPU_ISOLATION
diff --git a/include/linux/sched/smt.h b/include/linux/sched/smt.h
index 166b19a..cde6679 100644
--- a/include/linux/sched/smt.h
+++ b/include/linux/sched/smt.h
@@ -4,16 +4,12 @@
#include <linux/static_key.h>
-#ifdef CONFIG_SCHED_SMT
extern struct static_key_false sched_smt_present;
static __always_inline bool sched_smt_active(void)
{
return static_branch_likely(&sched_smt_present);
}
-#else
-static __always_inline bool sched_smt_active(void) { return false; }
-#endif
void arch_smt_update(void);
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 36553e1..b5d9d7c 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -67,7 +67,25 @@ struct sched_domain_shared {
atomic_t ref;
atomic_t nr_busy_cpus;
int has_idle_cores;
- int nr_idle_scan;
+ union {
+ int nr_idle_scan;
+ /*
+ * Used during allocation to claim the sched_domain_shared
+ * object at multiple levels.
+ *
+ * Note: between build and the first periodic LB tick, which
+ * rewrites the union via update_idle_cpu_scan(), readers of
+ * nr_idle_scan may observe the transient SD_* flag value as
+ * the scan bound. The flag bits are small positive integers,
+ * so the effect is just a slightly relaxed scan bound for one
+ * window and self-heals on the first tick.
+ */
+ int alloc_flags;
+ };
+#ifdef CONFIG_SCHED_CACHE
+ unsigned long util_avg;
+ unsigned long capacity;
+#endif
};
struct sched_domain {
@@ -99,6 +117,12 @@ struct sched_domain {
u64 max_newidle_lb_cost;
unsigned long last_decay_max_lb_cost;
+#ifdef CONFIG_SCHED_CACHE
+ unsigned int llc_max;
+ unsigned int *llc_counts __counted_by_ptr(llc_max);
+ unsigned long llc_bytes;
+#endif
+
#ifdef CONFIG_SCHEDSTATS
/* sched_balance_rq() stats */
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
@@ -256,4 +280,10 @@ static inline int task_node(const struct task_struct *p)
return cpu_to_node(task_cpu(p));
}
+#ifdef CONFIG_SCHED_CACHE
+extern void sched_update_llc_bytes(unsigned int cpu);
+#else
+static inline void sched_update_llc_bytes(unsigned int cpu) { }
+#endif
+
#endif /* _LINUX_SCHED_TOPOLOGY_H */
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 5a40252..f865491 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -1259,14 +1259,15 @@ static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst)
extern void __scoped_seqlock_invalid_target(void);
-#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN)
+#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \
+ defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT)
/*
* For some reason some GCC-8 architectures (nios2, alpha) have trouble
* determining that the ss_done state is impossible in __scoped_seqlock_next()
* below.
*
- * Similarly KASAN is known to confuse compilers enough to break this. But we
- * don't care about code quality for KASAN builds anyway.
+ * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough
+ * to break this. But we don't care about code quality for such builds anyway.
*/
static inline void __scoped_seqlock_bug(void) { }
#else
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index 666430b..110ad4e 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -1275,6 +1275,18 @@ static inline void uart_unlock_and_check_sysrq_irqrestore(struct uart_port *port
#endif /* CONFIG_MAGIC_SYSRQ_SERIAL */
/*
+ * Variant of guard(uart_port_lock_irqsave) for IRQ handlers that may capture
+ * a SysRq character via uart_prepare_sysrq_char(). The destructor uses the
+ * sysrq-aware unlock helper so that a captured port->sysrq_ch is dispatched
+ * to handle_sysrq() on scope exit. The plain guard variant silently drops
+ * sysrq_ch and must not be used by callers that process RX.
+ */
+DEFINE_LOCK_GUARD_1(uart_port_lock_check_sysrq_irqsave, struct uart_port,
+ uart_port_lock_irqsave(_T->lock, &_T->flags),
+ uart_unlock_and_check_sysrq_irqrestore(_T->lock, _T->flags),
+ unsigned long flags);
+
+/*
* We do the SysRQ and SAK checking like this...
*/
static inline int uart_handle_break(struct uart_port *port)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 2bcf78a..3f06254 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -821,6 +821,7 @@ enum skb_tstamp_type {
* @_sk_redir: socket redirection information for skmsg
* @_nfct: Associated connection, if any (with nfctinfo bits)
* @skb_iif: ifindex of device we arrived on
+ * @tc_depth: counter for packet duplication
* @tc_index: Traffic control index
* @hash: the packet hash
* @queue_mapping: Queue mapping for multiqueue devices
@@ -1030,6 +1031,7 @@ struct sk_buff {
__u8 csum_not_inet:1;
#endif
__u8 unreadable:1;
+ __u8 tc_depth:2;
#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
__u16 tc_index; /* traffic control index */
#endif
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 15a60b5..2b5ab48 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -1234,6 +1234,9 @@ void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long alig
extern void kvfree(const void *addr);
DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
+extern void kvfree_atomic(const void *addr);
+DEFINE_FREE(kvfree_atomic, void *, if (!IS_ERR_OR_NULL(_T)) kvfree_atomic(_T))
+
extern void kvfree_sensitive(const void *addr, size_t len);
unsigned int kmem_cache_size(struct kmem_cache *s);
diff --git a/fs/smb/smbdirect/public.h b/include/linux/smbdirect.h
similarity index 76%
rename from fs/smb/smbdirect/public.h
rename to include/linux/smbdirect.h
index 5008815..97f5ba7 100644
--- a/fs/smb/smbdirect/public.h
+++ b/include/linux/smbdirect.h
@@ -3,18 +3,56 @@
* Copyright (C) 2025, Stefan Metzmacher
*/
-#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__
-#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__
+#ifndef __LINUX_SMBDIRECT_H__
+#define __LINUX_SMBDIRECT_H__
-struct smbdirect_buffer_descriptor_v1;
-struct smbdirect_socket_parameters;
+#include <linux/types.h>
+
+/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */
+struct smbdirect_buffer_descriptor_v1 {
+ __le64 offset;
+ __le32 token;
+ __le32 length;
+} __packed;
+
+/*
+ * Connection parameters mostly from [MS-SMBD] 3.1.1.1
+ *
+ * These are setup and negotiated at the beginning of a
+ * connection and remain constant unless explicitly changed.
+ *
+ * Some values are important for the upper layer.
+ */
+struct smbdirect_socket_parameters {
+ __u64 flags;
+#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1)
+#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2)
+ __u32 resolve_addr_timeout_msec;
+ __u32 resolve_route_timeout_msec;
+ __u32 rdma_connect_timeout_msec;
+ __u32 negotiate_timeout_msec;
+ __u16 initiator_depth; /* limited to U8_MAX */
+ __u16 responder_resources; /* limited to U8_MAX */
+ __u16 recv_credit_max;
+ __u16 send_credit_target;
+ __u32 max_send_size;
+ __u32 max_fragmented_send_size;
+ __u32 max_recv_size;
+ __u32 max_fragmented_recv_size;
+ __u32 max_read_write_size;
+ __u32 max_frmr_depth;
+ __u32 keepalive_interval_msec;
+ __u32 keepalive_timeout_msec;
+} __packed;
+
+#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \
+ SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \
+ SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW)
struct smbdirect_socket;
struct smbdirect_send_batch;
struct smbdirect_mr_io;
-#define __SMBDIRECT_EXPORT_SYMBOL__(__sym) EXPORT_SYMBOL_FOR_MODULES(__sym, "cifs,ksmbd")
-
#include <rdma/rw.h>
u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev);
@@ -145,4 +183,4 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
unsigned int rdma_readwrite_threshold,
struct seq_file *m);
-#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ */
+#endif /* __LINUX_SMBDIRECT_H__ */
diff --git a/include/linux/soc/airoha/airoha_offload.h b/include/linux/soc/airoha/airoha_offload.h
index d01ef4a..7589fcc 100644
--- a/include/linux/soc/airoha/airoha_offload.h
+++ b/include/linux/soc/airoha/airoha_offload.h
@@ -71,9 +71,9 @@ static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev,
#define NPU_RX1_DESC_NUM 512
/* CTRL */
-#define NPU_RX_DMA_DESC_LAST_MASK BIT(27)
-#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(26, 14)
-#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(13, 1)
+#define NPU_RX_DMA_DESC_LAST_MASK BIT(29)
+#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(28, 15)
+#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(14, 1)
#define NPU_RX_DMA_DESC_DONE_MASK BIT(0)
/* INFO */
#define NPU_RX_DMA_PKT_COUNT_MASK GENMASK(31, 29)
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index f5639d5..4fb7291 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -247,6 +247,10 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event)
__diag_push(); \
__diag_ignore(GCC, 8, "-Wattribute-alias", \
"Type aliasing is used to sanitize syscall arguments");\
+ __diag_ignore(clang, 23, "-Wunknown-warning-option", \
+ "Avoid breaking versions without -Wattribute-alias");\
+ __diag_ignore(clang, 23, "-Wattribute-alias", \
+ "Type aliasing is used to sanitize syscall arguments");\
asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \
__attribute__((alias(__stringify(__se_sys##name)))); \
ALLOW_ERROR_INJECTION(sys##name, ERRNO); \
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 738007d..1cf4651f 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -139,8 +139,6 @@ extern bool tick_nohz_idle_got_tick(void);
extern ktime_t tick_nohz_get_next_hrtimer(void);
extern ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next);
extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu);
-extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
-extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
#else /* !CONFIG_NO_HZ_COMMON */
#define tick_nohz_enabled (0)
static inline bool tick_nohz_is_active(void) { return false; }
@@ -162,8 +160,6 @@ static inline ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
*delta_next = TICK_NSEC;
return *delta_next;
}
-static inline u64 get_cpu_idle_time_us(int cpu, u64 *unused) { return -1; }
-static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; }
#endif /* !CONFIG_NO_HZ_COMMON */
/*
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index e36d11e..4486dfd 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -190,7 +190,7 @@ struct timekeeper {
s32 tai_offset;
};
-#ifdef CONFIG_GENERIC_TIME_VSYSCALL
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
extern void update_vsyscall(struct timekeeper *tk);
extern void update_vsyscall_tz(void);
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index aee2c1a..984a866 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -276,37 +276,30 @@ static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { ret
#endif
/**
- * struct system_time_snapshot - simultaneous raw/real time capture with
- * counter value
- * @cycles: Clocksource counter value to produce the system times
- * @real: Realtime system time
- * @boot: Boot time
- * @raw: Monotonic raw system time
- * @cs_id: Clocksource ID
+ * struct system_time_snapshot - Simultaneous time capture of CLOCK_MONOTONIC_RAW,
+ * a selected CLOCK_* and the clocksource counter value
+ * @cycles: Clocksource counter value to produce the system times
+ * @hw_cycles: For derived clocksources, the hardware counter value from
+ * which @cycles was derived
+ * @systime: The system time of the selected CLOCK ID
+ * @monoraw: Monotonic raw system time
+ * @cs_id: Clocksource ID
+ * @hw_csid: Clocksource ID of the underlying hardware counter for derived
+ * clocksources which implement the read_snapshot() callback.
* @clock_was_set_seq: The sequence number of clock-was-set events
* @cs_was_changed_seq: The sequence number of clocksource change events
+ * @valid: True if the snapshot is valid
*/
struct system_time_snapshot {
u64 cycles;
- ktime_t real;
- ktime_t boot;
- ktime_t raw;
+ u64 hw_cycles;
+ ktime_t systime;
+ ktime_t monoraw;
enum clocksource_ids cs_id;
+ enum clocksource_ids hw_csid;
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
-};
-
-/**
- * struct system_device_crosststamp - system/device cross-timestamp
- * (synchronized capture)
- * @device: Device time
- * @sys_realtime: Realtime simultaneous with device time
- * @sys_monoraw: Monotonic raw simultaneous with device time
- */
-struct system_device_crosststamp {
- ktime_t device;
- ktime_t sys_realtime;
- ktime_t sys_monoraw;
+ u8 valid;
};
/**
@@ -325,6 +318,23 @@ struct system_counterval_t {
bool use_nsecs;
};
+/**
+ * struct system_device_crosststamp - system/device cross-timestamp
+ * (synchronized capture)
+ * @clock_id: System time Clock ID to capture
+ * @device: Device time
+ * @sys_counter: Clocksource counter value simultaneous with device time
+ * @sys_systime: System time for @clock_id
+ * @sys_monoraw: Monotonic raw simultaneous with device time
+ */
+struct system_device_crosststamp {
+ clockid_t clock_id;
+ ktime_t device;
+ struct system_counterval_t sys_counter;
+ ktime_t sys_systime;
+ ktime_t sys_monoraw;
+};
+
extern bool ktime_real_to_base_clock(ktime_t treal,
enum clocksource_ids base_id, u64 *cycles);
extern bool timekeeping_clocksource_has_base(enum clocksource_ids id);
@@ -341,9 +351,10 @@ extern int get_device_system_crosststamp(
struct system_device_crosststamp *xtstamp);
/*
- * Simultaneously snapshot realtime and monotonic raw clocks
+ * Simultaneously snapshot a given clock with MONOTONIC_RAW and the underlying
+ * clocksource counter value.
*/
-extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);
+extern void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot);
/*
* Persistent clock related interfaces
diff --git a/include/linux/topology.h b/include/linux/topology.h
index 6575af3..709a2dc 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -230,11 +230,24 @@ static inline int cpu_to_mem(int cpu)
#define topology_drawer_cpumask(cpu) cpumask_of(cpu)
#endif
-#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
+/*
+ * Defining cpu_smt_mask as cpumask_of that CPU helps to get
+ * rid of lot of ifdeffery all around the codebase in case of
+ * CONFIG_SCHED_SMT=n. It just means there are no other siblings, which
+ * is what is expected.
+ */
+#if defined(CONFIG_SCHED_SMT)
+# if !defined(cpu_smt_mask)
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
return topology_sibling_cpumask(cpu);
}
+# endif
+#else /* !CONFIG_SCHED_SMT */
+static inline const struct cpumask *cpu_smt_mask(int cpu)
+{
+ return cpumask_of(cpu);
+}
#endif
#ifndef topology_is_primary_thread
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 578e520..2d2b9f8 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -20,6 +20,7 @@
#include <linux/rcupdate_trace.h>
#include <linux/tracepoint-defs.h>
#include <linux/static_call.h>
+#include <linux/cfi.h>
struct module;
struct tracepoint;
@@ -202,7 +203,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
#define TP_CONDITION(args...) args
/*
- * Individual subsystem my have a separate configuration to
+ * Individual subsystem may have a separate configuration to
* enable their tracepoints. By default, this file will create
* the tracepoints if CONFIG_TRACEPOINTS is defined. If a subsystem
* wants to be able to disable its tracepoints from being created
@@ -389,6 +390,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
void __probestub_##_name(void *__data, proto) \
{ \
} \
+ /* \
+ * Annotate the probestub 'CFI_NOSEAL' to stop objtool from \
+ * requesting the kernel remove the ENDBR, because the only \
+ * references to the function are in the __tracepoint section, \
+ * that objtool doesn't scan. \
+ */ \
+ CFI_NOSEAL(__probestub_##_name); \
DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name); \
DEFINE_RUST_DO_TRACE(_name, TP_PROTO(proto), TP_ARGS(args))
diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h
index d2a7882..23cad40 100644
--- a/include/linux/tty_port.h
+++ b/include/linux/tty_port.h
@@ -6,10 +6,10 @@
#include <linux/kref.h>
#include <linux/mutex.h>
#include <linux/tty_buffer.h>
+#include <linux/tty_driver.h>
#include <linux/wait.h>
struct attribute_group;
-struct tty_driver;
struct tty_port;
struct tty_struct;
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 5632860..c6bd200 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -649,6 +649,17 @@ static inline void user_access_restore(unsigned long flags) { }
#define user_read_access_end user_access_end
#endif
+#ifndef unsafe_atomic_store_release_user
+# define unsafe_atomic_store_release_user(val, uptr, elbl) \
+ do { \
+ if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \
+ smp_mb(); \
+ else \
+ barrier(); \
+ unsafe_put_user(val, uptr, elbl); \
+ } while (0)
+#endif
+
/* Define RW variant so the below _mode macro expansion works */
#define masked_user_rw_access_begin(u) masked_user_access_begin(u)
#define user_rw_access_begin(u, s) user_access_begin(u, s)
diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
index 0b53042..3dfba95 100644
--- a/include/linux/vdso_datastore.h
+++ b/include/linux/vdso_datastore.h
@@ -2,12 +2,12 @@
#ifndef _LINUX_VDSO_DATASTORE_H
#define _LINUX_VDSO_DATASTORE_H
-#ifdef CONFIG_HAVE_GENERIC_VDSO
#include <linux/mm_types.h>
extern const struct vm_special_mapping vdso_vvar_mapping;
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
+#ifdef CONFIG_HAVE_GENERIC_VDSO
void __init vdso_setup_data_pages(void);
#else /* !CONFIG_HAVE_GENERIC_VDSO */
static inline void vdso_setup_data_pages(void) { }
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 2ebba74..89165b7 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -21,7 +21,7 @@
#define VFIO_PCI_CORE_H
#define VFIO_PCI_OFFSET_SHIFT 40
-#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
+#define VFIO_PCI_OFFSET_TO_INDEX(off) ((u64)(off) >> VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
diff --git a/include/linux/vtime.h b/include/linux/vtime.h
index 29dd5b9..9dc25b0 100644
--- a/include/linux/vtime.h
+++ b/include/linux/vtime.h
@@ -10,7 +10,6 @@
*/
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
extern void vtime_account_kernel(struct task_struct *tsk);
-extern void vtime_account_idle(struct task_struct *tsk);
#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
@@ -27,16 +26,33 @@ static inline void vtime_guest_exit(struct task_struct *tsk) { }
static inline void vtime_init_idle(struct task_struct *tsk, int cpu) { }
#endif
+static inline bool vtime_generic_enabled_cpu(int cpu)
+{
+ return context_tracking_enabled_cpu(cpu);
+}
+
+static inline bool vtime_generic_enabled_this_cpu(void)
+{
+ return context_tracking_enabled_this_cpu();
+}
+
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+extern void vtime_account_idle(struct task_struct *tsk);
extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset);
extern void vtime_account_softirq(struct task_struct *tsk);
extern void vtime_account_hardirq(struct task_struct *tsk);
extern void vtime_flush(struct task_struct *tsk);
+extern void vtime_reset(void);
+extern void vtime_dyntick_start(void);
+extern void vtime_dyntick_stop(void);
#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
static inline void vtime_account_irq(struct task_struct *tsk, unsigned int offset) { }
static inline void vtime_account_softirq(struct task_struct *tsk) { }
static inline void vtime_account_hardirq(struct task_struct *tsk) { }
static inline void vtime_flush(struct task_struct *tsk) { }
+static inline void vtime_reset(void) { }
+static inline void vtime_dyntick_start(void) { }
+static inline void vtime_dyntick_stop(void) { }
#endif
/*
@@ -74,12 +90,12 @@ static inline bool vtime_accounting_enabled(void)
static inline bool vtime_accounting_enabled_cpu(int cpu)
{
- return context_tracking_enabled_cpu(cpu);
+ return vtime_generic_enabled_cpu(cpu);
}
static inline bool vtime_accounting_enabled_this_cpu(void)
{
- return context_tracking_enabled_this_cpu();
+ return vtime_generic_enabled_this_cpu();
}
extern void vtime_task_switch_generic(struct task_struct *prev);
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index ab6cb70..61776245 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -534,8 +534,10 @@ alloc_workqueue_noprof(const char *fmt, unsigned int flags, int max_active, ...)
* Pointer to the allocated workqueue on success, %NULL on failure.
*/
__printf(2, 5) struct workqueue_struct *
-devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags,
- int max_active, ...);
+devm_alloc_workqueue_noprof(struct device *dev, const char *fmt,
+ unsigned int flags, int max_active, ...);
+#define devm_alloc_workqueue(...) \
+ alloc_hooks(devm_alloc_workqueue_noprof(__VA_ARGS__))
#ifdef CONFIG_LOCKDEP
/**
diff --git a/include/net/act_api.h b/include/net/act_api.h
index d11b791..fd2967e 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -45,6 +45,7 @@ struct tc_action {
struct tc_cookie __rcu *user_cookie;
struct tcf_chain __rcu *goto_chain;
u32 tcfa_flags;
+ struct rcu_head tcfa_rcu;
u8 hw_stats;
u8 used_hw_stats;
bool used_hw_stats_valid;
diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h
index 69eed69..3faea66 100644
--- a/include/net/bluetooth/bluetooth.h
+++ b/include/net/bluetooth/bluetooth.h
@@ -398,6 +398,7 @@ void baswap(bdaddr_t *dst, const bdaddr_t *src);
struct bt_sock {
struct sock sk;
struct list_head accept_q;
+ spinlock_t accept_q_lock; /* protects accept_q */
struct sock *parent;
unsigned long flags;
void (*skb_msg_name)(struct sk_buff *, void *, int *);
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index a7bffb9..aa600fb 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -2495,7 +2495,7 @@ void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle,
bdaddr_t *bdaddr, u8 addr_type);
int hci_abort_conn(struct hci_conn *conn, u8 reason);
-u8 hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency,
+void hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency,
u16 to_multiplier);
void hci_le_start_enc(struct hci_conn *conn, __le16 ediv, __le64 rand,
__u8 ltk[16], __u8 key_size);
diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h
index 5172afe..e0a1f22 100644
--- a/include/net/bluetooth/l2cap.h
+++ b/include/net/bluetooth/l2cap.h
@@ -33,6 +33,7 @@
/* L2CAP defaults */
#define L2CAP_DEFAULT_MTU 672
#define L2CAP_DEFAULT_MIN_MTU 48
+#define L2CAP_SIG_MTU 48 /* BR/EDR signaling MTU */
#define L2CAP_DEFAULT_FLUSH_TO 0xFFFF
#define L2CAP_EFS_DEFAULT_FLUSH_TO 0xFFFFFFFF
#define L2CAP_DEFAULT_TX_WINDOW 63
diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h
index c92d4a9..05572c1 100644
--- a/include/net/bond_3ad.h
+++ b/include/net/bond_3ad.h
@@ -243,7 +243,7 @@ typedef struct port {
churn_state_t sm_churn_actor_state;
churn_state_t sm_churn_partner_state;
struct slave *slave; /* pointer to the bond slave that this port belongs to */
- struct aggregator *aggregator; /* pointer to an aggregator that this port related to */
+ struct aggregator __rcu *aggregator; /* pointer to an aggregator that this port related to */
struct port *next_port_in_aggregator; /* Next port on the linked list of the parent aggregator */
u32 transaction_id; /* continuous number for identification of Marker PDU's; */
struct lacpdu lacpdu; /* the lacpdu that will be sent for this port */
diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h
index e0ca390..2f312d1 100644
--- a/include/net/dropreason-core.h
+++ b/include/net/dropreason-core.h
@@ -99,6 +99,7 @@
FN(FRAG_TOO_FAR) \
FN(TCP_MINTTL) \
FN(IPV6_BAD_EXTHDR) \
+ FN(IPV6_TOO_MANY_EXTHDRS) \
FN(IPV6_NDISC_FRAG) \
FN(IPV6_NDISC_HOP_LIMIT) \
FN(IPV6_NDISC_BAD_CODE) \
@@ -494,6 +495,11 @@ enum skb_drop_reason {
SKB_DROP_REASON_TCP_MINTTL,
/** @SKB_DROP_REASON_IPV6_BAD_EXTHDR: Bad IPv6 extension header. */
SKB_DROP_REASON_IPV6_BAD_EXTHDR,
+ /**
+ * @SKB_DROP_REASON_IPV6_TOO_MANY_EXTHDRS: Number of IPv6 extension
+ * headers in the packet exceeds IP6_MAX_EXT_HDRS_CNT.
+ */
+ SKB_DROP_REASON_IPV6_TOO_MANY_EXTHDRS,
/** @SKB_DROP_REASON_IPV6_NDISC_FRAG: invalid frag (suppress_frag_ndisc). */
SKB_DROP_REASON_IPV6_NDISC_FRAG,
/** @SKB_DROP_REASON_IPV6_NDISC_HOP_LIMIT: invalid hop limit. */
diff --git a/include/net/genetlink.h b/include/net/genetlink.h
index 7b84f2c..d70510a 100644
--- a/include/net/genetlink.h
+++ b/include/net/genetlink.h
@@ -489,8 +489,10 @@ genlmsg_multicast_netns_filtered(const struct genl_family *family,
netlink_filter_fn filter,
void *filter_data)
{
- if (WARN_ON_ONCE(group >= family->n_mcgrps))
+ if (WARN_ON_ONCE(group >= family->n_mcgrps)) {
+ nlmsg_free(skb);
return -EINVAL;
+ }
group = family->mcgrp_offset + group;
return nlmsg_multicast_filtered(net->genl_sock, skb, portid, group,
flags, filter, filter_data);
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h
index 72d325c..e517eaa 100644
--- a/include/net/ip_vs.h
+++ b/include/net/ip_vs.h
@@ -491,6 +491,7 @@ struct ip_vs_est_kt_data {
DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */
unsigned long est_timer; /* estimation timer (jiffies) */
struct ip_vs_stats *calc_stats; /* Used for calculation */
+ int needed; /* task is needed */
int tick_len[IPVS_EST_NTICKS]; /* est count */
int id; /* ktid per netns */
int chain_max; /* max ests per tick chain */
@@ -1185,8 +1186,9 @@ struct netns_ipvs {
struct timer_list dest_trash_timer; /* expiration timer */
struct mutex service_mutex; /* service reconfig */
struct rw_semaphore svc_resize_sem; /* svc_table resizing */
+ struct rw_semaphore svc_replace_sem; /* svc_table replace */
struct delayed_work svc_resize_work; /* resize svc_table */
- atomic_t svc_table_changes;/* ++ on new table */
+ atomic_t svc_table_changes;/* ++ on table changes */
/* Service counters */
atomic_t num_services[IP_VS_AF_MAX]; /* Services */
atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */
@@ -1411,7 +1413,7 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
return ipvs->sysctl_run_estimation;
}
-static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs)
+static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs)
{
if (ipvs->est_cpulist_valid)
return ipvs->sysctl_est_cpulist;
@@ -1529,7 +1531,7 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
return 1;
}
-static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs)
+static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs)
{
return housekeeping_cpumask(HK_TYPE_KTHREAD);
}
@@ -1564,6 +1566,18 @@ static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs)
return READ_ONCE(ipvs->sysctl_svc_lfactor);
}
+static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs)
+{
+ guard(rcu)();
+ return cpumask_empty(__sysctl_est_cpulist(ipvs));
+}
+
+static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs)
+{
+ guard(rcu)();
+ return cpumask_weight(__sysctl_est_cpulist(ipvs));
+}
+
/* IPVS core functions
* (from ip_vs_core.c)
*/
@@ -1810,8 +1824,7 @@ int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler);
int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler);
int ip_vs_bind_scheduler(struct ip_vs_service *svc,
struct ip_vs_scheduler *scheduler);
-void ip_vs_unbind_scheduler(struct ip_vs_service *svc,
- struct ip_vs_scheduler *sched);
+void ip_vs_unbind_scheduler(struct ip_vs_service *svc);
struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name);
void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler);
struct ip_vs_conn *
@@ -1884,18 +1897,26 @@ int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats);
void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats);
void ip_vs_zero_estimator(struct ip_vs_stats *stats);
void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats);
-void ip_vs_est_reload_start(struct netns_ipvs *ipvs);
+void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart);
int ip_vs_est_kthread_start(struct netns_ipvs *ipvs,
struct ip_vs_est_kt_data *kd);
void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd);
+static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs)
+{
+#ifdef CONFIG_SYSCTL
+ ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s);
+ ipvs->tot_stats->s.est.ktid = -2;
+#endif
+}
+
static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs)
{
#ifdef CONFIG_SYSCTL
/* Stop tasks while cpulist is empty or if disabled with flag */
ipvs->est_stopped = !sysctl_run_estimation(ipvs) ||
(ipvs->est_cpulist_valid &&
- cpumask_empty(sysctl_est_cpulist(ipvs)));
+ sysctl_est_cpulist_empty(ipvs));
#endif
}
@@ -1911,7 +1932,7 @@ static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs)
static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs)
{
unsigned int limit = IPVS_EST_CPU_KTHREADS *
- cpumask_weight(sysctl_est_cpulist(ipvs));
+ sysctl_est_cpulist_weight(ipvs);
return max(1U, limit);
}
diff --git a/include/net/ipv6.h b/include/net/ipv6.h
index d042afe..1dec81f 100644
--- a/include/net/ipv6.h
+++ b/include/net/ipv6.h
@@ -90,6 +90,9 @@ struct ip_tunnel_info;
#define IP6_DEFAULT_MAX_DST_OPTS_LEN INT_MAX /* No limit */
#define IP6_DEFAULT_MAX_HBH_OPTS_LEN INT_MAX /* No limit */
+/* Hard limit on traversed IPv6 extension headers */
+#define IP6_MAX_EXT_HDRS_CNT 12
+
/*
* Addr type
*
diff --git a/include/net/macsec.h b/include/net/macsec.h
index bc7de5b..d962093e 100644
--- a/include/net/macsec.h
+++ b/include/net/macsec.h
@@ -9,6 +9,7 @@
#include <linux/u64_stats_sync.h>
#include <linux/if_vlan.h>
+#include <linux/workqueue.h>
#include <uapi/linux/if_link.h>
#include <uapi/linux/if_macsec.h>
@@ -123,6 +124,7 @@ struct macsec_dev_stats {
* @key: key structure
* @ssci: short secure channel identifier
* @stats: per-SA stats
+ * @destroy_work: deferred work to free the SA in process context after RCU grace period
*/
struct macsec_rx_sa {
struct macsec_key key;
@@ -136,7 +138,7 @@ struct macsec_rx_sa {
bool active;
struct macsec_rx_sa_stats __percpu *stats;
struct macsec_rx_sc *sc;
- struct rcu_head rcu;
+ struct rcu_work destroy_work;
};
struct pcpu_rx_sc_stats {
@@ -174,6 +176,7 @@ struct macsec_rx_sc {
* @key: key structure
* @ssci: short secure channel identifier
* @stats: per-SA stats
+ * @destroy_work: deferred work to free the SA in process context after RCU grace period
*/
struct macsec_tx_sa {
struct macsec_key key;
@@ -186,7 +189,7 @@ struct macsec_tx_sa {
refcount_t refcnt;
bool active;
struct macsec_tx_sa_stats __percpu *stats;
- struct rcu_head rcu;
+ struct rcu_work destroy_work;
};
/**
diff --git a/include/net/mana/shm_channel.h b/include/net/mana/shm_channel.h
index 5199b41..dbabcfb 100644
--- a/include/net/mana/shm_channel.h
+++ b/include/net/mana/shm_channel.h
@@ -4,6 +4,12 @@
#ifndef _SHM_CHANNEL_H
#define _SHM_CHANNEL_H
+#define SMC_APERTURE_BITS 256
+#define SMC_BASIC_UNIT (sizeof(u32))
+#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
+#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
+#define SMC_APERTURE_SIZE (SMC_APERTURE_BITS / 8)
+
struct shm_channel {
struct device *dev;
void __iomem *base;
diff --git a/include/net/mptcp.h b/include/net/mptcp.h
index f7263fe..ee70f59 100644
--- a/include/net/mptcp.h
+++ b/include/net/mptcp.h
@@ -27,7 +27,9 @@ struct mptcp_ext {
u32 subflow_seq;
u16 data_len;
__sum16 csum;
- u8 use_map:1,
+
+ struct_group(flags,
+ u8 use_map:1,
dsn64:1,
data_fin:1,
use_ack:1,
@@ -35,9 +37,10 @@ struct mptcp_ext {
mpc_map:1,
frozen:1,
reset_transient:1;
- u8 reset_reason:4,
+ u8 reset_reason:4,
csum_reqd:1,
infinite_map:1;
+ ); /* end of flags group */
};
#define MPTCPOPT_HMAC_LEN 20
diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 2dfee6d..8860cc2 100644
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -489,11 +489,15 @@ static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb)
{
- unsigned int seq, hh_alen;
+ unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN);
+ int err;
+
+ err = skb_cow_head(skb, hh_alen);
+ if (err)
+ return err;
do {
seq = read_seqbegin(&hh->hh_lock);
- hh_alen = HH_DATA_ALIGN(ETH_HLEN);
memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN);
} while (read_seqretry(&hh->hh_lock, seq));
return 0;
diff --git a/include/net/net_shaper.h b/include/net/net_shaper.h
index 5c3f49b5..3939b81 100644
--- a/include/net/net_shaper.h
+++ b/include/net/net_shaper.h
@@ -53,6 +53,7 @@ struct net_shaper {
/* private: */
u32 leaves; /* accounted only for NODE scope */
+ bool valid;
struct rcu_head rcu;
};
diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h
index e9a8350..80f50fd 100644
--- a/include/net/netfilter/nf_conntrack_expect.h
+++ b/include/net/netfilter/nf_conntrack_expect.h
@@ -45,9 +45,12 @@ struct nf_conntrack_expect {
void (*expectfn)(struct nf_conn *new,
struct nf_conntrack_expect *this);
- /* Helper to assign to new connection */
+ /* Helper that created this expectation */
struct nf_conntrack_helper __rcu *helper;
+ /* Helper to assign to new connection */
+ struct nf_conntrack_helper __rcu *assign_helper;
+
/* The conntrack of the master connection */
struct nf_conn *master;
diff --git a/include/net/netfilter/nf_dup_netdev.h b/include/net/netfilter/nf_dup_netdev.h
index b175d271..609bcf4 100644
--- a/include/net/netfilter/nf_dup_netdev.h
+++ b/include/net/netfilter/nf_dup_netdev.h
@@ -3,10 +3,23 @@
#define _NF_DUP_NETDEV_H_
#include <net/netfilter/nf_tables.h>
+#include <linux/netdevice.h>
+#include <linux/sched.h>
void nf_dup_netdev_egress(const struct nft_pktinfo *pkt, int oif);
void nf_fwd_netdev_egress(const struct nft_pktinfo *pkt, int oif);
+#define NF_RECURSION_LIMIT 2
+
+static inline u8 *nf_get_nf_dup_skb_recursion(void)
+{
+#ifndef CONFIG_PREEMPT_RT
+ return this_cpu_ptr(&softnet_data.xmit.nf_dup_skb_recursion);
+#else
+ return ¤t->net_xmit.nf_dup_skb_recursion;
+#endif
+}
+
struct nft_offload_ctx;
struct nft_flow_rule;
diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
index b09c11c..7b23b24 100644
--- a/include/net/netfilter/nf_flow_table.h
+++ b/include/net/netfilter/nf_flow_table.h
@@ -148,9 +148,10 @@ struct flow_offload_tuple {
/* All members above are keys for lookups, see flow_offload_hash(). */
struct { } __hash;
- u8 dir:2,
+ u16 dir:2,
xmit_type:3,
encap_num:2,
+ needs_gso_segment:1,
tun_num:2,
in_vlan_ingress:2;
u16 mtu;
@@ -232,6 +233,7 @@ struct nf_flow_route {
u32 hw_ifindex;
u8 h_source[ETH_ALEN];
u8 h_dest[ETH_ALEN];
+ u8 needs_gso_segment:1;
} out;
enum flow_offload_xmit_type xmit_type;
} tuple[FLOW_OFFLOAD_DIR_MAX];
diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h
index d17035d..3978c31 100644
--- a/include/net/netfilter/nf_queue.h
+++ b/include/net/netfilter/nf_queue.h
@@ -14,6 +14,7 @@ struct nf_queue_entry {
struct list_head list;
struct rhash_head hash_node;
struct sk_buff *skb;
+ struct net_device *skb_dev;
unsigned int id;
unsigned int hook_index; /* index in hook_entries->hook[] */
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index 2c0173d..9d84435 100644
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -180,6 +180,13 @@ static inline u64 nft_reg_load64(const u32 *sreg)
return get_unaligned((u64 *)sreg);
}
+static inline bool nft_reg_overlap(u8 src, u8 dst, u32 len)
+{
+ unsigned int n = DIV_ROUND_UP(len, sizeof(u32));
+
+ return src != dst && src < dst + n && dst < src + n;
+}
+
static inline void nft_data_copy(u32 *dst, const struct nft_data *src,
unsigned int len)
{
@@ -1204,12 +1211,15 @@ struct nft_stats {
struct u64_stats_sync syncp;
};
+#define NFT_HOOK_REMOVE (1 << 0)
+
struct nft_hook {
struct list_head list;
struct list_head ops_list;
struct rcu_head rcu;
char ifname[IFNAMSIZ];
u8 ifnamelen;
+ u8 flags;
};
struct nf_hook_ops *nft_hook_find_ops(const struct nft_hook *hook,
@@ -1665,6 +1675,16 @@ struct nft_trans {
};
/**
+ * struct nft_trans_hook - nf_tables hook update in transaction
+ * @list: used internally
+ * @hook: struct nft_hook with the device hook
+ */
+struct nft_trans_hook {
+ struct list_head list;
+ struct nft_hook *hook;
+};
+
+/**
* struct nft_trans_binding - nf_tables object with binding support in transaction
* @nft_trans: base structure, MUST be first member
* @binding_list: list of objects with possible bindings
diff --git a/include/net/netmem.h b/include/net/netmem.h
index 507b74c..bccacd2 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -94,20 +94,10 @@ enum net_iov_type {
*/
struct net_iov {
struct netmem_desc desc;
- unsigned int page_type;
enum net_iov_type type;
struct net_iov_area *owner;
};
-/* Make sure 'the offset of page_type in struct page == the offset of
- * type in struct net_iov'.
- */
-#define NET_IOV_ASSERT_OFFSET(pg, iov) \
- static_assert(offsetof(struct page, pg) == \
- offsetof(struct net_iov, iov))
-NET_IOV_ASSERT_OFFSET(page_type, page_type);
-#undef NET_IOV_ASSERT_OFFSET
-
struct net_iov_area {
/* Array of net_iovs for this area. */
struct net_iov *niovs;
@@ -127,6 +117,16 @@ static inline unsigned int net_iov_idx(const struct net_iov *niov)
return niov - net_iov_owner(niov)->niovs;
}
+/* Initialize a niov: stamp the owning area, the memory provider type.
+ */
+static inline void net_iov_init(struct net_iov *niov,
+ struct net_iov_area *owner,
+ enum net_iov_type type)
+{
+ niov->owner = owner;
+ niov->type = type;
+}
+
/* netmem */
/**
@@ -230,7 +230,7 @@ static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
*/
#define pp_page_to_nmdesc(p) \
({ \
- DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p)); \
+ DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \
__pp_page_to_nmdesc(p); \
})
diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index 80ccd4d..6e27c56 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -275,7 +275,7 @@ struct netns_ipv4 {
#ifdef CONFIG_IP_MROUTE
#ifndef CONFIG_IP_MROUTE_MULTIPLE_TABLES
- struct mr_table *mrt;
+ struct mr_table __rcu *mrt;
#else
struct list_head mr_tables;
struct fib_rules_ops *mr_rules_ops;
diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h
index 499e428..875916d 100644
--- a/include/net/netns/ipv6.h
+++ b/include/net/netns/ipv6.h
@@ -119,6 +119,7 @@ struct netns_ipv6 {
struct fib_notifier_ops *notifier_ops;
struct fib_notifier_ops *ip6mr_notifier_ops;
atomic_t ipmr_seq;
+ int flowlabel_count;
struct {
struct hlist_head head;
spinlock_t lock;
diff --git a/include/net/nsh.h b/include/net/nsh.h
index 16a7510..15a26c5 100644
--- a/include/net/nsh.h
+++ b/include/net/nsh.h
@@ -247,10 +247,10 @@ struct nshhdr {
#define NSH_M_TYPE1_LEN 24
/* NSH header maximum Length. */
-#define NSH_HDR_MAX_LEN 256
+#define NSH_HDR_MAX_LEN ((NSH_LEN_MASK >> NSH_LEN_SHIFT) * 4)
/* NSH context headers maximum Length. */
-#define NSH_CTX_HDRS_MAX_LEN 248
+#define NSH_CTX_HDRS_MAX_LEN (NSH_HDR_MAX_LEN - NSH_BASE_HDR_LEN)
static inline struct nshhdr *nsh_hdr(struct sk_buff *skb)
{
diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h
index f58ee15..cb7b82f 100644
--- a/include/net/tc_act/tc_pedit.h
+++ b/include/net/tc_act/tc_pedit.h
@@ -15,7 +15,6 @@ struct tcf_pedit_parms {
struct tc_pedit_key *tcfp_keys;
struct tcf_pedit_key_ex *tcfp_keys_ex;
int action;
- u32 tcfp_off_max_hint;
unsigned char tcfp_nkeys;
unsigned char tcfp_flags;
struct rcu_head rcu;
diff --git a/include/net/tcp.h b/include/net/tcp.h
index ecbadcb..98848db 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -65,8 +65,6 @@ static inline void tcp_orphan_count_dec(void)
this_cpu_dec(tcp_orphan_count);
}
-DECLARE_PER_CPU(u32, tcp_tw_isn);
-
void tcp_time_wait(struct sock *sk, int state, int timeo);
#define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
@@ -1102,10 +1100,13 @@ struct tcp_skb_cb {
__u32 seq; /* Starting sequence number */
__u32 end_seq; /* SEQ + FIN + SYN + datalen */
union {
- /* Note :
+ /* Notes :
+ * tcp_tw_isn is used in input path only
+ * (isn chosen by tcp_timewait_state_process())
* tcp_gso_segs/size are used in write queue only,
* cf tcp_skb_pcount()/tcp_skb_mss()
*/
+ u32 tcp_tw_isn;
struct {
u16 tcp_gso_segs;
u16 tcp_gso_size;
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 10d3edd..8744091 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -715,6 +715,7 @@ struct xfrm_mgr {
const struct xfrm_migrate *m,
int num_bundles,
const struct xfrm_kmaddress *k,
+ struct net *net,
const struct xfrm_encap_tmpl *encap);
bool (*is_alive)(const struct km_event *c);
};
@@ -1891,7 +1892,7 @@ int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol);
#ifdef CONFIG_XFRM_MIGRATE
int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_bundles,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap);
struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
u32 if_id);
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index e2af17da..c894280 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -635,6 +635,7 @@ struct uverbs_attr_bundle {
struct ib_uverbs_file *ufile;
struct ib_ucontext *context;
struct ib_uobject *uobject;
+ const struct uverbs_api_ioctl_method *method_elm;
DECLARE_BITMAP(attr_present, UVERBS_API_ATTR_BKEY_LEN);
);
struct uverbs_attr attrs[];
diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h
index 8ad7a2d..ec1df8b 100644
--- a/include/trace/events/btrfs.h
+++ b/include/trace/events/btrfs.h
@@ -771,10 +771,8 @@ TRACE_EVENT(btrfs_sync_file,
TP_fast_assign(
struct dentry *dentry = file_dentry(file);
struct inode *inode = file_inode(file);
- struct dentry *parent = dget_parent(dentry);
- struct inode *parent_inode = d_inode(parent);
+ struct inode *parent_inode = d_inode(dentry->d_parent);
- dput(parent);
TP_fast_assign_fsid(btrfs_sb(inode->i_sb));
__entry->ino = btrfs_ino(BTRFS_I(inode));
__entry->parent = btrfs_ino(BTRFS_I(parent_inode));
diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h
index 24fc402..7e25f446 100644
--- a/include/trace/events/damon.h
+++ b/include/trace/events/damon.h
@@ -41,7 +41,7 @@ TRACE_EVENT(damos_stat_after_apply_interval,
),
TP_printk("ctx_idx=%u scheme_idx=%u nr_tried=%lu sz_tried=%lu "
- "nr_applied=%lu sz_tried=%lu sz_ops_filter_passed=%lu "
+ "nr_applied=%lu sz_applied=%lu sz_ops_filter_passed=%lu "
"qt_exceeds=%lu nr_snapshots=%lu",
__entry->context_idx, __entry->scheme_idx,
__entry->nr_tried, __entry->sz_tried,
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index 8c936fc..082cb03 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -177,7 +177,11 @@
EM(netfs_folio_is_uptodate, "mod-uptodate") \
EM(netfs_just_prefetch, "mod-prefetch") \
EM(netfs_whole_folio_modify, "mod-whole-f") \
+ EM(netfs_whole_folio_modify_efault, "mod-whole-f!") \
+ EM(netfs_whole_folio_modify_filled, "mod-whole-f+") \
+ EM(netfs_whole_folio_modify_filled_efault, "mod-whole-f+!") \
EM(netfs_modify_and_clear, "mod-n-clear") \
+ EM(netfs_modify_and_clear_rm_finfo, "mod-n-clear+") \
EM(netfs_streaming_write, "mod-streamw") \
EM(netfs_streaming_write_cont, "mod-streamw+") \
EM(netfs_flush_content, "flush") \
@@ -194,6 +198,10 @@
EM(netfs_folio_trace_copy_to_cache, "mark-copy") \
EM(netfs_folio_trace_end_copy, "end-copy") \
EM(netfs_folio_trace_filled_gaps, "filled-gaps") \
+ EM(netfs_folio_trace_invalidate_all, "inval-all") \
+ EM(netfs_folio_trace_invalidate_front, "inval-front") \
+ EM(netfs_folio_trace_invalidate_middle, "inval-mid") \
+ EM(netfs_folio_trace_invalidate_tail, "inval-tail") \
EM(netfs_folio_trace_kill, "kill") \
EM(netfs_folio_trace_kill_cc, "kill-cc") \
EM(netfs_folio_trace_kill_g, "kill-g") \
diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h
index 573f2df..704a10d 100644
--- a/include/trace/events/rxrpc.h
+++ b/include/trace/events/rxrpc.h
@@ -71,6 +71,7 @@
EM(rxkad_abort_resp_unknown_tkt, "rxkad-resp-unknown-tkt") \
EM(rxkad_abort_resp_version, "rxkad-resp-version") \
/* RxGK security errors */ \
+ EM(rxgk_abort_1_short_header, "rxgk1-short-hdr") \
EM(rxgk_abort_1_verify_mic_eproto, "rxgk1-vfy-mic-eproto") \
EM(rxgk_abort_2_decrypt_eproto, "rxgk2-dec-eproto") \
EM(rxgk_abort_2_short_data, "rxgk2-short-data") \
diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h
index 07cbb98..ca82fd6 100644
--- a/include/trace/events/timer.h
+++ b/include/trace/events/timer.h
@@ -299,6 +299,19 @@ DECLARE_EVENT_CLASS(hrtimer_class,
);
/**
+ * hrtimer_start_expired - Invoked when a expired timer was started
+ * @hrtimer: pointer to struct hrtimer
+ *
+ * Preceeded by a hrtimer_start tracepoint.
+ */
+DEFINE_EVENT(hrtimer_class, hrtimer_start_expired,
+
+ TP_PROTO(struct hrtimer *hrtimer),
+
+ TP_ARGS(hrtimer)
+);
+
+/**
* hrtimer_expire_exit - called immediately after the hrtimer callback returns
* @hrtimer: pointer to struct hrtimer
*
diff --git a/include/trace/events/timer_migration.h b/include/trace/events/timer_migration.h
index 61171b1..0b135e9 100644
--- a/include/trace/events/timer_migration.h
+++ b/include/trace/events/timer_migration.h
@@ -33,15 +33,16 @@ TRACE_EVENT(tmigr_group_set,
TRACE_EVENT(tmigr_connect_child_parent,
- TP_PROTO(struct tmigr_group *child),
+ TP_PROTO(struct tmigr_hierarchy *hier, struct tmigr_group *child),
- TP_ARGS(child),
+ TP_ARGS(hier, child),
TP_STRUCT__entry(
__field( void *, child )
__field( void *, parent )
__field( unsigned int, lvl )
__field( unsigned int, numa_node )
+ __field( unsigned int, capacity )
__field( unsigned int, num_children )
__field( u32, groupmask )
),
@@ -51,26 +52,28 @@ TRACE_EVENT(tmigr_connect_child_parent,
__entry->parent = child->parent;
__entry->lvl = child->parent->level;
__entry->numa_node = child->parent->numa_node;
+ __entry->capacity = hier->capacity;
__entry->num_children = child->parent->num_children;
__entry->groupmask = child->groupmask;
),
- TP_printk("group=%p groupmask=%0x parent=%p lvl=%d numa=%d num_children=%d",
- __entry->child, __entry->groupmask, __entry->parent,
- __entry->lvl, __entry->numa_node, __entry->num_children)
+ TP_printk("group=%p groupmask=%0x parent=%p lvl=%d numa=%d capacity=%d num_children=%d",
+ __entry->child, __entry->groupmask, __entry->parent, __entry->lvl,
+ __entry->numa_node, __entry->capacity, __entry->num_children)
);
TRACE_EVENT(tmigr_connect_cpu_parent,
- TP_PROTO(struct tmigr_cpu *tmc),
+ TP_PROTO(struct tmigr_hierarchy *hier, struct tmigr_cpu *tmc),
- TP_ARGS(tmc),
+ TP_ARGS(hier, tmc),
TP_STRUCT__entry(
__field( void *, parent )
__field( unsigned int, cpu )
__field( unsigned int, lvl )
__field( unsigned int, numa_node )
+ __field( unsigned int, capacity )
__field( unsigned int, num_children )
__field( u32, groupmask )
),
@@ -80,13 +83,14 @@ TRACE_EVENT(tmigr_connect_cpu_parent,
__entry->cpu = tmc->cpuevt.cpu;
__entry->lvl = tmc->tmgroup->level;
__entry->numa_node = tmc->tmgroup->numa_node;
+ __entry->capacity = hier->capacity;
__entry->num_children = tmc->tmgroup->num_children;
__entry->groupmask = tmc->groupmask;
),
- TP_printk("cpu=%d groupmask=%0x parent=%p lvl=%d numa=%d num_children=%d",
- __entry->cpu, __entry->groupmask, __entry->parent,
- __entry->lvl, __entry->numa_node, __entry->num_children)
+ TP_printk("cpu=%d groupmask=%0x parent=%p lvl=%d numa=%d capacity=%d num_children=%d",
+ __entry->cpu, __entry->groupmask, __entry->parent, __entry->lvl,
+ __entry->numa_node, __entry->capacity, __entry->num_children)
);
DECLARE_EVENT_CLASS(tmigr_group_and_cpu,
diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h
index 7e2744e..10a36c5 100644
--- a/include/uapi/linux/futex.h
+++ b/include/uapi/linux/futex.h
@@ -25,23 +25,49 @@
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
-#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)
+#define FUTEX_ROBUST_UNLOCK 512
+#define FUTEX_ROBUST_LIST32 1024
-#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
-#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
-#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
-#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG)
-#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG)
-#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
-#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG)
-#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
-#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME | \
+ FUTEX_ROBUST_UNLOCK | FUTEX_ROBUST_LIST32)
+
+#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG)
+#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG)
+#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG)
#define FUTEX_WAKE_BITSET_PRIVATE (FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG)
-#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \
- FUTEX_PRIVATE_FLAG)
-#define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | \
- FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | FUTEX_PRIVATE_FLAG)
+
+/*
+ * Operations to unlock a futex, clear the robust list pending op pointer and
+ * wake waiters.
+ */
+#define FUTEX_UNLOCK_PI_LIST64 (FUTEX_UNLOCK_PI | FUTEX_ROBUST_UNLOCK)
+#define FUTEX_UNLOCK_PI_LIST64_PRIVATE (FUTEX_UNLOCK_PI_LIST64 | FUTEX_PRIVATE_FLAG)
+#define FUTEX_UNLOCK_PI_LIST32 (FUTEX_UNLOCK_PI | FUTEX_ROBUST_UNLOCK | \
+ FUTEX_ROBUST_LIST32)
+#define FUTEX_UNLOCK_PI_LIST32_PRIVATE (FUTEX_UNLOCK_PI_LIST32 | FUTEX_PRIVATE_FLAG)
+
+#define FUTEX_UNLOCK_WAKE_LIST64 (FUTEX_WAKE | FUTEX_ROBUST_UNLOCK)
+#define FUTEX_UNLOCK_WAKE_LIST64_PRIVATE (FUTEX_UNLOCK_WAKE_LIST64 | FUTEX_PRIVATE_FLAG)
+
+#define FUTEX_UNLOCK_WAKE_LIST32 (FUTEX_WAKE | FUTEX_ROBUST_UNLOCK | \
+ FUTEX_ROBUST_LIST32)
+#define FUTEX_UNLOCK_WAKE_LIST32_PRIVATE (FUTEX_UNLOCK_WAKE_LIST32 | FUTEX_PRIVATE_FLAG)
+
+#define FUTEX_UNLOCK_BITSET_LIST64 (FUTEX_WAKE_BITSET | FUTEX_ROBUST_UNLOCK)
+#define FUTEX_UNLOCK_BITSET_LIST64_PRIVATE (FUTEX_UNLOCK_BITSET_LIST64 | FUTEX_PRIVATE_FLAG)
+
+#define FUTEX_UNLOCK_BITSET_LIST32 (FUTEX_WAKE_BITSET | FUTEX_ROBUST_UNLOCK | \
+ FUTEX_ROBUST_LIST32)
+#define FUTEX_UNLOCK_BITSET_LIST32_PRIVATE (FUTEX_UNLOCK_BITSET_LIST32 | FUTEX_PRIVATE_FLAG)
/*
* Flags for futex2 syscalls.
@@ -177,6 +203,10 @@ struct robust_list_head {
*/
#define ROBUST_LIST_LIMIT 2048
+/* Modifiers for robust_list_head::list_op_pending */
+#define FUTEX_ROBUST_MOD_PI (0x1UL)
+#define FUTEX_ROBUST_MOD_MASK (FUTEX_ROBUST_MOD_PI)
+
/*
* bitset with all bits set for the FUTEX_xxx_BITSET OPs to request a
* match of any bit.
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 17ac1b7..909fb7a 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -905,7 +905,8 @@ struct io_uring_buf_reg {
__u32 ring_entries;
__u16 bgid;
__u16 flags;
- __u64 resv[3];
+ __u32 min_left;
+ __u32 resv[5];
};
/* argument for IORING_REGISTER_PBUF_STATUS */
diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h
index f69344f..ca6fe1f 100644
--- a/include/uapi/linux/rseq.h
+++ b/include/uapi/linux/rseq.h
@@ -28,7 +28,7 @@ enum rseq_cs_flags_bit {
RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0,
RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1,
RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2,
- /* (3) Intentional gap to put new bits into a separate byte */
+ /* (3) Intentional gap to keep new bits separate */
/* User read only feature flags */
RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4,
@@ -161,6 +161,9 @@ struct rseq {
* - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT
* - RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL
* - RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE
+ *
+ * It is now used for feature status advertisement by the kernel.
+ * See: enum rseq_cs_flags_bit for further information.
*/
__u32 flags;
diff --git a/include/uapi/linux/tee.h b/include/uapi/linux/tee.h
index cab5cad..5203977 100644
--- a/include/uapi/linux/tee.h
+++ b/include/uapi/linux/tee.h
@@ -470,6 +470,7 @@ struct tee_ioctl_object_invoke_arg {
__u32 op;
__u32 ret;
__u32 num_params;
+ __u32 :32;
/* num_params tells the actual number of element in params */
struct tee_ioctl_param params[];
};
diff --git a/include/ufs/unipro.h b/include/ufs/unipro.h
index f849a2a..9c16870 100644
--- a/include/ufs/unipro.h
+++ b/include/ufs/unipro.h
@@ -333,6 +333,11 @@ enum ufs_eom_eye_mask {
#define DME_LocalTC0ReplayTimeOutVal 0xD042
#define DME_LocalAFC0ReqTimeOutVal 0xD043
+enum ufs_op_mode {
+ LS_MODE = 1,
+ HS_MODE = 2,
+};
+
/* PA power modes */
enum ufs_pa_pwr_mode {
FAST_MODE = 1,
diff --git a/include/vdso/futex.h b/include/vdso/futex.h
new file mode 100644
index 0000000..3cd175e
--- /dev/null
+++ b/include/vdso/futex.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _VDSO_FUTEX_H
+#define _VDSO_FUTEX_H
+
+#include <uapi/linux/types.h>
+
+/**
+ * __vdso_futex_robust_list64_try_unlock - Try to unlock an uncontended robust futex
+ * with a 64-bit pending op pointer
+ * @lock: Pointer to the futex lock object
+ * @tid: The TID of the calling task
+ * @pop: Pointer to the task's robust_list_head::list_pending_op
+ *
+ * Return: The content of *@lock. On success this is the same as @tid.
+ *
+ * The function implements:
+ * if (atomic_try_cmpxchg(lock, &tid, 0))
+ * *op = NULL;
+ * return tid;
+ *
+ * There is a race between a successful unlock and clearing the pending op
+ * pointer in the robust list head. If the calling task is interrupted in the
+ * race window and has to handle a (fatal) signal on return to user space then
+ * the kernel handles the clearing of @pending_op before attempting to deliver
+ * the signal. That ensures that a task cannot exit with a potentially invalid
+ * pending op pointer.
+ *
+ * User space uses it in the following way:
+ *
+ * if (__vdso_futex_robust_list64_try_unlock(lock, tid, &pending_op) != tid)
+ * err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....);
+ *
+ * If the unlock attempt fails due to the FUTEX_WAITERS bit set in the lock,
+ * then the syscall does the unlock, clears the pending op pointer and wakes the
+ * requested number of waiters.
+ */
+__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop);
+
+/**
+ * __vdso_futex_robust_list32_try_unlock - Try to unlock an uncontended robust futex
+ * with a 32-bit pending op pointer
+ * @lock: Pointer to the futex lock object
+ * @tid: The TID of the calling task
+ * @pop: Pointer to the task's robust_list_head::list_pending_op
+ *
+ * Return: The content of *@lock. On success this is the same as @tid.
+ *
+ * Same as __vdso_futex_robust_list64_try_unlock() just with a 32-bit @pop pointer.
+ */
+__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop);
+
+#endif
diff --git a/include/video/imx-ipu-image-convert.h b/include/video/imx-ipu-image-convert.h
index 003b392..6b77968 100644
--- a/include/video/imx-ipu-image-convert.h
+++ b/include/video/imx-ipu-image-convert.h
@@ -27,12 +27,13 @@ struct ipu_image_convert_run {
int status;
+ /* private: */
/* internal to image converter, callers don't touch */
struct list_head list;
};
/**
- * ipu_image_convert_cb_t - conversion callback function prototype
+ * typedef ipu_image_convert_cb_t - conversion callback function prototype
*
* @run: the completed conversion run pointer
* @ctx: a private context pointer for the callback
@@ -60,7 +61,7 @@ void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
* @out: output image format
* @rot_mode: rotation mode
*
- * Returns 0 if the formats and rotation mode meet IPU restrictions,
+ * Returns: 0 if the formats and rotation mode meet IPU restrictions,
* -EINVAL otherwise.
*/
int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
@@ -77,11 +78,11 @@ int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
* @complete: run completion callback
* @complete_context: a context pointer for the completion callback
*
- * Returns an opaque conversion context pointer on success, error pointer
+ * In V4L2, drivers should call ipu_image_convert_prepare() at streamon.
+ *
+ * Returns: an opaque conversion context pointer on success, error pointer
* on failure. The input/output formats and rotation mode must already meet
* IPU retrictions.
- *
- * In V4L2, drivers should call ipu_image_convert_prepare() at streamon.
*/
struct ipu_image_convert_ctx *
ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
@@ -122,6 +123,8 @@ void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx);
* In V4L2, drivers should call ipu_image_convert_queue() while
* streaming to queue the conversion of a received input buffer.
* For example mem2mem devices this would be called in .device_run.
+ *
+ * Returns: 0 on success or -errno on error.
*/
int ipu_image_convert_queue(struct ipu_image_convert_run *run);
@@ -155,6 +158,9 @@ void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx);
* On successful return the caller can queue more run requests if needed, using
* the prepared context in run->ctx. The caller is responsible for unpreparing
* the context when no more conversion requests are needed.
+ *
+ * Returns: pointer to the created &struct ipu_image_convert_run that has
+ * been queued on success; an ERR_PTR(errno) on error.
*/
struct ipu_image_convert_run *
ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
diff --git a/include/video/udlfb.h b/include/video/udlfb.h
index 58fb573..ab34790 100644
--- a/include/video/udlfb.h
+++ b/include/video/udlfb.h
@@ -56,6 +56,7 @@ struct dlfb_data {
spinlock_t damage_lock;
struct work_struct damage_work;
struct fb_ops ops;
+ atomic_t mmap_count;
/* blit-only rendering path metrics, exposed through sysfs */
atomic_t bytes_rendered; /* raw pixel-bytes driver asked to render */
atomic_t bytes_identical; /* saved effort with backbuffer comparison */
diff --git a/include/xen/arm/interface.h b/include/xen/arm/interface.h
index c3eada2..61360b8 100644
--- a/include/xen/arm/interface.h
+++ b/include/xen/arm/interface.h
@@ -30,7 +30,7 @@
#define __HYPERVISOR_platform_op_raw __HYPERVISOR_platform_op
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
/* Explicitly size integers that represent pfns in the interface with
* Xen so that we can have one ABI that works for 32 and 64 bit guests.
* Note that this means that the xen_pfn_t type may be capable of
diff --git a/init/Kconfig b/init/Kconfig
index 2937c4d..7e6d3ae 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1019,6 +1019,17 @@
This system will be inactive on UMA systems.
+config SCHED_CACHE
+ bool "Cache aware load balance"
+ default y
+ depends on SMP
+ help
+ When enabled, the scheduler will attempt to aggregate tasks from
+ the same process onto a single Last Level Cache (LLC) domain when
+ possible. This improves cache locality by keeping tasks that share
+ resources within the same cache domain, reducing cache misses and
+ lowering data access latency.
+
config NUMA_BALANCING_DEFAULT_ENABLED
bool "Automatically enable NUMA aware memory/task placement"
default y
@@ -1842,6 +1853,12 @@
depends on FUTEX && NUMA
default y
+config HAVE_FUTEX_ROBUST_UNLOCK
+ bool
+
+config FUTEX_ROBUST_UNLOCK
+ def_bool FUTEX && HAVE_GENERIC_VDSO && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK
+
config EPOLL
bool "Enable eventpoll support" if EXPERT
default y
diff --git a/init/init_task.c b/init/init_task.c
index b5f48eb..674d174 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -200,6 +200,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
&init_task.alloc_lock),
#endif
+ .blocked_donor = NULL,
#ifdef CONFIG_RT_MUTEXES
.pi_waiters = RB_ROOT_CACHED,
.pi_top_task = NULL,
@@ -215,6 +216,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.numa_group = NULL,
.numa_faults = NULL,
#endif
+#ifdef CONFIG_SCHED_CACHE
+ .preferred_llc = -1,
+ .pref_llc_queued = 0,
+#endif
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
.kasan_depth = 1,
#endif
diff --git a/io_uring/cancel.c b/io_uring/cancel.c
index 5e5eb9c..4aa3103 100644
--- a/io_uring/cancel.c
+++ b/io_uring/cancel.c
@@ -561,8 +561,8 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
ret |= io_waitid_remove_all(ctx, tctx, cancel_all);
ret |= io_futex_remove_all(ctx, tctx, cancel_all);
ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all);
- mutex_unlock(&ctx->uring_lock);
ret |= io_kill_timeouts(ctx, tctx, cancel_all);
+ mutex_unlock(&ctx->uring_lock);
if (tctx)
ret |= io_run_task_work() > 0;
else
diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c
index 3da0285..d656cc2 100644
--- a/io_uring/eventfd.c
+++ b/io_uring/eventfd.c
@@ -43,6 +43,7 @@ static void io_eventfd_do_signal(struct rcu_head *rcu)
{
struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
+ atomic_andnot(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops);
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
io_eventfd_put(ev_fd);
}
diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c
index c2d3e45..001fb54 100644
--- a/io_uring/fdinfo.c
+++ b/io_uring/fdinfo.c
@@ -190,8 +190,9 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
get_task_struct(tsk);
rcu_read_unlock();
usec = io_sq_cpu_usec(tsk);
+ sq_pid = task_pid_nr_ns(tsk,
+ proc_pid_ns(file_inode(m->file)->i_sb));
put_task_struct(tsk);
- sq_pid = sq->task_pid;
sq_cpu = sq->sq_cpu;
sq_total_time = usec;
sq_work_time = sq->work_time;
diff --git a/io_uring/futex.c b/io_uring/futex.c
index 9cc1788..906701b 100644
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -327,7 +327,7 @@ int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags)
* Strict flags - ensure that waking 0 futexes yields a 0 result.
* See commit 43adf8449510 ("futex: FLAGS_STRICT") for details.
*/
- ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags,
+ ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, NULL,
iof->futex_val, iof->futex_mask);
if (ret < 0)
req_set_fail(req);
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 7a9f94a..8cc7b47 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -1124,7 +1124,8 @@ static inline void io_wq_remove_pending(struct io_wq *wq,
if (io_wq_is_hashed(work) && work == wq->hash_tail[hash]) {
if (prev)
prev_work = container_of(prev, struct io_wq_work, list);
- if (prev_work && io_get_work_hash(prev_work) == hash)
+ if (prev_work && io_wq_is_hashed(prev_work) &&
+ io_get_work_hash(prev_work) == hash)
wq->hash_tail[hash] = prev_work;
else
wq->hash_tail[hash] = NULL;
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4ed998d6..103b6c8 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -687,12 +687,26 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
}
/*
+ * Compute queued CQEs for free-space calculation, clamped to cq_entries.
+ */
+static unsigned int io_cqring_queued(struct io_ring_ctx *ctx)
+{
+ struct io_rings *rings = io_get_rings(ctx);
+ int diff;
+
+ diff = (int)(ctx->cached_cq_tail - READ_ONCE(rings->cq.head));
+ if (diff >= 0)
+ return min((unsigned int)diff, ctx->cq_entries);
+ return 0;
+}
+
+/*
* Fill an empty dummy CQE, in case alignment is off for posting a 32b CQE
* because the ring is a single 16b entry away from wrapping.
*/
static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)
{
- if (__io_cqring_events(ctx) < ctx->cq_entries) {
+ if (io_cqring_queued(ctx) < ctx->cq_entries) {
struct io_uring_cqe *cqe = &ctx->rings->cqes[off];
cqe->user_data = 0;
@@ -713,7 +727,7 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
{
struct io_rings *rings = ctx->rings;
unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
- unsigned int free, queued, len;
+ unsigned int free, len;
/*
* Posting into the CQ when there are pending overflowed CQEs may break
@@ -733,9 +747,7 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
off = 0;
}
- /* userspace may cheat modifying the tail, be safe and do min */
- queued = min(__io_cqring_events(ctx), ctx->cq_entries);
- free = ctx->cq_entries - queued;
+ free = ctx->cq_entries - io_cqring_queued(ctx);
/* we need a contiguous range, limit based on the current array offset */
len = min(free, ctx->cq_entries - off);
if (len < (cqe32 + 1))
@@ -1452,8 +1464,13 @@ struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
struct io_kiocb *nxt = NULL;
if (req_ref_put_and_test_atomic(req)) {
- if (req->flags & IO_REQ_LINK_FLAGS)
+ if (req->flags & IO_REQ_LINK_FLAGS) {
+ struct io_ring_ctx *ctx = req->ctx;
+
+ mutex_lock(&ctx->uring_lock);
nxt = io_req_find_next(req);
+ mutex_unlock(&ctx->uring_lock);
+ }
io_free_req(req);
}
return nxt ? &nxt->work : NULL;
@@ -1721,10 +1738,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
const struct io_issue_def *def;
unsigned int sqe_flags;
int personality;
- u8 opcode;
req->ctx = ctx;
- req->opcode = opcode = READ_ONCE(sqe->opcode);
+ req->opcode = READ_ONCE(sqe->opcode);
/* same numerical values with corresponding REQ_F_*, safe to copy */
sqe_flags = READ_ONCE(sqe->flags);
req->flags = (__force io_req_flags_t) sqe_flags;
@@ -1734,13 +1750,13 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
req->cancel_seq_set = false;
req->async_data = NULL;
- if (unlikely(opcode >= IORING_OP_LAST)) {
+ if (unlikely(req->opcode >= IORING_OP_LAST)) {
req->opcode = 0;
return io_init_fail_req(req, -EINVAL);
}
- opcode = array_index_nospec(opcode, IORING_OP_LAST);
+ req->opcode = array_index_nospec(req->opcode, IORING_OP_LAST);
- def = &io_issue_defs[opcode];
+ def = &io_issue_defs[req->opcode];
if (def->is_128 && !(ctx->flags & IORING_SETUP_SQE128)) {
/*
* A 128b op on a non-128b SQ requires mixed SQE support as
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 8da2ff7..63061aa 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -47,7 +47,7 @@ static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len)
this_len = min_t(u32, len, buf_len);
buf_len -= this_len;
/* Stop looping for invalid buffer length of 0 */
- if (buf_len || !this_len) {
+ if (buf_len > bl->min_left_sub_one || !this_len) {
WRITE_ONCE(buf->addr, READ_ONCE(buf->addr) + this_len);
WRITE_ONCE(buf->len, buf_len);
return false;
@@ -637,6 +637,10 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
if (reg.ring_entries >= 65536)
return -EINVAL;
+ /* minimum left byte count is a property of incremental buffers */
+ if (!(reg.flags & IOU_PBUF_RING_INC) && reg.min_left)
+ return -EINVAL;
+
bl = io_buffer_get_list(ctx, reg.bgid);
if (bl) {
/* if mapped buffer ring OR classic exists, don't allow */
@@ -680,10 +684,11 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
}
#endif
- bl->nr_entries = reg.ring_entries;
bl->mask = reg.ring_entries - 1;
bl->flags |= IOBL_BUF_RING;
bl->buf_ring = br;
+ if (reg.min_left)
+ bl->min_left_sub_one = reg.min_left - 1;
if (reg.flags & IOU_PBUF_RING_INC)
bl->flags |= IOBL_INC;
ret = io_buffer_add_list(ctx, bl, reg.bgid);
diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index bf15e265..401773e 100644
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -27,12 +27,18 @@ struct io_buffer_list {
__u16 bgid;
/* below is for ring provided buffers */
- __u16 nr_entries;
__u16 head;
__u16 mask;
__u16 flags;
+ /*
+ * minimum required amount to be left to reuse an incrementally
+ * consumed buffer. If less than this is left at consumption time,
+ * buffer is done and head is incremented to the next buffer.
+ */
+ __u32 min_left_sub_one;
+
struct io_mapped_region region;
};
diff --git a/io_uring/napi.c b/io_uring/napi.c
index 4a10de0..bfc7714 100644
--- a/io_uring/napi.c
+++ b/io_uring/napi.c
@@ -38,7 +38,8 @@ static inline ktime_t net_to_ktime(unsigned long t)
return ns_to_ktime(t << 10);
}
-int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id)
+int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id,
+ unsigned int mode)
{
struct hlist_head *hash_list;
struct io_napi_entry *e;
@@ -69,6 +70,11 @@ int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id)
* kfree()
*/
spin_lock(&ctx->napi_lock);
+ if (unlikely(READ_ONCE(ctx->napi_track_mode) != mode)) {
+ spin_unlock(&ctx->napi_lock);
+ kfree(e);
+ return -EINVAL;
+ }
if (unlikely(io_napi_hash_find(hash_list, napi_id))) {
spin_unlock(&ctx->napi_lock);
kfree(e);
@@ -196,9 +202,14 @@ __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg)
{
- if (READ_ONCE(ctx->napi_track_mode) == IO_URING_NAPI_TRACKING_STATIC)
+ switch (READ_ONCE(ctx->napi_track_mode)) {
+ case IO_URING_NAPI_TRACKING_STATIC:
return static_tracking_do_busy_loop(ctx, loop_end, loop_end_arg);
- return dynamic_tracking_do_busy_loop(ctx, loop_end, loop_end_arg);
+ case IO_URING_NAPI_TRACKING_DYNAMIC:
+ return dynamic_tracking_do_busy_loop(ctx, loop_end, loop_end_arg);
+ default:
+ return false;
+ }
}
static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx,
@@ -273,11 +284,13 @@ static int io_napi_register_napi(struct io_ring_ctx *ctx,
default:
return -EINVAL;
}
- /* clean the napi list for new settings */
+ WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE);
io_napi_free(ctx);
- WRITE_ONCE(ctx->napi_track_mode, napi->op_param);
+ /* cap NAPI at 10 msec of spin time */
+ napi->busy_poll_to = min(10000, napi->busy_poll_to);
WRITE_ONCE(ctx->napi_busy_poll_dt, napi->busy_poll_to * NSEC_PER_USEC);
WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi->prefer_busy_poll);
+ WRITE_ONCE(ctx->napi_track_mode, napi->op_param);
return 0;
}
@@ -313,7 +326,8 @@ int io_register_napi(struct io_ring_ctx *ctx, void __user *arg)
case IO_URING_NAPI_STATIC_ADD_ID:
if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC)
return -EINVAL;
- return __io_napi_add_id(ctx, napi.op_param);
+ return __io_napi_add_id(ctx, napi.op_param,
+ IO_URING_NAPI_TRACKING_STATIC);
case IO_URING_NAPI_STATIC_DEL_ID:
if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC)
return -EINVAL;
@@ -341,9 +355,10 @@ int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg)
if (arg && copy_to_user(arg, &curr, sizeof(curr)))
return -EFAULT;
+ WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE);
WRITE_ONCE(ctx->napi_busy_poll_dt, 0);
WRITE_ONCE(ctx->napi_prefer_busy_poll, false);
- WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE);
+ io_napi_free(ctx);
return 0;
}
diff --git a/io_uring/napi.h b/io_uring/napi.h
index fa742f4..e0aeccc 100644
--- a/io_uring/napi.h
+++ b/io_uring/napi.h
@@ -15,7 +15,8 @@ void io_napi_free(struct io_ring_ctx *ctx);
int io_register_napi(struct io_ring_ctx *ctx, void __user *arg);
int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg);
-int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id);
+int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id,
+ unsigned int mode);
void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq);
int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx);
@@ -43,13 +44,14 @@ static inline void io_napi_add(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
struct socket *sock;
+ unsigned int mode = IO_URING_NAPI_TRACKING_DYNAMIC;
- if (READ_ONCE(ctx->napi_track_mode) != IO_URING_NAPI_TRACKING_DYNAMIC)
+ if (READ_ONCE(ctx->napi_track_mode) != mode)
return;
sock = sock_from_file(req->file);
if (sock && sock->sk)
- __io_napi_add_id(ctx, READ_ONCE(sock->sk->sk_napi_id));
+ __io_napi_add_id(ctx, READ_ONCE(sock->sk->sk_napi_id), mode);
}
#else
diff --git a/io_uring/net.c b/io_uring/net.c
index 30cd22c..8df15b6 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -4,6 +4,7 @@
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/net.h>
+#include <linux/un.h>
#include <linux/compat.h>
#include <net/compat.h>
#include <linux/io_uring.h>
@@ -1799,11 +1800,29 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags)
return IOU_COMPLETE;
}
+/*
+ * Check if bind request would potentially end up with filename_create(),
+ * which in turn end up in mnt_want_write() which will grab the fs
+ * percpu start write sem. This can trigger a lockdep warning.
+ */
+static int io_bind_file_create(const struct io_async_msghdr *io, int addr_len)
+{
+ const struct sockaddr_un *sun;
+
+ if (io->addr.ss_family != AF_UNIX)
+ return 0;
+ if (addr_len <= offsetof(struct sockaddr_un, sun_path))
+ return 0;
+ sun = (const struct sockaddr_un *) &io->addr;
+ return sun->sun_path[0] != '\0';
+}
+
int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
struct sockaddr __user *uaddr;
struct io_async_msghdr *io;
+ int ret;
if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
return -EINVAL;
@@ -1814,7 +1833,12 @@ int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
io = io_msg_alloc_async(req);
if (unlikely(!io))
return -ENOMEM;
- return move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
+ ret = move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
+ if (unlikely(ret))
+ return ret;
+ if (io_bind_file_create(io, bind->addr_len))
+ req->flags |= REQ_F_FORCE_ASYNC;
+ return 0;
}
int io_bind(struct io_kiocb *req, unsigned int issue_flags)
diff --git a/io_uring/nop.c b/io_uring/nop.c
index 3caf078..f5c9969 100644
--- a/io_uring/nop.c
+++ b/io_uring/nop.c
@@ -79,9 +79,9 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
if (ret < 0)
req_set_fail(req);
if (nop->flags & IORING_NOP_CQE32)
- io_req_set_res32(req, nop->result, 0, nop->extra1, nop->extra2);
+ io_req_set_res32(req, ret, 0, nop->extra1, nop->extra2);
else
- io_req_set_res(req, nop->result, 0);
+ io_req_set_res(req, ret, 0);
if (nop->flags & IORING_NOP_TW) {
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
diff --git a/io_uring/rw.c b/io_uring/rw.c
index e729e0e..0c48346 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -230,7 +230,7 @@ static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb)
}
static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
- u64 attr_ptr, u64 attr_type_mask)
+ u64 attr_ptr)
{
struct io_uring_attr_pi pi_attr;
struct io_async_rw *io;
@@ -305,7 +305,7 @@ static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return -EINVAL;
attr_ptr = READ_ONCE(sqe->attr_ptr);
- return io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask);
+ return io_prep_rw_pi(req, rw, ddir, attr_ptr);
}
return 0;
}
diff --git a/io_uring/tctx.c b/io_uring/tctx.c
index 6af62ca..42b219b 100644
--- a/io_uring/tctx.c
+++ b/io_uring/tctx.c
@@ -139,12 +139,14 @@ static int io_tctx_install_node(struct io_ring_ctx *ctx,
int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
{
struct io_uring_task *tctx = current->io_uring;
+ bool new_tctx = false;
int ret;
if (unlikely(!tctx)) {
tctx = io_uring_alloc_task_context(current, ctx);
if (IS_ERR(tctx))
return PTR_ERR(tctx);
+ new_tctx = true;
if (data_race(ctx->int_flags) & IO_RING_F_IOWQ_LIMITS_SET) {
unsigned int limits[2];
@@ -168,13 +170,15 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
if (tctx->io_wq)
io_wq_set_exit_on_idle(tctx->io_wq, false);
- ret = io_tctx_install_node(ctx, tctx);
- if (!ret) {
+ if (new_tctx)
current->io_uring = tctx;
+
+ ret = io_tctx_install_node(ctx, tctx);
+ if (!ret)
return 0;
- }
- if (!current->io_uring) {
err_free:
+ if (new_tctx) {
+ current->io_uring = NULL;
if (tctx->io_wq) {
io_wq_exit_start(tctx->io_wq);
io_wq_put_and_exit(tctx->io_wq);
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 4cfdfc51..c4dd26c 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -3,6 +3,7 @@
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/io_uring.h>
+#include <linux/time_namespace.h>
#include <trace/events/io_uring.h>
@@ -35,6 +36,22 @@ struct io_timeout_rem {
bool ltimeout;
};
+static clockid_t io_flags_to_clock(unsigned flags)
+{
+ switch (flags & IORING_TIMEOUT_CLOCK_MASK) {
+ case IORING_TIMEOUT_BOOTTIME:
+ return CLOCK_BOOTTIME;
+ case IORING_TIMEOUT_REALTIME:
+ return CLOCK_REALTIME;
+ default:
+ /* can't happen, vetted at prep time */
+ WARN_ON_ONCE(1);
+ fallthrough;
+ case 0:
+ return CLOCK_MONOTONIC;
+ }
+}
+
static int io_parse_user_time(ktime_t *time, u64 arg, unsigned flags)
{
struct timespec64 ts;
@@ -43,7 +60,7 @@ static int io_parse_user_time(ktime_t *time, u64 arg, unsigned flags)
*time = ns_to_ktime(arg);
if (*time < 0)
return -EINVAL;
- return 0;
+ goto out;
}
if (get_timespec64(&ts, u64_to_user_ptr(arg)))
@@ -51,6 +68,9 @@ static int io_parse_user_time(ktime_t *time, u64 arg, unsigned flags)
if (ts.tv_sec < 0 || ts.tv_nsec < 0)
return -EINVAL;
*time = timespec64_to_ktime(ts);
+out:
+ if (flags & IORING_TIMEOUT_ABS)
+ *time = timens_ktime_to_host(io_flags_to_clock(flags), *time);
return 0;
}
@@ -264,6 +284,10 @@ static struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout);
io_remove_next_linked(req);
+
+ /* If this is NULL, then timer already claimed it and will complete it */
+ if (!timeout->head)
+ return NULL;
timeout->head = NULL;
if (hrtimer_try_to_cancel(&io->timer) != -1) {
list_del(&timeout->list);
@@ -347,6 +371,14 @@ static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw)
int ret;
if (prev) {
+ /*
+ * splice the linked timeout out of prev's chain if the regular
+ * completion path didn't already do it.
+ */
+ if (prev->link == req)
+ prev->link = req->link;
+ req->link = NULL;
+
if (!tw.cancel) {
struct io_cancel_data cd = {
.ctx = req->ctx,
@@ -381,12 +413,14 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
/*
* We don't expect the list to be empty, that will only happen if we
- * race with the completion of the linked work.
+ * race with the completion of the linked work. Splice of prev is
+ * done in io_req_task_link_timeout(), if needed.
*/
if (prev) {
- io_remove_next_linked(prev);
- if (!req_ref_inc_not_zero(prev))
+ if (!req_ref_inc_not_zero(prev)) {
+ io_remove_next_linked(prev);
prev = NULL;
+ }
}
list_del(&timeout->list);
timeout->prev = prev;
@@ -399,18 +433,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
{
- switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
- case IORING_TIMEOUT_BOOTTIME:
- return CLOCK_BOOTTIME;
- case IORING_TIMEOUT_REALTIME:
- return CLOCK_REALTIME;
- default:
- /* can't happen, vetted at prep time */
- WARN_ON_ONCE(1);
- fallthrough;
- case 0:
- return CLOCK_MONOTONIC;
- }
+ return io_flags_to_clock(data->flags);
}
static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
diff --git a/io_uring/tw.c b/io_uring/tw.c
index fdff81e..023d5e6 100644
--- a/io_uring/tw.c
+++ b/io_uring/tw.c
@@ -273,8 +273,18 @@ void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags)
void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
{
- struct llist_node *node = llist_del_all(&ctx->work_llist);
+ struct llist_node *node;
+ /*
+ * Running the work items may utilize ->retry_llist as a means
+ * for capping the number of task_work entries run at the same
+ * time. But that list can potentially race with moving the work
+ * from here, if the task is exiting. As any normal task_work
+ * running holds ->uring_lock already, just guard this slow path
+ * with ->uring_lock to avoid racing on ->retry_llist.
+ */
+ guard(mutex)(&ctx->uring_lock);
+ node = llist_del_all(&ctx->work_llist);
__io_fallback_tw(node, false);
node = llist_del_all(&ctx->retry_llist);
__io_fallback_tw(node, false);
diff --git a/io_uring/wait.c b/io_uring/wait.c
index 91df86c..ec01e78 100644
--- a/io_uring/wait.c
+++ b/io_uring/wait.c
@@ -5,6 +5,7 @@
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/io_uring.h>
+#include <linux/time_namespace.h>
#include <trace/events/io_uring.h>
@@ -229,7 +230,10 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
if (ext_arg->ts_set) {
iowq.timeout = timespec64_to_ktime(ext_arg->ts);
- if (!(flags & IORING_ENTER_ABS_TIMER))
+ if (flags & IORING_ENTER_ABS_TIMER)
+ iowq.timeout = timens_ktime_to_host(ctx->clockid,
+ iowq.timeout);
+ else
iowq.timeout = ktime_add(iowq.timeout, start_time);
}
diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index d25d60a..32f68fd7 100644
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -275,6 +275,7 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
iw->options = READ_ONCE(sqe->file_index);
iw->head = NULL;
iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
+ memset(&iw->info, 0, sizeof(iw->info));
return 0;
}
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 7b93c87..19837e0 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -495,10 +495,9 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
for (i = 0; i < nr_iovs; i++) {
struct net_iov *niov = &area->nia.niovs[i];
- niov->owner = &area->nia;
+ net_iov_init(niov, &area->nia, NET_IOV_IOURING);
area->freelist[i] = i;
atomic_set(&area->user_refs[i], 0);
- niov->type = NET_IOV_IOURING;
}
if (ifq->dev) {
diff --git a/ipc/util.c b/ipc/util.c
index 9eb8982..1737d77 100644
--- a/ipc/util.c
+++ b/ipc/util.c
@@ -253,7 +253,7 @@ static inline int ipc_idr_alloc(struct ipc_ids *ids, struct kern_ipc_perm *new)
} else {
new->seq = ipcid_to_seqx(next_id);
idx = idr_alloc(&ids->ipcs_idr, new, ipcid_to_idx(next_id),
- 0, GFP_NOWAIT);
+ ipc_mni, GFP_NOWAIT);
}
if (idx >= 0)
new->id = (new->seq << ipcmni_seq_shift()) + idx;
diff --git a/kernel/audit.c b/kernel/audit.c
index e1d489b..34dc7cb 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -1468,6 +1468,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
err = audit_list_rules_send(skb, seq);
break;
case AUDIT_TRIM:
+ if (audit_enabled == AUDIT_LOCKED)
+ return -EPERM;
audit_trim_trees();
audit_log_common_recv_msg(audit_context(), &ab,
AUDIT_CONFIG_CHANGE);
@@ -1480,6 +1482,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
size_t msglen = data_len;
char *old, *new;
+ if (audit_enabled == AUDIT_LOCKED)
+ return -EPERM;
err = -EINVAL;
if (msglen < 2 * sizeof(u32))
break;
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index ab54fcc..abdf8da 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -2786,7 +2786,7 @@ void __audit_log_capset(const struct cred *new, const struct cred *old)
context->capset.pid = task_tgid_nr(current);
context->capset.cap.effective = new->cap_effective;
- context->capset.cap.inheritable = new->cap_effective;
+ context->capset.cap.inheritable = new->cap_inheritable;
context->capset.cap.permitted = new->cap_permitted;
context->capset.cap.ambient = new->cap_ambient;
context->type = AUDIT_CAPSET;
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 802656c6..49a8f7b 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -511,7 +511,7 @@ static int arena_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32
{
struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
- if ((u64)off > arena->user_vm_end - arena->user_vm_start)
+ if ((u64)off >= arena->user_vm_end - arena->user_vm_start)
return -ERANGE;
*imm = (unsigned long)arena->user_vm_start;
return 0;
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 5e25e03..dfb2110 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -827,7 +827,7 @@ const struct bpf_map_ops array_map_ops = {
};
const struct bpf_map_ops percpu_array_map_ops = {
- .map_meta_equal = bpf_map_meta_equal,
+ .map_meta_equal = array_map_meta_equal,
.map_alloc_check = array_map_alloc_check,
.map_alloc = array_map_alloc,
.map_free = array_map_free,
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 998f42a..26d3706 100644
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
subprog->might_sleep = true;
}
+static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off)
+{
+ struct bpf_subprog_info *subprog;
+
+ subprog = bpf_find_containing_subprog(env, off);
+ subprog->might_throw = true;
+}
+
/* 't' is an index of a call-site.
* 'w' is a callee entry point.
* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
* Rely on DFS traversal order and absence of recursive calls to guarantee that
- * callee's change_pkt_data marks would be correct at that moment.
+ * callee's effect marks would be correct at that moment.
*/
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
{
@@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
callee = bpf_find_containing_subprog(env, w);
caller->changes_pkt_data |= callee->changes_pkt_data;
caller->might_sleep |= callee->might_sleep;
+ caller->might_throw |= callee->might_throw;
}
enum {
@@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
mark_subprog_might_sleep(env, t);
if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
mark_subprog_changes_pkt_data(env, t);
+ if (ret == 0 && bpf_is_throw_kfunc(insn))
+ mark_subprog_might_throw(env, t);
}
return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 8b018ff..6aa2a8b 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x)
return x >= 0 ? (u32)x : -(u32)x;
}
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+ const struct bpf_insn *insn);
+
/**
* ___bpf_prog_run - run eBPF program on a given context
* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2080,9 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
CONT;
JMP_CALL_ARGS:
- BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
- BPF_R3, BPF_R4,
- BPF_R5,
- insn + insn->off + 1);
+ BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+ BPF_R4, BPF_R5,
+ insn + insn->imm + 1);
CONT;
JMP_TAIL_CALL: {
@@ -2394,13 +2396,22 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
#undef PROG_NAME_LIST
#ifdef CONFIG_BPF_SYSCALL
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
{
stack_depth = max_t(u32, stack_depth, 1);
- insn->off = (s16) insn->imm;
- insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
- __bpf_call_base_args;
+ /* Prevent out-of-bounds read to interpreters_args */
+ if (stack_depth > MAX_BPF_STACK)
+ return -EINVAL;
+ insn->off = (round_up(stack_depth, 32) / 32) - 1;
insn->code = BPF_JMP | BPF_CALL_ARGS;
+ return 0;
+}
+
+s32 bpf_call_args_imm(s16 idx)
+{
+ if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+ return 0;
+ return BPF_CALL_IMM(interpreters_args[idx]);
}
#endif
#endif
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index fba9e8c..3692adf 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
}
if (!bpf_pseudo_call(insn))
continue;
- insn->off = env->insn_aux_data[i].call_imm;
- subprog = bpf_find_subprog(env, i + insn->off + 1);
- insn->imm = subprog;
+ insn->imm = env->insn_aux_data[i].call_imm;
+ subprog = bpf_find_subprog(env, i + insn->imm + 1);
+ insn->off = subprog;
}
prog->jited = 1;
@@ -1416,7 +1416,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
depth = get_callee_stack_depth(env, insn, i);
if (depth < 0)
return depth;
- bpf_patch_call_args(insn, depth);
+ err = bpf_patch_call_args(insn, depth);
+ if (err) {
+ verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",
+ depth);
+ return err;
+ }
}
err = 0;
#endif
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 2bb6020..b5314c9 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -4241,8 +4241,13 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
data_len = __bpf_dynptr_size(data_ptr);
data = __bpf_dynptr_data(data_ptr, data_len);
+ if (!data)
+ return -EINVAL;
+
sig_len = __bpf_dynptr_size(sig_ptr);
sig = __bpf_dynptr_data(sig_ptr, sig_len);
+ if (!sig)
+ return -EINVAL;
return verify_pkcs7_signature(data, data_len, sig, sig_len,
trusted_keyring->key,
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 332e6e0..58197d7 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -1914,26 +1914,15 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)
return -ENOMEM;
}
- instance = call_instance(env, NULL, 0, 0);
- if (IS_ERR(instance)) {
- err = PTR_ERR(instance);
- goto out;
- }
- err = analyze_subprog(env, NULL, info, instance, callsites);
- if (err)
- goto out;
-
/*
- * Subprogs and callbacks that don't receive FP-derived arguments
- * cannot access ancestor stack frames, so they were skipped during
- * the recursive walk above. Async callbacks (timer, workqueue) are
- * also not reachable from the main program's call graph. Analyze
- * all unvisited subprogs as independent roots at depth 0.
+ * Analyze every subprog in reverse topological order (callers
+ * before callees) so that each subprog is analyzed before its
+ * callees, allowing the recursive walk inside analyze_subprog()
+ * to naturally reach callees that receive FP-derived args.
*
- * Use reverse topological order (callers before callees) so that
- * each subprog is analyzed before its callees, allowing the
- * recursive walk inside analyze_subprog() to naturally
- * reach nested callees that also lack FP-derived args.
+ * Subprogs and callbacks that don't receive FP-derived arguments
+ * cannot access ancestor stack frames are analyzed independently.
+ * Async callbacks (timer, workqueue) are handled the same way.
*/
for (k = env->subprog_cnt - 1; k >= 0; k--) {
int sub = env->subprog_topo_order[k];
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214..630d530 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4919,6 +4919,29 @@ static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
return map;
}
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+ s32 call_off = insn->imm;
+
+ /*
+ * BPF_CALL_ARGS only exists for interpreter fallback.
+ * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+ * interpreters_args array, so here using bpf_call_args_imm()
+ * to get the real address offset.
+ * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+ */
+ if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+ insn->imm = bpf_call_args_imm(insn->off);
+ else
+ insn->imm = insn->off;
+
+ /* Avoid dumping a truncated and misleading pc-relative offset. */
+ if (call_off > S16_MAX || call_off < S16_MIN)
+ insn->off = 0;
+ else
+ insn->off = call_off;
+}
+
static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
const struct cred *f_cred)
{
@@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
}
if (code == (BPF_JMP | BPF_CALL) ||
code == (BPF_JMP | BPF_CALL_ARGS)) {
+ /* Restore the legacy xlated dump layout. */
+ if (insns[i].src_reg == BPF_PSEUDO_CALL)
+ prepare_dump_pseudo_call(&insns[i]);
if (code == (BPF_JMP | BPF_CALL_ARGS))
insns[i].code = BPF_JMP | BPF_CALL;
if (!bpf_dump_raw_ok(f_cred))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 69d7551..7fb88e1 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id)
static bool is_sync_callback_calling_kfunc(u32 btf_id);
static bool is_async_callback_calling_kfunc(u32 btf_id);
static bool is_callback_calling_kfunc(u32 btf_id);
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
static bool is_task_work_add_kfunc(u32 func_id);
@@ -5405,7 +5404,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
bool err = false;
- if (!is_bpf_throw_kfunc(insn + i))
+ if (!bpf_is_throw_kfunc(insn + i))
continue;
for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
if (subprog[tmp].is_cb) {
@@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
return 0;
}
+static int process_bpf_exit_full(struct bpf_verifier_env *env,
+ bool *do_print_state, bool exception_exit);
+
static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int *insn_idx)
{
@@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
}
+ if (env->subprog_info[subprog].might_throw) {
+ struct bpf_verifier_state *branch;
+
+ branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
+ if (IS_ERR(branch)) {
+ verbose(env, "failed to push state for global subprog exception path\n");
+ return PTR_ERR(branch);
+ }
+ return process_bpf_exit_full(env, NULL, true);
+ }
+
/* continue with next insn after call */
return 0;
}
@@ -11263,7 +11276,11 @@ BTF_ID(func, bpf_task_work_schedule_resume)
BTF_ID(func, bpf_arena_alloc_pages)
BTF_ID(func, bpf_arena_free_pages)
BTF_ID(func, bpf_arena_reserve_pages)
+#ifdef CONFIG_BPF_EVENTS
BTF_ID(func, bpf_session_is_return)
+#else
+BTF_ID_UNUSED
+#endif
BTF_ID(func, bpf_stream_vprintk)
BTF_ID(func, bpf_stream_print_stack)
@@ -11778,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id)
is_task_work_add_kfunc(btf_id);
}
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
+bool bpf_is_throw_kfunc(struct bpf_insn *insn)
{
return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
insn->imm == special_kfunc_list[KF_bpf_throw];
@@ -12968,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca
}
static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
-static int process_bpf_exit_full(struct bpf_verifier_env *env,
- bool *do_print_state, bool exception_exit);
static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int *insn_idx_p)
@@ -13350,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
env->prog->call_session_cookie = true;
- if (is_bpf_throw_kfunc(insn))
+ if (bpf_is_throw_kfunc(insn))
return process_bpf_exit_full(env, NULL, true);
return 0;
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 43adc96..6152add 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -264,10 +264,12 @@ static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
static void css_task_iter_skip(struct css_task_iter *it,
struct task_struct *task);
static int cgroup_destroy_locked(struct cgroup *cgrp);
+static void cgroup_finish_destroy(struct cgroup *cgrp);
+static void kill_css_sync(struct cgroup_subsys_state *css);
+static void kill_css_finish(struct cgroup_subsys_state *css);
static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
struct cgroup_subsys *ss);
static void css_release(struct percpu_ref *ref);
-static void kill_css(struct cgroup_subsys_state *css);
static int cgroup_addrm_files(struct cgroup_subsys_state *css,
struct cgroup *cgrp, struct cftype cfts[],
bool is_add);
@@ -797,6 +799,16 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
if (was_populated == cgroup_is_populated(cgrp))
break;
+ /*
+ * Subtree just emptied below an offlined cgrp. Fire deferred
+ * destroy. The transition is one-shot.
+ */
+ if (was_populated && !css_is_online(&cgrp->self)) {
+ cgroup_get(cgrp);
+ WARN_ON_ONCE(!queue_work(cgroup_offline_wq,
+ &cgrp->finish_destroy_work));
+ }
+
cgroup1_check_for_release(cgrp);
TRACE_CGROUP_PATH(notify_populated, cgrp,
cgroup_is_populated(cgrp));
@@ -2039,6 +2051,16 @@ static int cgroup_reconfigure(struct fs_context *fc)
return 0;
}
+static void cgroup_finish_destroy_work_fn(struct work_struct *work)
+{
+ struct cgroup *cgrp = container_of(work, struct cgroup, finish_destroy_work);
+
+ cgroup_lock();
+ cgroup_finish_destroy(cgrp);
+ cgroup_unlock();
+ cgroup_put(cgrp);
+}
+
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
struct cgroup_subsys *ss;
@@ -2065,7 +2087,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
#endif
init_waitqueue_head(&cgrp->offline_waitq);
- init_waitqueue_head(&cgrp->dying_populated_waitq);
+ INIT_WORK(&cgrp->finish_destroy_work, cgroup_finish_destroy_work_fn);
INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);
}
@@ -3375,7 +3397,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp)
if (css->parent &&
!(cgroup_ss_mask(dsct) & (1 << ss->id))) {
- kill_css(css);
+ kill_css_sync(css);
+ kill_css_finish(css);
} else if (!css_visible(css)) {
css_clear_dir(css);
if (ss->css_reset)
@@ -3934,33 +3957,41 @@ static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
size_t nbytes, enum psi_res res)
{
- struct cgroup_file_ctx *ctx = of->priv;
+ struct cgroup_file_ctx *ctx;
struct psi_trigger *new;
struct cgroup *cgrp;
struct psi_group *psi;
+ ssize_t ret = 0;
cgrp = cgroup_kn_lock_live(of->kn, false);
if (!cgrp)
return -ENODEV;
- cgroup_get(cgrp);
- cgroup_kn_unlock(of->kn);
+ ctx = of->priv;
+ if (!ctx) {
+ ret = -ENODEV;
+ goto out_unlock;
+ }
/* Allow only one trigger per file descriptor */
if (ctx->psi.trigger) {
- cgroup_put(cgrp);
- return -EBUSY;
+ ret = -EBUSY;
+ goto out_unlock;
}
psi = cgroup_psi(cgrp);
new = psi_trigger_create(psi, buf, res, of->file, of);
if (IS_ERR(new)) {
- cgroup_put(cgrp);
- return PTR_ERR(new);
+ ret = PTR_ERR(new);
+ goto out_unlock;
}
smp_store_release(&ctx->psi.trigger, new);
- cgroup_put(cgrp);
+
+out_unlock:
+ cgroup_kn_unlock(of->kn);
+ if (ret)
+ return ret;
return nbytes;
}
@@ -5059,10 +5090,12 @@ static void css_task_iter_advance(struct css_task_iter *it)
task = list_entry(it->task_pos, struct task_struct, cg_list);
/*
- * Hide tasks that are exiting but not yet removed. Keep zombie
- * leaders with live threads visible.
+ * Hide tasks that are exiting but not yet removed by default. Keep
+ * zombie leaders with live threads visible. Usages that need to walk
+ * every existing task can opt out via CSS_TASK_ITER_WITH_DEAD.
*/
- if ((task->flags & PF_EXITING) && !atomic_read(&task->signal->live))
+ if (!(it->flags & CSS_TASK_ITER_WITH_DEAD) &&
+ (task->flags & PF_EXITING) && !atomic_read(&task->signal->live))
goto repeat;
if (it->flags & CSS_TASK_ITER_PROCS) {
@@ -5506,7 +5539,7 @@ static struct cftype cgroup_psi_files[] = {
* css destruction is four-stage process.
*
* 1. Destruction starts. Killing of the percpu_ref is initiated.
- * Implemented in kill_css().
+ * Implemented in kill_css_finish().
*
* 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
* and thus css_tryget_online() is guaranteed to fail, the css can be
@@ -5716,16 +5749,6 @@ static void offline_css(struct cgroup_subsys_state *css)
RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
wake_up_all(&css->cgroup->offline_waitq);
-
- css->cgroup->nr_dying_subsys[ss->id]++;
- /*
- * Parent css and cgroup cannot be freed until after the freeing
- * of child css, see css_free_rwork_fn().
- */
- while ((css = css->parent)) {
- css->nr_descendants--;
- css->cgroup->nr_dying_subsys[ss->id]++;
- }
}
/**
@@ -5995,7 +6018,7 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
/*
* This is called when the refcnt of a css is confirmed to be killed.
* css_tryget_online() is now guaranteed to fail. Tell the subsystem to
- * initiate destruction and put the css ref from kill_css().
+ * initiate destruction and put the css ref from kill_css_finish().
*/
static void css_killed_work_fn(struct work_struct *work)
{
@@ -6028,16 +6051,15 @@ static void css_killed_ref_fn(struct percpu_ref *ref)
}
/**
- * kill_css - destroy a css
- * @css: css to destroy
+ * kill_css_sync - synchronous half of css teardown
+ * @css: css being killed
*
- * This function initiates destruction of @css by removing cgroup interface
- * files and putting its base reference. ->css_offline() will be invoked
- * asynchronously once css_tryget_online() is guaranteed to fail and when
- * the reference count reaches zero, @css will be released.
+ * See cgroup_destroy_locked().
*/
-static void kill_css(struct cgroup_subsys_state *css)
+static void kill_css_sync(struct cgroup_subsys_state *css)
{
+ struct cgroup_subsys *ss = css->ss;
+
lockdep_assert_held(&cgroup_mutex);
if (css->flags & CSS_DYING)
@@ -6057,64 +6079,100 @@ static void kill_css(struct cgroup_subsys_state *css)
*/
css_clear_dir(css);
+ css->cgroup->nr_dying_subsys[ss->id]++;
/*
- * Killing would put the base ref, but we need to keep it alive
- * until after ->css_offline().
+ * Parent css and cgroup cannot be freed until after the freeing
+ * of child css, see css_free_rwork_fn().
+ */
+ while ((css = css->parent)) {
+ css->nr_descendants--;
+ css->cgroup->nr_dying_subsys[ss->id]++;
+ }
+}
+
+/**
+ * kill_css_finish - deferred half of css teardown
+ * @css: css being killed
+ *
+ * See cgroup_destroy_locked().
+ */
+static void kill_css_finish(struct cgroup_subsys_state *css)
+{
+ lockdep_assert_held(&cgroup_mutex);
+
+ /*
+ * Skip on re-entry: cgroup_apply_control_disable() may have killed @css
+ * earlier. cgroup_destroy_locked() can still walk it because
+ * offline_css() (which NULLs cgrp->subsys[ssid]) runs async.
+ */
+ if (percpu_ref_is_dying(&css->refcnt))
+ return;
+
+ /*
+ * Killing would put the base ref, but we need to keep it alive until
+ * after ->css_offline().
*/
css_get(css);
/*
- * cgroup core guarantees that, by the time ->css_offline() is
- * invoked, no new css reference will be given out via
- * css_tryget_online(). We can't simply call percpu_ref_kill() and
- * proceed to offlining css's because percpu_ref_kill() doesn't
- * guarantee that the ref is seen as killed on all CPUs on return.
+ * cgroup core guarantees that, by the time ->css_offline() is invoked,
+ * no new css reference will be given out via css_tryget_online(). We
+ * can't simply call percpu_ref_kill() and proceed to offlining css's
+ * because percpu_ref_kill() doesn't guarantee that the ref is seen as
+ * killed on all CPUs on return.
*
- * Use percpu_ref_kill_and_confirm() to get notifications as each
- * css is confirmed to be seen as killed on all CPUs.
+ * Use percpu_ref_kill_and_confirm() to get notifications as each css is
+ * confirmed to be seen as killed on all CPUs.
*/
percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
}
/**
- * cgroup_destroy_locked - the first stage of cgroup destruction
+ * cgroup_destroy_locked - destroy @cgrp (called on rmdir)
* @cgrp: cgroup to be destroyed
*
- * css's make use of percpu refcnts whose killing latency shouldn't be
- * exposed to userland and are RCU protected. Also, cgroup core needs to
- * guarantee that css_tryget_online() won't succeed by the time
- * ->css_offline() is invoked. To satisfy all the requirements,
- * destruction is implemented in the following two steps.
+ * Tear down @cgrp on behalf of rmdir. Constraints:
*
- * s1. Verify @cgrp can be destroyed and mark it dying. Remove all
- * userland visible parts and start killing the percpu refcnts of
- * css's. Set up so that the next stage will be kicked off once all
- * the percpu refcnts are confirmed to be killed.
+ * - Userspace: rmdir must succeed when cgroup.procs and friends are empty.
*
- * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
- * rest of destruction. Once all cgroup references are gone, the
- * cgroup is RCU-freed.
+ * - Kernel: subsystem ->css_offline() must not run while any task in @cgrp's
+ * subtree is still doing kernel work. A task hidden from cgroup.procs (past
+ * exit_signals() with signal->live cleared) can still schedule, allocate, and
+ * consume resources until its final context switch. Dying descendants in the
+ * subtree can host such tasks too.
*
- * This function implements s1. After this step, @cgrp is gone as far as
- * the userland is concerned and a new cgroup with the same name may be
- * created. As cgroup doesn't care about the names internally, this
- * doesn't cause any problem.
+ * - Kernel: css_tryget_online() must fail by the time ->css_offline() runs.
+ *
+ * The destruction runs in three parts:
+ *
+ * - This function: synchronous user-visible state teardown plus kill_css_sync()
+ * on each subsystem css.
+ *
+ * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on
+ * each subsystem css. Fires once @cgrp's subtree is fully drained, either
+ * inline here or from cgroup_update_populated().
+ *
+ * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn ->
+ * ->css_offline() -> release/free.
+ *
+ * Return 0 on success, -EBUSY if a userspace-visible task or an online child
+ * remains.
*/
static int cgroup_destroy_locked(struct cgroup *cgrp)
- __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
{
struct cgroup *tcgrp, *parent = cgroup_parent(cgrp);
struct cgroup_subsys_state *css;
struct cgrp_cset_link *link;
+ struct css_task_iter it;
+ struct task_struct *task;
int ssid, ret;
lockdep_assert_held(&cgroup_mutex);
- /*
- * Only migration can raise populated from zero and we're already
- * holding cgroup_mutex.
- */
- if (cgroup_is_populated(cgrp))
+ css_task_iter_start(&cgrp->self, 0, &it);
+ task = css_task_iter_next(&it);
+ css_task_iter_end(&it);
+ if (task)
return -EBUSY;
/*
@@ -6138,9 +6196,8 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
link->cset->dead = true;
spin_unlock_irq(&css_set_lock);
- /* initiate massacre of all css's */
for_each_css(css, ssid, cgrp)
- kill_css(css);
+ kill_css_sync(css);
/* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */
css_clear_dir(&cgrp->self);
@@ -6171,79 +6228,27 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
/* put the base reference */
percpu_ref_kill(&cgrp->self.refcnt);
+ if (!cgroup_is_populated(cgrp))
+ cgroup_finish_destroy(cgrp);
+
return 0;
};
/**
- * cgroup_drain_dying - wait for dying tasks to leave before rmdir
- * @cgrp: the cgroup being removed
+ * cgroup_finish_destroy - deferred half of @cgrp destruction
+ * @cgrp: cgroup whose subtree just became empty
*
- * cgroup.procs and cgroup.threads use css_task_iter which filters out
- * PF_EXITING tasks so that userspace doesn't see tasks that have already been
- * reaped via waitpid(). However, cgroup_has_tasks() - which tests whether the
- * cgroup has non-empty css_sets - is only updated when dying tasks pass through
- * cgroup_task_dead() in finish_task_switch(). This creates a window where
- * cgroup.procs reads empty but cgroup_has_tasks() is still true, making rmdir
- * fail with -EBUSY from cgroup_destroy_locked() even though userspace sees no
- * tasks.
- *
- * This function aligns cgroup_has_tasks() with what userspace can observe. If
- * cgroup_has_tasks() but the task iterator sees nothing (all remaining tasks are
- * PF_EXITING), we wait for cgroup_task_dead() to finish processing them. As the
- * window between PF_EXITING and cgroup_task_dead() is short, the wait is brief.
- *
- * This function only concerns itself with this cgroup's own dying tasks.
- * Whether the cgroup has children is cgroup_destroy_locked()'s problem.
- *
- * Each cgroup_task_dead() kicks the waitqueue via cset->cgrp_links, and we
- * retry the full check from scratch.
- *
- * Must be called with cgroup_mutex held.
+ * See cgroup_destroy_locked() for the rationale.
*/
-static int cgroup_drain_dying(struct cgroup *cgrp)
- __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
+static void cgroup_finish_destroy(struct cgroup *cgrp)
{
- struct css_task_iter it;
- struct task_struct *task;
- DEFINE_WAIT(wait);
+ struct cgroup_subsys_state *css;
+ int ssid;
lockdep_assert_held(&cgroup_mutex);
-retry:
- if (!cgroup_has_tasks(cgrp))
- return 0;
- /* Same iterator as cgroup.threads - if any task is visible, it's busy */
- css_task_iter_start(&cgrp->self, 0, &it);
- task = css_task_iter_next(&it);
- css_task_iter_end(&it);
-
- if (task)
- return -EBUSY;
-
- /*
- * All remaining tasks are PF_EXITING and will pass through
- * cgroup_task_dead() shortly. Wait for a kick and retry.
- *
- * cgroup_has_tasks() can't transition from false to true while we're
- * holding cgroup_mutex, but the true to false transition happens
- * under css_set_lock (via cgroup_task_dead()). We must retest and
- * prepare_to_wait() under css_set_lock. Otherwise, the transition
- * can happen between our first test and prepare_to_wait(), and we
- * sleep with no one to wake us.
- */
- spin_lock_irq(&css_set_lock);
- if (!cgroup_has_tasks(cgrp)) {
- spin_unlock_irq(&css_set_lock);
- return 0;
- }
- prepare_to_wait(&cgrp->dying_populated_waitq, &wait,
- TASK_UNINTERRUPTIBLE);
- spin_unlock_irq(&css_set_lock);
- mutex_unlock(&cgroup_mutex);
- schedule();
- finish_wait(&cgrp->dying_populated_waitq, &wait);
- mutex_lock(&cgroup_mutex);
- goto retry;
+ for_each_css(css, ssid, cgrp)
+ kill_css_finish(css);
}
int cgroup_rmdir(struct kernfs_node *kn)
@@ -6255,12 +6260,9 @@ int cgroup_rmdir(struct kernfs_node *kn)
if (!cgrp)
return 0;
- ret = cgroup_drain_dying(cgrp);
- if (!ret) {
- ret = cgroup_destroy_locked(cgrp);
- if (!ret)
- TRACE_CGROUP_PATH(rmdir, cgrp);
- }
+ ret = cgroup_destroy_locked(cgrp);
+ if (!ret)
+ TRACE_CGROUP_PATH(rmdir, cgrp);
cgroup_kn_unlock(kn);
return ret;
@@ -7020,7 +7022,6 @@ void cgroup_task_exit(struct task_struct *tsk)
static void do_cgroup_task_dead(struct task_struct *tsk)
{
- struct cgrp_cset_link *link;
struct css_set *cset;
unsigned long flags;
@@ -7034,11 +7035,6 @@ static void do_cgroup_task_dead(struct task_struct *tsk)
if (thread_group_leader(tsk) && atomic_read(&tsk->signal->live))
list_add_tail(&tsk->cg_list, &cset->dying_tasks);
- /* kick cgroup_drain_dying() waiters, see cgroup_rmdir() */
- list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
- if (waitqueue_active(&link->cgrp->dying_populated_waitq))
- wake_up(&link->cgrp->dying_populated_waitq);
-
if (dl_task(tsk))
dec_dl_tasks_cs(tsk);
diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h
index fd7d198..f7aaf01 100644
--- a/kernel/cgroup/cpuset-internal.h
+++ b/kernel/cgroup/cpuset-internal.h
@@ -167,7 +167,13 @@ struct cpuset {
*/
int nr_deadline_tasks;
int nr_migrate_dl_tasks;
+ /* DL bandwidth that needs destination reservation for this attach. */
u64 sum_migrate_dl_bw;
+ /*
+ * CPU used for temporary DL bandwidth allocation during attach;
+ * -1 if no DL bandwidth was allocated in the current attach.
+ */
+ int dl_bw_cpu;
/* Invalid partition error code, not lock protected */
enum prs_errcode prs_err;
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 1335e437..c9e14fd 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -288,6 +288,7 @@ struct cpuset top_cpuset = {
.flags = BIT(CS_CPU_EXCLUSIVE) |
BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE),
.partition_root_state = PRS_ROOT,
+ .dl_bw_cpu = -1,
};
/**
@@ -579,6 +580,8 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs)
if (!trial)
return NULL;
+ trial->dl_bw_cpu = -1;
+
/* Setup cpumask pointer array */
cpumask_var_t *pmask[4] = {
&trial->cpus_allowed,
@@ -1715,7 +1718,8 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
*/
if (is_partition_valid(parent))
adding = cpumask_and(tmp->addmask,
- xcpus, parent->effective_xcpus);
+ cs->effective_xcpus,
+ parent->effective_xcpus);
if (old_prs > 0)
new_prs = -old_prs;
@@ -1807,9 +1811,9 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
* Compute add/delete mask to/from effective_cpus
*
* For valid partition:
- * addmask = exclusive_cpus & ~newmask
+ * addmask = effective_xcpus & ~newmask
* & parent->effective_xcpus
- * delmask = newmask & ~exclusive_cpus
+ * delmask = newmask & ~effective_xcpus
* & parent->effective_xcpus
*
* For invalid partition:
@@ -1821,11 +1825,11 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
deleting = cpumask_and(tmp->delmask,
newmask, parent->effective_xcpus);
} else {
- cpumask_andnot(tmp->addmask, xcpus, newmask);
+ cpumask_andnot(tmp->addmask, cs->effective_xcpus, newmask);
adding = cpumask_and(tmp->addmask, tmp->addmask,
parent->effective_xcpus);
- cpumask_andnot(tmp->delmask, newmask, xcpus);
+ cpumask_andnot(tmp->delmask, newmask, cs->effective_xcpus);
deleting = cpumask_and(tmp->delmask, tmp->delmask,
parent->effective_xcpus);
}
@@ -1864,7 +1868,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
part_error = PERR_NOCPUS;
deleting = false;
adding = cpumask_and(tmp->addmask,
- xcpus, parent->effective_xcpus);
+ cs->effective_xcpus, parent->effective_xcpus);
}
} else {
/*
@@ -1886,7 +1890,8 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
part_error = PERR_NOCPUS;
if (is_partition_valid(cs))
adding = cpumask_and(tmp->addmask,
- xcpus, parent->effective_xcpus);
+ cs->effective_xcpus,
+ parent->effective_xcpus);
} else if (is_partition_invalid(cs) && !cpumask_empty(xcpus) &&
cpumask_subset(xcpus, parent->effective_xcpus)) {
struct cgroup_subsys_state *css;
@@ -2980,6 +2985,7 @@ static void reset_migrate_dl_data(struct cpuset *cs)
{
cs->nr_migrate_dl_tasks = 0;
cs->sum_migrate_dl_bw = 0;
+ cs->dl_bw_cpu = -1;
}
/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
@@ -2989,7 +2995,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
struct cpuset *cs, *oldcs;
struct task_struct *task;
bool setsched_check;
- int ret;
+ int cpu, ret;
/* used later by cpuset_attach() */
cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
@@ -3034,37 +3040,42 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
}
if (dl_task(task)) {
+ /*
+ * Count all migrating DL tasks for cpuset task accounting.
+ * Only tasks that need a root-domain bandwidth move
+ * contribute to sum_migrate_dl_bw.
+ */
cs->nr_migrate_dl_tasks++;
- cs->sum_migrate_dl_bw += task->dl.dl_bw;
+ if (dl_task_needs_bw_move(task, cs->effective_cpus))
+ cs->sum_migrate_dl_bw += task->dl.dl_bw;
}
}
- if (!cs->nr_migrate_dl_tasks)
+ if (!cs->sum_migrate_dl_bw)
goto out_success;
- if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
- int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
-
- if (unlikely(cpu >= nr_cpu_ids)) {
- reset_migrate_dl_data(cs);
- ret = -EINVAL;
- goto out_unlock;
- }
-
- ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
- if (ret) {
- reset_migrate_dl_data(cs);
- goto out_unlock;
- }
+ cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
+ if (unlikely(cpu >= nr_cpu_ids)) {
+ ret = -EINVAL;
+ goto out_unlock;
}
+ ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
+ if (ret)
+ goto out_unlock;
+
+ cs->dl_bw_cpu = cpu;
+
out_success:
/*
* Mark attach is in progress. This makes validate_change() fail
* changes which zero cpus/mems_allowed.
*/
cs->attach_in_progress++;
+
out_unlock:
+ if (ret)
+ reset_migrate_dl_data(cs);
mutex_unlock(&cpuset_mutex);
return ret;
}
@@ -3080,12 +3091,11 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset)
mutex_lock(&cpuset_mutex);
dec_attach_in_progress_locked(cs);
- if (cs->nr_migrate_dl_tasks) {
- int cpu = cpumask_any(cs->effective_cpus);
+ if (cs->dl_bw_cpu >= 0)
+ dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw);
- dl_bw_free(cpu, cs->sum_migrate_dl_bw);
+ if (cs->nr_migrate_dl_tasks)
reset_migrate_dl_data(cs);
- }
mutex_unlock(&cpuset_mutex);
}
@@ -4171,11 +4181,11 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
* current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this
* node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
* yes. If current has access to memory reserves as an oom victim, yes.
- * Otherwise, no.
+ * If the current task is PF_EXITING, yes. Otherwise, no.
*
* GFP_USER allocations are marked with the __GFP_HARDWALL bit,
* and do not allow allocations outside the current tasks cpuset
- * unless the task has been OOM killed.
+ * unless the task has been OOM killed or is exiting.
* GFP_KERNEL allocations are not so marked, so can escape to the
* nearest enclosing hardwalled ancestor cpuset.
*
@@ -4189,7 +4199,9 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
* The first call here from mm/page_alloc:get_page_from_freelist()
* has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
* so no allocation on a node outside the cpuset is allowed (unless
- * in interrupt, of course).
+ * in interrupt, of course). The PF_EXITING check must therefore
+ * come before the __GFP_HARDWALL check, otherwise a dying task
+ * would be blocked on the fast path.
*
* The second pass through get_page_from_freelist() doesn't even call
* here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
@@ -4199,6 +4211,7 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
* in_interrupt - any node ok (current task context irrelevant)
* GFP_ATOMIC - any node ok
* tsk_is_oom_victim - any node ok
+ * PF_EXITING - any node ok (let dying task exit quickly)
* GFP_KERNEL - any node in enclosing hardwalled cpuset ok
* GFP_USER - only nodes in current tasks mems allowed ok.
*/
@@ -4218,11 +4231,10 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask)
*/
if (unlikely(tsk_is_oom_victim(current)))
return true;
- if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
- return false;
-
if (current->flags & PF_EXITING) /* Let dying task have memory */
return true;
+ if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
+ return false;
/* Not hardwall and node outside mems_allowed: scan up cpusets */
spin_lock_irqsave(&callback_lock, flags);
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 1ab1fb4..4753a67 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -602,6 +602,7 @@ get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region)
pool = NULL;
continue;
}
+ pool = ERR_PTR(-ENOMEM);
}
}
diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c
index 9967fb2..4fdab4c 100644
--- a/kernel/cgroup/rdma.c
+++ b/kernel/cgroup/rdma.c
@@ -283,7 +283,7 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg,
ret = PTR_ERR(rpool);
goto err;
} else {
- new = rpool->resources[index].usage + 1;
+ new = (s64)rpool->resources[index].usage + 1;
if (new > rpool->resources[index].max) {
ret = -EAGAIN;
goto err;
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index 150e587..de816a4 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "cgroup-internal.h"
+#include <linux/cpumask.h>
#include <linux/sched/cputime.h>
#include <linux/bpf.h>
@@ -53,7 +54,7 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)
}
/**
- * css_rstat_updated - keep track of updated rstat_cpu
+ * __css_rstat_updated - keep track of updated rstat_cpu
* @css: target cgroup subsystem state
* @cpu: cpu on which rstat_cpu was updated
*
@@ -63,31 +64,27 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)
*
* NOTE: if the user needs the guarantee that the updater either add itself in
* the lockless list or the concurrent flusher flushes its updated stats, a
- * memory barrier is needed before the call to css_rstat_updated() i.e. a
+ * memory barrier is needed before the call to __css_rstat_updated() i.e. a
* barrier after updating the per-cpu stats and before calling
- * css_rstat_updated().
+ * __css_rstat_updated().
*/
-__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
{
struct llist_head *lhead;
struct css_rstat_cpu *rstatc;
struct llist_node *self;
- /*
- * Since bpf programs can call this function, prevent access to
- * uninitialized rstat pointers.
- */
+ /* Prevent access to uninitialized rstat pointers. */
if (!css_uses_rstat(css))
return;
lockdep_assert_preemption_disabled();
/*
- * For archs withnot nmi safe cmpxchg or percpu ops support, ignore
- * the requests from nmi context.
+ * The lockless insertion below relies on NMI-safe cmpxchg;
+ * bail out in NMI on archs that don't provide it.
*/
- if ((!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) ||
- !IS_ENABLED(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS)) && in_nmi())
+ if (!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) && in_nmi())
return;
rstatc = css_rstat_cpu(css, cpu);
@@ -125,6 +122,18 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
llist_add(&rstatc->lnode, lhead);
}
+/*
+ * BPF-facing wrapper for __css_rstat_updated(). Validate the caller-provided
+ * CPU before passing it to the internal rstat updater.
+ */
+__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
+{
+ if (unlikely(cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu)))
+ return;
+
+ __css_rstat_updated(css, cpu);
+}
+
static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)
{
/* put @css and all ancestors on the corresponding updated lists */
@@ -170,7 +179,7 @@ static void css_process_update_tree(struct cgroup_subsys *ss, int cpu)
* flusher flush the stats updated by the updater who have
* observed that they are already on the list. The
* corresponding barrier pair for this one should be before
- * css_rstat_updated() by the user.
+ * __css_rstat_updated() by the user.
*
* For now, there aren't any such user, so not adding the
* barrier here but if such a use-case arise, please add
@@ -614,7 +623,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
unsigned long flags)
{
u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);
- css_rstat_updated(&cgrp->self, smp_processor_id());
+ __css_rstat_updated(&cgrp->self, smp_processor_id());
put_cpu_ptr(rstatbc);
}
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 1a725ed..3248f8b 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -1251,7 +1251,14 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
entry->direction = direction;
entry->map_err_type = MAP_ERR_NOT_CHECKED;
- if (!(attrs & DMA_ATTR_MMIO)) {
+ if (attrs & DMA_ATTR_MMIO) {
+ unsigned long pfn = PHYS_PFN(phys);
+
+ if (pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
+ err_printk(dev, entry,
+ "dma_map_resource called for RAM address %pa\n",
+ &phys);
+ } else {
check_for_stack(dev, phys);
if (!PhysHighMem(phys))
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index ec887f4..583c592 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -39,7 +39,7 @@ static inline struct page *dma_direct_to_page(struct device *dev,
u64 dma_direct_get_required_mask(struct device *dev)
{
- phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT;
+ phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;
u64 max_dma = phys_to_dma_direct(dev, phys);
return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
@@ -553,7 +553,7 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
int dma_direct_supported(struct device *dev, u64 mask)
{
- u64 min_mask = (max_pfn - 1) << PAGE_SHIFT;
+ u64 min_mask = ((u64)max_pfn << PAGE_SHIFT) - 1;
/*
* Because 32-bit DMA masks are so common we expect every architecture
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 23ed8eb..e6b07f1 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -365,10 +365,6 @@ EXPORT_SYMBOL(dma_unmap_sg_attrs);
dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
- if (IS_ENABLED(CONFIG_DMA_API_DEBUG) &&
- WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
- return DMA_MAPPING_ERROR;
-
return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);
}
EXPORT_SYMBOL(dma_map_resource);
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 19d2244..e3d381f 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -1,11 +1,12 @@
// SPDX-License-Identifier: GPL-2.0
-#include <linux/irq-entry-common.h>
-#include <linux/resume_user_mode.h>
+#include <linux/futex.h>
#include <linux/highmem.h>
+#include <linux/irq-entry-common.h>
#include <linux/jump_label.h>
#include <linux/kmsan.h>
#include <linux/livepatch.h>
+#include <linux/resume_user_mode.h>
#include <linux/tick.h>
/* Workaround to allow gradual conversion of architecture code */
@@ -60,8 +61,10 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re
if (ti_work & _TIF_PATCH_PENDING)
klp_update_patch_state(current);
- if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
+ if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) {
+ futex_fixup_robust_unlock(regs);
arch_do_signal_or_restart(regs);
+ }
if (ti_work & _TIF_NOTIFY_RESUME)
resume_user_mode_work(regs);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 6d1f8ba..7935d56 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7006,6 +7006,7 @@ static void perf_mmap_open(struct vm_area_struct *vma)
}
static void perf_pmu_output_stop(struct perf_event *event);
+static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb);
/*
* A buffer can be mmap()ed multiple times; either directly through the same
@@ -7021,8 +7022,6 @@ static void perf_mmap_close(struct vm_area_struct *vma)
mapped_f unmapped = get_mapped(event, event_unmapped);
struct perf_buffer *rb = ring_buffer_get(event);
struct user_struct *mmap_user = rb->mmap_user;
- int mmap_locked = rb->mmap_locked;
- unsigned long size = perf_data_size(rb);
bool detach_rest = false;
/* FIXIES vs perf_pmu_unregister() */
@@ -7117,11 +7116,7 @@ static void perf_mmap_close(struct vm_area_struct *vma)
* Aside from that, this buffer is 'fully' detached and unmapped,
* undo the VM accounting.
*/
-
- atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,
- &mmap_user->locked_vm);
- atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
- free_uid(mmap_user);
+ perf_mmap_unaccount(vma, rb);
out_put:
ring_buffer_put(rb); /* could be last */
@@ -7261,6 +7256,15 @@ static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long
atomic64_add(extra, &vma->vm_mm->pinned_vm);
}
+static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb)
+{
+ struct user_struct *user = rb->mmap_user;
+
+ atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked,
+ &user->locked_vm);
+ atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm);
+}
+
static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event,
unsigned long nr_pages)
{
@@ -7323,8 +7327,6 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event,
if (!rb)
return -ENOMEM;
- refcount_set(&rb->mmap_count, 1);
- rb->mmap_user = get_current_user();
rb->mmap_locked = extra;
ring_buffer_attach(event, rb);
@@ -7474,16 +7476,54 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
mapped(event, vma->vm_mm);
/*
- * Try to map it into the page table. On fail, invoke
- * perf_mmap_close() to undo the above, as the callsite expects
- * full cleanup in this case and therefore does not invoke
- * vmops::close().
+ * Try to map it into the page table. On fail undo the above,
+ * as the callsite expects full cleanup in this case and
+ * therefore does not invoke vmops::close().
*/
ret = map_range(event->rb, vma);
- if (ret)
- perf_mmap_close(vma);
+ if (likely(!ret))
+ return 0;
+
+ /* Error path */
+
+ /*
+ * If this is the first mmap(), then event->mmap_count should
+ * be stable at 1. It is only modified by:
+ * perf_mmap_{open,close}() and perf_mmap().
+ *
+ * The former are not possible because this mmap() hasn't been
+ * successful yet, and the latter is serialized by
+ * event->mmap_mutex which we still hold (note that mmap_lock
+ * is not strictly sufficient here, because the event fd can
+ * be passed to another process through trivial means like
+ * fork(), leading to concurrent mmap() from different mm).
+ *
+ * Make sure to remove event->rb before releasing
+ * event->mmap_mutex, such that any concurrent mmap() will not
+ * attempt use this failed buffer.
+ */
+ if (refcount_read(&event->mmap_count) == 1) {
+ /*
+ * Minimal perf_mmap_close(); there can't be AUX or
+ * other events on account of this being the first.
+ */
+ mapped = get_mapped(event, event_unmapped);
+ if (mapped)
+ mapped(event, vma->vm_mm);
+ perf_mmap_unaccount(vma, event->rb);
+ ring_buffer_attach(event, NULL); /* drops last rb->refcount */
+ refcount_set(&event->mmap_count, 0);
+ return ret;
+ }
+
+ /*
+ * Otherwise this is an already existing buffer, and there is
+ * no race vs first exposure, so fall-through and call
+ * perf_mmap_close().
+ */
}
+ perf_mmap_close(vma);
return ret;
}
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index d9cc570..c03c4f2 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -67,6 +67,7 @@ static inline void rb_free_rcu(struct rcu_head *rcu_head)
struct perf_buffer *rb;
rb = container_of(rcu_head, struct perf_buffer, rcu_head);
+ free_uid(rb->mmap_user);
rb_free(rb);
}
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 3e7de26..9fe9216 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -340,6 +340,8 @@ ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)
rb->paused = 1;
mutex_init(&rb->aux_mutex);
+ rb->mmap_user = get_current_user();
+ refcount_set(&rb->mmap_count, 1);
}
void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)
diff --git a/kernel/exit.c b/kernel/exit.c
index 25e9cb6..9821922 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -543,6 +543,32 @@ void mm_update_next_owner(struct mm_struct *mm)
}
#endif /* CONFIG_MEMCG */
+#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
+/*
+ * Subtract the memory footprint of the current task from
+ * mm.
+ */
+static void exit_mm_sched_cache(struct mm_struct *mm)
+{
+ unsigned long fp, sub;
+
+ if (!current->total_numa_faults)
+ return;
+ /*
+ * No lock protection due to performance considerations.
+ * Make sure mm->sc_stat.footprint does not become
+ * negative.
+ */
+ fp = READ_ONCE(mm->sc_stat.footprint);
+ sub = min(fp, current->total_numa_faults);
+ WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
+}
+#else
+static inline void exit_mm_sched_cache(struct mm_struct *mm)
+{
+}
+#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
+
/*
* Turn us into a lazy TLB process if we
* aren't already..
@@ -554,6 +580,9 @@ static void exit_mm(void)
exit_mm_release(current, mm);
if (!mm)
return;
+
+ exit_mm_sched_cache(mm);
+
mmap_read_lock(mm);
mmgrab_lazy_tlb(mm);
BUG_ON(mm != current->active_mm);
@@ -571,6 +600,7 @@ static void exit_mm(void)
*/
smp_mb__after_spinlock();
local_irq_disable();
+ current->user_dumpable = (get_dumpable(mm) == SUID_DUMP_USER);
current->mm = NULL;
membarrier_update_current_mm(NULL);
enter_lazy_tlb(mm, current);
@@ -988,8 +1018,8 @@ void __noreturn do_exit(long code)
proc_exit_connector(tsk);
mpol_put_task_policy(tsk);
#ifdef CONFIG_FUTEX
- if (unlikely(current->pi_state_cache))
- kfree(current->pi_state_cache);
+ if (unlikely(current->futex.pi_state_cache))
+ kfree(current->futex.pi_state_cache);
#endif
/*
* Make sure we are holding no locks:
@@ -1073,6 +1103,7 @@ void __noreturn make_task_dead(int signr)
futex_exit_recursive(tsk);
tsk->exit_state = EXIT_DEAD;
refcount_inc(&tsk->rcu_users);
+ preempt_disable();
do_task_dead();
}
diff --git a/kernel/fork.c b/kernel/fork.c
index f1ad69c..ac1adbd5 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -726,6 +726,7 @@ void __mmdrop(struct mm_struct *mm)
cleanup_lazy_tlbs(mm);
WARN_ON_ONCE(mm == current->active_mm);
+ mm_destroy_sched(mm);
mm_free_pgd(mm);
mm_free_id(mm);
destroy_context(mm);
@@ -1101,6 +1102,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
#endif
mm_init_uprobes_state(mm);
hugetlb_count_init(mm);
+ futex_mm_init(mm);
mm_flags_clear_all(mm);
if (current->mm) {
@@ -1113,11 +1115,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
mm->def_flags = 0;
}
- if (futex_mm_init(mm))
- goto fail_mm_init;
-
if (mm_alloc_pgd(mm))
- goto fail_nopgd;
+ goto fail_mm_init;
if (mm_alloc_id(mm))
goto fail_noid;
@@ -1128,6 +1127,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
if (mm_alloc_cid(mm, p))
goto fail_cid;
+ if (mm_alloc_sched(mm))
+ goto fail_sched;
+
if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
NR_MM_COUNTERS))
goto fail_pcpu;
@@ -1137,6 +1139,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
return mm;
fail_pcpu:
+ mm_destroy_sched(mm);
+fail_sched:
mm_destroy_cid(mm);
fail_cid:
destroy_context(mm);
@@ -1144,8 +1148,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
mm_free_id(mm);
fail_noid:
mm_free_pgd(mm);
-fail_nopgd:
- futex_hash_free(mm);
fail_mm_init:
free_mm(mm);
return NULL;
@@ -1951,9 +1953,11 @@ static void rv_task_fork(struct task_struct *p)
static bool need_futex_hash_allocate_default(u64 clone_flags)
{
- if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM))
- return false;
- return true;
+ /*
+ * Allocate a default futex hash for any sibling that will
+ * share the parent's mm, except vfork.
+ */
+ return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM;
}
/*
@@ -2216,6 +2220,7 @@ __latent_entropy struct task_struct *copy_process(
lockdep_init_task(p);
p->blocked_on = NULL; /* not blocked yet */
+ p->blocked_donor = NULL; /* nobody is boosting p yet */
#ifdef CONFIG_BCACHE
p->sequential_io = 0;
@@ -2380,10 +2385,6 @@ __latent_entropy struct task_struct *copy_process(
if (retval)
goto bad_fork_cancel_cgroup;
- /*
- * Allocate a default futex hash for the user process once the first
- * thread spawns.
- */
if (need_futex_hash_allocate_default(clone_flags)) {
retval = futex_hash_allocate_default();
if (retval)
@@ -2666,8 +2667,6 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
*
* It copies the process, and if successful kick-starts
* it and waits for it to finish using the VM if required.
- *
- * args->exit_signal is expected to be checked for sanity by the caller.
*/
pid_t kernel_clone(struct kernel_clone_args *args)
{
@@ -2702,6 +2701,9 @@ pid_t kernel_clone(struct kernel_clone_args *args)
(args->pidfd == args->parent_tid))
return -EINVAL;
+ if (!valid_signal(args->exit_signal))
+ return -EINVAL;
+
/*
* Determine whether and which event to report to ptracer. When
* called from kernel_thread or CLONE_UNTRACED is explicitly
@@ -2900,11 +2902,9 @@ static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
return -EINVAL;
/*
- * Verify that higher 32bits of exit_signal are unset and that
- * it is a valid signal
+ * Verify that higher 32bits of exit_signal are unset
*/
- if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
- !valid_signal(args.exit_signal)))
+ if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
return -EINVAL;
if ((args.flags & CLONE_INTO_CGROUP) &&
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index ff2a4fb..6ea4a97 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -32,18 +32,21 @@
* "But they come in a choice of three flavours!"
*/
#include <linux/compat.h>
-#include <linux/jhash.h>
-#include <linux/pagemap.h>
#include <linux/debugfs.h>
-#include <linux/plist.h>
-#include <linux/gfp.h>
-#include <linux/vmalloc.h>
-#include <linux/memblock.h>
#include <linux/fault-inject.h>
-#include <linux/slab.h>
-#include <linux/prctl.h>
+#include <linux/gfp.h>
+#include <linux/jhash.h>
+#include <linux/memblock.h>
#include <linux/mempolicy.h>
#include <linux/mmap_lock.h>
+#include <linux/pagemap.h>
+#include <linux/plist.h>
+#include <linux/prctl.h>
+#include <linux/rseq.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include <vdso/futex.h>
#include "futex.h"
#include "../locking/rtmutex_common.h"
@@ -187,13 +190,13 @@ __futex_hash_private(union futex_key *key, struct futex_private_hash *fph)
return NULL;
if (!fph)
- fph = rcu_dereference(key->private.mm->futex_phash);
+ fph = rcu_dereference(key->private.mm->futex.phash.hash);
if (!fph || !fph->hash_mask)
return NULL;
- hash = jhash2((void *)&key->private.address,
- sizeof(key->private.address) / 4,
+ hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4,
key->both.offset);
+
return &fph->queues[hash & fph->hash_mask];
}
@@ -232,18 +235,17 @@ static void futex_rehash_private(struct futex_private_hash *old,
}
}
-static bool __futex_pivot_hash(struct mm_struct *mm,
- struct futex_private_hash *new)
+static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *new)
{
+ struct futex_mm_phash *mmph = &mm->futex.phash;
struct futex_private_hash *fph;
- WARN_ON_ONCE(mm->futex_phash_new);
+ WARN_ON_ONCE(mmph->hash_new);
- fph = rcu_dereference_protected(mm->futex_phash,
- lockdep_is_held(&mm->futex_hash_lock));
+ fph = rcu_dereference_protected(mmph->hash, lockdep_is_held(&mmph->lock));
if (fph) {
if (!futex_ref_is_dead(fph)) {
- mm->futex_phash_new = new;
+ mmph->hash_new = new;
return false;
}
@@ -251,8 +253,8 @@ static bool __futex_pivot_hash(struct mm_struct *mm,
}
new->state = FR_PERCPU;
scoped_guard(rcu) {
- mm->futex_batches = get_state_synchronize_rcu();
- rcu_assign_pointer(mm->futex_phash, new);
+ mmph->batches = get_state_synchronize_rcu();
+ rcu_assign_pointer(mmph->hash, new);
}
kvfree_rcu(fph, rcu);
return true;
@@ -260,12 +262,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm,
static void futex_pivot_hash(struct mm_struct *mm)
{
- scoped_guard(mutex, &mm->futex_hash_lock) {
+ scoped_guard(mutex, &mm->futex.phash.lock) {
struct futex_private_hash *fph;
- fph = mm->futex_phash_new;
+ fph = mm->futex.phash.hash_new;
if (fph) {
- mm->futex_phash_new = NULL;
+ mm->futex.phash.hash_new = NULL;
__futex_pivot_hash(mm, fph);
}
}
@@ -288,7 +290,7 @@ struct futex_private_hash *futex_private_hash(void)
scoped_guard(rcu) {
struct futex_private_hash *fph;
- fph = rcu_dereference(mm->futex_phash);
+ fph = rcu_dereference(mm->futex.phash.hash);
if (!fph)
return NULL;
@@ -411,8 +413,7 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr)
* private hash) is returned if existing. Otherwise a hash bucket from the
* global hash is returned.
*/
-static struct futex_hash_bucket *
-__futex_hash(union futex_key *key, struct futex_private_hash *fph)
+static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph)
{
int node = key->both.node;
u32 hash;
@@ -425,8 +426,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph)
return hb;
}
- hash = jhash2((u32 *)key,
- offsetof(typeof(*key), both.offset) / sizeof(u32),
+ hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32),
key->both.offset);
if (node == FUTEX_NO_NODE) {
@@ -441,8 +441,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph)
*/
node = (hash >> futex_hashshift) % nr_node_ids;
if (!node_possible(node)) {
- node = find_next_bit_wrap(node_possible_map.bits,
- nr_node_ids, node);
+ node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node);
}
}
@@ -459,9 +458,8 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph)
* Return: Initialized hrtimer_sleeper structure or NULL if no timeout
* value given
*/
-struct hrtimer_sleeper *
-futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
- int flags, u64 range_ns)
+struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
+ int flags, u64 range_ns)
{
if (!time)
return NULL;
@@ -829,7 +827,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting)
if (WARN_ON_ONCE(ret == -EBUSY && !exiting))
return;
- mutex_lock(&exiting->futex_exit_mutex);
+ mutex_lock(&exiting->futex.exit_mutex);
/*
* No point in doing state checking here. If the waiter got here
* while the task was in exec()->exec_futex_release() then it can
@@ -838,7 +836,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting)
* already. Highly unlikely and not a problem. Just one more round
* through the futex maze.
*/
- mutex_unlock(&exiting->futex_exit_mutex);
+ mutex_unlock(&exiting->futex.exit_mutex);
put_task_struct(exiting);
}
@@ -1012,8 +1010,9 @@ void futex_unqueue_pi(struct futex_q *q)
* dying task, and do notification if so:
*/
static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
- bool pi, bool pending_op)
+ unsigned int mod, bool pending_op)
{
+ bool pi = !!(mod & FUTEX_ROBUST_MOD_PI);
u32 uval, nval, mval;
pid_t owner;
int err;
@@ -1047,7 +1046,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
*
* In both cases the following conditions are met:
*
- * 1) task->robust_list->list_op_pending != NULL
+ * 1) task->futex.robust_list->list_op_pending != NULL
* @pending_op == true
* 2) The owner part of user space futex value == 0
* 3) Regular futex: @pi == false
@@ -1065,7 +1064,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
owner = uval & FUTEX_TID_MASK;
if (pending_op && !pi && !owner) {
- futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1,
+ futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
FUTEX_BITSET_MATCH_ANY);
return 0;
}
@@ -1119,7 +1118,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
* PI futexes happens in exit_pi_state():
*/
if (!pi && (uval & FUTEX_WAITERS)) {
- futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1,
+ futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
FUTEX_BITSET_MATCH_ANY);
}
@@ -1131,31 +1130,30 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
*/
static inline int fetch_robust_entry(struct robust_list __user **entry,
struct robust_list __user * __user *head,
- unsigned int *pi)
+ unsigned int *mod)
{
unsigned long uentry;
if (get_user(uentry, (unsigned long __user *)head))
return -EFAULT;
- *entry = (void __user *)(uentry & ~1UL);
- *pi = uentry & 1;
+ *entry = (void __user *)(uentry & ~FUTEX_ROBUST_MOD_MASK);
+ *mod = uentry & FUTEX_ROBUST_MOD_MASK;
return 0;
}
/*
- * Walk curr->robust_list (very carefully, it's a userspace list!)
+ * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
* and mark any locks found there dead, and notify any waiters.
*
* We silently return on any sign of list-walking problem.
*/
static void exit_robust_list(struct task_struct *curr)
{
- struct robust_list_head __user *head = curr->robust_list;
+ struct robust_list_head __user *head = curr->futex.robust_list;
+ unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod;
struct robust_list __user *entry, *next_entry, *pending;
- unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
- unsigned int next_pi;
unsigned long futex_offset;
int rc;
@@ -1163,7 +1161,7 @@ static void exit_robust_list(struct task_struct *curr)
* Fetch the list head (which was registered earlier, via
* sys_set_robust_list()):
*/
- if (fetch_robust_entry(&entry, &head->list.next, &pi))
+ if (fetch_robust_entry(&entry, &head->list.next, &cur_mod))
return;
/*
* Fetch the relative futex offset:
@@ -1174,7 +1172,7 @@ static void exit_robust_list(struct task_struct *curr)
* Fetch any possibly pending lock-add first, and handle it
* if it exists:
*/
- if (fetch_robust_entry(&pending, &head->list_op_pending, &pip))
+ if (fetch_robust_entry(&pending, &head->list_op_pending, &pend_mod))
return;
next_entry = NULL; /* avoid warning with gcc */
@@ -1183,20 +1181,20 @@ static void exit_robust_list(struct task_struct *curr)
* Fetch the next entry in the list before calling
* handle_futex_death:
*/
- rc = fetch_robust_entry(&next_entry, &entry->next, &next_pi);
+ rc = fetch_robust_entry(&next_entry, &entry->next, &next_mod);
/*
* A pending lock might already be on the list, so
* don't process it twice:
*/
if (entry != pending) {
if (handle_futex_death((void __user *)entry + futex_offset,
- curr, pi, HANDLE_DEATH_LIST))
+ curr, cur_mod, HANDLE_DEATH_LIST))
return;
}
if (rc)
return;
entry = next_entry;
- pi = next_pi;
+ cur_mod = next_mod;
/*
* Avoid excessively long or circular lists:
*/
@@ -1208,10 +1206,31 @@ static void exit_robust_list(struct task_struct *curr)
if (pending) {
handle_futex_death((void __user *)pending + futex_offset,
- curr, pip, HANDLE_DEATH_PENDING);
+ curr, pend_mod, HANDLE_DEATH_PENDING);
}
}
+static bool robust_list_clear_pending(unsigned long __user *pop)
+{
+ struct robust_list_head __user *head = current->futex.robust_list;
+
+ if (!put_user(0UL, pop))
+ return true;
+
+ /*
+ * Just give up. The robust list head is usually part of TLS, so the
+ * chance that this gets resolved is close to zero.
+ *
+ * If @pop_addr is the robust_list_head::list_op_pending pointer then
+ * clear the robust list head pointer to prevent further damage when the
+ * task exits. Better a few stale futexes than corrupted memory. But
+ * that's mostly an academic exercise.
+ */
+ if (pop == (unsigned long __user *)&head->list_op_pending)
+ current->futex.robust_list = NULL;
+ return false;
+}
+
#ifdef CONFIG_COMPAT
static void __user *futex_uaddr(struct robust_list __user *entry,
compat_long_t futex_offset)
@@ -1227,29 +1246,28 @@ static void __user *futex_uaddr(struct robust_list __user *entry,
*/
static inline int
compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry,
- compat_uptr_t __user *head, unsigned int *pi)
+ compat_uptr_t __user *head, unsigned int *pflags)
{
if (get_user(*uentry, head))
return -EFAULT;
- *entry = compat_ptr((*uentry) & ~1);
- *pi = (unsigned int)(*uentry) & 1;
+ *entry = compat_ptr((*uentry) & ~FUTEX_ROBUST_MOD_MASK);
+ *pflags = (unsigned int)(*uentry) & FUTEX_ROBUST_MOD_MASK;
return 0;
}
/*
- * Walk curr->robust_list (very carefully, it's a userspace list!)
+ * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
* and mark any locks found there dead, and notify any waiters.
*
* We silently return on any sign of list-walking problem.
*/
static void compat_exit_robust_list(struct task_struct *curr)
{
- struct compat_robust_list_head __user *head = curr->compat_robust_list;
+ struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
+ unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod;
struct robust_list __user *entry, *next_entry, *pending;
- unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
- unsigned int next_pi;
compat_uptr_t uentry, next_uentry, upending;
compat_long_t futex_offset;
int rc;
@@ -1258,7 +1276,7 @@ static void compat_exit_robust_list(struct task_struct *curr)
* Fetch the list head (which was registered earlier, via
* sys_set_robust_list()):
*/
- if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &pi))
+ if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &cur_mod))
return;
/*
* Fetch the relative futex offset:
@@ -1269,8 +1287,7 @@ static void compat_exit_robust_list(struct task_struct *curr)
* Fetch any possibly pending lock-add first, and handle it
* if it exists:
*/
- if (compat_fetch_robust_entry(&upending, &pending,
- &head->list_op_pending, &pip))
+ if (compat_fetch_robust_entry(&upending, &pending, &head->list_op_pending, &pend_mod))
return;
next_entry = NULL; /* avoid warning with gcc */
@@ -1280,7 +1297,7 @@ static void compat_exit_robust_list(struct task_struct *curr)
* handle_futex_death:
*/
rc = compat_fetch_robust_entry(&next_uentry, &next_entry,
- (compat_uptr_t __user *)&entry->next, &next_pi);
+ (compat_uptr_t __user *)&entry->next, &next_mod);
/*
* A pending lock might already be on the list, so
* dont process it twice:
@@ -1288,15 +1305,14 @@ static void compat_exit_robust_list(struct task_struct *curr)
if (entry != pending) {
void __user *uaddr = futex_uaddr(entry, futex_offset);
- if (handle_futex_death(uaddr, curr, pi,
- HANDLE_DEATH_LIST))
+ if (handle_futex_death(uaddr, curr, cur_mod, HANDLE_DEATH_LIST))
return;
}
if (rc)
return;
uentry = next_uentry;
entry = next_entry;
- pi = next_pi;
+ cur_mod = next_mod;
/*
* Avoid excessively long or circular lists:
*/
@@ -1308,9 +1324,24 @@ static void compat_exit_robust_list(struct task_struct *curr)
if (pending) {
void __user *uaddr = futex_uaddr(pending, futex_offset);
- handle_futex_death(uaddr, curr, pip, HANDLE_DEATH_PENDING);
+ handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING);
}
}
+
+static bool compat_robust_list_clear_pending(u32 __user *pop)
+{
+ struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
+
+ if (!put_user(0U, pop))
+ return true;
+
+ /* See comment in robust_list_clear_pending(). */
+ if (pop == &head->list_op_pending)
+ current->futex.compat_robust_list = NULL;
+ return false;
+}
+#else
+static bool compat_robust_list_clear_pending(u32 __user *pop_addr) { return false; }
#endif
#ifdef CONFIG_FUTEX_PI
@@ -1322,7 +1353,7 @@ static void compat_exit_robust_list(struct task_struct *curr)
*/
static void exit_pi_state_list(struct task_struct *curr)
{
- struct list_head *next, *head = &curr->pi_state_list;
+ struct list_head *next, *head = &curr->futex.pi_state_list;
struct futex_pi_state *pi_state;
union futex_key key = FUTEX_KEY_INIT;
@@ -1404,21 +1435,50 @@ static void exit_pi_state_list(struct task_struct *curr)
static inline void exit_pi_state_list(struct task_struct *curr) { }
#endif
+bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags)
+{
+ bool size32bit = !!(flags & FLAGS_ROBUST_LIST32);
+
+ if (!IS_ENABLED(CONFIG_64BIT) && !size32bit)
+ return false;
+
+ if (IS_ENABLED(CONFIG_64BIT) && size32bit)
+ return compat_robust_list_clear_pending(pop);
+
+ return robust_list_clear_pending(pop);
+}
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr)
+{
+ /*
+ * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from
+ * @regs if the try_cmpxchg() succeeded.
+ */
+ void __user *pop = arch_futex_robust_unlock_get_pop(regs);
+
+ if (!pop)
+ return;
+
+ futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0);
+}
+#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */
+
static void futex_cleanup(struct task_struct *tsk)
{
- if (unlikely(tsk->robust_list)) {
+ if (unlikely(tsk->futex.robust_list)) {
exit_robust_list(tsk);
- tsk->robust_list = NULL;
+ tsk->futex.robust_list = NULL;
}
#ifdef CONFIG_COMPAT
- if (unlikely(tsk->compat_robust_list)) {
+ if (unlikely(tsk->futex.compat_robust_list)) {
compat_exit_robust_list(tsk);
- tsk->compat_robust_list = NULL;
+ tsk->futex.compat_robust_list = NULL;
}
#endif
- if (unlikely(!list_empty(&tsk->pi_state_list)))
+ if (unlikely(!list_empty(&tsk->futex.pi_state_list)))
exit_pi_state_list(tsk);
}
@@ -1442,23 +1502,23 @@ static void futex_cleanup(struct task_struct *tsk)
void futex_exit_recursive(struct task_struct *tsk)
{
/* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */
- if (tsk->futex_state == FUTEX_STATE_EXITING) {
- __assume_ctx_lock(&tsk->futex_exit_mutex);
- mutex_unlock(&tsk->futex_exit_mutex);
+ if (tsk->futex.state == FUTEX_STATE_EXITING) {
+ __assume_ctx_lock(&tsk->futex.exit_mutex);
+ mutex_unlock(&tsk->futex.exit_mutex);
}
- tsk->futex_state = FUTEX_STATE_DEAD;
+ tsk->futex.state = FUTEX_STATE_DEAD;
}
static void futex_cleanup_begin(struct task_struct *tsk)
- __acquires(&tsk->futex_exit_mutex)
+ __acquires(&tsk->futex.exit_mutex)
{
/*
* Prevent various race issues against a concurrent incoming waiter
* including live locks by forcing the waiter to block on
- * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in
+ * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in
* attach_to_pi_owner().
*/
- mutex_lock(&tsk->futex_exit_mutex);
+ mutex_lock(&tsk->futex.exit_mutex);
/*
* Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock.
@@ -1472,23 +1532,23 @@ static void futex_cleanup_begin(struct task_struct *tsk)
* be observed in exit_pi_state_list().
*/
raw_spin_lock_irq(&tsk->pi_lock);
- tsk->futex_state = FUTEX_STATE_EXITING;
+ tsk->futex.state = FUTEX_STATE_EXITING;
raw_spin_unlock_irq(&tsk->pi_lock);
}
static void futex_cleanup_end(struct task_struct *tsk, int state)
- __releases(&tsk->futex_exit_mutex)
+ __releases(&tsk->futex.exit_mutex)
{
/*
* Lockless store. The only side effect is that an observer might
* take another loop until it becomes visible.
*/
- tsk->futex_state = state;
+ tsk->futex.state = state;
/*
* Drop the exit protection. This unblocks waiters which observed
* FUTEX_STATE_EXITING to reevaluate the state.
*/
- mutex_unlock(&tsk->futex_exit_mutex);
+ mutex_unlock(&tsk->futex.exit_mutex);
}
void futex_exec_release(struct task_struct *tsk)
@@ -1553,17 +1613,17 @@ static void __futex_ref_atomic_begin(struct futex_private_hash *fph)
* otherwise it would be impossible for it to have reported success
* from futex_ref_is_dead().
*/
- WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0);
+ WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0);
/*
* Set the atomic to the bias value such that futex_ref_{get,put}()
* will never observe 0. Will be fixed up in __futex_ref_atomic_end()
* when folding in the percpu count.
*/
- atomic_long_set(&mm->futex_atomic, LONG_MAX);
+ atomic_long_set(&mm->futex.phash.atomic, LONG_MAX);
smp_store_release(&fph->state, FR_ATOMIC);
- call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu);
+ call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
}
static void __futex_ref_atomic_end(struct futex_private_hash *fph)
@@ -1584,7 +1644,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph)
* Therefore the per-cpu counter is now stable, sum and reset.
*/
for_each_possible_cpu(cpu) {
- unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu);
+ unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu);
count += *ptr;
*ptr = 0;
}
@@ -1592,7 +1652,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph)
/*
* Re-init for the next cycle.
*/
- this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */
+ this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */
/*
* Add actual count, subtract bias and initial refcount.
@@ -1600,7 +1660,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph)
* The moment this atomic operation happens, futex_ref_is_dead() can
* become true.
*/
- ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic);
+ ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic);
if (!ret)
wake_up_var(mm);
@@ -1610,8 +1670,8 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph)
static void futex_ref_rcu(struct rcu_head *head)
{
- struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu);
- struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash);
+ struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu);
+ struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash);
if (fph->state == FR_PERCPU) {
/*
@@ -1640,7 +1700,7 @@ static void futex_ref_drop(struct futex_private_hash *fph)
/*
* Can only transition the current fph;
*/
- WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph);
+ WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph);
/*
* We enqueue at least one RCU callback. Ensure mm stays if the task
* exits before the transition is completed.
@@ -1651,9 +1711,9 @@ static void futex_ref_drop(struct futex_private_hash *fph)
* In order to avoid the following scenario:
*
* futex_hash() __futex_pivot_hash()
- * guard(rcu); guard(mm->futex_hash_lock);
- * fph = mm->futex_phash;
- * rcu_assign_pointer(&mm->futex_phash, new);
+ * guard(rcu); guard(mm->futex.phash.lock);
+ * fph = mm->futex.phash.hash;
+ * rcu_assign_pointer(&mm->futex.phash.hash, new);
* futex_hash_allocate()
* futex_ref_drop()
* fph->state = FR_ATOMIC;
@@ -1668,7 +1728,7 @@ static void futex_ref_drop(struct futex_private_hash *fph)
* There must be at least one full grace-period between publishing a
* new fph and trying to replace it.
*/
- if (poll_state_synchronize_rcu(mm->futex_batches)) {
+ if (poll_state_synchronize_rcu(mm->futex.phash.batches)) {
/*
* There was a grace-period, we can begin now.
*/
@@ -1676,7 +1736,7 @@ static void futex_ref_drop(struct futex_private_hash *fph)
return;
}
- call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu);
+ call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
}
static bool futex_ref_get(struct futex_private_hash *fph)
@@ -1686,11 +1746,11 @@ static bool futex_ref_get(struct futex_private_hash *fph)
guard(preempt)();
if (READ_ONCE(fph->state) == FR_PERCPU) {
- __this_cpu_inc(*mm->futex_ref);
+ __this_cpu_inc(*mm->futex.phash.ref);
return true;
}
- return atomic_long_inc_not_zero(&mm->futex_atomic);
+ return atomic_long_inc_not_zero(&mm->futex.phash.atomic);
}
static bool futex_ref_put(struct futex_private_hash *fph)
@@ -1700,11 +1760,11 @@ static bool futex_ref_put(struct futex_private_hash *fph)
guard(preempt)();
if (READ_ONCE(fph->state) == FR_PERCPU) {
- __this_cpu_dec(*mm->futex_ref);
+ __this_cpu_dec(*mm->futex.phash.ref);
return false;
}
- return atomic_long_dec_and_test(&mm->futex_atomic);
+ return atomic_long_dec_and_test(&mm->futex.phash.atomic);
}
static bool futex_ref_is_dead(struct futex_private_hash *fph)
@@ -1716,28 +1776,23 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph)
if (smp_load_acquire(&fph->state) == FR_PERCPU)
return false;
- return atomic_long_read(&mm->futex_atomic) == 0;
+ return atomic_long_read(&mm->futex.phash.atomic) == 0;
}
-int futex_mm_init(struct mm_struct *mm)
+static void futex_hash_init_mm(struct futex_mm_data *fd)
{
- mutex_init(&mm->futex_hash_lock);
- RCU_INIT_POINTER(mm->futex_phash, NULL);
- mm->futex_phash_new = NULL;
- /* futex-ref */
- mm->futex_ref = NULL;
- atomic_long_set(&mm->futex_atomic, 0);
- mm->futex_batches = get_state_synchronize_rcu();
- return 0;
+ memset(&fd->phash, 0, sizeof(fd->phash));
+ mutex_init(&fd->phash.lock);
+ fd->phash.batches = get_state_synchronize_rcu();
}
void futex_hash_free(struct mm_struct *mm)
{
struct futex_private_hash *fph;
- free_percpu(mm->futex_ref);
- kvfree(mm->futex_phash_new);
- fph = rcu_dereference_raw(mm->futex_phash);
+ free_percpu(mm->futex.phash.ref);
+ kvfree(mm->futex.phash.hash_new);
+ fph = rcu_dereference_raw(mm->futex.phash.hash);
if (fph)
kvfree(fph);
}
@@ -1748,10 +1803,10 @@ static bool futex_pivot_pending(struct mm_struct *mm)
guard(rcu)();
- if (!mm->futex_phash_new)
+ if (!mm->futex.phash.hash_new)
return true;
- fph = rcu_dereference(mm->futex_phash);
+ fph = rcu_dereference(mm->futex.phash.hash);
return futex_ref_is_dead(fph);
}
@@ -1793,7 +1848,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
* Once we've disabled the global hash there is no way back.
*/
scoped_guard(rcu) {
- fph = rcu_dereference(mm->futex_phash);
+ fph = rcu_dereference(mm->futex.phash.hash);
if (fph && !fph->hash_mask) {
if (custom)
return -EBUSY;
@@ -1801,15 +1856,15 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
}
}
- if (!mm->futex_ref) {
+ if (!mm->futex.phash.ref) {
/*
* This will always be allocated by the first thread and
* therefore requires no locking.
*/
- mm->futex_ref = alloc_percpu(unsigned int);
- if (!mm->futex_ref)
+ mm->futex.phash.ref = alloc_percpu(unsigned int);
+ if (!mm->futex.phash.ref)
return -ENOMEM;
- this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */
+ this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */
}
fph = kvzalloc(struct_size(fph, queues, hash_slots),
@@ -1832,14 +1887,14 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
wait_var_event(mm, futex_pivot_pending(mm));
}
- scoped_guard(mutex, &mm->futex_hash_lock) {
+ scoped_guard(mutex, &mm->futex.phash.lock) {
struct futex_private_hash *free __free(kvfree) = NULL;
struct futex_private_hash *cur, *new;
- cur = rcu_dereference_protected(mm->futex_phash,
- lockdep_is_held(&mm->futex_hash_lock));
- new = mm->futex_phash_new;
- mm->futex_phash_new = NULL;
+ cur = rcu_dereference_protected(mm->futex.phash.hash,
+ lockdep_is_held(&mm->futex.phash.lock));
+ new = mm->futex.phash.hash_new;
+ mm->futex.phash.hash_new = NULL;
if (fph) {
if (cur && !cur->hash_mask) {
@@ -1849,7 +1904,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
* the second one returns here.
*/
free = fph;
- mm->futex_phash_new = new;
+ mm->futex.phash.hash_new = new;
return -EBUSY;
}
if (cur && !new) {
@@ -1879,7 +1934,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
if (new) {
/*
- * Will set mm->futex_phash_new on failure;
+ * Will set mm->futex.phash.new_hash on failure;
* futex_private_hash_get() will try again.
*/
if (!__futex_pivot_hash(mm, new) && custom)
@@ -1898,11 +1953,9 @@ int futex_hash_allocate_default(void)
return 0;
scoped_guard(rcu) {
- threads = min_t(unsigned int,
- get_nr_threads(current),
- num_online_cpus());
+ threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus());
- fph = rcu_dereference(current->mm->futex_phash);
+ fph = rcu_dereference(current->mm->futex.phash.hash);
if (fph) {
if (fph->custom)
return 0;
@@ -1929,24 +1982,52 @@ static int futex_hash_get_slots(void)
struct futex_private_hash *fph;
guard(rcu)();
- fph = rcu_dereference(current->mm->futex_phash);
+ fph = rcu_dereference(current->mm->futex.phash.hash);
if (fph && fph->hash_mask)
return fph->hash_mask + 1;
return 0;
}
+#else /* CONFIG_FUTEX_PRIVATE_HASH */
+static inline int futex_hash_allocate(unsigned int hslots, unsigned int flags) { return -EINVAL; }
+static inline int futex_hash_get_slots(void) { return 0; }
+static inline void futex_hash_init_mm(struct futex_mm_data *fd) { }
+#endif /* !CONFIG_FUTEX_PRIVATE_HASH */
-#else
-
-static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+static void futex_invalidate_cs_ranges(struct futex_mm_data *fd)
{
- return -EINVAL;
+ /*
+ * Invalidate start_ip so that the quick check fails for ip >= start_ip
+ * if VDSO is not mapped or the second slot is not available for compat
+ * tasks as they use VDSO32 which does not provide the 64-bit pointer
+ * variant.
+ */
+ for (int i = 0; i < FUTEX_ROBUST_MAX_CS_RANGES; i++)
+ fd->unlock.cs_ranges[i].start_ip = ~0UL;
}
-static int futex_hash_get_slots(void)
+void futex_reset_cs_ranges(struct futex_mm_data *fd)
{
- return 0;
+ memset(fd->unlock.cs_ranges, 0, sizeof(fd->unlock.cs_ranges));
+ futex_invalidate_cs_ranges(fd);
}
+static void futex_robust_unlock_init_mm(struct futex_mm_data *fd)
+{
+ /* mm_dup() preserves the range, mm_alloc() clears it */
+ if (!fd->unlock.cs_ranges[0].start_ip)
+ futex_invalidate_cs_ranges(fd);
+}
+#else /* CONFIG_FUTEX_ROBUST_UNLOCK */
+static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { }
+#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
+
+#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK)
+void futex_mm_init(struct mm_struct *mm)
+{
+ futex_hash_init_mm(&mm->futex);
+ futex_robust_unlock_init_mm(&mm->futex);
+}
#endif
int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 9f6bf6f..79ef2c7 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -40,6 +40,8 @@
#define FLAGS_NUMA 0x0080
#define FLAGS_STRICT 0x0100
#define FLAGS_MPOL 0x0200
+#define FLAGS_ROBUST_UNLOCK 0x0400
+#define FLAGS_ROBUST_LIST32 0x0800
/* FUTEX_ to FLAGS_ */
static inline unsigned int futex_to_flags(unsigned int op)
@@ -52,6 +54,12 @@ static inline unsigned int futex_to_flags(unsigned int op)
if (op & FUTEX_CLOCK_REALTIME)
flags |= FLAGS_CLOCKRT;
+ if (op & FUTEX_ROBUST_UNLOCK)
+ flags |= FLAGS_ROBUST_UNLOCK;
+
+ if (op & FUTEX_ROBUST_LIST32)
+ flags |= FLAGS_ROBUST_LIST32;
+
return flags;
}
@@ -449,13 +457,16 @@ extern int futex_unqueue_multiple(struct futex_vector *v, int count);
extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
struct hrtimer_sleeper *to);
-extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
+extern int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop,
+ int nr_wake, u32 bitset);
extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
-extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
+extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop);
extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
+bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags);
+
#endif /* _FUTEX_H */
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index 643199f..9dd5c0b 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -14,7 +14,7 @@ int refill_pi_state_cache(void)
{
struct futex_pi_state *pi_state;
- if (likely(current->pi_state_cache))
+ if (likely(current->futex.pi_state_cache))
return 0;
pi_state = kzalloc_obj(*pi_state);
@@ -28,17 +28,17 @@ int refill_pi_state_cache(void)
refcount_set(&pi_state->refcount, 1);
pi_state->key = FUTEX_KEY_INIT;
- current->pi_state_cache = pi_state;
+ current->futex.pi_state_cache = pi_state;
return 0;
}
static struct futex_pi_state *alloc_pi_state(void)
{
- struct futex_pi_state *pi_state = current->pi_state_cache;
+ struct futex_pi_state *pi_state = current->futex.pi_state_cache;
WARN_ON(!pi_state);
- current->pi_state_cache = NULL;
+ current->futex.pi_state_cache = NULL;
return pi_state;
}
@@ -60,7 +60,7 @@ static void pi_state_update_owner(struct futex_pi_state *pi_state,
if (new_owner) {
raw_spin_lock(&new_owner->pi_lock);
WARN_ON(!list_empty(&pi_state->list));
- list_add(&pi_state->list, &new_owner->pi_state_list);
+ list_add(&pi_state->list, &new_owner->futex.pi_state_list);
pi_state->owner = new_owner;
raw_spin_unlock(&new_owner->pi_lock);
}
@@ -96,7 +96,7 @@ void put_pi_state(struct futex_pi_state *pi_state)
raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags);
}
- if (current->pi_state_cache) {
+ if (current->futex.pi_state_cache) {
kfree(pi_state);
} else {
/*
@@ -106,7 +106,7 @@ void put_pi_state(struct futex_pi_state *pi_state)
*/
pi_state->owner = NULL;
refcount_set(&pi_state->refcount, 1);
- current->pi_state_cache = pi_state;
+ current->futex.pi_state_cache = pi_state;
}
}
@@ -179,7 +179,7 @@ void put_pi_state(struct futex_pi_state *pi_state)
*
* p->pi_lock:
*
- * p->pi_state_list -> pi_state->list, relation
+ * p->futex.pi_state_list -> pi_state->list, relation
* pi_mutex->owner -> pi_state->owner, relation
*
* pi_state->refcount:
@@ -327,7 +327,7 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval,
* If the futex exit state is not yet FUTEX_STATE_DEAD, tell the
* caller that the alleged owner is busy.
*/
- if (tsk && tsk->futex_state != FUTEX_STATE_DEAD)
+ if (tsk && tsk->futex.state != FUTEX_STATE_DEAD)
return -EBUSY;
/*
@@ -346,8 +346,8 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval,
* *uaddr = 0xC0000000; tsk = get_task(PID);
* } if (!tsk->flags & PF_EXITING) {
* ... attach();
- * tsk->futex_state = } else {
- * FUTEX_STATE_DEAD; if (tsk->futex_state !=
+ * tsk->futex.state = } else {
+ * FUTEX_STATE_DEAD; if (tsk->futex.state !=
* FUTEX_STATE_DEAD)
* return -EAGAIN;
* return -ESRCH; <--- FAIL
@@ -396,7 +396,7 @@ static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key,
pi_state->key = *key;
WARN_ON(!list_empty(&pi_state->list));
- list_add(&pi_state->list, &p->pi_state_list);
+ list_add(&pi_state->list, &p->futex.pi_state_list);
/*
* Assignment without holding pi_state->pi_mutex.wait_lock is safe
* because there is no concurrency as the object is not published yet.
@@ -440,7 +440,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
* in futex_exit_release(), we do this protected by p->pi_lock:
*/
raw_spin_lock_irq(&p->pi_lock);
- if (unlikely(p->futex_state != FUTEX_STATE_OK)) {
+ if (unlikely(p->futex.state != FUTEX_STATE_OK)) {
/*
* The task is on the way out. When the futex state is
* FUTEX_STATE_DEAD, we know that the task has finished
@@ -1139,7 +1139,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
* This is the in-kernel slowpath: we look up the PI state (if any),
* and do the rt-mutex unlock.
*/
-int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
+static int __futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
{
u32 curval, uval, vpid = task_pid_vnr(current);
union futex_key key = FUTEX_KEY_INIT;
@@ -1148,7 +1148,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
if (!IS_ENABLED(CONFIG_FUTEX_PI))
return -ENOSYS;
-
retry:
if (get_user(uval, uaddr))
return -EFAULT;
@@ -1302,3 +1301,15 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
return ret;
}
+int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop)
+{
+ int ret = __futex_unlock_pi(uaddr, flags);
+
+ if (ret || !(flags & FLAGS_ROBUST_UNLOCK))
+ return ret;
+
+ if (!futex_robust_list_clear_pending(pop, flags))
+ return -EFAULT;
+
+ return 0;
+}
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index d818b4d..1d99a84 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -319,8 +319,11 @@ futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1,
return -EINVAL;
/* Ensure that this does not race against an early wakeup */
- if (!futex_requeue_pi_prepare(top_waiter, NULL))
+ if (!futex_requeue_pi_prepare(top_waiter, NULL)) {
+ plist_del(&top_waiter->list, &hb1->chain);
+ futex_hb_waiters_dec(hb1);
return -EAGAIN;
+ }
/*
* Try to take the lock for top_waiter and set the FUTEX_WAITERS bit
@@ -640,6 +643,12 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
continue;
}
+ /* Self-deadlock: non-top waiter already owns the PI futex. */
+ if (rt_mutex_owner(&pi_state->pi_mutex) == this->task) {
+ ret = -EDEADLK;
+ break;
+ }
+
ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
this->rt_waiter,
this->task);
@@ -722,10 +731,12 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
/*
* We were woken prior to requeue by a timeout or a signal.
- * Unqueue the futex_q and determine which it was.
+ * Conditionally unqueue the futex_q and determine which it was.
*/
- plist_del(&q->list, &hb->chain);
- futex_hb_waiters_dec(hb);
+ if (!plist_node_empty(&q->list)) {
+ plist_del(&q->list, &hb->chain);
+ futex_hb_waiters_dec(hb);
+ }
/* Handle spurious wakeups gracefully */
ret = -EWOULDBLOCK;
diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c
index 77ad969..2fa19d9 100644
--- a/kernel/futex/syscalls.c
+++ b/kernel/futex/syscalls.c
@@ -25,17 +25,13 @@
* @head: pointer to the list-head
* @len: length of the list-head, as userspace expects
*/
-SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
- size_t, len)
+SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, size_t, len)
{
- /*
- * The kernel knows only one size for now:
- */
+ /* The kernel knows only one size for now. */
if (unlikely(len != sizeof(*head)))
return -EINVAL;
- current->robust_list = head;
-
+ current->futex.robust_list = head;
return 0;
}
@@ -43,9 +39,9 @@ static inline void __user *futex_task_robust_list(struct task_struct *p, bool co
{
#ifdef CONFIG_COMPAT
if (compat)
- return p->compat_robust_list;
+ return p->futex.compat_robust_list;
#endif
- return p->robust_list;
+ return p->futex.robust_list;
}
static void __user *futex_get_robust_list_common(int pid, bool compat)
@@ -122,6 +118,13 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
return -ENOSYS;
}
+ if (flags & FLAGS_ROBUST_UNLOCK) {
+ if (cmd != FUTEX_WAKE &&
+ cmd != FUTEX_WAKE_BITSET &&
+ cmd != FUTEX_UNLOCK_PI)
+ return -ENOSYS;
+ }
+
switch (cmd) {
case FUTEX_WAIT:
val3 = FUTEX_BITSET_MATCH_ANY;
@@ -132,7 +135,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
val3 = FUTEX_BITSET_MATCH_ANY;
fallthrough;
case FUTEX_WAKE_BITSET:
- return futex_wake(uaddr, flags, val, val3);
+ return futex_wake(uaddr, flags, uaddr2, val, val3);
case FUTEX_REQUEUE:
return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0);
case FUTEX_CMP_REQUEUE:
@@ -145,7 +148,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
case FUTEX_LOCK_PI2:
return futex_lock_pi(uaddr, flags, timeout, 0);
case FUTEX_UNLOCK_PI:
- return futex_unlock_pi(uaddr, flags);
+ return futex_unlock_pi(uaddr, flags, uaddr2);
case FUTEX_TRYLOCK_PI:
return futex_lock_pi(uaddr, flags, NULL, 1);
case FUTEX_WAIT_REQUEUE_PI:
@@ -379,7 +382,7 @@ SYSCALL_DEFINE4(futex_wake,
if (!futex_validate_input(flags, mask))
return -EINVAL;
- return futex_wake(uaddr, FLAGS_STRICT | flags, nr, mask);
+ return futex_wake(uaddr, FLAGS_STRICT | flags, NULL, nr, mask);
}
/*
@@ -475,15 +478,13 @@ SYSCALL_DEFINE4(futex_requeue,
}
#ifdef CONFIG_COMPAT
-COMPAT_SYSCALL_DEFINE2(set_robust_list,
- struct compat_robust_list_head __user *, head,
- compat_size_t, len)
+COMPAT_SYSCALL_DEFINE2(set_robust_list, struct compat_robust_list_head __user *, head,
+ compat_size_t, len)
{
if (unlikely(len != sizeof(*head)))
return -EINVAL;
- current->compat_robust_list = head;
-
+ current->futex.compat_robust_list = head;
return 0;
}
@@ -523,4 +524,3 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
}
#endif /* CONFIG_COMPAT_32BIT_TIME */
-
diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c
index ceed9d8..8f5e5d3 100644
--- a/kernel/futex/waitwake.c
+++ b/kernel/futex/waitwake.c
@@ -150,12 +150,35 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
}
/*
+ * If requested, clear the robust list pending op and unlock the futex
+ */
+static bool futex_robust_unlock(u32 __user *uaddr, unsigned int flags, void __user *pop)
+{
+ if (!(flags & FLAGS_ROBUST_UNLOCK))
+ return true;
+
+ /* First unlock the futex, which requires release semantics. */
+ scoped_user_write_access(uaddr, efault)
+ unsafe_atomic_store_release_user(0, uaddr, efault);
+
+ /*
+ * Clear the pending list op now. If that fails, then the task is in
+ * deeper trouble as the robust list head is usually part of the TLS.
+ * The chance of survival is close to zero.
+ */
+ return futex_robust_list_clear_pending(pop, flags);
+
+efault:
+ return false;
+}
+
+/*
* Wake up waiters matching bitset queued on this futex (uaddr).
*/
-int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
+int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, int nr_wake, u32 bitset)
{
- struct futex_q *this, *next;
union futex_key key = FUTEX_KEY_INIT;
+ struct futex_q *this, *next;
DEFINE_WAKE_Q(wake_q);
int ret;
@@ -166,6 +189,9 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
if (unlikely(ret != 0))
return ret;
+ if (!futex_robust_unlock(uaddr, flags, pop))
+ return -EFAULT;
+
if ((flags & FLAGS_STRICT) && !nr_wake)
return 0;
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 6c9b1dc4..de754db 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -14,6 +14,7 @@
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/irqdomain.h>
+#include <linux/preempt.h>
#include <linux/random.h>
#include <trace/events/irq.h>
@@ -47,9 +48,11 @@ int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
scoped_irqdesc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
ret = 0;
}
- /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */
- if (!ret)
+ if (!ret) {
+ /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */
irq_mark_irq(irq);
+ irq_proc_update_chip(chip);
+ }
return ret;
}
EXPORT_SYMBOL(irq_set_chip);
@@ -893,7 +896,10 @@ void handle_percpu_irq(struct irq_desc *desc)
*
* action->percpu_dev_id is a pointer to percpu variables which
* contain the real device id for the cpu on which this handler is
- * called
+ * called.
+ *
+ * May be used for NMI interrupt lines, and so may be called in IRQ or NMI
+ * context.
*/
void handle_percpu_devid_irq(struct irq_desc *desc)
{
@@ -930,7 +936,8 @@ void handle_percpu_devid_irq(struct irq_desc *desc)
enabled ? " and unmasked" : "", irq, cpu);
}
- add_interrupt_randomness(irq);
+ if (!in_nmi())
+ add_interrupt_randomness(irq);
if (chip->irq_eoi)
chip->irq_eoi(&desc->irq_data);
@@ -1007,6 +1014,7 @@ __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc)));
irq_activate_and_startup(desc, IRQ_RESEND);
}
+ irq_proc_update_valid(desc);
}
void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
@@ -1067,6 +1075,7 @@ void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
trigger = tmp;
irqd_set(&desc->irq_data, trigger);
+ irq_proc_update_valid(desc);
}
}
EXPORT_SYMBOL_GPL(irq_modify_status);
diff --git a/kernel/irq/debugfs.h b/kernel/irq/debugfs.h
new file mode 100644
index 0000000..8a9360d
--- /dev/null
+++ b/kernel/irq/debugfs.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _KERNEL_IRQ_DEBUGFS_H
+#define _KERNEL_IRQ_DEBUGFS_H
+
+#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
+#include <linux/debugfs.h>
+
+struct irq_bit_descr {
+ unsigned int mask;
+ char *name;
+};
+
+#define BIT_MASK_DESCR(m) { .mask = m, .name = #m }
+
+void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state,
+ const struct irq_bit_descr *sd, int size);
+
+void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
+static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
+{
+ debugfs_remove(desc->debugfs_file);
+ kfree(desc->dev_name);
+}
+void irq_debugfs_copy_devname(int irq, struct device *dev);
+# ifdef CONFIG_IRQ_DOMAIN
+void irq_domain_debugfs_init(struct dentry *root);
+# else
+static inline void irq_domain_debugfs_init(struct dentry *root)
+{
+}
+# endif
+#else /* CONFIG_GENERIC_IRQ_DEBUGFS */
+static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
+{
+}
+static inline void irq_remove_debugfs_entry(struct irq_desc *d)
+{
+}
+static inline void irq_debugfs_copy_devname(int irq, struct device *dev)
+{
+}
+#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */
+
+#endif
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index 9412e57..0ce21dd 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -9,8 +9,12 @@
#include <linux/irqdesc.h>
#include <linux/kernel_stat.h>
#include <linux/pm_runtime.h>
+#include <linux/rcuref.h>
#include <linux/sched/clock.h>
+#include "debugfs.h"
+#include "proc.h"
+
#ifdef CONFIG_SPARSE_IRQ
# define MAX_SPARSE_IRQS INT_MAX
#else
@@ -21,6 +25,7 @@
extern bool noirqdebug;
extern int irq_poll_cpu;
+extern unsigned int total_nr_irqs;
extern struct irqaction chained_action;
@@ -100,9 +105,23 @@ extern void unmask_irq(struct irq_desc *desc);
extern void unmask_threaded_irq(struct irq_desc *desc);
#ifdef CONFIG_SPARSE_IRQ
-static inline void irq_mark_irq(unsigned int irq) { }
+static __always_inline void irq_mark_irq(unsigned int irq) { }
+void irq_desc_free_rcu(struct irq_desc *desc);
+
+static __always_inline bool irq_desc_get_ref(struct irq_desc *desc)
+{
+ return rcuref_get(&desc->refcnt);
+}
+
+static __always_inline void irq_desc_put_ref(struct irq_desc *desc)
+{
+ if (rcuref_put(&desc->refcnt))
+ irq_desc_free_rcu(desc);
+}
#else
extern void irq_mark_irq(unsigned int irq);
+static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; }
+static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { }
#endif
irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);
@@ -122,6 +141,7 @@ extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);
extern void register_handler_proc(unsigned int irq, struct irqaction *action);
extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);
+void irq_proc_update_valid(struct irq_desc *desc);
#else
static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }
static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }
@@ -129,8 +149,11 @@ static inline void register_handler_proc(unsigned int irq,
struct irqaction *action) { }
static inline void unregister_handler_proc(unsigned int irq,
struct irqaction *action) { }
+static inline void irq_proc_update_valid(struct irq_desc *desc) { }
#endif
+struct irq_desc *irq_find_desc_at_or_after(unsigned int offset);
+
extern bool irq_can_set_affinity_usr(unsigned int irq);
extern int irq_do_set_affinity(struct irq_data *data,
@@ -171,7 +194,7 @@ void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
__DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true);
__DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc,
- __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus),
+ if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus),
unsigned long flags; bool bus);
static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus,
@@ -372,42 +395,3 @@ static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
return NULL;
#endif
}
-
-#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
-#include <linux/debugfs.h>
-
-struct irq_bit_descr {
- unsigned int mask;
- char *name;
-};
-
-#define BIT_MASK_DESCR(m) { .mask = m, .name = #m }
-
-void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state,
- const struct irq_bit_descr *sd, int size);
-
-void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
-static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
-{
- debugfs_remove(desc->debugfs_file);
- kfree(desc->dev_name);
-}
-void irq_debugfs_copy_devname(int irq, struct device *dev);
-# ifdef CONFIG_IRQ_DOMAIN
-void irq_domain_debugfs_init(struct dentry *root);
-# else
-static inline void irq_domain_debugfs_init(struct dentry *root)
-{
-}
-# endif
-#else /* CONFIG_GENERIC_IRQ_DEBUGFS */
-static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
-{
-}
-static inline void irq_remove_debugfs_entry(struct irq_desc *d)
-{
-}
-static inline void irq_debugfs_copy_devname(int irq, struct device *dev)
-{
-}
-#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 7173b8b..80ef4e2 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -137,17 +137,18 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
desc->tot_count = 0;
desc->name = NULL;
desc->owner = owner;
+ rcuref_init(&desc->refcnt, 1);
desc_smp_init(desc, node, affinity);
}
-static unsigned int nr_irqs = NR_IRQS;
+unsigned int total_nr_irqs __read_mostly = NR_IRQS;
/**
* irq_get_nr_irqs() - Number of interrupts supported by the system.
*/
unsigned int irq_get_nr_irqs(void)
{
- return nr_irqs;
+ return total_nr_irqs;
}
EXPORT_SYMBOL_GPL(irq_get_nr_irqs);
@@ -157,13 +158,12 @@ EXPORT_SYMBOL_GPL(irq_get_nr_irqs);
*
* Return: @nr.
*/
-unsigned int irq_set_nr_irqs(unsigned int nr)
+unsigned int __init irq_set_nr_irqs(unsigned int nr)
{
- nr_irqs = nr;
-
+ total_nr_irqs = nr;
+ irq_proc_calc_prec();
return nr;
}
-EXPORT_SYMBOL_GPL(irq_set_nr_irqs);
static DEFINE_MUTEX(sparse_irq_lock);
static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs,
@@ -181,15 +181,12 @@ static int irq_find_free_area(unsigned int from, unsigned int cnt)
return mas.index;
}
-static unsigned int irq_find_at_or_after(unsigned int offset)
+struct irq_desc *irq_find_desc_at_or_after(unsigned int offset)
{
unsigned long index = offset;
- struct irq_desc *desc;
- guard(rcu)();
- desc = mt_find(&sparse_irqs, &index, nr_irqs);
-
- return desc ? irq_desc_get_irq(desc) : nr_irqs;
+ lockdep_assert_in_rcu_read_lock();
+ return mt_find(&sparse_irqs, &index, total_nr_irqs);
}
static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
@@ -466,6 +463,17 @@ static void delayed_free_desc(struct rcu_head *rhp)
kobject_put(&desc->kobj);
}
+void irq_desc_free_rcu(struct irq_desc *desc)
+{
+ /*
+ * We free the descriptor, masks and stat fields via RCU. That
+ * allows demultiplex interrupts to do rcu based management of
+ * the child interrupts.
+ * This also allows us to use rcu in kstat_irqs_usr().
+ */
+ call_rcu(&desc->rcu, delayed_free_desc);
+}
+
static void free_desc(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
@@ -484,14 +492,7 @@ static void free_desc(unsigned int irq)
*/
irq_sysfs_del(desc);
delete_irq_desc(irq);
-
- /*
- * We free the descriptor, masks and stat fields via RCU. That
- * allows demultiplex interrupts to do rcu based management of
- * the child interrupts.
- * This also allows us to use rcu in kstat_irqs_usr().
- */
- call_rcu(&desc->rcu, delayed_free_desc);
+ irq_desc_put_ref(desc);
}
static int alloc_descs(unsigned int start, unsigned int cnt, int node,
@@ -543,7 +544,8 @@ static bool irq_expand_nr_irqs(unsigned int nr)
{
if (nr > MAX_SPARSE_IRQS)
return false;
- nr_irqs = nr;
+ total_nr_irqs = nr;
+ irq_proc_calc_prec();
return true;
}
@@ -557,21 +559,22 @@ int __init early_irq_init(void)
/* Let arch update nr_irqs and return the nr of preallocated irqs */
initcnt = arch_probe_nr_irqs();
printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n",
- NR_IRQS, nr_irqs, initcnt);
+ NR_IRQS, total_nr_irqs, initcnt);
- if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS))
- nr_irqs = MAX_SPARSE_IRQS;
+ if (WARN_ON(total_nr_irqs > MAX_SPARSE_IRQS))
+ total_nr_irqs = MAX_SPARSE_IRQS;
if (WARN_ON(initcnt > MAX_SPARSE_IRQS))
initcnt = MAX_SPARSE_IRQS;
- if (initcnt > nr_irqs)
- nr_irqs = initcnt;
+ if (initcnt > total_nr_irqs)
+ total_nr_irqs = initcnt;
for (i = 0; i < initcnt; i++) {
desc = alloc_desc(i, node, 0, NULL, NULL);
irq_insert_desc(i, desc);
}
+ irq_proc_calc_prec();
return arch_early_irq_init();
}
@@ -592,7 +595,7 @@ int __init early_irq_init(void)
init_irq_default_affinity();
- printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS);
+ pr_info("NR_IRQS: %d\n", NR_IRQS);
count = ARRAY_SIZE(irq_desc);
@@ -602,6 +605,7 @@ int __init early_irq_init(void)
goto __free_desc_res;
}
+ irq_proc_calc_prec();
return arch_early_irq_init();
__free_desc_res:
@@ -862,7 +866,7 @@ void irq_free_descs(unsigned int from, unsigned int cnt)
{
int i;
- if (from >= nr_irqs || (from + cnt) > nr_irqs)
+ if (from >= total_nr_irqs || (from + cnt) > total_nr_irqs)
return;
guard(mutex)(&sparse_irq_lock);
@@ -911,7 +915,7 @@ int __ref __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int no
if (irq >=0 && start != irq)
return -EEXIST;
- if (start + cnt > nr_irqs) {
+ if (start + cnt > total_nr_irqs) {
if (!irq_expand_nr_irqs(start + cnt))
return -ENOMEM;
}
@@ -923,11 +927,15 @@ EXPORT_SYMBOL_GPL(__irq_alloc_descs);
* irq_get_next_irq - get next allocated irq number
* @offset: where to start the search
*
- * Returns next irq number after offset or nr_irqs if none is found.
+ * Returns next irq number after offset or total_nr_irqs if none is found.
*/
unsigned int irq_get_next_irq(unsigned int offset)
{
- return irq_find_at_or_after(offset);
+ struct irq_desc *desc;
+
+ guard(rcu)();
+ desc = irq_find_desc_at_or_after(offset);
+ return desc ? irq_desc_get_irq(desc) : total_nr_irqs;
}
struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index cc93abf..f15c9f1 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -20,6 +20,8 @@
#include <linux/smp.h>
#include <linux/fs.h>
+#include "proc.h"
+
static LIST_HEAD(irq_domain_list);
static DEFINE_MUTEX(irq_domain_mutex);
@@ -1532,6 +1534,7 @@ int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq,
irq_data->chip = (struct irq_chip *)(chip ? chip : &no_irq_chip);
irq_data->chip_data = chip_data;
+ irq_proc_update_chip(chip);
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_set_hwirq_and_chip);
@@ -2081,7 +2084,7 @@ static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq
#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
-#include "internals.h"
+#include "debugfs.h"
static struct dentry *domain_dir;
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 2e80724..7eb07e3 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -1802,6 +1802,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
__enable_irq(desc);
}
+ irq_proc_update_valid(desc);
raw_spin_unlock_irqrestore(&desc->lock, flags);
chip_bus_sync_unlock(desc);
mutex_unlock(&desc->request_mutex);
@@ -1906,6 +1907,7 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
desc->affinity_hint = NULL;
#endif
+ irq_proc_update_valid(desc);
raw_spin_unlock_irqrestore(&desc->lock, flags);
/*
* Drop bus_lock here so the changes which were done in the chip
@@ -2026,24 +2028,32 @@ const void *free_irq(unsigned int irq, void *dev_id)
}
EXPORT_SYMBOL(free_irq);
-/* This function must be called with desc->lock held */
static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
{
+ struct irqaction *action = NULL;
const char *devname = NULL;
- desc->istate &= ~IRQS_NMI;
+ scoped_guard(raw_spinlock_irqsave, &desc->lock) {
+ irq_nmi_teardown(desc);
- if (!WARN_ON(desc->action == NULL)) {
- irq_pm_remove_action(desc, desc->action);
- devname = desc->action->name;
- unregister_handler_proc(irq, desc->action);
+ desc->istate &= ~IRQS_NMI;
- kfree(desc->action);
+ if (!WARN_ON(desc->action == NULL)) {
+ action = desc->action;
+ irq_pm_remove_action(desc, action);
+ devname = action->name;
+ }
desc->action = NULL;
+
+ irq_settings_clr_disable_unlazy(desc);
+ irq_shutdown_and_deactivate(desc);
}
- irq_settings_clr_disable_unlazy(desc);
- irq_shutdown_and_deactivate(desc);
+ irq_proc_update_valid(desc);
+
+ if (action)
+ unregister_handler_proc(irq, action);
+ kfree(action);
irq_release_resources(desc);
@@ -2067,8 +2077,6 @@ const void *free_nmi(unsigned int irq, void *dev_id)
if (WARN_ON(desc->depth == 0))
disable_nmi_nosync(irq);
- guard(raw_spinlock_irqsave)(&desc->lock);
- irq_nmi_teardown(desc);
return __cleanup_nmi(irq, desc);
}
@@ -2318,13 +2326,14 @@ int request_nmi(unsigned int irq, irq_handler_t handler,
/* Setup NMI state */
desc->istate |= IRQS_NMI;
retval = irq_nmi_setup(desc);
- if (retval) {
- __cleanup_nmi(irq, desc);
- return -EINVAL;
- }
- return 0;
}
+ if (retval) {
+ __cleanup_nmi(irq, desc);
+ return -EINVAL;
+ }
+ return 0;
+
err_irq_setup:
irq_chip_pm_put(&desc->irq_data);
err_out:
@@ -2428,8 +2437,10 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_
*action_ptr = action->next;
/* Demote from NMI if we killed the last action */
- if (!desc->action)
+ if (!desc->action) {
desc->istate &= ~IRQS_NMI;
+ irq_proc_update_valid(desc);
+ }
}
unregister_handler_proc(irq, action);
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index b0999a4..1b83572 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -10,6 +10,7 @@
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/interrupt.h>
+#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/mutex.h>
#include <linux/string.h>
@@ -326,7 +327,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action)
#undef MAX_NAMELEN
-#define MAX_NAMELEN 10
+#define MAX_NAMELEN 11
void register_irq_proc(unsigned int irq, struct irq_desc *desc)
{
@@ -348,7 +349,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc)
return;
/* create /proc/irq/1234 */
- sprintf(name, "%u", irq);
+ snprintf(name, MAX_NAMELEN, "%u", irq);
desc->dir = proc_mkdir(name, root_irq_dir);
if (!desc->dir)
return;
@@ -401,7 +402,7 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc)
#endif
remove_proc_entry("spurious", desc->dir);
- sprintf(name, "%u", irq);
+ snprintf(name, MAX_NAMELEN, "%u", irq);
remove_proc_entry(name, root_irq_dir);
}
@@ -439,77 +440,159 @@ void init_irq_proc(void)
register_irq_proc(irq, desc);
}
+void irq_proc_update_valid(struct irq_desc *desc)
+{
+ u32 set = _IRQ_PROC_VALID;
+
+ if (irq_settings_is_hidden(desc) || irq_desc_is_chained(desc) || !desc->action)
+ set = 0;
+
+ irq_settings_update_proc_valid(desc, set);
+}
+
#ifdef CONFIG_GENERIC_IRQ_SHOW
+#define ARCH_PROC_IRQDESC ((void *)0x00001111)
+
int __weak arch_show_interrupts(struct seq_file *p, int prec)
{
return 0;
}
+static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock);
+
+static struct irq_proc_constraints {
+ bool print_header;
+ unsigned int num_prec;
+ unsigned int chip_width;
+} irq_proc_constraints __read_mostly = {
+ .num_prec = 4,
+ .chip_width = 8,
+};
+
#ifndef ACTUAL_NR_IRQS
-# define ACTUAL_NR_IRQS irq_get_nr_irqs()
+# define ACTUAL_NR_IRQS total_nr_irqs
#endif
-int show_interrupts(struct seq_file *p, void *v)
+void irq_proc_calc_prec(void)
{
- const unsigned int nr_irqs = irq_get_nr_irqs();
- static int prec;
+ unsigned int prec, n;
- int i = *(loff_t *) v, j;
+ for (prec = 4, n = 10000; prec < 10 && n <= total_nr_irqs; ++prec)
+ n *= 10;
+
+ guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock);
+ if (prec > irq_proc_constraints.num_prec)
+ WRITE_ONCE(irq_proc_constraints.num_prec, prec);
+}
+
+void irq_proc_update_chip(const struct irq_chip *chip)
+{
+ unsigned int len = chip && chip->name ? strlen(chip->name) : 0;
+
+ if (!len || len <= READ_ONCE(irq_proc_constraints.chip_width))
+ return;
+
+ /* Can be invoked from interrupt disabled contexts */
+ guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock);
+ if (len > irq_proc_constraints.chip_width)
+ WRITE_ONCE(irq_proc_constraints.chip_width, len);
+}
+
+/* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */
+#define ZSTR1 " 0"
+#define ZSTR1_LEN (sizeof(ZSTR1) - 1)
+#define ZSTR16 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 \
+ ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1
+#define ZSTR256 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 \
+ ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16
+
+static inline void irq_proc_emit_zero_counts(struct seq_file *p, unsigned int zeros)
+{
+ if (!zeros)
+ return;
+
+ for (unsigned int n = min(zeros, 256); n; zeros -= n, n = min(zeros, 256))
+ seq_write(p, ZSTR256, n * ZSTR1_LEN);
+}
+
+static inline unsigned int irq_proc_emit_count(struct seq_file *p, unsigned int cnt,
+ unsigned int zeros)
+{
+ if (!cnt)
+ return zeros + 1;
+
+ irq_proc_emit_zero_counts(p, zeros);
+ seq_put_decimal_ull_width(p, " ", cnt, 10);
+ return 0;
+}
+
+void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts)
+{
+ unsigned int cpu, zeros = 0;
+
+ for_each_online_cpu(cpu)
+ zeros = irq_proc_emit_count(p, per_cpu(*cnts, cpu), zeros);
+ irq_proc_emit_zero_counts(p, zeros);
+}
+
+static int irq_seq_show(struct seq_file *p, void *v)
+{
+ struct irq_proc_constraints *constr = p->private;
+ struct irq_desc *desc = v;
struct irqaction *action;
- struct irq_desc *desc;
- if (i > ACTUAL_NR_IRQS)
- return 0;
+ /* Print header for the first interrupt? */
+ if (constr->print_header) {
+ unsigned int cpu;
- if (i == ACTUAL_NR_IRQS)
- return arch_show_interrupts(p, prec);
-
- /* print header and calculate the width of the first column */
- if (i == 0) {
- for (prec = 3, j = 1000; prec < 10 && j <= nr_irqs; ++prec)
- j *= 10;
-
- seq_printf(p, "%*s", prec + 8, "");
- for_each_online_cpu(j)
- seq_printf(p, "CPU%-8d", j);
+ seq_printf(p, "%*s", constr->num_prec + 8, "");
+ for_each_online_cpu(cpu)
+ seq_printf(p, "CPU%-8d", cpu);
seq_putc(p, '\n');
+ constr->print_header = false;
}
- guard(rcu)();
- desc = irq_to_desc(i);
- if (!desc || irq_settings_is_hidden(desc))
- return 0;
+ if (desc == ARCH_PROC_IRQDESC)
+ return arch_show_interrupts(p, constr->num_prec);
- if (!desc->action || irq_desc_is_chained(desc) || !desc->kstat_irqs)
- return 0;
+ seq_put_decimal_ull_width(p, "", irq_desc_get_irq(desc), constr->num_prec);
+ seq_putc(p, ':');
- seq_printf(p, "%*d:", prec, i);
- for_each_online_cpu(j) {
- unsigned int cnt = desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, j) : 0;
+ /*
+ * Always output per CPU interrupts. Output device interrupts only when
+ * desc::tot_count is not zero.
+ */
+ if (irq_settings_is_per_cpu(desc) || irq_settings_is_per_cpu_devid(desc) ||
+ data_race(desc->tot_count))
+ irq_proc_emit_counts(p, &desc->kstat_irqs->cnt);
+ else
+ irq_proc_emit_zero_counts(p, num_online_cpus());
- seq_put_decimal_ull_width(p, " ", cnt, 10);
- }
- seq_putc(p, ' ');
+ /* Enforce a visual gap */
+ seq_write(p, " ", 2);
guard(raw_spinlock_irq)(&desc->lock);
if (desc->irq_data.chip) {
if (desc->irq_data.chip->irq_print_chip)
desc->irq_data.chip->irq_print_chip(&desc->irq_data, p);
else if (desc->irq_data.chip->name)
- seq_printf(p, "%8s", desc->irq_data.chip->name);
+ seq_printf(p, "%-*s", constr->chip_width, desc->irq_data.chip->name);
else
- seq_printf(p, "%8s", "-");
+ seq_printf(p, "%-*s", constr->chip_width, "-");
} else {
- seq_printf(p, "%8s", "None");
+ seq_printf(p, "%-*s", constr->chip_width, "None");
}
+
+ seq_putc(p, ' ');
if (desc->irq_data.domain)
- seq_printf(p, " %*lu", prec, desc->irq_data.hwirq);
+ seq_put_decimal_ull_width(p, "", desc->irq_data.hwirq, constr->num_prec);
else
- seq_printf(p, " %*s", prec, "");
-#ifdef CONFIG_GENERIC_IRQ_SHOW_LEVEL
- seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge");
-#endif
+ seq_printf(p, " %*s", constr->num_prec, "");
+
+ if (IS_ENABLED(CONFIG_GENERIC_IRQ_SHOW_LEVEL))
+ seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge");
+
if (desc->name)
seq_printf(p, "-%-8s", desc->name);
@@ -523,4 +606,73 @@ int show_interrupts(struct seq_file *p, void *v)
seq_putc(p, '\n');
return 0;
}
+
+static void *irq_seq_next_desc(loff_t *pos)
+{
+ if (*pos > total_nr_irqs)
+ return NULL;
+
+ guard(rcu)();
+ for (;;) {
+ struct irq_desc *desc = irq_find_desc_at_or_after((unsigned int) *pos);
+
+ if (desc) {
+ *pos = irq_desc_get_irq(desc);
+ /*
+ * If valid for output then try to acquire a reference
+ * count on the descriptor so that it can't be freed
+ * after dropping RCU read lock on return.
+ */
+ if (irq_settings_proc_valid(desc) && irq_desc_get_ref(desc))
+ return desc;
+ (*pos)++;
+ } else {
+ *pos = total_nr_irqs;
+ return ARCH_PROC_IRQDESC;
+ }
+ }
+}
+
+static void *irq_seq_start(struct seq_file *f, loff_t *pos)
+{
+ if (!*pos) {
+ struct irq_proc_constraints *constr = f->private;
+
+ constr->num_prec = READ_ONCE(irq_proc_constraints.num_prec);
+ constr->chip_width = READ_ONCE(irq_proc_constraints.chip_width);
+ constr->print_header = true;
+ }
+ return irq_seq_next_desc(pos);
+}
+
+static void *irq_seq_next(struct seq_file *f, void *v, loff_t *pos)
+{
+ if (v && v != ARCH_PROC_IRQDESC)
+ irq_desc_put_ref(v);
+
+ (*pos)++;
+ return irq_seq_next_desc(pos);
+}
+
+static void irq_seq_stop(struct seq_file *f, void *v)
+{
+ if (v && v != ARCH_PROC_IRQDESC)
+ irq_desc_put_ref(v);
+}
+
+static const struct seq_operations irq_seq_ops = {
+ .start = irq_seq_start,
+ .next = irq_seq_next,
+ .stop = irq_seq_stop,
+ .show = irq_seq_show,
+};
+
+static int __init irq_proc_init(void)
+{
+ proc_create_seq_private("interrupts", 0, NULL, &irq_seq_ops,
+ sizeof(irq_proc_constraints), NULL);
+ return 0;
+}
+fs_initcall(irq_proc_init);
+
#endif
diff --git a/kernel/irq/proc.h b/kernel/irq/proc.h
new file mode 100644
index 0000000..0631d57
--- /dev/null
+++ b/kernel/irq/proc.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _KERNEL_IRQ_PROC_H
+#define _KERNEL_IRQ_PROC_H
+
+#if defined(CONFIG_PROC_FS) && defined(CONFIG_GENERIC_IRQ_SHOW)
+void irq_proc_calc_prec(void);
+void irq_proc_update_chip(const struct irq_chip *chip);
+#else
+static inline void irq_proc_calc_prec(void) { }
+static inline void irq_proc_update_chip(const struct irq_chip *chip) { }
+#endif
+
+#endif
diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h
index 00b3bd1..0a0c027 100644
--- a/kernel/irq/settings.h
+++ b/kernel/irq/settings.h
@@ -18,6 +18,7 @@ enum {
_IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY,
_IRQ_HIDDEN = IRQ_HIDDEN,
_IRQ_NO_DEBUG = IRQ_NO_DEBUG,
+ _IRQ_PROC_VALID = IRQ_RESERVED,
_IRQF_MODIFY_MASK = IRQF_MODIFY_MASK,
};
@@ -34,6 +35,7 @@ enum {
#define IRQ_DISABLE_UNLAZY GOT_YOU_MORON
#define IRQ_HIDDEN GOT_YOU_MORON
#define IRQ_NO_DEBUG GOT_YOU_MORON
+#define IRQ_RESERVED GOT_YOU_MORON
#undef IRQF_MODIFY_MASK
#define IRQF_MODIFY_MASK GOT_YOU_MORON
@@ -180,3 +182,14 @@ static inline bool irq_settings_no_debug(struct irq_desc *desc)
{
return desc->status_use_accessors & _IRQ_NO_DEBUG;
}
+
+static inline bool irq_settings_proc_valid(struct irq_desc *desc)
+{
+ return desc->status_use_accessors & _IRQ_PROC_VALID;
+}
+
+static inline void irq_settings_update_proc_valid(struct irq_desc *desc, u32 set)
+{
+ desc->status_use_accessors &= ~_IRQ_PROC_VALID;
+ desc->status_use_accessors |= (set & _IRQ_PROC_VALID);
+}
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 120fd73..f7e2dc2 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -292,6 +292,12 @@ void irq_work_sync(struct irq_work *work)
!arch_irq_work_has_interrupt()) {
rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work),
TASK_UNINTERRUPTIBLE);
+ /*
+ * Ensure irq_work_single() does not access @work
+ * after removing IRQ_WORK_BUSY. It is always
+ * accessed within a RCU-read section.
+ */
+ synchronize_rcu();
return;
}
@@ -302,6 +308,7 @@ EXPORT_SYMBOL_GPL(irq_work_sync);
static void run_irq_workd(unsigned int cpu)
{
+ guard(rcu)();
irq_work_run_list(this_cpu_ptr(&lazy_list));
}
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 94762de..1b592d8 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -357,20 +357,6 @@ int kho_radix_walk_tree(struct kho_radix_tree *tree,
}
EXPORT_SYMBOL_GPL(kho_radix_walk_tree);
-static void __kho_unpreserve(struct kho_radix_tree *tree,
- unsigned long pfn, unsigned long end_pfn)
-{
- unsigned int order;
-
- while (pfn < end_pfn) {
- order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
-
- kho_radix_del_page(tree, pfn, order);
-
- pfn += 1 << order;
- }
-}
-
/* For physically contiguous 0-order pages. */
static void kho_init_pages(struct page *page, unsigned long nr_pages)
{
@@ -762,19 +748,24 @@ int kho_add_subtree(const char *name, void *blob, size_t size)
goto out_pack;
}
- err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_PROP_NAME,
- &phys, sizeof(phys));
- if (err < 0)
- goto out_pack;
+ fdt_err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_PROP_NAME,
+ &phys, sizeof(phys));
+ if (fdt_err < 0)
+ goto out_del_node;
- err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_SIZE_PROP_NAME,
- &size_u64, sizeof(size_u64));
- if (err < 0)
- goto out_pack;
+ fdt_err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_SIZE_PROP_NAME,
+ &size_u64, sizeof(size_u64));
+ if (fdt_err < 0)
+ goto out_del_node;
WARN_ON_ONCE(kho_debugfs_blob_add(&kho_out.dbg, name, blob,
size, false));
+ err = 0;
+ goto out_pack;
+
+out_del_node:
+ fdt_del_node(root_fdt, off);
out_pack:
fdt_pack(root_fdt);
@@ -855,6 +846,37 @@ void kho_unpreserve_folio(struct folio *folio)
}
EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
+static unsigned int __kho_preserve_pages_order(unsigned long start_pfn,
+ unsigned long end_pfn)
+{
+ unsigned int order = min(count_trailing_zeros(start_pfn),
+ ilog2(end_pfn - start_pfn));
+
+ /*
+ * Make sure all the pages in a single preservation are in the same NUMA
+ * node. The restore machinery can not cope with a preservation spanning
+ * multiple NUMA nodes.
+ */
+ while (pfn_to_nid(start_pfn) != pfn_to_nid(start_pfn + (1UL << order) - 1))
+ order--;
+
+ return order;
+}
+
+static void __kho_unpreserve(struct kho_radix_tree *tree,
+ unsigned long pfn, unsigned long end_pfn)
+{
+ unsigned int order;
+
+ while (pfn < end_pfn) {
+ order = __kho_preserve_pages_order(pfn, end_pfn);
+
+ kho_radix_del_page(tree, pfn, order);
+
+ pfn += 1 << order;
+ }
+}
+
/**
* kho_preserve_pages - preserve contiguous pages across kexec
* @page: first page in the list.
@@ -880,16 +902,7 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
}
while (pfn < end_pfn) {
- unsigned int order =
- min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
-
- /*
- * Make sure all the pages in a single preservation are in the
- * same NUMA node. The restore machinery can not cope with a
- * preservation spanning multiple NUMA nodes.
- */
- while (pfn_to_nid(pfn) != pfn_to_nid(pfn + (1UL << order) - 1))
- order--;
+ unsigned int order = __kho_preserve_pages_order(pfn, end_pfn);
err = kho_radix_add_page(tree, pfn, order);
if (err) {
@@ -1702,7 +1715,7 @@ int kho_fill_kimage(struct kimage *image)
int err = 0;
struct kexec_buf scratch;
- if (!kho_enable)
+ if (!kho_enable || image->type == KEXEC_TYPE_CRASH)
return 0;
image->kho.fdt = virt_to_phys(kho_out.fdt);
diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c
index a3327a2..7a42385 100644
--- a/kernel/liveupdate/luo_session.c
+++ b/kernel/liveupdate/luo_session.c
@@ -514,11 +514,12 @@ int luo_session_deserialize(void)
{
struct luo_session_header *sh = &luo_session_global.incoming;
static bool is_deserialized;
- static int err;
+ static int saved_err;
+ int err;
/* If has been deserialized, always return the same error code */
if (is_deserialized)
- return err;
+ return saved_err;
is_deserialized = true;
if (!sh->active)
@@ -547,7 +548,8 @@ int luo_session_deserialize(void)
pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n",
(int)sizeof(sh->ser[i].name),
sh->ser[i].name, session);
- return PTR_ERR(session);
+ err = PTR_ERR(session);
+ goto save_err;
}
err = luo_session_insert(sh, session);
@@ -555,7 +557,7 @@ int luo_session_deserialize(void)
pr_warn("Failed to insert session [%s] %pe\n",
session->name, ERR_PTR(err));
luo_session_free(session);
- return err;
+ goto save_err;
}
scoped_guard(mutex, &session->mutex) {
@@ -565,7 +567,7 @@ int luo_session_deserialize(void)
if (err) {
pr_warn("Failed to deserialize files for session [%s] %pe\n",
session->name, ERR_PTR(err));
- return err;
+ goto save_err;
}
}
@@ -574,6 +576,9 @@ int luo_session_deserialize(void)
sh->ser = NULL;
return 0;
+save_err:
+ saved_err = err;
+ return err;
}
int luo_session_serialize(void)
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 0953462..89d01f7 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -763,6 +763,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
raw_spin_lock_irqsave(&lock->wait_lock, flags);
raw_spin_lock(¤t->blocked_lock);
__set_task_blocked_on(current, lock);
+ set_current_state(state);
if (opt_acquired)
break;
@@ -980,9 +981,8 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
__releases(lock)
{
- struct task_struct *next = NULL;
+ struct task_struct *donor, *next = NULL;
struct mutex_waiter *waiter;
- DEFINE_WAKE_Q(wake_q);
unsigned long owner;
unsigned long flags;
@@ -990,6 +990,14 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
__release(lock);
/*
+ * Ensures the proxy donor stack is stable across unlock and handoff.
+ * Specifically, it avoids the case where current->blocked_donor is
+ * NULL when it is inspected while doing the unlock, but a preemption
+ * before taking the wake_lock would make it set and a hand-off is
+ * missed.
+ */
+ guard(preempt)();
+ /*
* Release the lock before (potentially) taking the spinlock such that
* other contenders can get on with things ASAP.
*
@@ -1001,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
MUTEX_WARN_ON(__owner_task(owner) != current);
MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
+ if (sched_proxy_exec() && current->blocked_donor) {
+ /* force handoff if we have a blocked_donor */
+ owner = MUTEX_FLAG_HANDOFF;
+ break;
+ }
+
if (owner & MUTEX_FLAG_HANDOFF)
break;
@@ -1013,20 +1027,53 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
}
raw_spin_lock_irqsave(&lock->wait_lock, flags);
+ raw_spin_lock(¤t->blocked_lock);
debug_mutex_unlock(lock);
- waiter = lock->first_waiter;
- if (waiter) {
- next = waiter->task;
+ if (sched_proxy_exec()) {
+ /*
+ * If we have a task boosting current, and that task was boosting
+ * current through this lock, hand the lock to that task, as that
+ * is the highest waiter, as selected by the scheduling function.
+ */
+ donor = current->blocked_donor;
+ if (donor) {
+ struct mutex *next_lock;
+
+ raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
+ next_lock = __get_task_blocked_on(donor);
+ if (next_lock == lock) {
+ next = get_task_struct(donor);
+ __clear_task_blocked_on(next, lock);
+ current->blocked_donor = NULL;
+ }
+ raw_spin_unlock(&donor->blocked_lock);
+ }
+ }
+
+ /*
+ * Failing that, pick first on the wait list.
+ */
+ waiter = lock->first_waiter;
+ if (!next && waiter) {
+ next = get_task_struct(waiter->task);
+
+ raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
debug_mutex_wake_waiter(lock, waiter);
- set_task_blocked_on_waking(next, lock);
- wake_q_add(&wake_q, next);
+ __clear_task_blocked_on(next, lock);
+ raw_spin_unlock(&next->blocked_lock);
+
}
if (owner & MUTEX_FLAG_HANDOFF)
__mutex_handoff(lock, next);
- raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
+ raw_spin_unlock(¤t->blocked_lock);
+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+ if (next) {
+ wake_up_process(next);
+ put_task_struct(next);
+ }
}
#ifndef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4f386ea..22b9d5c 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -484,6 +484,7 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod
static __always_inline void
rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -492,6 +493,7 @@ rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
static __always_inline void
rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -1092,6 +1094,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
static int __sched
try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -1319,6 +1322,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
*/
static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
struct rt_mutex_base *lock)
+ __must_hold(&lock->wait_lock)
{
struct rt_mutex_waiter *waiter;
@@ -1558,6 +1562,9 @@ static void __sched remove_waiter(struct rt_mutex_base *lock,
lockdep_assert_held(&lock->wait_lock);
+ if (!waiter_task) /* never enqueued */
+ return;
+
scoped_guard(raw_spinlock, &waiter_task->pi_lock) {
rt_mutex_dequeue(lock, waiter);
waiter_task->pi_blocked_on = NULL;
diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index 124219a..5d48d64 100644
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -41,6 +41,7 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
unsigned int state,
struct lockdep_map *nest_lock,
unsigned int subclass)
+ __cond_acquires(0, lock)
{
int ret;
@@ -67,13 +68,27 @@ EXPORT_SYMBOL(rt_mutex_base_init);
*/
void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
@@ -86,7 +101,14 @@ EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
*/
void __sched rt_mutex_lock(struct rt_mutex *lock)
{
- __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);
+ if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0) == 0)
+ return;
+ /*
+ * The code below is never reached because __rt_mutex_lock_common() only
+ * returns an error code if interrupted by a signal or upon a timeout.
+ */
+ WARN_ON_ONCE(true);
+ __acquire(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock);
#endif
@@ -157,6 +179,7 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock)
{
mutex_release(&lock->dep_map, _RET_IP_);
__rt_mutex_unlock(&lock->rtmutex);
+ __release(lock);
}
EXPORT_SYMBOL_GPL(rt_mutex_unlock);
@@ -182,6 +205,7 @@ int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
*/
bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
struct rt_wake_q_head *wqh)
+ __must_hold(&lock->wait_lock)
{
lockdep_assert_held(&lock->wait_lock);
@@ -312,6 +336,7 @@ int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
struct wake_q_head *wake_q)
+ __must_hold(&lock->wait_lock)
{
int ret;
@@ -365,7 +390,7 @@ int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
raw_spin_lock_irq(&lock->wait_lock);
ret = __rt_mutex_start_proxy_lock(lock, waiter, task, &wake_q);
- if (unlikely(ret))
+ if (unlikely(ret < 0))
remove_waiter(lock, waiter);
preempt_disable();
raw_spin_unlock_irq(&lock->wait_lock);
diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h
index 6c12452..d62b49b 100644
--- a/kernel/locking/ww_mutex.h
+++ b/kernel/locking/ww_mutex.h
@@ -324,7 +324,7 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter,
* blocked_on to PROXY_WAKING. Otherwise we can see
* circular blocked_on relationships that can't resolve.
*/
- set_task_blocked_on_waking(waiter->task, lock);
+ clear_task_blocked_on(waiter->task, lock);
wake_q_add(wake_q, waiter->task);
}
@@ -383,7 +383,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock,
* are waking the mutex owner, who may be currently
* blocked on a different mutex.
*/
- set_task_blocked_on_waking(owner, NULL);
+ clear_task_blocked_on(owner, NULL);
wake_q_add(wake_q, owner);
}
return true;
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 68c17da..130043b 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -272,11 +272,24 @@ static bool ptrace_has_cap(struct user_namespace *ns, unsigned int mode)
return ns_capable(ns, CAP_SYS_PTRACE);
}
+static bool task_still_dumpable(struct task_struct *task, unsigned int mode)
+{
+ struct mm_struct *mm = task->mm;
+ if (mm) {
+ if (get_dumpable(mm) == SUID_DUMP_USER)
+ return true;
+ return ptrace_has_cap(mm->user_ns, mode);
+ }
+
+ if (task->user_dumpable)
+ return true;
+ return ptrace_has_cap(&init_user_ns, mode);
+}
+
/* Returns 0 on success, -errno on denial. */
static int __ptrace_may_access(struct task_struct *task, unsigned int mode)
{
const struct cred *cred = current_cred(), *tcred;
- struct mm_struct *mm;
kuid_t caller_uid;
kgid_t caller_gid;
@@ -337,11 +350,8 @@ static int __ptrace_may_access(struct task_struct *task, unsigned int mode)
* Pairs with a write barrier in commit_creds().
*/
smp_rmb();
- mm = task->mm;
- if (mm &&
- ((get_dumpable(mm) != SUID_DUMP_USER) &&
- !ptrace_has_cap(mm->user_ns, mode)))
- return -EPERM;
+ if (!task_still_dumpable(task, mode))
+ return -EPERM;
return security_ptrace_access_check(task, mode);
}
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 0d01cd8..7c2f7cc 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -897,11 +897,9 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp
{
int cpu;
- for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {
- if (!(mask & (1UL << (cpu - snp->grplo))))
- continue;
- srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);
- }
+ for (cpu = snp->grplo; cpu <= snp->grphi; cpu++)
+ if ((mask & (1UL << (cpu - snp->grplo))) && rcu_cpu_beenfullyonline(cpu))
+ srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);
}
/*
@@ -1322,7 +1320,9 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
*/
idx = __srcu_read_lock_nmisafe(ssp);
ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);
- if (ss_state < SRCU_SIZE_WAIT_CALL)
+ // If !rcu_cpu_beenfullyonline(), interrupts are still disabled,
+ // so no migration is possible in either direction from this CPU.
+ if (ss_state < SRCU_SIZE_WAIT_CALL || !rcu_cpu_beenfullyonline(raw_smp_processor_id()))
sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());
else
sdp = raw_cpu_ptr(ssp->sda);
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 55df6d3..3cbf79b 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -969,14 +969,11 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp)
if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) {
int cpu = rdp->cpu;
struct rcu_snap_record *rsrp;
- struct kernel_cpustat *kcsp;
-
- kcsp = &kcpustat_cpu(cpu);
rsrp = &rdp->snap_record;
- rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu);
- rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu);
- rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu);
+ rsrp->cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu);
+ rsrp->cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu);
+ rsrp->cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu);
rsrp->nr_hardirqs = kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu);
rsrp->nr_softirqs = kstat_cpu_softirqs_sum(cpu);
rsrp->nr_csw = nr_context_switches_cpu(cpu);
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index b67532c..cf7ae51 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -479,7 +479,6 @@ static void print_cpu_stat_info(int cpu)
{
struct rcu_snap_record rsr, *rsrp;
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
- struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu);
if (!rcu_cpu_stall_cputime)
return;
@@ -488,9 +487,9 @@ static void print_cpu_stat_info(int cpu)
if (rsrp->gp_seq != rdp->gp_seq)
return;
- rsr.cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu);
- rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu);
- rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu);
+ rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu);
+ rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu);
+ rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu);
pr_err("\t hardirqs softirqs csw/system\n");
pr_err("\t number: %8lld %10d %12lld\n",
diff --git a/kernel/rseq.c b/kernel/rseq.c
index 38d3ef5..e75e3a5e 100644
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -236,11 +236,6 @@ static int __init rseq_debugfs_init(void)
}
__initcall(rseq_debugfs_init);
-static bool rseq_set_ids(struct task_struct *t, struct rseq_ids *ids, u32 node_id)
-{
- return rseq_set_ids_get_csaddr(t, ids, node_id, NULL);
-}
-
static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs)
{
struct rseq __user *urseq = t->rseq.usrptr;
@@ -258,14 +253,16 @@ static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs)
static void rseq_slowpath_update_usr(struct pt_regs *regs)
{
/*
- * Preserve rseq state and user_irq state. The generic entry code
- * clears user_irq on the way out, the non-generic entry
- * architectures are not having user_irq.
+ * Preserve has_rseq and user_irq state. The generic entry code clears
+ * user_irq on the way out, the non-generic entry architectures are not
+ * setting user_irq.
*/
- const struct rseq_event evt_mask = { .has_rseq = true, .user_irq = true, };
+ const struct rseq_event evt_mask = {
+ .has_rseq = RSEQ_HAS_RSEQ_VERSION_MASK,
+ .user_irq = true,
+ };
struct task_struct *t = current;
struct rseq_ids ids;
- u32 node_id;
bool event;
if (unlikely(t->flags & PF_EXITING))
@@ -301,9 +298,9 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
if (!event)
return;
- node_id = cpu_to_node(ids.cpu_id);
+ ids.node_id = cpu_to_node(ids.cpu_id);
- if (unlikely(!rseq_update_usr(t, regs, &ids, node_id))) {
+ if (unlikely(!rseq_update_usr(t, regs, &ids))) {
/*
* Clear the errors just in case this might survive magically, but
* leave the rest intact.
@@ -335,8 +332,9 @@ void __rseq_handle_slowpath(struct pt_regs *regs)
void __rseq_signal_deliver(int sig, struct pt_regs *regs)
{
rseq_stat_inc(rseq_stats.signal);
+
/*
- * Don't update IDs, they are handled on exit to user if
+ * Don't update IDs yet, they are handled on exit to user if
* necessary. The important thing is to abort a critical section of
* the interrupted context as after this point the instruction
* pointer in @regs points to the signal handler.
@@ -349,6 +347,13 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs)
current->rseq.event.error = 0;
force_sigsegv(sig);
}
+
+ /*
+ * In legacy mode, force the update of IDs before returning to user
+ * space to stay compatible.
+ */
+ if (!rseq_v2(current))
+ rseq_force_update();
}
/*
@@ -384,19 +389,22 @@ void rseq_syscall(struct pt_regs *regs)
static bool rseq_reset_ids(void)
{
- struct rseq_ids ids = {
- .cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
- .mm_cid = 0,
- };
+ struct rseq __user *rseq = current->rseq.usrptr;
/*
* If this fails, terminate it because this leaves the kernel in
* stupid state as exit to user space will try to fixup the ids
* again.
*/
- if (rseq_set_ids(current, &ids, 0))
- return true;
+ scoped_user_rw_access(rseq, efault) {
+ unsafe_put_user(0, &rseq->cpu_id_start, efault);
+ unsafe_put_user(RSEQ_CPU_ID_UNINITIALIZED, &rseq->cpu_id, efault);
+ unsafe_put_user(0, &rseq->node_id, efault);
+ unsafe_put_user(0, &rseq->mm_cid, efault);
+ }
+ return true;
+efault:
force_sig(SIGSEGV);
return false;
}
@@ -404,70 +412,29 @@ static bool rseq_reset_ids(void)
/* The original rseq structure size (including padding) is 32 bytes. */
#define ORIG_RSEQ_SIZE 32
-/*
- * sys_rseq - setup restartable sequences for caller thread.
- */
-SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig)
+static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u32 sig)
{
u32 rseqfl = 0;
+ u8 version = 1;
- if (flags & RSEQ_FLAG_UNREGISTER) {
- if (flags & ~RSEQ_FLAG_UNREGISTER)
- return -EINVAL;
- /* Unregister rseq for current thread. */
- if (current->rseq.usrptr != rseq || !current->rseq.usrptr)
- return -EINVAL;
- if (rseq_len != current->rseq.len)
- return -EINVAL;
- if (current->rseq.sig != sig)
- return -EPERM;
- if (!rseq_reset_ids())
- return -EFAULT;
- rseq_reset(current);
- return 0;
- }
-
- if (unlikely(flags & ~(RSEQ_FLAG_SLICE_EXT_DEFAULT_ON)))
- return -EINVAL;
-
- if (current->rseq.usrptr) {
- /*
- * If rseq is already registered, check whether
- * the provided address differs from the prior
- * one.
- */
- if (current->rseq.usrptr != rseq || rseq_len != current->rseq.len)
- return -EINVAL;
- if (current->rseq.sig != sig)
- return -EPERM;
- /* Already registered. */
- return -EBUSY;
- }
-
- /*
- * If there was no rseq previously registered, ensure the provided rseq
- * is properly aligned, as communcated to user-space through the ELF
- * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq
- * size, the required alignment is the original struct rseq alignment.
- *
- * The rseq_len is required to be greater or equal to the original rseq
- * size. In order to be valid, rseq_len is either the original rseq size,
- * or large enough to contain all supported fields, as communicated to
- * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE.
- */
- if (rseq_len < ORIG_RSEQ_SIZE ||
- (rseq_len == ORIG_RSEQ_SIZE && !IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE)) ||
- (rseq_len != ORIG_RSEQ_SIZE && (!IS_ALIGNED((unsigned long)rseq, rseq_alloc_align()) ||
- rseq_len < offsetof(struct rseq, end))))
- return -EINVAL;
if (!access_ok(rseq, rseq_len))
return -EFAULT;
- if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION)) {
- rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE;
- if (rseq_slice_extension_enabled() &&
- (flags & RSEQ_FLAG_SLICE_EXT_DEFAULT_ON))
- rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED;
+ /*
+ * Architectures, which use the generic IRQ entry code (at least) enable
+ * registrations with a size greater than the original v1 fixed sized
+ * @rseq_len, which has been validated already to utilize the optimized
+ * v2 ABI mode which also enables extended RSEQ features beyond MMCID.
+ */
+ if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && rseq_len > ORIG_RSEQ_SIZE)
+ version = 2;
+
+ if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION) && version > 1) {
+ if (rseq_slice_extension_enabled()) {
+ rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE;
+ if (flags & RSEQ_FLAG_SLICE_EXT_DEFAULT_ON)
+ rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED;
+ }
}
scoped_user_write_access(rseq, efault) {
@@ -485,7 +452,15 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32
unsafe_put_user(RSEQ_CPU_ID_UNINITIALIZED, &rseq->cpu_id, efault);
unsafe_put_user(0U, &rseq->node_id, efault);
unsafe_put_user(0U, &rseq->mm_cid, efault);
- unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
+
+ /*
+ * All fields past mm_cid are only valid for non-legacy v2
+ * registrations.
+ */
+ if (version > 1) {
+ if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION))
+ unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
+ }
}
/*
@@ -501,11 +476,10 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32
#endif
/*
- * If rseq was previously inactive, and has just been
- * registered, ensure the cpu_id_start and cpu_id fields
- * are updated before returning to user-space.
+ * Ensure the cpu_id_start and cpu_id fields are updated before
+ * returning to user-space.
*/
- current->rseq.event.has_rseq = true;
+ current->rseq.event.has_rseq = version;
rseq_force_update();
return 0;
@@ -513,6 +487,80 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32
return -EFAULT;
}
+static long rseq_unregister(struct rseq __user * rseq, u32 rseq_len, int flags, u32 sig)
+{
+ if (flags & ~RSEQ_FLAG_UNREGISTER)
+ return -EINVAL;
+ if (current->rseq.usrptr != rseq || !current->rseq.usrptr)
+ return -EINVAL;
+ if (rseq_len != current->rseq.len)
+ return -EINVAL;
+ if (current->rseq.sig != sig)
+ return -EPERM;
+ if (!rseq_reset_ids())
+ return -EFAULT;
+ rseq_reset(current);
+ return 0;
+}
+
+static long rseq_reregister(struct rseq __user * rseq, u32 rseq_len, u32 sig)
+{
+ /*
+ * If rseq is already registered, check whether the provided address
+ * differs from the prior one.
+ */
+ if (current->rseq.usrptr != rseq || rseq_len != current->rseq.len)
+ return -EINVAL;
+ if (current->rseq.sig != sig)
+ return -EPERM;
+ /* Already registered. */
+ return -EBUSY;
+}
+
+static bool rseq_length_valid(struct rseq __user *rseq, unsigned int rseq_len)
+{
+ /*
+ * Ensure the provided rseq is properly aligned, as communicated to
+ * user-space through the ELF auxiliary vector AT_RSEQ_ALIGN. If
+ * rseq_len is the original rseq size, the required alignment is the
+ * original struct rseq alignment.
+ *
+ * In order to be valid, rseq_len is either the original rseq size, or
+ * large enough to contain all supported fields, as communicated to
+ * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE.
+ */
+ if (rseq_len < ORIG_RSEQ_SIZE)
+ return false;
+
+ if (rseq_len == ORIG_RSEQ_SIZE)
+ return IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE);
+
+ return IS_ALIGNED((unsigned long)rseq, rseq_alloc_align()) &&
+ rseq_len >= offsetof(struct rseq, end);
+}
+
+#define RSEQ_FLAGS_SUPPORTED (RSEQ_FLAG_SLICE_EXT_DEFAULT_ON)
+
+/*
+ * sys_rseq - Register or unregister restartable sequences for the caller thread.
+ */
+SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig)
+{
+ if (flags & RSEQ_FLAG_UNREGISTER)
+ return rseq_unregister(rseq, rseq_len, flags, sig);
+
+ if (unlikely(flags & ~RSEQ_FLAGS_SUPPORTED))
+ return -EINVAL;
+
+ if (current->rseq.usrptr)
+ return rseq_reregister(rseq, rseq_len, sig);
+
+ if (!rseq_length_valid(rseq, rseq_len))
+ return -EINVAL;
+
+ return rseq_register(rseq, rseq_len, flags, sig);
+}
+
#ifdef CONFIG_RSEQ_SLICE_EXTENSION
struct slice_timer {
struct hrtimer timer;
@@ -713,6 +761,8 @@ int rseq_slice_extension_prctl(unsigned long arg2, unsigned long arg3)
return -ENOTSUPP;
if (!current->rseq.usrptr)
return -ENXIO;
+ if (!rseq_v2(current))
+ return -ENOTSUPP;
/* No change? */
if (enable == !!current->rseq.slice.state.enabled)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index da20fb6..a2f8680 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -537,13 +537,22 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
/* need a wrapper since we may need to trace from modules */
EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
-/* Call via the helper macro trace_set_current_state. */
+/*
+ * Call via the helper macro trace_set_current_state.
+ * Calls to this function MUST be guarded by a
+ * tracepoint_enabled(sched_set_state_tp)
+ */
void __trace_set_current_state(int state_value)
{
- trace_sched_set_state_tp(current, state_value);
+ trace_call__sched_set_state_tp(current, state_value);
}
EXPORT_SYMBOL(__trace_set_current_state);
+int task_llc(const struct task_struct *p)
+{
+ return per_cpu(sd_llc_id, task_cpu(p));
+}
+
/*
* Serialization rules:
*
@@ -615,6 +624,12 @@ EXPORT_SYMBOL(__trace_set_current_state);
* [ The astute reader will observe that it is possible for two tasks on one
* CPU to have ->on_cpu = 1 at the same time. ]
*
+ * p->is_blocked <- { 0, 1 }:
+ *
+ * is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks
+ * if the task is blocked. Traditionally this would mirror p->on_rq, however
+ * due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge.
+ *
* task_cpu(p): is changed by set_task_cpu(), the rules are:
*
* - Don't call set_task_cpu() on a blocked task:
@@ -1203,9 +1218,13 @@ static void __resched_curr(struct rq *rq, int tif)
}
}
+/*
+ * Calls to this function MUST be guarded by a
+ * tracepoint_enabled(sched_set_need_resched_tp)
+ */
void __trace_set_need_resched(struct task_struct *curr, int tif)
{
- trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif);
+ trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif);
}
EXPORT_SYMBOL_GPL(__trace_set_need_resched);
@@ -2223,8 +2242,29 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
dequeue_task(rq, p, flags);
}
-static void block_task(struct rq *rq, struct task_struct *p, int flags)
+static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
{
+ int flags = DEQUEUE_NOCLOCK;
+
+ p->sched_contributes_to_load =
+ (task_state & TASK_UNINTERRUPTIBLE) &&
+ !(task_state & TASK_NOLOAD) &&
+ !(task_state & TASK_FROZEN);
+
+ if (unlikely(is_special_task_state(task_state)))
+ flags |= DEQUEUE_SPECIAL;
+
+ /*
+ * __schedule() ttwu()
+ * prev_state = prev->state; if (p->on_rq && ...)
+ * if (prev_state) goto out;
+ * p->on_rq = 0; smp_acquire__after_ctrl_dep();
+ * p->state = TASK_WAKING
+ *
+ * Where __schedule() and ttwu() have matching control dependencies.
+ *
+ * After this, schedule() must not care about p->state any more.
+ */
if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags))
__block_task(rq, p);
}
@@ -3685,6 +3725,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
*/
static inline void ttwu_do_wakeup(struct task_struct *p)
{
+ p->is_blocked = 0;
WRITE_ONCE(p->__state, TASK_RUNNING);
trace_sched_wakeup(p);
}
@@ -3701,6 +3742,65 @@ void update_rq_avg_idle(struct rq *rq)
rq->idle_stamp = 0;
}
+#ifdef CONFIG_SCHED_PROXY_EXEC
+static void zap_balance_callbacks(struct rq *rq);
+
+static inline void proxy_reset_donor(struct rq *rq)
+{
+ WARN_ON_ONCE(rq->donor == rq->curr);
+
+ put_prev_set_next_task(rq, rq->donor, rq->curr);
+ rq_set_donor(rq, rq->curr);
+ zap_balance_callbacks(rq);
+ resched_curr(rq);
+}
+
+/*
+ * Checks to see if task p has been proxy-migrated to another rq
+ * and needs to be returned. If so, we deactivate the task here
+ * so that it can be properly woken up on the p->wake_cpu
+ * (or whichever cpu select_task_rq() picks at the bottom of
+ * try_to_wake_up()
+ */
+static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
+{
+ /*
+ * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
+ *
+ * However, proxy_set_task_cpu() is such that it preserves the
+ * original cpu in p->wake_cpu while migrating p for proxy reasons
+ * (possibly outside of the allowed p->cpus_ptr).
+ *
+ * Furthermore, migration_cpu_stop() / __migrate_swap_task(), will
+ * only set p->wake_cpu when !p->on_rq, and since here p->on_rq, this
+ * will not apply. But if it did, this check is the safe way around
+ * and would migrate.
+ */
+ if (task_cpu(p) == p->wake_cpu)
+ return false;
+
+ scoped_guard(raw_spinlock, &p->blocked_lock) {
+ /* Task is waking up; clear any blocked_on relationship */
+ __clear_task_blocked_on(p, NULL);
+
+ /* If already current, don't need to return migrate */
+ if (task_current(rq, p))
+ return false;
+
+ /* If we're return migrating the rq->donor, switch it out for idle */
+ if (task_current_donor(rq, p))
+ proxy_reset_donor(rq);
+ }
+ block_task(rq, p, TASK_WAKING);
+ return true;
+}
+#else /* !CONFIG_SCHED_PROXY_EXEC */
+static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
+{
+ return false;
+}
+#endif /* CONFIG_SCHED_PROXY_EXEC */
+
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
struct rq_flags *rf)
@@ -3765,28 +3865,28 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
*/
static int ttwu_runnable(struct task_struct *p, int wake_flags)
{
- struct rq_flags rf;
- struct rq *rq;
- int ret = 0;
+ ACQUIRE(__task_rq_lock, guard)(p);
+ struct rq *rq = guard.rq;
- rq = __task_rq_lock(p, &rf);
- if (task_on_rq_queued(p)) {
- update_rq_clock(rq);
+ if (!task_on_rq_queued(p))
+ return 0;
+
+ update_rq_clock(rq);
+ if (p->is_blocked) {
if (p->se.sched_delayed)
enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED);
- if (!task_on_cpu(rq, p)) {
- /*
- * When on_rq && !on_cpu the task is preempted, see if
- * it should preempt the task that is current now.
- */
- wakeup_preempt(rq, p, wake_flags);
- }
- ttwu_do_wakeup(p);
- ret = 1;
+ if (proxy_needs_return(rq, p))
+ return 0;
}
- __task_rq_unlock(rq, p, &rf);
-
- return ret;
+ if (!task_on_cpu(rq, p)) {
+ /*
+ * When on_rq && !on_cpu the task is preempted, see if
+ * it should preempt the task that is current now.
+ */
+ wakeup_preempt(rq, p, wake_flags);
+ }
+ ttwu_do_wakeup(p);
+ return 1;
}
void sched_ttwu_pending(void *arg)
@@ -4173,6 +4273,9 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
* it disabling IRQs (this allows not taking ->pi_lock).
*/
WARN_ON_ONCE(p->se.sched_delayed);
+ WARN_ON_ONCE(p->is_blocked);
+ /* If p is current, we know we can run here, so clear blocked_on */
+ clear_task_blocked_on(p, NULL);
if (!ttwu_state_match(p, state, &success))
goto out;
@@ -4189,6 +4292,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
smp_mb__after_spinlock();
+
if (!ttwu_state_match(p, state, &success))
break;
@@ -4297,6 +4401,16 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
wake_flags |= WF_MIGRATED;
psi_ttwu_dequeue(p);
set_task_cpu(p, cpu);
+ } else if (cpu != p->wake_cpu) {
+ /*
+ * If we were proxy-migrated to cpu, then
+ * select_task_rq() picks cpu instead of wake_cpu
+ * to return to, we won't call set_task_cpu(),
+ * leaving a stale wake_cpu pointing to where we
+ * proxy-migrated from. So just fixup wake_cpu here
+ * if its not correct
+ */
+ p->wake_cpu = cpu;
}
ttwu_queue(p, cpu, wake_flags);
@@ -4458,10 +4572,12 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
p->se.nr_migrations = 0;
p->se.vruntime = 0;
p->se.vlag = 0;
+ p->se.rel_deadline = 0;
INIT_LIST_HEAD(&p->se.group_node);
/* A delayed task cannot be in clone(). */
WARN_ON_ONCE(p->se.sched_delayed);
+ WARN_ON_ONCE(p->is_blocked);
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
@@ -4497,6 +4613,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
+ init_sched_mm(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -4709,6 +4826,7 @@ int sched_fork(u64 clone_flags, struct task_struct *p)
p->policy = SCHED_NORMAL;
p->static_prio = NICE_TO_PRIO(0);
p->rt_priority = 0;
+ p->timer_slack_ns = p->default_timer_slack_ns;
} else if (PRIO_TO_NICE(p->static_prio) < 0)
p->static_prio = NICE_TO_PRIO(0);
@@ -5517,7 +5635,11 @@ void sched_exec(void)
}
DEFINE_PER_CPU(struct kernel_stat, kstat);
-DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
+DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat) = {
+#ifdef CONFIG_NO_HZ_COMMON
+ .idle_sleeptime_seq = SEQCNT_ZERO(kernel_cpustat.idle_sleeptime_seq)
+#endif
+};
EXPORT_PER_CPU_SYMBOL(kstat);
EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
@@ -5971,10 +6093,9 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)
schedstat_inc(this_rq()->sched_count);
}
-static void prev_balance(struct rq *rq, struct task_struct *prev,
- struct rq_flags *rf)
+static void prev_balance(struct rq *rq, struct rq_flags *rf)
{
- const struct sched_class *start_class = prev->sched_class;
+ const struct sched_class *start_class = rq->donor->sched_class;
const struct sched_class *class;
/*
@@ -5986,7 +6107,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev,
* a runnable task of @class priority or higher.
*/
for_active_class_range(class, start_class, &idle_sched_class) {
- if (class->balance && class->balance(rq, prev, rf))
+ if (class->balance && class->balance(rq, rf))
break;
}
}
@@ -5995,7 +6116,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev,
* Pick up the highest-prio task:
*/
static inline struct task_struct *
-__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+__pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
const struct sched_class *class;
@@ -6012,40 +6133,31 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
* higher scheduling class, because otherwise those lose the
* opportunity to pull in more work from other CPUs.
*/
- if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
+ if (likely(!sched_class_above(rq->donor->sched_class, &fair_sched_class) &&
rq->nr_running == rq->cfs.h_nr_queued)) {
- p = pick_next_task_fair(rq, prev, rf);
+ p = pick_task_fair(rq, rf);
if (unlikely(p == RETRY_TASK))
goto restart;
/* Assume the next prioritized class is idle_sched_class */
- if (!p) {
+ if (!p)
p = pick_task_idle(rq, rf);
- put_prev_set_next_task(rq, prev, p);
- }
+ put_prev_set_next_task(rq, rq->donor, p);
return p;
}
restart:
- prev_balance(rq, prev, rf);
+ prev_balance(rq, rf);
for_each_active_class(class) {
- if (class->pick_next_task) {
- p = class->pick_next_task(rq, prev, rf);
- if (unlikely(p == RETRY_TASK))
- goto restart;
- if (p)
- return p;
- } else {
- p = class->pick_task(rq, rf);
- if (unlikely(p == RETRY_TASK))
- goto restart;
- if (p) {
- put_prev_set_next_task(rq, prev, p);
- return p;
- }
+ p = class->pick_task(rq, rf);
+ if (unlikely(p == RETRY_TASK))
+ goto restart;
+ if (p) {
+ put_prev_set_next_task(rq, rq->donor, p);
+ return p;
}
}
@@ -6096,7 +6208,7 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f
static void queue_core_balance(struct rq *rq);
static struct task_struct *
-pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
struct task_struct *next, *p, *max;
@@ -6109,7 +6221,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
bool need_sync;
if (!sched_core_enabled(rq))
- return __pick_next_task(rq, prev, rf);
+ return __pick_next_task(rq, rf);
cpu = cpu_of(rq);
@@ -6122,7 +6234,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
*/
rq->core_pick = NULL;
rq->core_dl_server = NULL;
- return __pick_next_task(rq, prev, rf);
+ return __pick_next_task(rq, rf);
}
/*
@@ -6146,7 +6258,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
goto out_set_next;
}
- prev_balance(rq, prev, rf);
+ prev_balance(rq, rf);
smt_mask = cpu_smt_mask(cpu);
need_sync = !!rq->core->core_cookie;
@@ -6328,7 +6440,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
}
out_set_next:
- put_prev_set_next_task(rq, prev, next);
+ put_prev_set_next_task(rq, rq->donor, next);
if (rq->core->core_forceidle_count && next == rq->idle)
queue_core_balance(rq);
@@ -6551,10 +6663,10 @@ static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
static inline void sched_core_cpu_dying(unsigned int cpu) {}
static struct task_struct *
-pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+pick_next_task(struct rq *rq, struct rq_flags *rf)
__must_hold(__rq_lockp(rq))
{
- return __pick_next_task(rq, prev, rf);
+ return __pick_next_task(rq, rf);
}
#endif /* !CONFIG_SCHED_CORE */
@@ -6582,16 +6694,19 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
unsigned long *task_state_p, bool should_block)
{
unsigned long task_state = *task_state_p;
- int flags = DEQUEUE_NOCLOCK;
+
+ WARN_ON_ONCE(p->is_blocked);
if (signal_pending_state(task_state, p)) {
WRITE_ONCE(p->__state, TASK_RUNNING);
*task_state_p = TASK_RUNNING;
- set_task_blocked_on_waking(p, NULL);
+ clear_task_blocked_on(p, NULL);
return false;
}
+ p->is_blocked = 1;
+
/*
* We check should_block after signal_pending because we
* will want to wake the task in that case. But if
@@ -6602,26 +6717,7 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
if (!should_block)
return false;
- p->sched_contributes_to_load =
- (task_state & TASK_UNINTERRUPTIBLE) &&
- !(task_state & TASK_NOLOAD) &&
- !(task_state & TASK_FROZEN);
-
- if (unlikely(is_special_task_state(task_state)))
- flags |= DEQUEUE_SPECIAL;
-
- /*
- * __schedule() ttwu()
- * prev_state = prev->state; if (p->on_rq && ...)
- * if (prev_state) goto out;
- * p->on_rq = 0; smp_acquire__after_ctrl_dep();
- * p->state = TASK_WAKING
- *
- * Where __schedule() and ttwu() have matching control dependencies.
- *
- * After this, schedule() must not care about p->state any more.
- */
- block_task(rq, p, flags);
+ block_task(rq, p, task_state);
return true;
}
@@ -6644,18 +6740,18 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
static inline struct task_struct *proxy_resched_idle(struct rq *rq)
{
put_prev_set_next_task(rq, rq->donor, rq->idle);
+ rq->next_class = &idle_sched_class;
rq_set_donor(rq, rq->idle);
set_tsk_need_resched(rq->idle);
return rq->idle;
}
-static bool proxy_deactivate(struct rq *rq, struct task_struct *donor)
+static void proxy_deactivate(struct rq *rq, struct task_struct *donor)
{
unsigned long state = READ_ONCE(donor->__state);
- /* Don't deactivate if the state has been changed to TASK_RUNNING */
- if (state == TASK_RUNNING)
- return false;
+ WARN_ON_ONCE(state == TASK_RUNNING);
+ WARN_ON_ONCE(donor->blocked_on);
/*
* Because we got donor from pick_next_task(), it is *crucial*
* that we call proxy_resched_idle() before we deactivate it.
@@ -6666,7 +6762,7 @@ static bool proxy_deactivate(struct rq *rq, struct task_struct *donor)
* need to be changed from next *before* we deactivate.
*/
proxy_resched_idle(rq);
- return try_to_block_task(rq, donor, &state, true);
+ block_task(rq, donor, state);
}
static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf)
@@ -6740,76 +6836,21 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
proxy_reacquire_rq_lock(rq, rf);
}
-static void proxy_force_return(struct rq *rq, struct rq_flags *rf,
- struct task_struct *p)
- __must_hold(__rq_lockp(rq))
-{
- struct rq *task_rq, *target_rq = NULL;
- int cpu, wake_flag = WF_TTWU;
-
- lockdep_assert_rq_held(rq);
- WARN_ON(p == rq->curr);
-
- if (p == rq->donor)
- proxy_resched_idle(rq);
-
- proxy_release_rq_lock(rq, rf);
- /*
- * We drop the rq lock, and re-grab task_rq_lock to get
- * the pi_lock (needed for select_task_rq) as well.
- */
- scoped_guard (task_rq_lock, p) {
- task_rq = scope.rq;
-
- /*
- * Since we let go of the rq lock, the task may have been
- * woken or migrated to another rq before we got the
- * task_rq_lock. So re-check we're on the same RQ. If
- * not, the task has already been migrated and that CPU
- * will handle any futher migrations.
- */
- if (task_rq != rq)
- break;
-
- /*
- * Similarly, if we've been dequeued, someone else will
- * wake us
- */
- if (!task_on_rq_queued(p))
- break;
-
- /*
- * Since we should only be calling here from __schedule()
- * -> find_proxy_task(), no one else should have
- * assigned current out from under us. But check and warn
- * if we see this, then bail.
- */
- if (task_current(task_rq, p) || task_on_cpu(task_rq, p)) {
- WARN_ONCE(1, "%s rq: %i current/on_cpu task %s %d on_cpu: %i\n",
- __func__, cpu_of(task_rq),
- p->comm, p->pid, p->on_cpu);
- break;
- }
-
- update_rq_clock(task_rq);
- deactivate_task(task_rq, p, DEQUEUE_NOCLOCK);
- cpu = select_task_rq(p, p->wake_cpu, &wake_flag);
- set_task_cpu(p, cpu);
- target_rq = cpu_rq(cpu);
- clear_task_blocked_on(p, NULL);
- }
-
- if (target_rq)
- attach_one_task(target_rq, p);
-
- proxy_reacquire_rq_lock(rq, rf);
-}
-
/*
* Find runnable lock owner to proxy for mutex blocked donor
*
* Follow the blocked-on relation:
- * task->blocked_on -> mutex->owner -> task...
+ *
+ * ,-> task
+ * | | blocked-on
+ * | v
+ * blocked_donor | mutex
+ * | | owner
+ * | v
+ * `-- task
+ *
+ * and set the blocked_donor relation, this latter is used by the mutex
+ * code to find which (blocked) task to hand-off to.
*
* Lock order:
*
@@ -6829,18 +6870,19 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
bool curr_in_chain = false;
int this_cpu = cpu_of(rq);
struct task_struct *p;
- struct mutex *mutex;
int owner_cpu;
/* Follow blocked_on chain. */
- for (p = donor; (mutex = p->blocked_on); p = owner) {
+ for (p = donor; p->is_blocked; p = owner) {
/* if its PROXY_WAKING, do return migration or run if current */
- if (mutex == PROXY_WAKING) {
+ struct mutex *mutex = p->blocked_on;
+ if (!mutex) {
+ clear_task_blocked_on(p, mutex);
if (task_current(rq, p)) {
- clear_task_blocked_on(p, PROXY_WAKING);
+ p->is_blocked = 0;
return p;
}
- goto force_return;
+ goto deactivate;
}
/*
@@ -6871,17 +6913,19 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* and return p (if it is current and safe to
* just run on this rq), or return-migrate the task.
*/
+ __clear_task_blocked_on(p, NULL);
if (task_current(rq, p)) {
- __clear_task_blocked_on(p, NULL);
+ p->is_blocked = 0;
return p;
}
- goto force_return;
+ goto deactivate;
}
if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
/* XXX Don't handle blocked owners/delayed dequeue yet */
if (curr_in_chain)
return proxy_resched_idle(rq);
+ __clear_task_blocked_on(p, NULL);
goto deactivate;
}
@@ -6949,17 +6993,13 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* rq, therefore holding @rq->lock is sufficient to
* guarantee its existence, as per ttwu_remote().
*/
+ owner->blocked_donor = p;
}
WARN_ON_ONCE(owner && !owner->on_rq);
return owner;
deactivate:
- if (proxy_deactivate(rq, donor))
- return NULL;
- /* If deactivate fails, force return */
- p = donor;
-force_return:
- proxy_force_return(rq, rf, p);
+ proxy_deactivate(rq, p);
return NULL;
migrate_task:
proxy_migrate_task(rq, rf, p, owner_cpu);
@@ -7101,13 +7141,14 @@ static void __sched notrace __schedule(int sched_mode)
pick_again:
assert_balance_callbacks_empty(rq);
- next = pick_next_task(rq, rq->donor, &rf);
+ next = pick_next_task(rq, &rf);
rq->next_class = next->sched_class;
if (sched_proxy_exec()) {
struct task_struct *prev_donor = rq->donor;
rq_set_donor(rq, next);
- if (unlikely(next->blocked_on)) {
+ next->blocked_donor = NULL;
+ if (unlikely(next->is_blocked)) {
next = find_proxy_task(rq, next, &rf);
if (!next) {
zap_balance_callbacks(rq);
@@ -7963,7 +8004,7 @@ static void __sched_dynamic_update(int mode)
break;
}
- preempt_dynamic_mode = mode;
+ WRITE_ONCE(preempt_dynamic_mode, mode);
}
void sched_dynamic_update(int mode)
@@ -8004,12 +8045,13 @@ static void __init preempt_dynamic_init(void)
}
}
-# define PREEMPT_MODEL_ACCESSOR(mode) \
- bool preempt_model_##mode(void) \
- { \
- WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \
- return preempt_dynamic_mode == preempt_dynamic_##mode; \
- } \
+# define PREEMPT_MODEL_ACCESSOR(mode) \
+ bool preempt_model_##mode(void) \
+ { \
+ int mode = READ_ONCE(preempt_dynamic_mode); \
+ WARN_ON_ONCE(mode == preempt_dynamic_undefined); \
+ return mode == preempt_dynamic_##mode; \
+ } \
EXPORT_SYMBOL_GPL(preempt_model_##mode)
PREEMPT_MODEL_ACCESSOR(none);
@@ -8603,18 +8645,14 @@ static void cpuset_cpu_inactive(unsigned int cpu)
static inline void sched_smt_present_inc(int cpu)
{
-#ifdef CONFIG_SCHED_SMT
if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
static_branch_inc_cpuslocked(&sched_smt_present);
-#endif
}
static inline void sched_smt_present_dec(int cpu)
{
-#ifdef CONFIG_SCHED_SMT
if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
static_branch_dec_cpuslocked(&sched_smt_present);
-#endif
}
int sched_cpu_activate(unsigned int cpu)
@@ -8669,7 +8707,8 @@ int sched_cpu_deactivate(unsigned int cpu)
* Remove CPU from nohz.idle_cpus_mask to prevent participating in
* load balancing when not active
*/
- nohz_balance_exit_idle(rq);
+ scoped_guard (rcu)
+ nohz_balance_exit_idle(rq);
set_cpu_active(cpu, false);
@@ -8693,6 +8732,8 @@ int sched_cpu_deactivate(unsigned int cpu)
*/
synchronize_rcu();
+ sched_domains_free_llc_id(cpu);
+
sched_set_rq_offline(rq, cpu);
scx_rq_deactivate(rq);
@@ -8702,9 +8743,7 @@ int sched_cpu_deactivate(unsigned int cpu)
*/
sched_smt_present_dec(cpu);
-#ifdef CONFIG_SCHED_SMT
sched_core_cpu_deactivate(cpu);
-#endif
if (!sched_smp_initialized)
return 0;
@@ -8872,7 +8911,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
void __init sched_init(void)
{
- unsigned long ptr = 0;
+ unsigned long __maybe_unused ptr = 0;
int i;
/* Make sure the linker didn't screw up */
@@ -8888,36 +8927,24 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
- ptr += 2 * nr_cpu_ids * sizeof(void **);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
- ptr += 2 * nr_cpu_ids * sizeof(void **);
-#endif
- if (ptr) {
- ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+ root_task_group.cfs_rq = &runqueues.cfs;
-#ifdef CONFIG_FAIR_GROUP_SCHED
- root_task_group.se = (struct sched_entity **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
-
- root_task_group.cfs_rq = (struct cfs_rq **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
-
- root_task_group.shares = ROOT_TASK_GROUP_LOAD;
- init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
+ root_task_group.shares = ROOT_TASK_GROUP_LOAD;
+ init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_EXT_GROUP_SCHED
- scx_tg_init(&root_task_group);
+ scx_tg_init(&root_task_group);
#endif /* CONFIG_EXT_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
- root_task_group.rt_se = (struct sched_rt_entity **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
+ ptr += 2 * nr_cpu_ids * sizeof(void **);
+ ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+ root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+ ptr += nr_cpu_ids * sizeof(void **);
- root_task_group.rt_rq = (struct rt_rq **)ptr;
- ptr += nr_cpu_ids * sizeof(void **);
+ root_task_group.rt_rq = (struct rt_rq **)ptr;
+ ptr += nr_cpu_ids * sizeof(void **);
#endif /* CONFIG_RT_GROUP_SCHED */
- }
init_defrootdomain();
@@ -9026,6 +9053,11 @@ void __init sched_init(void)
rq->core_cookie = 0UL;
#endif
+#ifdef CONFIG_SCHED_CACHE
+ raw_spin_lock_init(&rq->cpu_epoch_lock);
+ rq->cpu_epoch_next = jiffies;
+#endif
+
zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
}
@@ -9827,15 +9859,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
}
for_each_online_cpu(i) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[i];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
struct rq *rq = cfs_rq->rq;
guard(rq_lock_irq)(rq);
+
cfs_rq->runtime_enabled = runtime_enabled;
cfs_rq->runtime_remaining = 1;
- if (cfs_rq->throttled)
+ if (cfs_rq->throttled) {
+ update_rq_clock(rq);
unthrottle_cfs_rq(cfs_rq);
+ }
}
if (runtime_was_enabled && !runtime_enabled)
@@ -9976,7 +10011,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
int i;
for_each_possible_cpu(i) {
- stats = __schedstats_from_se(tg->se[i]);
+ stats = __schedstats_from_se(tg_se(tg, i));
ws += schedstat_val(stats->wait_sum);
}
@@ -9995,7 +10030,7 @@ static u64 throttled_time_self(struct task_group *tg)
u64 total = 0;
for_each_possible_cpu(i) {
- total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
+ total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
}
return total;
diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c
index 73b6b24..43e0bde 100644
--- a/kernel/sched/core_sched.c
+++ b/kernel/sched/core_sched.c
@@ -136,7 +136,7 @@ int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
struct pid *grp;
int err = 0;
- if (!static_branch_likely(&sched_smt_present))
+ if (!sched_smt_active())
return -ENODEV;
BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index fbf31db..679ac65 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -2,6 +2,7 @@
/*
* Simple CPU accounting cgroup controller
*/
+#include <linux/sched/clock.h>
#include <linux/sched/cputime.h>
#include <linux/tsacct_kern.h>
#include "sched.h"
@@ -46,7 +47,8 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
u64_stats_update_begin(&irqtime->sync);
cpustat[idx] += delta;
irqtime->total += delta;
- irqtime->tick_delta += delta;
+ if (!kcpustat_idle_dyntick())
+ irqtime->tick_delta += delta;
u64_stats_update_end(&irqtime->sync);
}
@@ -414,16 +416,219 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
}
}
-static void irqtime_account_idle_ticks(int ticks)
-{
- irqtime_account_process_tick(current, 0, ticks);
-}
#else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
-static inline void irqtime_account_idle_ticks(int ticks) { }
static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
int nr_ticks) { }
#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
+#ifdef CONFIG_NO_HZ_COMMON
+static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now)
+{
+ u64 *cpustat = kc->cpustat;
+ u64 delta, steal, steal_delta;
+ int iowait;
+
+ if (!kc->idle_elapse)
+ return;
+
+ iowait = nr_iowait_cpu(smp_processor_id()) > 0;
+ delta = now - kc->idle_entrytime;
+ steal = steal_account_process_time(delta);
+
+ /*
+ * Record the idle time after substracting the steal time from
+ * previous update sequence. Don't substract the steal time from
+ * the current update sequence to avoid readers moving backward.
+ */
+ write_seqcount_begin(&kc->idle_sleeptime_seq);
+ steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta);
+ delta -= steal_delta;
+ kc->idle_stealtime[iowait] -= steal_delta;
+
+ if (iowait)
+ cpustat[CPUTIME_IOWAIT] += delta;
+ else
+ cpustat[CPUTIME_IDLE] += delta;
+
+ kc->idle_stealtime[iowait] += steal;
+ kc->idle_entrytime = now;
+ kc->idle_elapse = false;
+ write_seqcount_end(&kc->idle_sleeptime_seq);
+}
+
+static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now)
+{
+ /* Irqtime accounting might have been enabled in the middle of the IRQ */
+ if (kc->idle_elapse)
+ return;
+
+ write_seqcount_begin(&kc->idle_sleeptime_seq);
+ kc->idle_entrytime = now;
+ kc->idle_elapse = true;
+ write_seqcount_end(&kc->idle_sleeptime_seq);
+}
+
+void kcpustat_dyntick_stop(u64 now)
+{
+ struct kernel_cpustat *kc = kcpustat_this_cpu;
+
+ if (!vtime_generic_enabled_this_cpu()) {
+ WARN_ON_ONCE(!kc->idle_dyntick);
+ kcpustat_idle_stop(kc, now);
+ kc->idle_dyntick = false;
+ vtime_dyntick_stop();
+ }
+}
+
+void kcpustat_dyntick_start(u64 now)
+{
+ struct kernel_cpustat *kc = kcpustat_this_cpu;
+
+ if (!vtime_generic_enabled_this_cpu()) {
+ vtime_dyntick_start();
+ kc->idle_dyntick = true;
+ kcpustat_idle_start(kc, now);
+ }
+}
+
+void kcpustat_irq_enter(u64 now)
+{
+ struct kernel_cpustat *kc = kcpustat_this_cpu;
+
+ if (!vtime_generic_enabled_this_cpu() &&
+ (irqtime_enabled() || vtime_accounting_enabled_this_cpu()))
+ kcpustat_idle_stop(kc, now);
+}
+
+void kcpustat_irq_exit(u64 now)
+{
+ struct kernel_cpustat *kc = kcpustat_this_cpu;
+
+ /*
+ * Generic vtime already does its own idle accounting.
+ * But irqtime accounting or arch vtime which also accounts IRQs
+ * need to pause nohz accounting. Resume nohz accounting as long
+ * as the irqtime config is enabled to handle case where irqtime
+ * accounting got runtime disabled in the middle of an IRQ.
+ */
+ if (!vtime_generic_enabled_this_cpu() &&
+ (IS_ENABLED(CONFIG_IRQ_TIME_ACCOUNTING) || vtime_accounting_enabled_this_cpu()))
+ kcpustat_idle_start(kc, now);
+}
+
+static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
+ bool compute_delta, u64 now)
+{
+ struct kernel_cpustat *kc = &kcpustat_cpu(cpu);
+ int iowait = idx == CPUTIME_IOWAIT;
+ u64 *cpustat = kc->cpustat;
+ unsigned int seq;
+ u64 idle;
+
+ do {
+ seq = read_seqcount_begin(&kc->idle_sleeptime_seq);
+
+ idle = cpustat[idx];
+
+ if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) {
+ u64 delta = now - kc->idle_entrytime;
+
+ delta -= min_t(u64, kc->idle_stealtime[iowait], delta);
+ idle += delta;
+ }
+ } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq));
+
+ return idle;
+}
+
+u64 kcpustat_field_idle(int cpu)
+{
+ return kcpustat_field_dyntick(cpu, CPUTIME_IDLE,
+ !nr_iowait_cpu(cpu), ktime_get());
+}
+EXPORT_SYMBOL_GPL(kcpustat_field_idle);
+
+u64 kcpustat_field_iowait(int cpu)
+{
+ return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT,
+ nr_iowait_cpu(cpu), ktime_get());
+}
+EXPORT_SYMBOL_GPL(kcpustat_field_iowait);
+#else
+static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
+ bool compute_delta, ktime_t now)
+{
+ return kcpustat_cpu(cpu).cpustat[idx];
+}
+#endif /* CONFIG_NO_HZ_COMMON */
+
+static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx,
+ bool compute_delta, u64 *last_update_time)
+{
+ ktime_t now = ktime_get();
+ u64 res;
+
+ if (vtime_generic_enabled_cpu(cpu))
+ res = kcpustat_field(idx, cpu);
+ else
+ res = kcpustat_field_dyntick(cpu, idx, compute_delta, now);
+
+ do_div(res, NSEC_PER_USEC);
+
+ if (last_update_time)
+ *last_update_time = ktime_to_us(now);
+
+ return res;
+}
+
+/**
+ * get_cpu_idle_time_us - get the total idle time of a CPU
+ * @cpu: CPU number to query
+ * @last_update_time: variable to store update time in. Do not update
+ * counters if NULL.
+ *
+ * Return the cumulative idle time (since boot) for a given
+ * CPU, in microseconds. Note that this is partially broken due to
+ * the counter of iowait tasks that can be remotely updated without
+ * any synchronization. Therefore it is possible to observe backward
+ * values within two consecutive reads.
+ *
+ * This time is measured via accounting rather than sampling,
+ * and is as accurate as ktime_get() is.
+ *
+ * Return: total idle time of the @cpu
+ */
+u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
+{
+ return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE,
+ !nr_iowait_cpu(cpu), last_update_time);
+}
+EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
+
+/**
+ * get_cpu_iowait_time_us - get the total iowait time of a CPU
+ * @cpu: CPU number to query
+ * @last_update_time: variable to store update time in. Do not update
+ * counters if NULL.
+ *
+ * Return the cumulative iowait time (since boot) for a given
+ * CPU, in microseconds. Note this is partially broken due to
+ * the counter of iowait tasks that can be remotely updated without
+ * any synchronization. Therefore it is possible to observe backward
+ * values within two consecutive reads.
+ *
+ * This time is measured via accounting rather than sampling,
+ * and is as accurate as ktime_get() is.
+ *
+ * Return: total iowait time of @cpu
+ */
+u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
+{
+ return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT,
+ nr_iowait_cpu(cpu), last_update_time);
+}
+EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
+
/*
* Use precise platform statistics if available:
*/
@@ -437,11 +642,15 @@ void vtime_account_irq(struct task_struct *tsk, unsigned int offset)
vtime_account_hardirq(tsk);
} else if (pc & SOFTIRQ_OFFSET) {
vtime_account_softirq(tsk);
- } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
- is_idle_task(tsk)) {
- vtime_account_idle(tsk);
+ } else if (!kcpustat_idle_dyntick()) {
+ if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
+ is_idle_task(tsk)) {
+ vtime_account_idle(tsk);
+ } else {
+ vtime_account_kernel(tsk);
+ }
} else {
- vtime_account_kernel(tsk);
+ vtime_reset();
}
}
@@ -483,6 +692,9 @@ void account_process_tick(struct task_struct *p, int user_tick)
if (vtime_accounting_enabled_this_cpu())
return;
+ if (kcpustat_idle_dyntick())
+ return;
+
if (irqtime_enabled()) {
irqtime_account_process_tick(p, user_tick, 1);
return;
@@ -505,29 +717,6 @@ void account_process_tick(struct task_struct *p, int user_tick)
}
/*
- * Account multiple ticks of idle time.
- * @ticks: number of stolen ticks
- */
-void account_idle_ticks(unsigned long ticks)
-{
- u64 cputime, steal;
-
- if (irqtime_enabled()) {
- irqtime_account_idle_ticks(ticks);
- return;
- }
-
- cputime = ticks * TICK_NSEC;
- steal = steal_account_process_time(ULONG_MAX);
-
- if (steal >= cputime)
- return;
-
- cputime -= steal;
- account_idle_time(cputime);
-}
-
-/*
* Adjust tick based cputime random precision against scheduler runtime
* accounting.
*
@@ -587,12 +776,6 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
}
stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
- /*
- * Because mul_u64_u64_div_u64() can approximate on some
- * achitectures; enforce the constraint that: a*b/(b+c) <= a.
- */
- if (unlikely(stime > rtime))
- stime = rtime;
update:
/*
@@ -773,9 +956,9 @@ void vtime_guest_exit(struct task_struct *tsk)
}
EXPORT_SYMBOL_GPL(vtime_guest_exit);
-void vtime_account_idle(struct task_struct *tsk)
+static void __vtime_account_idle(struct vtime *vtime)
{
- account_idle_time(get_vtime_delta(&tsk->vtime));
+ account_idle_time(get_vtime_delta(vtime));
}
void vtime_task_switch_generic(struct task_struct *prev)
@@ -784,7 +967,7 @@ void vtime_task_switch_generic(struct task_struct *prev)
write_seqcount_begin(&vtime->seqcount);
if (vtime->state == VTIME_IDLE)
- vtime_account_idle(prev);
+ __vtime_account_idle(vtime);
else
__vtime_account_kernel(prev, vtime);
vtime->state = VTIME_INACTIVE;
@@ -926,6 +1109,7 @@ static int kcpustat_field_vtime(u64 *cpustat,
int cpu, u64 *val)
{
struct vtime *vtime = &tsk->vtime;
+ struct rq *rq = cpu_rq(cpu);
unsigned int seq;
do {
@@ -967,6 +1151,14 @@ static int kcpustat_field_vtime(u64 *cpustat,
if (state == VTIME_GUEST && task_nice(tsk) > 0)
*val += vtime->gtime + vtime_delta(vtime);
break;
+ case CPUTIME_IDLE:
+ if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait))
+ *val += vtime_delta(vtime);
+ break;
+ case CPUTIME_IOWAIT:
+ if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0)
+ *val += vtime_delta(vtime);
+ break;
default:
break;
}
@@ -975,16 +1167,15 @@ static int kcpustat_field_vtime(u64 *cpustat,
return 0;
}
-u64 kcpustat_field(struct kernel_cpustat *kcpustat,
- enum cpu_usage_stat usage, int cpu)
+u64 kcpustat_field(enum cpu_usage_stat usage, int cpu)
{
- u64 *cpustat = kcpustat->cpustat;
+ u64 *cpustat = kcpustat_cpu(cpu).cpustat;
u64 val = cpustat[usage];
struct rq *rq;
int err;
- if (!vtime_accounting_enabled_cpu(cpu))
- return val;
+ if (!vtime_generic_enabled_cpu(cpu))
+ return kcpustat_field_default(usage, cpu);
rq = cpu_rq(cpu);
@@ -1030,8 +1221,8 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
*dst = *src;
cpustat = dst->cpustat;
- /* Task is sleeping, dead or idle, nothing to add */
- if (state < VTIME_SYS)
+ /* Task is sleeping or dead, nothing to add */
+ if (state < VTIME_IDLE)
continue;
delta = vtime_delta(vtime);
@@ -1040,15 +1231,17 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
* Task runs either in user (including guest) or kernel space,
* add pending nohz time to the right place.
*/
- if (state == VTIME_SYS) {
+ switch (state) {
+ case VTIME_SYS:
cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
- } else if (state == VTIME_USER) {
+ break;
+ case VTIME_USER:
if (task_nice(tsk) > 0)
cpustat[CPUTIME_NICE] += vtime->utime + delta;
else
cpustat[CPUTIME_USER] += vtime->utime + delta;
- } else {
- WARN_ON_ONCE(state != VTIME_GUEST);
+ break;
+ case VTIME_GUEST:
if (task_nice(tsk) > 0) {
cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
cpustat[CPUTIME_NICE] += vtime->gtime + delta;
@@ -1056,6 +1249,15 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
cpustat[CPUTIME_USER] += vtime->gtime + delta;
}
+ break;
+ case VTIME_IDLE:
+ if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0)
+ cpustat[CPUTIME_IOWAIT] += delta;
+ else
+ cpustat[CPUTIME_IDLE] += delta;
+ break;
+ default:
+ WARN_ON_ONCE(1);
}
} while (read_seqcount_retry(&vtime->seqcount, seq));
@@ -1068,8 +1270,8 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
struct rq *rq;
int err;
- if (!vtime_accounting_enabled_cpu(cpu)) {
- *dst = *src;
+ if (!vtime_generic_enabled_cpu(cpu)) {
+ kcpustat_cpu_fetch_default(dst, cpu);
return;
}
@@ -1082,7 +1284,7 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
curr = rcu_dereference(rq->curr);
if (WARN_ON_ONCE(!curr)) {
rcu_read_unlock();
- *dst = *src;
+ kcpustat_cpu_fetch_default(dst, cpu);
return;
}
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index edca784..d2c3437 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1515,8 +1515,12 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64
if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se))) {
if (dl_server(dl_se)) {
- replenish_dl_new_period(dl_se, rq);
- start_dl_timer(dl_se);
+ if (dl_se->dl_defer) {
+ replenish_dl_new_period(dl_se, rq);
+ start_dl_timer(dl_se);
+ } else {
+ enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
+ }
} else {
enqueue_task_dl(rq, dl_task_of(dl_se), ENQUEUE_REPLENISH);
}
@@ -1793,7 +1797,8 @@ void dl_server_start(struct sched_dl_entity *dl_se)
struct rq *rq = dl_se->rq;
dl_se->dl_defer_idle = 0;
- if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime)
+ if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime ||
+ !dl_se->dl_bw_attached)
return;
/*
@@ -1868,6 +1873,13 @@ void sched_init_dl_servers(void)
dl_se->dl_server = 1;
dl_se->dl_defer = 1;
setup_new_dl_entity(dl_se);
+
+ /*
+ * No BPF scheduler is loaded at boot, so the ext_server has no
+ * tasks to protect. Detach its bandwidth reservation, it will
+ * be attached when a BPF scheduler is loaded.
+ */
+ dl_server_detach_bw(dl_se);
#endif
}
}
@@ -1878,6 +1890,9 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
int cpu = cpu_of(rq);
struct dl_bw *dl_b;
+ if (!dl_se->dl_bw_attached)
+ return;
+
dl_b = dl_bw_of(cpu_of(rq));
guard(raw_spinlock)(&dl_b->lock);
@@ -1889,7 +1904,8 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq)
int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init)
{
- u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime);
+ u64 old_bw = (init || !dl_se->dl_bw_attached) ? 0 :
+ to_ratio(dl_se->dl_period, dl_se->dl_runtime);
u64 new_bw = to_ratio(period, runtime);
struct rq *rq = dl_se->rq;
int cpu = cpu_of(rq);
@@ -1909,7 +1925,8 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio
if (init) {
__add_rq_bw(new_bw, &rq->dl);
__dl_add(dl_b, new_bw, cpus);
- } else {
+ dl_se->dl_bw_attached = 1;
+ } else if (dl_se->dl_bw_attached) {
__dl_sub(dl_b, dl_se->dl_bw, cpus);
__dl_add(dl_b, new_bw, cpus);
@@ -1930,6 +1947,181 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio
}
/*
+ * Add @dl_se's bw to the root-domain accounting.
+ *
+ * Return -EBUSY if attaching would overflow root domain capacity.
+ */
+static int __dl_server_attach_bw_locked(struct sched_dl_entity *dl_se,
+ struct dl_bw *dl_b, int cpus)
+{
+ struct rq *rq = dl_se->rq;
+ unsigned long cap;
+
+ /*
+ * Always update @rq->dl.this_bw, but only update @dl_b->total_bw
+ * (and run the overflow check it gates) while this CPU is active.
+ *
+ * This mirrors dl_server_add_bw() during root-domain rebuilds, which
+ * only publishes bandwidth from active CPUs into @dl_b.
+ */
+ if (cpu_active(cpu_of(rq))) {
+ cap = dl_bw_capacity(cpu_of(rq));
+ if (__dl_overflow(dl_b, cap, 0, dl_se->dl_bw))
+ return -EBUSY;
+ __dl_add(dl_b, dl_se->dl_bw, cpus);
+ }
+ __add_rq_bw(dl_se->dl_bw, &rq->dl);
+ dl_se->dl_bw_attached = 1;
+
+ return 0;
+}
+
+/*
+ * Drain @dl_se and remove its bw from the root-domain accounting.
+ */
+static void __dl_server_detach_bw_locked(struct sched_dl_entity *dl_se,
+ struct dl_bw *dl_b, int cpus)
+{
+ struct rq *rq = dl_se->rq;
+
+ /*
+ * If the server is still active (on_rq), dequeue it via
+ * dl_server_stop(); task_non_contending() will either subtract
+ * @dl_bw from running_bw immediately (0-lag passed) or set
+ * dl_non_contending and arm the inactive_timer.
+ */
+ if (dl_se->dl_server_active)
+ dl_server_stop(dl_se);
+
+ /*
+ * Drop @dl_se's contribution from this rq's bandwidth accounting,
+ * mirroring the __add_rq_bw() done at attach time.
+ */
+ dl_rq_change_utilization(rq, dl_se, 0);
+
+ /*
+ * Update @dl_b only while this CPU is active, matching
+ * dl_server_add_bw() during root-domain rebuilds.
+ *
+ * If this CPU is inactive, its bandwidth is not currently accounted in
+ * @dl_b->total_bw: either attach skipped adding it, or a rebuild
+ * already dropped it while re-publishing active CPUs only.
+ *
+ * In that case there is nothing to subtract from @dl_b. Just clear
+ * @dl_se->dl_bw_attached; if the CPU becomes active again, the next
+ * rebuild will re-publish its bandwidth.
+ */
+ if (cpu_active(cpu_of(rq)))
+ __dl_sub(dl_b, dl_se->dl_bw, cpus);
+ dl_se->dl_bw_attached = 0;
+}
+
+/*
+ * Attach @dl_se's bandwidth to the root domain's total_bw accounting.
+ *
+ * Use to dynamically register a dl_server's bandwidth reservation while
+ * preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is
+ * already attached.
+ *
+ * Returns -EBUSY if attaching would overflow the root domain capacity.
+ */
+int dl_server_attach_bw(struct sched_dl_entity *dl_se)
+{
+ struct rq *rq = dl_se->rq;
+ int cpu = cpu_of(rq);
+ struct dl_bw *dl_b;
+ int cpus, ret;
+
+ if (dl_se->dl_bw_attached)
+ return 0;
+
+ scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) {
+ dl_b = dl_bw_of(cpu);
+ cpus = dl_bw_cpus(cpu);
+ ret = __dl_server_attach_bw_locked(dl_se, dl_b, cpus);
+ }
+ if (ret)
+ return ret;
+
+ /*
+ * The natural 0->nr_running transition that triggers dl_server_start()
+ * may have happened while @dl_se was still detached (e.g., between
+ * scx_bypass(false) and the scx_enable() re-balance loop), so kick a
+ * start here.
+ *
+ * dl_server_start() bails out cleanly if there's nothing to schedule or
+ * it's already active. Skip if @cpu is offline; the server will be
+ * started naturally on the first enqueue once @cpu comes back.
+ */
+ if (cpu_online(cpu))
+ dl_server_start(dl_se);
+
+ return 0;
+}
+
+/*
+ * Detach @dl_se's bandwidth from the root domain's total_bw accounting.
+ *
+ * Use to dynamically unregister a dl_server's bandwidth reservation while
+ * preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is
+ * not currently attached.
+ */
+void dl_server_detach_bw(struct sched_dl_entity *dl_se)
+{
+ int cpu = cpu_of(dl_se->rq);
+ struct dl_bw *dl_b;
+ int cpus;
+
+ if (!dl_se->dl_bw_attached)
+ return;
+
+ dl_b = dl_bw_of(cpu);
+ guard(raw_spinlock)(&dl_b->lock);
+ cpus = dl_bw_cpus(cpu);
+ __dl_server_detach_bw_locked(dl_se, dl_b, cpus);
+}
+
+/*
+ * Atomically detach @detach_se and attach @attach_se on the same rq, holding
+ * @dl_b->lock across both operations so a concurrent sched_setattr() cannot
+ * steal the bandwidth freed by the detach before the attach can claim it.
+ *
+ * Both entities must live on the same rq (same root domain). Returns the
+ * result of the attach: -EBUSY if attaching @attach_se would overflow root
+ * domain capacity (in which case both servers end up detached).
+ */
+int dl_server_swap_bw(struct sched_dl_entity *detach_se,
+ struct sched_dl_entity *attach_se)
+{
+ struct rq *rq = detach_se->rq;
+ int cpu = cpu_of(rq);
+ struct dl_bw *dl_b;
+ int cpus, ret;
+
+ WARN_ON_ONCE(attach_se->rq != rq);
+
+ scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) {
+ dl_b = dl_bw_of(cpu);
+ cpus = dl_bw_cpus(cpu);
+
+ if (detach_se->dl_bw_attached)
+ __dl_server_detach_bw_locked(detach_se, dl_b, cpus);
+
+ if (attach_se->dl_bw_attached)
+ ret = 0;
+ else
+ ret = __dl_server_attach_bw_locked(attach_se, dl_b, cpus);
+ }
+ if (ret)
+ return ret;
+
+ if (cpu_online(cpu))
+ dl_server_start(attach_se);
+
+ return 0;
+}
+
+/*
* Update the current task's runtime statistics (provided it is still
* a -deadline task and has not been removed from the dl_rq).
*/
@@ -2292,7 +2484,10 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags)
static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
{
- if (is_dl_boosted(&p->dl)) {
+ struct sched_dl_entity *dl_se = &p->dl;
+ struct dl_rq *dl_rq = &rq->dl;
+
+ if (is_dl_boosted(dl_se)) {
/*
* Because of delays in the detection of the overrun of a
* thread's runtime, it might be the case that a thread
@@ -2305,14 +2500,14 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
*
* In this case, the boost overrides the throttle.
*/
- if (p->dl.dl_throttled) {
+ if (dl_se->dl_throttled) {
/*
* The replenish timer needs to be canceled. No
* problem if it fires concurrently: boosted threads
* are ignored in dl_task_timer().
*/
- cancel_replenish_timer(&p->dl);
- p->dl.dl_throttled = 0;
+ cancel_replenish_timer(dl_se);
+ dl_se->dl_throttled = 0;
}
} else if (!dl_prio(p->normal_prio)) {
/*
@@ -2324,7 +2519,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
* being boosted again with no means to replenish the runtime and clear
* the throttle.
*/
- p->dl.dl_throttled = 0;
+ dl_se->dl_throttled = 0;
if (!(flags & ENQUEUE_REPLENISH))
printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n",
task_pid_nr(p));
@@ -2333,20 +2528,23 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
}
check_schedstat_required();
- update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl);
+ update_stats_wait_start_dl(dl_rq, dl_se);
if (p->on_rq == TASK_ON_RQ_MIGRATING)
flags |= ENQUEUE_MIGRATING;
- enqueue_dl_entity(&p->dl, flags);
+ enqueue_dl_entity(dl_se, flags);
- if (dl_server(&p->dl))
+ if (dl_server(dl_se))
return;
if (task_is_blocked(p))
return;
- if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1)
+ if (dl_rq->curr == dl_se)
+ return;
+
+ if (!task_current(rq, p) && !dl_se->dl_throttled && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -2506,8 +2704,14 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
resched_curr(rq);
}
-static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
+static int balance_dl(struct rq *rq, struct rq_flags *rf)
{
+ /*
+ * Note, rq->donor may change during rq lock drops,
+ * so don't re-use prev across lock drops
+ */
+ struct task_struct *p = rq->donor;
+
if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) {
/*
* This is OK, because current is on_cpu, which avoids it being
@@ -2562,6 +2766,10 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
}
#endif /* !CONFIG_SCHED_HRTICK */
+/*
+ * DL keeps current in tree, because ->deadline is not typically changed while
+ * a task is runnable.
+ */
static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
{
struct sched_dl_entity *dl_se = &p->dl;
@@ -2574,6 +2782,9 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
/* You can't push away the running task */
dequeue_pushable_dl_task(rq, p);
+ WARN_ON_ONCE(dl_rq->curr);
+ dl_rq->curr = dl_se;
+
if (!first)
return;
@@ -2637,17 +2848,20 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s
struct sched_dl_entity *dl_se = &p->dl;
struct dl_rq *dl_rq = &rq->dl;
- if (on_dl_rq(&p->dl))
+ if (on_dl_rq(dl_se))
update_stats_wait_start_dl(dl_rq, dl_se);
update_curr_dl(rq);
update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1);
+ WARN_ON_ONCE(dl_rq->curr != dl_se);
+ dl_rq->curr = NULL;
+
if (task_is_blocked(p))
return;
- if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
+ if (on_dl_rq(dl_se) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -3107,20 +3321,18 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
static void set_cpus_allowed_dl(struct task_struct *p,
struct affinity_context *ctx)
{
- struct root_domain *src_rd;
struct rq *rq;
WARN_ON_ONCE(!dl_task(p));
rq = task_rq(p);
- src_rd = rq->rd;
/*
* Migrating a SCHED_DEADLINE task between exclusive
* cpusets (different root_domains) entails a bandwidth
* update. We already made space for us in the destination
* domain (see cpuset_can_attach()).
*/
- if (!cpumask_intersects(src_rd->span, ctx->new_mask)) {
+ if (dl_task_needs_bw_move(p, ctx->new_mask)) {
struct dl_bw *src_dl_b;
src_dl_b = dl_bw_of(cpu_of(rq));
@@ -3137,6 +3349,15 @@ static void set_cpus_allowed_dl(struct task_struct *p,
set_cpus_allowed_common(p, ctx);
}
+bool dl_task_needs_bw_move(struct task_struct *p,
+ const struct cpumask *new_mask)
+{
+ if (!dl_task(p))
+ return false;
+
+ return !cpumask_intersects(task_rq(p)->rd->span, new_mask);
+}
+
/* Assumes rq->lock is held */
static void rq_online_dl(struct rq *rq)
{
@@ -3229,12 +3450,12 @@ static void dl_server_add_bw(struct root_domain *rd, int cpu)
struct sched_dl_entity *dl_se;
dl_se = &cpu_rq(cpu)->fair_server;
- if (dl_server(dl_se) && cpu_active(cpu))
+ if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu))
__dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu));
#ifdef CONFIG_SCHED_CLASS_EXT
dl_se = &cpu_rq(cpu)->ext_server;
- if (dl_server(dl_se) && cpu_active(cpu))
+ if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu))
__dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu));
#endif
}
@@ -3243,11 +3464,13 @@ static u64 dl_server_read_bw(int cpu)
{
u64 dl_bw = 0;
- if (cpu_rq(cpu)->fair_server.dl_server)
+ if (cpu_rq(cpu)->fair_server.dl_server &&
+ cpu_rq(cpu)->fair_server.dl_bw_attached)
dl_bw += cpu_rq(cpu)->fair_server.dl_bw;
#ifdef CONFIG_SCHED_CLASS_EXT
- if (cpu_rq(cpu)->ext_server.dl_server)
+ if (cpu_rq(cpu)->ext_server.dl_server &&
+ cpu_rq(cpu)->ext_server.dl_bw_attached)
dl_bw += cpu_rq(cpu)->ext_server.dl_bw;
#endif
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 74c1617..40584b2 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -136,7 +136,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
if (cnt > 63)
cnt = 63;
- if (copy_from_user(&buf, ubuf, cnt))
+ if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@@ -210,6 +210,48 @@ static const struct file_operations sched_scaling_fops = {
.release = single_release,
};
+#ifdef CONFIG_SCHED_CACHE
+static ssize_t
+sched_cache_enable_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ bool val;
+ int ret;
+
+ ret = kstrtobool_from_user(ubuf, cnt, &val);
+ if (ret)
+ return ret;
+
+ sysctl_sched_cache_user = val;
+
+ sched_cache_active_set();
+
+ *ppos += cnt;
+
+ return cnt;
+}
+
+static int sched_cache_enable_show(struct seq_file *m, void *v)
+{
+ seq_printf(m, "%d\n", sysctl_sched_cache_user);
+ return 0;
+}
+
+static int sched_cache_enable_open(struct inode *inode,
+ struct file *filp)
+{
+ return single_open(filp, sched_cache_enable_show, NULL);
+}
+
+static const struct file_operations sched_cache_enable_fops = {
+ .open = sched_cache_enable_open,
+ .write = sched_cache_enable_write,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+#endif
+
#ifdef CONFIG_PREEMPT_DYNAMIC
static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -221,7 +263,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
if (cnt > 15)
cnt = 15;
- if (copy_from_user(&buf, ubuf, cnt))
+ if (copy_from_user(buf, ubuf, cnt))
return -EFAULT;
buf[cnt] = 0;
@@ -239,6 +281,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
static int sched_dynamic_show(struct seq_file *m, void *v)
{
int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2;
+ int mode = READ_ONCE(preempt_dynamic_mode);
int j;
/* Count entries in NULL terminated preempt_modes */
@@ -247,10 +290,10 @@ static int sched_dynamic_show(struct seq_file *m, void *v)
j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY);
for (; i < j; i++) {
- if (preempt_dynamic_mode == i)
+ if (mode == i)
seq_puts(m, "(");
seq_puts(m, preempt_modes[i]);
- if (preempt_dynamic_mode == i)
+ if (mode == i)
seq_puts(m, ")");
seq_puts(m, " ");
@@ -373,6 +416,9 @@ static ssize_t sched_server_write_common(struct file *filp, const char __user *u
return -EINVAL;
}
+ if (!cpu_online(cpu_of(rq)))
+ return -EBUSY;
+
update_rq_clock(rq);
dl_server_stop(dl_se);
retval = dl_server_apply_params(dl_se, runtime, period, 0);
@@ -445,6 +491,8 @@ static const struct file_operations fair_server_runtime_fops = {
.release = single_release,
};
+static struct dentry *debugfs_sched;
+
#ifdef CONFIG_SCHED_CLASS_EXT
static ssize_t
sched_ext_server_runtime_write(struct file *filp, const char __user *ubuf,
@@ -477,6 +525,59 @@ static const struct file_operations ext_server_runtime_fops = {
.llseek = seq_lseek,
.release = single_release,
};
+
+static ssize_t
+sched_ext_server_period_write(struct file *filp, const char __user *ubuf,
+ size_t cnt, loff_t *ppos)
+{
+ long cpu = (long) ((struct seq_file *) filp->private_data)->private;
+ struct rq *rq = cpu_rq(cpu);
+
+ return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD,
+ &rq->ext_server);
+}
+
+static int sched_ext_server_period_show(struct seq_file *m, void *v)
+{
+ unsigned long cpu = (unsigned long) m->private;
+ struct rq *rq = cpu_rq(cpu);
+
+ return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server);
+}
+
+static int sched_ext_server_period_open(struct inode *inode, struct file *filp)
+{
+ return single_open(filp, sched_ext_server_period_show, inode->i_private);
+}
+
+static const struct file_operations ext_server_period_fops = {
+ .open = sched_ext_server_period_open,
+ .write = sched_ext_server_period_write,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static void debugfs_ext_server_init(void)
+{
+ struct dentry *d_ext;
+ unsigned long cpu;
+
+ d_ext = debugfs_create_dir("ext_server", debugfs_sched);
+ if (!d_ext)
+ return;
+
+ for_each_possible_cpu(cpu) {
+ struct dentry *d_cpu;
+ char buf[32];
+
+ snprintf(buf, sizeof(buf), "cpu%lu", cpu);
+ d_cpu = debugfs_create_dir(buf, d_ext);
+
+ debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops);
+ debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops);
+ }
+}
#endif /* CONFIG_SCHED_CLASS_EXT */
static ssize_t
@@ -511,42 +612,6 @@ static const struct file_operations fair_server_period_fops = {
.release = single_release,
};
-#ifdef CONFIG_SCHED_CLASS_EXT
-static ssize_t
-sched_ext_server_period_write(struct file *filp, const char __user *ubuf,
- size_t cnt, loff_t *ppos)
-{
- long cpu = (long) ((struct seq_file *) filp->private_data)->private;
- struct rq *rq = cpu_rq(cpu);
-
- return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD,
- &rq->ext_server);
-}
-
-static int sched_ext_server_period_show(struct seq_file *m, void *v)
-{
- unsigned long cpu = (unsigned long) m->private;
- struct rq *rq = cpu_rq(cpu);
-
- return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server);
-}
-
-static int sched_ext_server_period_open(struct inode *inode, struct file *filp)
-{
- return single_open(filp, sched_ext_server_period_show, inode->i_private);
-}
-
-static const struct file_operations ext_server_period_fops = {
- .open = sched_ext_server_period_open,
- .write = sched_ext_server_period_write,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
-#endif /* CONFIG_SCHED_CLASS_EXT */
-
-static struct dentry *debugfs_sched;
-
static void debugfs_fair_server_init(void)
{
struct dentry *d_fair;
@@ -568,32 +633,9 @@ static void debugfs_fair_server_init(void)
}
}
-#ifdef CONFIG_SCHED_CLASS_EXT
-static void debugfs_ext_server_init(void)
-{
- struct dentry *d_ext;
- unsigned long cpu;
-
- d_ext = debugfs_create_dir("ext_server", debugfs_sched);
- if (!d_ext)
- return;
-
- for_each_possible_cpu(cpu) {
- struct dentry *d_cpu;
- char buf[32];
-
- snprintf(buf, sizeof(buf), "cpu%lu", cpu);
- d_cpu = debugfs_create_dir(buf, d_ext);
-
- debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops);
- debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops);
- }
-}
-#endif /* CONFIG_SCHED_CLASS_EXT */
-
static __init int sched_init_debug(void)
{
- struct dentry __maybe_unused *numa;
+ struct dentry __maybe_unused *numa, *llc;
debugfs_sched = debugfs_create_dir("sched", NULL);
@@ -626,6 +668,22 @@ static __init int sched_init_debug(void)
debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
#endif /* CONFIG_NUMA_BALANCING */
+#ifdef CONFIG_SCHED_CACHE
+ llc = debugfs_create_dir("llc_balancing", debugfs_sched);
+ debugfs_create_file("enabled", 0644, llc, NULL,
+ &sched_cache_enable_fops);
+ debugfs_create_u32("aggr_tolerance", 0644, llc,
+ &llc_aggr_tolerance);
+ debugfs_create_u32("epoch_period", 0644, llc,
+ &llc_epoch_period);
+ debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
+ &llc_epoch_affinity_timeout);
+ debugfs_create_u32("overaggr_pct", 0644, llc,
+ &llc_overaggr_pct);
+ debugfs_create_u32("imb_pct", 0644, llc,
+ &llc_imb_pct);
+#endif
+
debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
debugfs_fair_server_init();
@@ -750,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
{
- struct sched_entity *se = tg->se[cpu];
+ struct sched_entity *se = tg_se(tg, cpu);
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index e426e27..f5a3233 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -32,6 +32,7 @@ static const struct rhashtable_params scx_sched_hash_params = {
.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),
.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),
.head_offset = offsetof(struct scx_sched, hash_node),
+ .insecure_elasticity = true, /* inserted under scx_sched_lock */
};
static struct rhashtable scx_sched_hash;
@@ -52,8 +53,6 @@ DEFINE_STATIC_KEY_FALSE(__scx_enabled);
DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
-static cpumask_var_t scx_bypass_lb_donee_cpumask;
-static cpumask_var_t scx_bypass_lb_resched_cpumask;
static bool scx_init_task_enabled;
static bool scx_switching_all;
DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
@@ -298,7 +297,6 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)
#else /* CONFIG_EXT_SUB_SCHED */
static struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
-static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return NULL; }
static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
#endif /* CONFIG_EXT_SUB_SCHED */
@@ -469,24 +467,35 @@ static inline void update_locked_rq(struct rq *rq)
__this_cpu_write(scx_locked_rq_state, rq);
}
-#define SCX_CALL_OP(sch, op, rq, args...) \
+/*
+ * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
+ * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
+ */
+#define SCX_CALL_OP(sch, op, locked_rq, args...) \
do { \
- if (rq) \
- update_locked_rq(rq); \
+ struct rq *__prev_locked_rq; \
+ \
+ if (locked_rq) { \
+ __prev_locked_rq = scx_locked_rq(); \
+ update_locked_rq(locked_rq); \
+ } \
(sch)->ops.op(args); \
- if (rq) \
- update_locked_rq(NULL); \
+ if (locked_rq) \
+ update_locked_rq(__prev_locked_rq); \
} while (0)
-#define SCX_CALL_OP_RET(sch, op, rq, args...) \
+#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
({ \
+ struct rq *__prev_locked_rq; \
__typeof__((sch)->ops.op(args)) __ret; \
\
- if (rq) \
- update_locked_rq(rq); \
+ if (locked_rq) { \
+ __prev_locked_rq = scx_locked_rq(); \
+ update_locked_rq(locked_rq); \
+ } \
__ret = (sch)->ops.op(args); \
- if (rq) \
- update_locked_rq(NULL); \
+ if (locked_rq) \
+ update_locked_rq(__prev_locked_rq); \
__ret; \
})
@@ -498,39 +507,39 @@ do { \
* those subject tasks.
*
* Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
- * either via the @rq argument here, or (for ops.select_cpu()) via @p's pi_lock
- * held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. So if
- * kf_tasks[] is set, @p's scheduler-protected fields are stable.
+ * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
+ * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
+ * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
*
* kf_tasks[] can not stack, so task-based SCX ops must not nest. The
* WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
* while a previous one is still in progress.
*/
-#define SCX_CALL_OP_TASK(sch, op, rq, task, args...) \
+#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
do { \
WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task; \
- SCX_CALL_OP((sch), op, rq, task, ##args); \
+ SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
current->scx.kf_tasks[0] = NULL; \
} while (0)
-#define SCX_CALL_OP_TASK_RET(sch, op, rq, task, args...) \
+#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
({ \
__typeof__((sch)->ops.op(task, ##args)) __ret; \
WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task; \
- __ret = SCX_CALL_OP_RET((sch), op, rq, task, ##args); \
+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
current->scx.kf_tasks[0] = NULL; \
__ret; \
})
-#define SCX_CALL_OP_2TASKS_RET(sch, op, rq, task0, task1, args...) \
+#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
({ \
__typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task0; \
current->scx.kf_tasks[1] = task1; \
- __ret = SCX_CALL_OP_RET((sch), op, rq, task0, task1, ##args); \
+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
current->scx.kf_tasks[0] = NULL; \
current->scx.kf_tasks[1] = NULL; \
__ret; \
@@ -702,6 +711,51 @@ struct bpf_iter_scx_dsq {
} __attribute__((aligned(8)));
+static u32 scx_get_task_state(const struct task_struct *p)
+{
+ return p->scx.flags & SCX_TASK_STATE_MASK;
+}
+
+static void scx_set_task_state(struct task_struct *p, u32 state)
+{
+ u32 prev_state = scx_get_task_state(p);
+ bool warn = false;
+
+ switch (state) {
+ case SCX_TASK_NONE:
+ warn = prev_state == SCX_TASK_DEAD;
+ break;
+ case SCX_TASK_INIT_BEGIN:
+ warn = prev_state != SCX_TASK_NONE;
+ break;
+ case SCX_TASK_INIT:
+ warn = prev_state != SCX_TASK_INIT_BEGIN;
+ p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
+ break;
+ case SCX_TASK_READY:
+ warn = !(prev_state == SCX_TASK_INIT ||
+ prev_state == SCX_TASK_ENABLED);
+ break;
+ case SCX_TASK_ENABLED:
+ warn = prev_state != SCX_TASK_READY;
+ break;
+ case SCX_TASK_DEAD:
+ warn = !(prev_state == SCX_TASK_NONE ||
+ prev_state == SCX_TASK_INIT_BEGIN);
+ break;
+ default:
+ WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
+ prev_state, state, p->comm, p->pid);
+ return;
+ }
+
+ WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
+ prev_state, state, p->comm, p->pid);
+
+ p->scx.flags &= ~SCX_TASK_STATE_MASK;
+ p->scx.flags |= state;
+}
+
/*
* SCX task iterator.
*/
@@ -756,7 +810,8 @@ static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp)
lockdep_assert_held(&cgroup_mutex);
iter->cgrp = cgrp;
iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self);
- css_task_iter_start(iter->css_pos, 0, &iter->css_iter);
+ css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
+ &iter->css_iter);
return;
}
#endif
@@ -856,7 +911,8 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
iter->css_pos = css_next_descendant_pre(iter->css_pos,
&iter->cgrp->self);
if (iter->css_pos)
- css_task_iter_start(iter->css_pos, 0, &iter->css_iter);
+ css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
+ &iter->css_iter);
}
return NULL;
}
@@ -916,16 +972,27 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
*
* Test for idle_sched_class as only init_tasks are on it.
*/
- if (p->sched_class != &idle_sched_class)
- break;
+ if (p->sched_class == &idle_sched_class)
+ continue;
+
+ iter->rq = task_rq_lock(p, &iter->rf);
+ iter->locked_task = p;
+
+ /*
+ * cgroup_task_dead() removes the dead tasks from cset->tasks
+ * after sched_ext_dead() and cgroup iteration may see tasks
+ * which already finished sched_ext_dead(). %SCX_TASK_DEAD is
+ * set by sched_ext_dead() under @p's rq lock. Test it to
+ * avoid visiting tasks which are already dead from SCX POV.
+ */
+ if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+ __scx_task_iter_rq_unlock(iter);
+ continue;
+ }
+
+ return p;
}
- if (!p)
- return NULL;
-
- iter->rq = task_rq_lock(p, &iter->rf);
- iter->locked_task = p;
-
- return p;
+ return NULL;
}
/**
@@ -1388,18 +1455,55 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
}
-static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p,
- u64 enq_flags)
+static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
+ struct task_struct *p, u64 enq_flags)
{
struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
- bool preempt = false;
- call_task_dequeue(scx_root, rq, p, 0);
+ call_task_dequeue(sch, rq, p, 0);
+
+ /*
+ * Note that @rq's lock may be dropped between this enqueue and @p
+ * actually getting on CPU. This gives higher-class tasks (e.g. RT)
+ * an opportunity to wake up on @rq and prevent @p from running.
+ * Here are some concrete examples:
+ *
+ * Example 1:
+ *
+ * We dispatch two tasks from a single ops.dispatch():
+ * - First, a local task to this CPU's local DSQ;
+ * - Second, a local/remote task to a remote CPU's local DSQ.
+ * We must drop the local rq lock in order to finish the second
+ * dispatch. In that time, an RT task can wake up on the local rq.
+ *
+ * Example 2:
+ *
+ * We dispatch a local/remote task to a remote CPU's local DSQ.
+ * We must drop the remote rq lock before the dispatched task can run,
+ * which gives an RT task an opportunity to wake up on the remote rq.
+ *
+ * Both examples work the same if we replace dispatching with moving
+ * the tasks from a user-created DSQ.
+ *
+ * We must detect these wakeups so that we can re-enqueue IMMED tasks
+ * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
+ * purpose, but for it to be invoked, we must ensure that we bump
+ * @rq->next_class to &ext_sched_class if it's currently idle.
+ *
+ * wakeup_preempt() does the bumping, and since we only invoke it if
+ * @rq->next_class is below &ext_sched_class, it will also
+ * resched_curr(rq).
+ */
+ if (sched_class_above(p->sched_class, rq->next_class))
+ wakeup_preempt(rq, p, 0);
/*
* If @rq is in balance, the CPU is already vacant and looking for the
* next task to run. No need to preempt or trigger resched after moving
* @p into its local DSQ.
+ * Note that the wakeup_preempt() above may have already triggered
+ * a resched if @rq->next_class was idle. It's harmless, since
+ * need_resched is cleared immediately after task pick.
*/
if (rq->scx.flags & SCX_RQ_IN_BALANCE)
return;
@@ -1407,11 +1511,8 @@ static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p
if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
rq->curr->sched_class == &ext_sched_class) {
rq->curr->scx.slice = 0;
- preempt = true;
- }
-
- if (preempt || sched_class_above(&ext_sched_class, rq->curr->sched_class))
resched_curr(rq);
+ }
}
static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
@@ -1494,11 +1595,13 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN))
rcu_assign_pointer(dsq->first_task, p);
} else {
- bool was_empty;
-
- was_empty = list_empty(&dsq->list);
+ /*
+ * dsq->list can contain parked BPF iterator cursors, so
+ * list_empty() here isn't a reliable proxy for "no real
+ * task in the DSQ". Test dsq->first_task directly.
+ */
list_add_tail(&p->scx.dsq_list.node, &dsq->list);
- if (was_empty && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
+ if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
rcu_assign_pointer(dsq->first_task, p);
}
}
@@ -1518,7 +1621,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
* concurrently in a non-atomic way.
*/
if (is_local) {
- local_dsq_post_enq(dsq, p, enq_flags);
+ local_dsq_post_enq(sch, dsq, p, enq_flags);
} else {
/*
* Task on global/bypass DSQ: leave custody, task on
@@ -1975,6 +2078,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
/* dequeue is always temporary, don't reset runnable_at */
clr_task_runnable(p, false);
+retry:
/* acquire ensures that we see the preceding updates on QUEUED */
opss = atomic_long_read_acquire(&p->scx.ops_state);
@@ -1988,8 +2092,20 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
*/
BUG();
case SCX_OPSS_QUEUED:
- /* A queued task must always be in BPF scheduler's custody */
- WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY));
+ /*
+ * A queued task must always be in BPF scheduler's custody. If
+ * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another
+ * CPU has already passed call_task_dequeue() (which clears the
+ * flag), but has not yet written SCX_OPSS_NONE. That final
+ * store does not require this rq's lock, so retrying with
+ * cpu_relax() is bounded: we will observe NONE (or DISPATCHING,
+ * handled by the fallthrough) on a subsequent iteration.
+ */
+ if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) {
+ cpu_relax();
+ goto retry;
+ }
+
if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
SCX_OPSS_NONE))
break;
@@ -2129,7 +2245,8 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
schedule_reenq_local(rq, 0);
}
-static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
+static void move_local_task_to_local_dsq(struct scx_sched *sch,
+ struct task_struct *p, u64 enq_flags,
struct scx_dispatch_q *src_dsq,
struct rq *dst_rq)
{
@@ -2149,7 +2266,7 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
dsq_inc_nr(dst_dsq, p, enq_flags);
p->scx.dsq = dst_dsq;
- local_dsq_post_enq(dst_dsq, p, enq_flags);
+ local_dsq_post_enq(sch, dst_dsq, p, enq_flags);
}
/**
@@ -2370,7 +2487,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
/* @p is going from a non-local DSQ to a local DSQ */
if (src_rq == dst_rq) {
task_unlink_from_dsq(p, src_dsq);
- move_local_task_to_local_dsq(p, enq_flags,
+ move_local_task_to_local_dsq(sch, p, enq_flags,
src_dsq, dst_rq);
raw_spin_unlock(&src_dsq->lock);
} else {
@@ -2423,7 +2540,7 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
if (rq == task_rq) {
task_unlink_from_dsq(p, dsq);
- move_local_task_to_local_dsq(p, enq_flags, dsq, rq);
+ move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
raw_spin_unlock(&dsq->lock);
return true;
}
@@ -3183,7 +3300,7 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) &&
!scx_bypassing(sch_a, task_cpu(a)))
return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before,
- NULL,
+ task_rq(a),
(struct task_struct *)a,
(struct task_struct *)b);
else
@@ -3440,41 +3557,6 @@ static struct cgroup *tg_cgrp(struct task_group *tg)
#endif /* CONFIG_EXT_GROUP_SCHED */
-static u32 scx_get_task_state(const struct task_struct *p)
-{
- return p->scx.flags & SCX_TASK_STATE_MASK;
-}
-
-static void scx_set_task_state(struct task_struct *p, u32 state)
-{
- u32 prev_state = scx_get_task_state(p);
- bool warn = false;
-
- switch (state) {
- case SCX_TASK_NONE:
- break;
- case SCX_TASK_INIT:
- warn = prev_state != SCX_TASK_NONE;
- break;
- case SCX_TASK_READY:
- warn = prev_state == SCX_TASK_NONE;
- break;
- case SCX_TASK_ENABLED:
- warn = prev_state != SCX_TASK_READY;
- break;
- default:
- WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
- prev_state, state, p->comm, p->pid);
- return;
- }
-
- WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
- prev_state, state, p->comm, p->pid);
-
- p->scx.flags &= ~SCX_TASK_STATE_MASK;
- p->scx.flags |= state;
-}
-
static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
{
int ret;
@@ -3526,22 +3608,6 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo
return 0;
}
-static int scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
-{
- int ret;
-
- ret = __scx_init_task(sch, p, fork);
- if (!ret) {
- /*
- * While @p's rq is not locked. @p is not visible to the rest of
- * SCX yet and it's safe to update the flags and state.
- */
- p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
- scx_set_task_state(p, SCX_TASK_INIT);
- }
- return ret;
-}
-
static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
{
struct rq *rq = task_rq(p);
@@ -3631,6 +3697,22 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch,
SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
}
+/*
+ * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
+ * ran. The task state has not been transitioned, so this mirrors the
+ * SCX_TASK_INIT branch in __scx_disable_and_exit_task().
+ */
+static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p)
+{
+ struct scx_exit_task_args args = { .cancelled = true };
+
+ lockdep_assert_held(&p->pi_lock);
+ lockdep_assert_rq_held(task_rq(p));
+
+ if (SCX_HAS_OP(sch, exit_task))
+ SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
+}
+
static void scx_disable_and_exit_task(struct scx_sched *sch,
struct task_struct *p)
{
@@ -3639,11 +3721,13 @@ static void scx_disable_and_exit_task(struct scx_sched *sch,
/*
* If set, @p exited between __scx_init_task() and scx_enable_task() in
* scx_sub_enable() and is initialized for both the associated sched and
- * its parent. Disable and exit for the child too.
+ * its parent. Exit for the child too - scx_enable_task() never ran for
+ * it, so undo only init_task. The flag is only set on the sub-enable
+ * path, so it's always clear when @p arrives here in %SCX_TASK_NONE.
*/
- if ((p->scx.flags & SCX_TASK_SUB_INIT) &&
- !WARN_ON_ONCE(!scx_enabling_sub_sched)) {
- __scx_disable_and_exit_task(scx_enabling_sub_sched, p);
+ if (p->scx.flags & SCX_TASK_SUB_INIT) {
+ if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
+ scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT;
}
@@ -3687,10 +3771,14 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs)
#else
struct scx_sched *sch = scx_root;
#endif
- ret = scx_init_task(sch, p, true);
- if (!ret)
- scx_set_task_sched(p, sch);
- return ret;
+ scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
+ ret = __scx_init_task(sch, p, true);
+ if (unlikely(ret)) {
+ scx_set_task_state(p, SCX_TASK_NONE);
+ return ret;
+ }
+ scx_set_task_state(p, SCX_TASK_INIT);
+ scx_set_task_sched(p, sch);
}
return 0;
@@ -3784,13 +3872,24 @@ void sched_ext_dead(struct task_struct *p)
/*
* @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
* ENABLED transitions can't race us. Disable ops for @p.
+ *
+ * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see
+ * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup
+ * iteration is only used from sub-sched paths, which require root
+ * enabled. Root enable transitions every live task to at least READY.
+ *
+ * %INIT_BEGIN means ops.init_task() is running for @p. Don't call
+ * into ops; transition to %DEAD so the post-init recheck unwinds
+ * via scx_sub_init_cancel_task().
*/
if (scx_get_task_state(p) != SCX_TASK_NONE) {
struct rq_flags rf;
struct rq *rq;
rq = task_rq_lock(p, &rf);
- scx_disable_and_exit_task(scx_task_sched(p), p);
+ if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN)
+ scx_disable_and_exit_task(scx_task_sched(p), p);
+ scx_set_task_state(p, SCX_TASK_DEAD);
task_rq_unlock(rq, p, &rf);
}
}
@@ -3836,6 +3935,16 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p)
if (task_dead_and_done(p))
return;
+ /*
+ * %NONE means SCX is no longer tracking @p at the task level (e.g.
+ * scx_fail_parent() handed @p back to the parent at NONE pending the
+ * parent's own teardown). There is nothing to disable; calling
+ * scx_disable_task() would WARN on the non-%ENABLED state and trigger a
+ * NONE -> READY validation failure.
+ */
+ if (scx_get_task_state(p) == SCX_TASK_NONE)
+ return;
+
scx_disable_task(scx_task_sched(p), p);
}
@@ -4293,11 +4402,13 @@ void scx_cgroup_move_task(struct task_struct *p)
return;
/*
- * @p must have ops.cgroup_prep_move() called on it and thus
- * cgrp_moving_from set.
+ * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's
+ * cgroup changes. Migration keys off css rather than cgroup identity,
+ * so it can hand an unchanged-cgroup task here with cgrp_moving_from
+ * NULL. Nothing to report to the BPF scheduler then, so skip it and
+ * keep prep_move and move paired.
*/
- if (SCX_HAS_OP(sch, cgroup_move) &&
- !WARN_ON_ONCE(!p->scx.cgrp_moving_from))
+ if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from)
SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p),
p, p->scx.cgrp_moving_from,
tg_cgrp(task_group(p)));
@@ -4324,9 +4435,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset)
void scx_group_set_weight(struct task_group *tg, unsigned long weight)
{
- struct scx_sched *sch = scx_root;
+ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem);
+ sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) &&
tg->scx.weight != weight)
@@ -4339,9 +4451,10 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight)
void scx_group_set_idle(struct task_group *tg, bool idle)
{
- struct scx_sched *sch = scx_root;
+ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem);
+ sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle))
SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
@@ -4355,9 +4468,10 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
void scx_group_set_bandwidth(struct task_group *tg,
u64 period_us, u64 quota_us, u64 burst_us)
{
- struct scx_sched *sch = scx_root;
+ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem);
+ sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&
(tg->scx.bw_period_us != period_us ||
@@ -4380,21 +4494,6 @@ static struct cgroup *root_cgroup(void)
return &cgrp_dfl_root.cgrp;
}
-static struct cgroup *sch_cgroup(struct scx_sched *sch)
-{
- return sch->cgrp;
-}
-
-/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
-static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
-{
- struct cgroup *pos;
- struct cgroup_subsys_state *css;
-
- cgroup_for_each_live_descendant_pre(pos, css, cgrp)
- rcu_assign_pointer(pos->scx_sched, sch);
-}
-
static void scx_cgroup_lock(void)
{
#ifdef CONFIG_EXT_GROUP_SCHED
@@ -4412,12 +4511,30 @@ static void scx_cgroup_unlock(void)
}
#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
static struct cgroup *root_cgroup(void) { return NULL; }
-static struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
-static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
static void scx_cgroup_lock(void) {}
static void scx_cgroup_unlock(void) {}
#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
+#ifdef CONFIG_EXT_SUB_SCHED
+static struct cgroup *sch_cgroup(struct scx_sched *sch)
+{
+ return sch->cgrp;
+}
+
+/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
+static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
+{
+ struct cgroup *pos;
+ struct cgroup_subsys_state *css;
+
+ cgroup_for_each_live_descendant_pre(pos, css, cgrp)
+ rcu_assign_pointer(pos->scx_sched, sch);
+}
+#else /* CONFIG_EXT_SUB_SCHED */
+static struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
+static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
+#endif /* CONFIG_EXT_SUB_SCHED */
+
/*
* Omitted operations:
*
@@ -4712,11 +4829,15 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
irq_work_sync(&sch->disable_irq_work);
kthread_destroy_worker(sch->helper);
timer_shutdown_sync(&sch->bypass_lb_timer);
+ free_cpumask_var(sch->bypass_lb_donee_cpumask);
+ free_cpumask_var(sch->bypass_lb_resched_cpumask);
#ifdef CONFIG_EXT_SUB_SCHED
kfree(sch->cgrp_path);
if (sch_cgroup(sch))
cgroup_put(sch_cgroup(sch));
+ if (sch->sub_kset)
+ kobject_put(&sch->sub_kset->kobj);
#endif /* CONFIG_EXT_SUB_SCHED */
for_each_possible_cpu(cpu) {
@@ -4840,10 +4961,30 @@ static const struct kset_uevent_ops scx_uevent_ops = {
*/
bool task_should_scx(int policy)
{
- if (!scx_enabled() || unlikely(scx_enable_state() == SCX_DISABLING))
+ /* if disabled, nothing should be on it */
+ if (!scx_enabled())
return false;
+
+ /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */
if (READ_ONCE(scx_switching_all))
return true;
+
+ /*
+ * scx is tearing down - keep new SCHED_EXT tasks out.
+ *
+ * Must come after scx_switching_all test, which serves as a proxy
+ * for __scx_switched_all. While __scx_switched_all is set, we must
+ * return true via the branch above: a fork routed to fair would
+ * stall because next_active_class() skips fair.
+ *
+ * This can develop into a deadlock - scx holds scx_enable_mutex across
+ * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is
+ * the stalled task, the disable path can never grab the mutex to clear
+ * scx_switching_all.
+ */
+ if (unlikely(scx_enable_state() == SCX_DISABLING))
+ return false;
+
return policy == SCHED_EXT;
}
@@ -4938,6 +5079,25 @@ void scx_softlockup(u32 dur_s)
smp_processor_id(), dur_s);
}
+/*
+ * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(),
+ * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing
+ * it from NMI context can lead to deadlocks. Defer via irq_work; the
+ * disable path runs off irq_work anyway.
+ */
+static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1);
+
+static void scx_hardlockup_irq_workfn(struct irq_work *work)
+{
+ int cpu = atomic_xchg(&scx_hardlockup_cpu, -1);
+
+ if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu))
+ printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
+ cpu);
+}
+
+static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn);
+
/**
* scx_hardlockup - sched_ext hardlockup handler
*
@@ -4946,17 +5106,19 @@ void scx_softlockup(u32 dur_s)
* Try kicking out the current scheduler in an attempt to recover the system to
* a good state before taking more drastic actions.
*
- * Returns %true if sched_ext is enabled and abort was initiated, which may
- * resolve the reported hardlockup. %false if sched_ext is not enabled or
- * someone else already initiated abort.
+ * Queues an irq_work; the handle_lockup() call happens in IRQ context (see
+ * scx_hardlockup_irq_workfn).
+ *
+ * Returns %true if sched_ext is enabled and the work was queued, %false
+ * otherwise.
*/
bool scx_hardlockup(int cpu)
{
- if (!handle_lockup("hard lockup - CPU %d", cpu))
+ if (!rcu_access_pointer(scx_root))
return false;
- printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
- cpu);
+ atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu);
+ irq_work_queue(&scx_hardlockup_irq_work);
return true;
}
@@ -5000,6 +5162,15 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
if (cpumask_empty(donee_mask))
break;
+ /*
+ * If an earlier pass placed @p on @donor_dsq from a different
+ * CPU and the donee hasn't consumed it yet, @p is still on the
+ * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved
+ * without its rq locked. Skip.
+ */
+ if (task_rq(p) != donor_rq)
+ continue;
+
donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);
if (donee >= nr_cpu_ids)
continue;
@@ -5058,8 +5229,8 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
static void bypass_lb_node(struct scx_sched *sch, int node)
{
const struct cpumask *node_mask = cpumask_of_node(node);
- struct cpumask *donee_mask = scx_bypass_lb_donee_cpumask;
- struct cpumask *resched_mask = scx_bypass_lb_resched_cpumask;
+ struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask;
+ struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask;
u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;
u32 nr_target, nr_donor_target;
u32 before_min = U32_MAX, before_max = 0;
@@ -5464,10 +5635,12 @@ static void refresh_watchdog(void)
static s32 scx_link_sched(struct scx_sched *sch)
{
+ const char *err_msg = "";
+ s32 ret = 0;
+
scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_sched *parent = scx_parent(sch);
- s32 ret;
if (parent) {
/*
@@ -5477,15 +5650,16 @@ static s32 scx_link_sched(struct scx_sched *sch)
* parent can shoot us down.
*/
if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) {
- scx_error(sch, "parent disabled");
- return -ENOENT;
+ err_msg = "parent disabled";
+ ret = -ENOENT;
+ break;
}
ret = rhashtable_lookup_insert_fast(&scx_sched_hash,
&sch->hash_node, scx_sched_hash_params);
if (ret) {
- scx_error(sch, "failed to insert into scx_sched_hash (%d)", ret);
- return ret;
+ err_msg = "failed to insert into scx_sched_hash";
+ break;
}
list_add_tail(&sch->sibling, &parent->children);
@@ -5495,6 +5669,15 @@ static s32 scx_link_sched(struct scx_sched *sch)
list_add_tail_rcu(&sch->all, &scx_sched_all);
}
+ /*
+ * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after
+ * the guard above is released.
+ */
+ if (ret) {
+ scx_error(sch, "%s (%d)", err_msg, ret);
+ return ret;
+ }
+
refresh_watchdog();
return 0;
}
@@ -5564,7 +5747,7 @@ static void scx_fail_parent(struct scx_sched *sch,
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
scx_disable_and_exit_task(sch, p);
- rcu_assign_pointer(p->scx.sched, parent);
+ scx_set_task_sched(p, parent);
}
}
scx_task_iter_stop(&sti);
@@ -5642,6 +5825,21 @@ static void scx_sub_disable(struct scx_sched *sch)
}
rq = task_rq_lock(p, &rf);
+
+ if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+ /*
+ * sched_ext_dead() raced us between __scx_init_task()
+ * and this rq lock and ran exit_task() on @sch (the
+ * sched @p was on at that point), not on $parent.
+ * $parent's just-completed init is owed an exit_task()
+ * and we issue it here.
+ */
+ scx_sub_init_cancel_task(parent, p);
+ task_rq_unlock(rq, p, &rf);
+ put_task_struct(p);
+ continue;
+ }
+
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
/*
* $p is initialized for $parent and still attached to
@@ -5650,13 +5848,14 @@ static void scx_sub_disable(struct scx_sched *sch)
* $p having already been initialized, and then enable.
*/
scx_disable_and_exit_task(sch, p);
+ scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_set_task_state(p, SCX_TASK_INIT);
- rcu_assign_pointer(p->scx.sched, parent);
+ scx_set_task_sched(p, parent);
scx_set_task_state(p, SCX_TASK_READY);
scx_enable_task(parent, p);
}
- task_rq_unlock(rq, p, &rf);
+ task_rq_unlock(rq, p, &rf);
put_task_struct(p);
}
scx_task_iter_stop(&sti);
@@ -5698,6 +5897,8 @@ static void scx_sub_disable(struct scx_sched *sch)
if (sch->ops.exit)
SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
+ if (sch->sub_kset)
+ kobject_del(&sch->sub_kset->kobj);
kobject_del(&sch->kobj);
}
#else /* CONFIG_EXT_SUB_SCHED */
@@ -5710,6 +5911,7 @@ static void scx_root_disable(struct scx_sched *sch)
struct scx_exit_info *ei = sch->exit_info;
struct scx_task_iter sti;
struct task_struct *p;
+ bool was_switched_all;
int cpu;
/* guarantee forward progress and wait for descendants to be disabled */
@@ -5736,6 +5938,8 @@ static void scx_root_disable(struct scx_sched *sch)
*/
mutex_lock(&scx_enable_mutex);
+ was_switched_all = scx_switched_all();
+
static_branch_disable(&__scx_switched_all);
WRITE_ONCE(scx_switching_all, false);
@@ -5785,10 +5989,34 @@ static void scx_root_disable(struct scx_sched *sch)
/*
* Invalidate all the rq clocks to prevent getting outdated
* rq clocks from a previous scx scheduler.
+ *
+ * Also re-balance the dl_server bandwidth reservations: detach
+ * ext_server (no more sched_ext tasks) and reinstate fair_server if it
+ * was previously detached because we were running in full mode.
+ *
+ * Unlike the enable path, this runs on a recovery path that cannot
+ * fail, so we use dl_server_swap_bw() to atomically free ext_server's
+ * bandwidth and reclaim it for fair_server under the same dl_b lock.
+ *
+ * The swap can still fail with -EBUSY if someone bumped ext_server's
+ * runtime via debugfs between enable and disable; in that narrow case
+ * both servers end up detached and we just WARN.
*/
for_each_possible_cpu(cpu) {
struct rq *rq = cpu_rq(cpu);
+
scx_rq_clock_invalidate(rq);
+
+ scoped_guard(rq_lock_irqsave, rq) {
+ update_rq_clock(rq);
+ if (was_switched_all) {
+ if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
+ &rq->fair_server)))
+ pr_warn("failed to re-attach fair_server on CPU %d\n", cpu);
+ } else {
+ dl_server_detach_bw(&rq->ext_server);
+ }
+ }
}
/* no task is on scx, turn off all the switches and flush in-progress calls */
@@ -5829,6 +6057,10 @@ static void scx_root_disable(struct scx_sched *sch)
* could observe an object of the same name still in the hierarchy when
* the next scheduler is loaded.
*/
+#ifdef CONFIG_EXT_SUB_SCHED
+ if (sch->sub_kset)
+ kobject_del(&sch->sub_kset->kobj);
+#endif
kobject_del(&sch->kobj);
free_kick_syncs();
@@ -5921,6 +6153,25 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)
irq_work_queue(&sch->disable_irq_work);
}
+/**
+ * scx_flush_disable_work - flush the disable work and wait for it to finish
+ * @sch: the scheduler
+ *
+ * sch->disable_work might still not queued, causing kthread_flush_work()
+ * as a noop. Syncing the irq_work first is required to guarantee the
+ * kthread work has been queued before waiting for it.
+ */
+static void scx_flush_disable_work(struct scx_sched *sch)
+{
+ int kind;
+
+ do {
+ irq_work_sync(&sch->disable_irq_work);
+ kthread_flush_work(&sch->disable_work);
+ kind = atomic_read(&sch->exit_kind);
+ } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
+}
+
static void dump_newline(struct seq_buf *s)
{
trace_sched_ext_dump("");
@@ -6032,9 +6283,8 @@ static void ops_dump_exit(void)
scx_dump_data.cpu = -1;
}
-static void scx_dump_task(struct scx_sched *sch,
- struct seq_buf *s, struct scx_dump_ctx *dctx,
- struct task_struct *p, char marker)
+static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx,
+ struct rq *rq, struct task_struct *p, char marker)
{
static unsigned long bt[SCX_EXIT_BT_LEN];
struct scx_sched *task_sch = scx_task_sched(p);
@@ -6075,7 +6325,7 @@ static void scx_dump_task(struct scx_sched *sch,
if (SCX_HAS_OP(sch, dump_task)) {
ops_dump_init(s, " ");
- SCX_CALL_OP(sch, dump_task, NULL, dctx, p);
+ SCX_CALL_OP(sch, dump_task, rq, dctx, p);
ops_dump_exit();
}
@@ -6199,8 +6449,7 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
used = seq_buf_used(&ns);
if (SCX_HAS_OP(sch, dump_cpu)) {
ops_dump_init(&ns, " ");
- SCX_CALL_OP(sch, dump_cpu, NULL,
- &dctx, cpu, idle);
+ SCX_CALL_OP(sch, dump_cpu, rq, &dctx, cpu, idle);
ops_dump_exit();
}
@@ -6223,11 +6472,11 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
if (rq->curr->sched_class == &ext_sched_class &&
(dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
- scx_dump_task(sch, &s, &dctx, rq->curr, '*');
+ scx_dump_task(sch, &s, &dctx, rq, rq->curr, '*');
list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
if (dump_all_tasks || scx_task_on_sched(sch, p))
- scx_dump_task(sch, &s, &dctx, p, ' ');
+ scx_dump_task(sch, &s, &dctx, rq, p, ' ');
next:
rq_unlock_irqrestore(rq, &rf);
}
@@ -6434,26 +6683,36 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
sch->slice_dfl = SCX_SLICE_DFL;
atomic_set(&sch->exit_kind, SCX_EXIT_NONE);
- init_irq_work(&sch->disable_irq_work, scx_disable_irq_workfn);
+ sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn);
kthread_init_work(&sch->disable_work, scx_disable_workfn);
timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
+
+ if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto err_stop_helper;
+ }
+ if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto err_free_lb_cpumask;
+ }
sch->ops = *ops;
rcu_assign_pointer(ops->priv, sch);
sch->kobj.kset = scx_kset;
+ INIT_LIST_HEAD(&sch->all);
#ifdef CONFIG_EXT_SUB_SCHED
char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
if (!buf) {
ret = -ENOMEM;
- goto err_stop_helper;
+ goto err_free_lb_resched;
}
cgroup_path(cgrp, buf, PATH_MAX);
sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
kfree(buf);
if (!sch->cgrp_path) {
ret = -ENOMEM;
- goto err_stop_helper;
+ goto err_free_lb_resched;
}
sch->cgrp = cgrp;
@@ -6468,6 +6727,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
if (ret < 0) {
+ RCU_INIT_POINTER(ops->priv, NULL);
kobject_put(&sch->kobj);
return ERR_PTR(ret);
}
@@ -6475,6 +6735,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
if (ops->sub_attach) {
sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
if (!sch->sub_kset) {
+ RCU_INIT_POINTER(ops->priv, NULL);
kobject_put(&sch->kobj);
return ERR_PTR(-ENOMEM);
}
@@ -6482,6 +6743,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
#else /* CONFIG_EXT_SUB_SCHED */
ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
if (ret < 0) {
+ RCU_INIT_POINTER(ops->priv, NULL);
kobject_put(&sch->kobj);
return ERR_PTR(ret);
}
@@ -6489,9 +6751,14 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
return sch;
#ifdef CONFIG_EXT_SUB_SCHED
+err_free_lb_resched:
+ RCU_INIT_POINTER(ops->priv, NULL);
+ free_cpumask_var(sch->bypass_lb_resched_cpumask);
+#endif
+err_free_lb_cpumask:
+ free_cpumask_var(sch->bypass_lb_donee_cpumask);
err_stop_helper:
kthread_destroy_worker(sch->helper);
-#endif
err_free_pcpu:
for_each_possible_cpu(cpu) {
if (cpu == bypass_fail_cpu)
@@ -6510,7 +6777,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
err_free_sch:
kfree(sch);
err_put_cgrp:
-#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
+#ifdef CONFIG_EXT_SUB_SCHED
cgroup_put(cgrp);
#endif
return ERR_PTR(ret);
@@ -6597,11 +6864,24 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_unlock;
}
+ /*
+ * @ops->priv binds @ops to its scx_sched instance. It is set here by
+ * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(),
+ * which runs after scx_root_disable() has dropped scx_enable_mutex. If
+ * it's still non-NULL here, a previous attachment on @ops has not
+ * finished tearing down; proceeding would let the in-flight unreg's
+ * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign.
+ */
+ if (rcu_access_pointer(ops->priv)) {
+ ret = -EBUSY;
+ goto err_unlock;
+ }
+
ret = alloc_kick_syncs();
if (ret)
goto err_unlock;
-#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
+#ifdef CONFIG_EXT_SUB_SCHED
cgroup_get(cgrp);
#endif
sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
@@ -6639,8 +6919,10 @@ static void scx_root_enable_workfn(struct kthread_work *work)
rcu_assign_pointer(scx_root, sch);
ret = scx_link_sched(sch);
- if (ret)
+ if (ret) {
+ cpus_read_unlock();
goto err_disable;
+ }
scx_idle_enable(ops);
@@ -6673,6 +6955,31 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
/*
+ * Attach the ext_server bandwidth reservation before anything is
+ * committed so that we can fail the enable if the root domain cannot
+ * accommodate it. The matching fair_server detach is deferred to the
+ * tail of this function, after the switch is fully committed and can no
+ * longer fail.
+ *
+ * On failure, err_disable funnels into scx_root_disable() which
+ * detaches ext_server, so partially-attached state is cleaned up
+ * automatically.
+ */
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+
+ scoped_guard(rq_lock_irqsave, rq) {
+ update_rq_clock(rq);
+ ret = dl_server_attach_bw(&rq->ext_server);
+ }
+ if (ret) {
+ pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n",
+ cpu, ret);
+ goto err_disable;
+ }
+ }
+
+ /*
* Once __scx_enabled is set, %current can be switched to SCX anytime.
* This can lead to stalls as some BPF schedulers (e.g. userspace
* scheduling) may not function correctly before all tasks are switched.
@@ -6717,6 +7024,9 @@ static void scx_root_enable_workfn(struct kthread_work *work)
scx_task_iter_start(&sti, NULL);
while ((p = scx_task_iter_next_locked(&sti))) {
+ struct rq_flags rf;
+ struct rq *rq;
+
/*
* @p may already be dead, have lost all its usages counts and
* be waiting for RCU grace period before being freed. @p can't
@@ -6725,20 +7035,47 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (!tryget_task_struct(p))
continue;
+ /*
+ * Set %INIT_BEGIN under the iter's rq lock so that a concurrent
+ * sched_ext_dead() does not call ops.exit_task() on @p while
+ * ops.init_task() is running. If sched_ext_dead() runs before
+ * this store, it has already removed @p from scx_tasks and the
+ * iter won't visit @p; if it runs after, it observes
+ * %INIT_BEGIN and transitions to %DEAD without calling ops,
+ * leaving the post-init recheck below to unwind.
+ */
+ scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_task_iter_unlock(&sti);
- ret = scx_init_task(sch, p, false);
- if (ret) {
- put_task_struct(p);
+ ret = __scx_init_task(sch, p, false);
+
+ rq = task_rq_lock(p, &rf);
+
+ if (unlikely(ret)) {
+ if (scx_get_task_state(p) != SCX_TASK_DEAD)
+ scx_set_task_state(p, SCX_TASK_NONE);
+ task_rq_unlock(rq, p, &rf);
scx_task_iter_stop(&sti);
scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
ret, p->comm, p->pid);
+ put_task_struct(p);
goto err_disable_unlock_all;
}
- scx_set_task_sched(p, sch);
- scx_set_task_state(p, SCX_TASK_READY);
+ if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+ /*
+ * sched_ext_dead() observed %INIT_BEGIN and set %DEAD.
+ * ops.exit_task() is owed to the sched __scx_init_task()
+ * ran against; call it now.
+ */
+ scx_sub_init_cancel_task(sch, p);
+ } else {
+ scx_set_task_state(p, SCX_TASK_INIT);
+ scx_set_task_sched(p, sch);
+ scx_set_task_state(p, SCX_TASK_READY);
+ }
+ task_rq_unlock(rq, p, &rf);
put_task_struct(p);
}
scx_task_iter_stop(&sti);
@@ -6788,6 +7125,25 @@ static void scx_root_enable_workfn(struct kthread_work *work)
if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
static_branch_enable(&__scx_switched_all);
+ /*
+ * Detach the fair_server bandwidth reservation now that the switch
+ * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
+ * task will ever run in the fair class, so give that bandwidth
+ * back to the RT class. The matching ext_server attach already
+ * happened earlier; this only releases bandwidth and cannot fail.
+ *
+ * In partial mode keep fair_server attached.
+ */
+ if (scx_switched_all()) {
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+
+ guard(rq_lock_irqsave)(rq);
+ update_rq_clock(rq);
+ dl_server_detach_bw(&rq->fair_server);
+ }
+ }
+
pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
sch->ops.name, scx_switched_all() ? "" : " (partial)");
kobject_uevent(&sch->kobj, KOBJ_ADD);
@@ -6821,7 +7177,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
* completion. sch's base reference will be put by bpf_scx_unreg().
*/
scx_error(sch, "scx_root_enable() failed (%d)", ret);
- kthread_flush_work(&sch->disable_work);
+ scx_flush_disable_work(sch);
cmd->ret = 0;
}
@@ -6882,6 +7238,12 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
goto out_unlock;
}
+ /* See scx_root_enable_workfn() for the @ops->priv check. */
+ if (rcu_access_pointer(ops->priv)) {
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
cgrp = cgroup_get_from_id(ops->sub_cgroup_id);
if (IS_ERR(cgrp)) {
ret = PTR_ERR(cgrp);
@@ -7008,6 +7370,21 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
goto abort;
rq = task_rq_lock(p, &rf);
+
+ if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+ /*
+ * sched_ext_dead() raced us between __scx_init_task()
+ * and this rq lock and ran exit_task() on $parent (the
+ * sched @p was on at that point), not on @sch. @sch's
+ * just-completed init is owed an exit_task() and we
+ * issue it here.
+ */
+ scx_sub_init_cancel_task(sch, p);
+ task_rq_unlock(rq, p, &rf);
+ put_task_struct(p);
+ continue;
+ }
+
p->scx.flags |= SCX_TASK_SUB_INIT;
task_rq_unlock(rq, p, &rf);
@@ -7042,7 +7419,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
* $p is now only initialized for @sch and READY, which
* is what we want. Assign it to @sch and enable.
*/
- rcu_assign_pointer(p->scx.sched, sch);
+ scx_set_task_sched(p, sch);
scx_enable_task(sch, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT;
@@ -7072,23 +7449,30 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
abort:
put_task_struct(p);
scx_task_iter_stop(&sti);
- scx_enabling_sub_sched = NULL;
+ /*
+ * Undo __scx_init_task() for tasks we marked. scx_enable_task() never
+ * ran for @sch on them, so calling scx_disable_task() here would invoke
+ * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched
+ * must stay set until SUB_INIT is cleared from every marked task -
+ * scx_disable_and_exit_task() reads it when a task exits concurrently.
+ */
scx_task_iter_start(&sti, sch->cgrp);
while ((p = scx_task_iter_next_locked(&sti))) {
if (p->scx.flags & SCX_TASK_SUB_INIT) {
- __scx_disable_and_exit_task(sch, p);
+ scx_sub_init_cancel_task(sch, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT;
}
}
scx_task_iter_stop(&sti);
+ scx_enabling_sub_sched = NULL;
err_unlock_and_disable:
/* we'll soon enter disable path, keep bypass on */
scx_cgroup_unlock();
percpu_up_write(&scx_fork_rwsem);
err_disable:
mutex_unlock(&scx_enable_mutex);
- kthread_flush_work(&sch->disable_work);
+ scx_flush_disable_work(sch);
cmd->ret = 0;
}
@@ -7137,8 +7521,7 @@ static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
static DEFINE_MUTEX(helper_mutex);
struct scx_enable_cmd cmd;
- if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN),
- cpu_possible_mask)) {
+ if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) {
pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n");
return -EINVAL;
}
@@ -7349,7 +7732,7 @@ static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
scx_disable(sch, SCX_EXIT_UNREG);
- kthread_flush_work(&sch->disable_work);
+ scx_flush_disable_work(sch);
RCU_INIT_POINTER(ops->priv, NULL);
kobject_put(&sch->kobj);
}
@@ -8033,12 +8416,22 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
struct task_struct *p, u64 dsq_id, u64 enq_flags)
{
struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
- struct scx_sched *sch = src_dsq->sched;
+ struct scx_sched *sch;
struct rq *this_rq, *src_rq, *locked_rq;
bool dispatched = false;
bool in_balance;
unsigned long flags;
+ /*
+ * The verifier considers an iterator slot initialized on any
+ * KF_ITER_NEW return, so a BPF program may legally reach here after
+ * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL.
+ */
+ if (unlikely(!src_dsq))
+ return false;
+
+ sch = src_dsq->sched;
+
if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
return false;
@@ -8526,7 +8919,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
guard(rcu)();
sch = scx_prog_sched(aux);
- if (unlikely(!scx_task_on_sched(sch, p)))
+ if (unlikely(!sch || !scx_task_on_sched(sch, p)))
return false;
p->scx.slice = slice;
@@ -8549,7 +8942,7 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
guard(rcu)();
sch = scx_prog_sched(aux);
- if (unlikely(!scx_task_on_sched(sch, p)))
+ if (unlikely(!sch || !scx_task_on_sched(sch, p)))
return false;
p->scx.dsq_vtime = vtime;
@@ -8633,11 +9026,12 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux
/**
* scx_bpf_dsq_nr_queued - Return the number of queued tasks
* @dsq_id: id of the DSQ
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Return the number of tasks in the DSQ matching @dsq_id. If not found,
* -%ENOENT is returned.
*/
-__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
+__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
{
struct scx_sched *sch;
struct scx_dispatch_q *dsq;
@@ -8645,7 +9039,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
preempt_disable();
- sch = rcu_dereference_sched(scx_root);
+ sch = scx_prog_sched(aux);
if (unlikely(!sch)) {
ret = -ENODEV;
goto out;
@@ -8677,21 +9071,21 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
/**
* scx_bpf_destroy_dsq - Destroy a custom DSQ
* @dsq_id: DSQ to destroy
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
* scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
* empty and no further tasks are dispatched to it. Ignored if called on a DSQ
* which doesn't exist. Can be called from any online scx_ops operations.
*/
-__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)
+__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
{
struct scx_sched *sch;
- rcu_read_lock();
- sch = rcu_dereference(scx_root);
+ guard(rcu)();
+ sch = scx_prog_sched(aux);
if (sch)
destroy_dsq(sch, dsq_id);
- rcu_read_unlock();
}
/**
@@ -9445,8 +9839,8 @@ BTF_KFUNCS_START(scx_kfunc_ids_any)
BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
-BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
-BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
+BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL)
BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
@@ -9479,6 +9873,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_any)
static const struct btf_kfunc_id_set scx_kfunc_set_any = {
.owner = THIS_MODULE,
.set = &scx_kfunc_ids_any,
+ .filter = scx_kfunc_context_filter,
};
/*
@@ -9526,13 +9921,12 @@ static const u32 scx_kf_allow_flags[] = {
};
/*
- * Verifier-time filter for context-sensitive SCX kfuncs. Registered via the
- * .filter field on each per-group btf_kfunc_id_set. The BPF core invokes this
- * for every kfunc call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
+ * Verifier-time filter for SCX kfuncs. Registered via the .filter field on
+ * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc
+ * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
* BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the
- * kfunc - so the filter must short-circuit on kfuncs it doesn't govern (e.g.
- * scx_kfunc_ids_any) by falling through to "allow" when none of the
- * context-sensitive sets contain the kfunc.
+ * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by
+ * falling through to "allow" when none of the SCX sets contain the kfunc.
*/
int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
{
@@ -9541,18 +9935,21 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
+ bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
+ bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
u32 moff, flags;
- /* Not a context-sensitive kfunc (e.g. from scx_kfunc_ids_any) - allow. */
- if (!(in_unlocked || in_select_cpu || in_enqueue || in_dispatch || in_cpu_release))
+ /* Not an SCX kfunc - allow. */
+ if (!(in_unlocked || in_select_cpu || in_enqueue || in_dispatch ||
+ in_cpu_release || in_idle || in_any))
return 0;
/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */
if (prog->type == BPF_PROG_TYPE_SYSCALL)
- return (in_unlocked || in_select_cpu) ? 0 : -EACCES;
+ return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES;
if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
- return -EACCES;
+ return (in_any || in_idle) ? 0 : -EACCES;
/*
* add_subprog_and_kfunc() collects all kfunc calls, including dead code
@@ -9565,14 +9962,15 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
return 0;
/*
- * Non-SCX struct_ops: only unlocked kfuncs are safe. The other
- * context-sensitive kfuncs assume the rq lock is held by the SCX
- * dispatch path, which doesn't apply to other struct_ops users.
+ * Non-SCX struct_ops: SCX kfuncs are not permitted.
*/
if (prog->aux->st_ops != &bpf_sched_ext_ops)
- return in_unlocked ? 0 : -EACCES;
+ return -EACCES;
/* SCX struct_ops: check the per-op allow list. */
+ if (in_any || in_idle)
+ return 0;
+
moff = prog->aux->attach_st_ops_member_off;
flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
@@ -9656,12 +10054,6 @@ static int __init scx_init(void)
return ret;
}
- if (!alloc_cpumask_var(&scx_bypass_lb_donee_cpumask, GFP_KERNEL) ||
- !alloc_cpumask_var(&scx_bypass_lb_resched_cpumask, GFP_KERNEL)) {
- pr_err("sched_ext: Failed to allocate cpumasks\n");
- return -ENOMEM;
- }
-
return 0;
}
__initcall(scx_init);
diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c
index 443d12a..9f5ad6b 100644
--- a/kernel/sched/ext_idle.c
+++ b/kernel/sched/ext_idle.c
@@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
int node = scx_cpu_node_if_enabled(cpu);
struct cpumask *idle_cpus = idle_cpumask(node)->cpu;
-#ifdef CONFIG_SCHED_SMT
/*
* SMT mask should be cleared whether we can claim @cpu or not. The SMT
* cluster is not wholly idle either way. This also prevents
@@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
else if (cpumask_test_cpu(cpu, idle_smts))
__cpumask_clear_cpu(cpu, idle_smts);
}
-#endif
return cpumask_test_and_clear_cpu(cpu, idle_cpus);
}
@@ -466,12 +464,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
preempt_disable();
/*
- * Check whether @prev_cpu is still within the allowed set. If not,
- * we can still try selecting a nearby CPU.
- */
- is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed);
-
- /*
* Determine the subset of CPUs usable by @p within @cpus_allowed.
*/
if (allowed != p->cpus_ptr) {
@@ -488,6 +480,12 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
}
/*
+ * Check whether @prev_cpu is still within the allowed set. If not,
+ * we can still try selecting a nearby CPU.
+ */
+ is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed);
+
+ /*
* This is necessary to protect llc_cpus.
*/
rcu_read_lock();
@@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
goto out_unlock;
}
-#ifdef CONFIG_SCHED_SMT
/*
* Use @prev_cpu's sibling if it's idle.
*/
@@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
goto out_unlock;
}
}
-#endif
/*
* Search for any idle CPU in the same LLC domain.
@@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle)
assign_cpu(cpu, idle_cpus, idle);
-#ifdef CONFIG_SCHED_SMT
if (sched_smt_active()) {
const struct cpumask *smt = cpu_smt_mask(cpu);
struct cpumask *idle_smts = idle_cpumask(node)->smt;
@@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle)
cpumask_andnot(idle_smts, idle_smts, smt);
}
}
-#endif
}
/*
@@ -927,14 +921,24 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
* Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq
* lock or @p's pi_lock. Three cases:
*
- * - inside ops.select_cpu(): try_to_wake_up() holds @p's pi_lock.
+ * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up
+ * task's pi_lock; the wake-up task is recorded in kf_tasks[0]
+ * by SCX_CALL_OP_TASK_RET().
* - other rq-locked SCX op: scx_locked_rq() points at the held rq.
* - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops):
* nothing held, take pi_lock ourselves.
+ *
+ * In the first two cases, BPF schedulers may pass an arbitrary task
+ * that the held lock doesn't cover. Refuse those.
*/
if (this_rq()->scx.in_select_cpu) {
+ if (!scx_kf_arg_task_ok(sch, p))
+ return -EINVAL;
lockdep_assert_held(&p->pi_lock);
- } else if (!scx_locked_rq()) {
+ } else if (scx_locked_rq()) {
+ if (task_rq(p) != scx_locked_rq())
+ goto cross_task;
+ } else {
raw_spin_lock_irqsave(&p->pi_lock, irq_flags);
we_locked = true;
}
@@ -960,6 +964,11 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags);
return cpu;
+
+cross_task:
+ scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]",
+ p->comm, p->pid);
+ return -EINVAL;
}
/**
@@ -1467,6 +1476,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_idle)
static const struct btf_kfunc_id_set scx_kfunc_set_idle = {
.owner = THIS_MODULE,
.set = &scx_kfunc_ids_idle,
+ .filter = scx_kfunc_context_filter,
};
/*
diff --git a/kernel/sched/ext_idle.h b/kernel/sched/ext_idle.h
index dc35f850..8d169d3 100644
--- a/kernel/sched/ext_idle.h
+++ b/kernel/sched/ext_idle.h
@@ -12,6 +12,7 @@
struct sched_ext_ops;
+extern struct btf_id_set8 scx_kfunc_ids_idle;
extern struct btf_id_set8 scx_kfunc_ids_select_cpu;
void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops);
diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h
index 62ce4ea..a075732 100644
--- a/kernel/sched/ext_internal.h
+++ b/kernel/sched/ext_internal.h
@@ -1075,6 +1075,8 @@ struct scx_sched {
struct irq_work disable_irq_work;
struct kthread_work disable_work;
struct timer_list bypass_lb_timer;
+ cpumask_var_t bypass_lb_donee_cpumask;
+ cpumask_var_t bypass_lb_resched_cpumask;
struct rcu_work rcu_work;
/* all ancestors including self */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 69361c6..f4ed841 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -334,7 +334,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* to a tree or when we reach the top of the tree
*/
if (cfs_rq->tg->parent &&
- cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
+ tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) {
/*
* If parent is already on the list, we add the child
* just before. Thanks to circular linked property of
@@ -342,7 +342,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* of the list that starts by parent.
*/
list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
- &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
+ &(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list));
/*
* The branch is now connected to its tree so we can
* reset tmp_alone_branch to the beginning of the
@@ -525,7 +525,7 @@ static int se_is_idle(struct sched_entity *se)
#endif /* !CONFIG_FAIR_GROUP_SCHED */
static __always_inline
-void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
+bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
/**************************************************************
* Scheduling class tree data structure manipulation methods:
@@ -847,13 +847,19 @@ static s64 entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 avrunt
* Similarly, check that the entity didn't gain positive lag when DELAY_ZERO
* is set.
*
- * Return true if the lag has been adjusted.
+ * Return true if the vlag has been modified. Specifically:
+ *
+ * se->vlag != avg_vruntime() - se->vruntime
+ *
+ * This can be due to clamping in entity_lag() or clamping due to
+ * sched_delayed. Either way, when vlag is modified and the entity is
+ * retained, the tree needs to be adjusted.
*/
static __always_inline
bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
- s64 vlag = entity_lag(cfs_rq, se, avg_vruntime(cfs_rq));
- bool ret;
+ u64 avruntime = avg_vruntime(cfs_rq);
+ s64 vlag = entity_lag(cfs_rq, se, avruntime);
WARN_ON_ONCE(!se->on_rq);
@@ -863,10 +869,9 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
if (sched_feat(DELAY_ZERO))
vlag = min(vlag, 0);
}
- ret = (vlag == se->vlag);
se->vlag = vlag;
- return ret;
+ return avruntime - vlag != se->vruntime;
}
/*
@@ -877,11 +882,11 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
*
* lag_i >= 0 -> V >= v_i
*
- * \Sum (v_i - v)*w_i
- * V = ------------------ + v
+ * \Sum (v_i - v0)*w_i
+ * V = ------------------- + v0
* \Sum w_i
*
- * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i)
+ * lag_i >= 0 -> \Sum (v_i - v0)*w_i >= (v_i - v0)*(\Sum w_i)
*
* Note: using 'avg_vruntime() > se->vruntime' is inaccurate due
* to the loss in precision caused by the division.
@@ -889,7 +894,7 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
{
struct sched_entity *curr = cfs_rq->curr;
- s64 avg = cfs_rq->sum_w_vruntime;
+ s64 key, avg = cfs_rq->sum_w_vruntime;
long load = cfs_rq->sum_weight;
if (curr && curr->on_rq) {
@@ -899,7 +904,36 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
load += weight;
}
- return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load;
+ key = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime);
+
+ /*
+ * The worst case term for @key includes 'NSEC_TICK * NICE_0_LOAD'
+ * and @load obviously includes NICE_0_LOAD. NSEC_TICK is around 24
+ * bits, while NICE_0_LOAD is 20 on 64bit and 10 otherwise.
+ *
+ * This gives that on 64bit the product will be at least 64bit which
+ * overflows s64, while on 32bit it will only be 44bits and should fit
+ * comfortably.
+ */
+#ifdef CONFIG_64BIT
+#ifdef CONFIG_ARCH_SUPPORTS_INT128
+ /* This often results in simpler code than __builtin_mul_overflow(). */
+ return avg >= (__int128)key * load;
+#else
+ s64 rhs;
+ /*
+ * On overflow, the sign of key tells us the correct answer: a large
+ * positive key means vruntime >> V, so not eligible; a large negative
+ * key means vruntime << V, so eligible.
+ */
+ if (check_mul_overflow(key, load, &rhs))
+ return key <= 0;
+
+ return avg >= rhs;
+#endif
+#else /* 32bit */
+ return avg >= key * load;
+#endif
}
int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -1099,7 +1133,7 @@ static inline void cancel_protect_slice(struct sched_entity *se)
*
* Which allows tree pruning through eligibility.
*/
-static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect)
+static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect)
{
struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node;
struct sched_entity *se = __pick_first_entity(cfs_rq);
@@ -1170,11 +1204,6 @@ static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect)
return best;
}
-static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
-{
- return __pick_eevdf(cfs_rq, true);
-}
-
struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
{
struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
@@ -1321,6 +1350,8 @@ void post_init_entity_util_avg(struct task_struct *p)
sa->runnable_avg = sa->util_avg;
}
+static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec);
+
static s64 update_se(struct rq *rq, struct sched_entity *se)
{
u64 now = rq_clock_task(rq);
@@ -1343,6 +1374,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se)
trace_sched_stat_runtime(running, delta_exec);
account_group_exec_runtime(running, delta_exec);
+ account_mm_sched(rq, running, delta_exec);
/* cgroup time is always accounted against the donor */
cgroup_account_cputime(donor, delta_exec);
@@ -1364,6 +1396,581 @@ static s64 update_se(struct rq *rq, struct sched_entity *se)
static void set_next_buddy(struct sched_entity *se);
+#ifdef CONFIG_SCHED_CACHE
+
+/*
+ * XXX numbers come from a place the sun don't shine -- probably wants to be SD
+ * tunable or so.
+ */
+#define EPOCH_PERIOD (HZ / 100) /* 10 ms */
+#define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */
+__read_mostly unsigned int llc_aggr_tolerance = 1;
+__read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD;
+__read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
+__read_mostly unsigned int llc_imb_pct = 20;
+__read_mostly unsigned int llc_overaggr_pct = 50;
+
+static int llc_id(int cpu)
+{
+ if (cpu < 0)
+ return -1;
+
+ return per_cpu(sd_llc_id, cpu);
+}
+
+static inline int get_sched_cache_scale(int mul)
+{
+ unsigned int tol = READ_ONCE(llc_aggr_tolerance);
+
+ if (!tol)
+ return 0;
+
+ if (tol >= 100)
+ return INT_MAX;
+
+ return (1 + (tol - 1) * mul);
+}
+
+static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
+{
+#ifdef CONFIG_NUMA_BALANCING
+ unsigned long llc, footprint;
+ struct sched_domain *sd;
+ int scale;
+
+ guard(rcu)();
+
+ sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd);
+ if (!sd)
+ return true;
+
+ if (static_branch_likely(&sched_numa_balancing)) {
+ /*
+ * TBD: RDT exclusive LLC ways reserved should be
+ * excluded.
+ */
+ llc = sd->llc_bytes;
+ footprint = READ_ONCE(mm->sc_stat.footprint);
+
+ /*
+ * Scale the LLC size by 256*llc_aggr_tolerance
+ * and compare it to the task's footprint.
+ *
+ * Suppose the L3 size is 32MB. If the
+ * llc_aggr_tolerance is 1:
+ * When the footprint is larger than 32MB, the
+ * process is regarded as exceeding the LLC
+ * capacity. If the llc_aggr_tolerance is 99:
+ * When the footprint is larger than 784GB, the
+ * process is regarded as exceeding the LLC
+ * capacity:
+ * 784GB = (1 + (99 - 1) * 256) * 32MB
+ * If the llc_aggr_tolerance is 100:
+ * ignore the footprint and do the aggregation
+ * anyway.
+ */
+ scale = get_sched_cache_scale(256);
+ if (scale == INT_MAX)
+ return false;
+
+ return ((llc * (u64)scale) < (footprint * PAGE_SIZE));
+ }
+#endif
+ return false;
+}
+
+static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
+ int cpu)
+{
+ int scale;
+
+ if (get_nr_threads(p) <= 1)
+ return true;
+
+ /*
+ * Scale the number of 'cores' in a LLC by llc_aggr_tolerance
+ * and compare it to the task's active threads.
+ */
+ scale = get_sched_cache_scale(1);
+ if (scale == INT_MAX)
+ return false;
+
+ return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads),
+ (scale * per_cpu(sd_llc_size, cpu)));
+}
+
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
+{
+ int pref_llc, pref_llc_queued;
+ struct sched_domain *sd;
+
+ pref_llc = p->preferred_llc;
+ if (pref_llc < 0)
+ return;
+
+ pref_llc_queued = (pref_llc == task_llc(p));
+ rq->nr_llc_running++;
+ rq->nr_pref_llc_running += pref_llc_queued;
+
+ /*
+ * Record whether p is enqueued on its preferred
+ * LLC, in order to pair with account_llc_dequeue()
+ * to maintain a consistent nr_pref_llc_running per
+ * runqueue.
+ * This is necessary because a race condition exists:
+ * after a task is enqueued on a runqueue, task_llc(p)
+ * may change due to CPU hotplug. Therefore, checking
+ * task_llc(p) to determine whether the task is being
+ * dequeued from its preferred LLC is unreliable and
+ * can cause inconsistent values - checking the
+ * p->pref_llc_queued in account_llc_dequeue() would
+ * be reliable.
+ */
+ p->pref_llc_queued = pref_llc_queued;
+
+ sd = rcu_dereference_all(rq->sd);
+ if (sd && (unsigned int)pref_llc < sd->llc_max)
+ sd->llc_counts[pref_llc]++;
+}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
+{
+ struct sched_domain *sd;
+ int pref_llc;
+
+ pref_llc = p->preferred_llc;
+ if (pref_llc < 0)
+ return;
+
+ rq->nr_llc_running--;
+ if (p->pref_llc_queued) {
+ rq->nr_pref_llc_running--;
+ /*
+ * Update the status in case
+ * other logic might query
+ * this.
+ */
+ p->pref_llc_queued = 0;
+ }
+
+ sd = rcu_dereference_all(rq->sd);
+ if (sd && (unsigned int)pref_llc < sd->llc_max) {
+ /*
+ * There is a race condition between dequeue
+ * and CPU hotplug. After a task has been enqueued
+ * on CPUx, a CPU hotplug event occurs, and all online
+ * CPUs (including CPUx) rebuild their sched_domains
+ * and reset statistics to zero(including sd->llc_counts).
+ * This can cause temporary undercount and we have to
+ * check for such underflow in sd->llc_counts.
+ *
+ * This undercount is temporary and accurate accounting
+ * will resume once the rq has a chance to be idle.
+ */
+ if (sd->llc_counts[pref_llc])
+ sd->llc_counts[pref_llc]--;
+ }
+}
+
+void mm_init_sched(struct mm_struct *mm,
+ struct sched_cache_time __percpu *_pcpu_sched)
+{
+ unsigned long epoch = 0;
+ int i;
+
+ for_each_possible_cpu(i) {
+ struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
+ struct rq *rq = cpu_rq(i);
+
+ pcpu_sched->runtime = 0;
+ /* a slightly stale cpu epoch is acceptible */
+ pcpu_sched->epoch = rq->cpu_epoch;
+ epoch = rq->cpu_epoch;
+ }
+
+ raw_spin_lock_init(&mm->sc_stat.lock);
+ mm->sc_stat.epoch = epoch;
+ mm->sc_stat.cpu = -1;
+ mm->sc_stat.next_scan = jiffies;
+ mm->sc_stat.nr_running_avg = 0;
+ mm->sc_stat.footprint = 0;
+ /*
+ * The update to mm->sc_stat should not be reordered
+ * before initialization to mm's other fields, in case
+ * the readers may get invalid mm_sched_epoch, etc.
+ */
+ smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
+}
+
+/* because why would C be fully specified */
+static __always_inline void __shr_u64(u64 *val, unsigned int n)
+{
+ if (n >= 64) {
+ *val = 0;
+ return;
+ }
+ *val >>= n;
+}
+
+static inline void __update_mm_sched(struct rq *rq,
+ struct sched_cache_time *pcpu_sched)
+{
+ lockdep_assert_held(&rq->cpu_epoch_lock);
+
+ unsigned int period = max(READ_ONCE(llc_epoch_period), 1U);
+ unsigned long n, now = jiffies;
+ long delta = now - rq->cpu_epoch_next;
+
+ if (delta > 0) {
+ n = (delta + period - 1) / period;
+ rq->cpu_epoch += n;
+ rq->cpu_epoch_next += n * period;
+ __shr_u64(&rq->cpu_runtime, n);
+ }
+
+ n = rq->cpu_epoch - pcpu_sched->epoch;
+ if (n) {
+ pcpu_sched->epoch += n;
+ __shr_u64(&pcpu_sched->runtime, n);
+ }
+}
+
+static unsigned long fraction_mm_sched(struct rq *rq,
+ struct sched_cache_time *pcpu_sched)
+{
+ guard(raw_spinlock_irqsave)(&rq->cpu_epoch_lock);
+
+ __update_mm_sched(rq, pcpu_sched);
+
+ /*
+ * Runtime is a geometric series (r=0.5) and as such will sum to twice
+ * the accumulation period, this means the multiplcation here should
+ * not overflow.
+ */
+ return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
+}
+
+static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+{
+ int mm_sched_llc = -1, mm_sched_cpu;
+
+ if (!mm)
+ return -1;
+
+ mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu);
+ if (mm_sched_cpu != -1) {
+ mm_sched_llc = llc_id(mm_sched_cpu);
+
+#ifdef CONFIG_NUMA_BALANCING
+ /*
+ * Don't assign preferred LLC if it
+ * conflicts with NUMA balancing.
+ * This can happen when sched_setnuma() gets
+ * called, however it is not much of an issue
+ * because we expect account_mm_sched() to get
+ * called fairly regularly -- at a higher rate
+ * than sched_setnuma() at least -- and thus the
+ * conflict only exists for a short period of time.
+ */
+ if (static_branch_likely(&sched_numa_balancing) &&
+ p->numa_preferred_nid >= 0 &&
+ cpu_to_node(mm_sched_cpu) != p->numa_preferred_nid)
+ mm_sched_llc = -1;
+#endif
+ }
+
+ return mm_sched_llc;
+}
+
+static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
+
+static inline
+void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
+{
+ struct sched_cache_time *pcpu_sched;
+ struct mm_struct *mm = p->mm;
+ int mm_sched_llc = -1;
+ unsigned long epoch;
+
+ if (!sched_cache_enabled())
+ return;
+
+ if (p->sched_class != &fair_sched_class)
+ return;
+ /*
+ * init_task, kthreads and user thread created
+ * by user_mode_thread() don't have mm.
+ */
+ if (!mm || !mm->sc_stat.pcpu_sched)
+ return;
+
+ pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq));
+
+ scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
+ __update_mm_sched(rq, pcpu_sched);
+ pcpu_sched->runtime += delta_exec;
+ rq->cpu_runtime += delta_exec;
+ epoch = rq->cpu_epoch;
+ }
+
+ /*
+ * If this process hasn't hit task_cache_work() for a while invalidate
+ * its preferred state.
+ */
+ if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout ||
+ invalid_llc_nr(mm, p, cpu_of(rq)) ||
+ exceed_llc_capacity(mm, cpu_of(rq))) {
+ if (READ_ONCE(mm->sc_stat.cpu) != -1)
+ WRITE_ONCE(mm->sc_stat.cpu, -1);
+ }
+
+ mm_sched_llc = get_pref_llc(p, mm);
+
+ /* task not on rq accounted later in account_entity_enqueue() */
+ if (task_running_on_cpu(rq->cpu, p) &&
+ READ_ONCE(p->preferred_llc) != mm_sched_llc) {
+ account_llc_dequeue(rq, p);
+ WRITE_ONCE(p->preferred_llc, mm_sched_llc);
+ account_llc_enqueue(rq, p);
+ }
+}
+
+static void task_tick_cache(struct rq *rq, struct task_struct *p)
+{
+ struct callback_head *work = &p->cache_work;
+ struct mm_struct *mm = p->mm;
+ unsigned long epoch;
+
+ if (!sched_cache_enabled())
+ return;
+
+ if (!mm || p->flags & PF_KTHREAD ||
+ !mm->sc_stat.pcpu_sched)
+ return;
+
+ epoch = rq->cpu_epoch;
+ /* avoid moving backwards */
+ if (time_after_eq(mm->sc_stat.epoch, epoch))
+ return;
+
+ guard(raw_spinlock)(&mm->sc_stat.lock);
+
+ if (work->next == work) {
+ task_work_add(p, work, TWA_RESUME);
+ WRITE_ONCE(mm->sc_stat.epoch, epoch);
+ }
+}
+
+static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
+{
+#ifdef CONFIG_NUMA_BALANCING
+ int cpu, curr_cpu, nid, pref_nid;
+
+ if (!static_branch_likely(&sched_numa_balancing))
+ goto out;
+
+ cpu = READ_ONCE(p->mm->sc_stat.cpu);
+ if (cpu != -1)
+ nid = cpu_to_node(cpu);
+ curr_cpu = task_cpu(p);
+
+ /*
+ * Scanning in the preferred NUMA node is ideal. However, the NUMA
+ * preferred node is per-task rather than per-process. It is possible
+ * for different threads of the process to have distinct preferred
+ * nodes; consequently, the process-wide preferred LLC may bounce
+ * between different nodes. As a workaround, maintain the scan
+ * CPU mask to also cover the process's current preferred LLC and the
+ * current running node to mitigate the bouncing risk.
+ * TBD: numa_group should be considered during task aggregation.
+ */
+ pref_nid = p->numa_preferred_nid;
+ /* honor the task's preferred node */
+ if (pref_nid == NUMA_NO_NODE)
+ goto out;
+
+ cpumask_or(cpus, cpus, cpumask_of_node(pref_nid));
+
+ /* honor the task's preferred LLC CPU */
+ if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE)
+ cpumask_or(cpus, cpus, cpumask_of_node(nid));
+
+ /* make sure the task's current running node is included */
+ if (!cpumask_test_cpu(curr_cpu, cpus))
+ cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu)));
+
+ return;
+
+out:
+#endif
+ cpumask_copy(cpus, cpu_online_mask);
+}
+
+static inline void update_avg_scale(u64 *avg, u64 sample)
+{
+ int factor = per_cpu(sd_llc_size, raw_smp_processor_id());
+ s64 diff = sample - *avg;
+ u32 divisor;
+
+ /*
+ * Scale the divisor based on the number of CPUs contained
+ * in the LLC. This scaling ensures smaller LLC domains use
+ * a smaller divisor to achieve more precise sensitivity to
+ * changes in nr_running, while larger LLC domains are capped
+ * at a maximum divisor of 8 which is the default smoothing
+ * factor of EWMA in update_avg().
+ */
+ divisor = clamp_t(u32, (factor >> 2), 2, 8);
+ *avg += div64_s64(diff, divisor);
+}
+
+static void task_cache_work(struct callback_head *work)
+{
+ int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
+ unsigned long next_scan, now = jiffies;
+ struct task_struct *p = current, *cur;
+ unsigned long curr_m_a_occ = 0;
+ struct mm_struct *mm = p->mm;
+ unsigned long m_a_occ = 0;
+ cpumask_var_t cpus;
+
+ WARN_ON_ONCE(work != &p->cache_work);
+
+ work->next = work;
+
+ if (p->flags & PF_EXITING)
+ return;
+
+ next_scan = READ_ONCE(mm->sc_stat.next_scan);
+ if (time_before(now, next_scan))
+ return;
+
+ /* only 1 thread is allowed to scan */
+ if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
+ now + max_t(unsigned long,
+ READ_ONCE(llc_epoch_period), 1)))
+ return;
+
+ curr_cpu = task_cpu(p);
+ if (invalid_llc_nr(mm, p, curr_cpu) ||
+ exceed_llc_capacity(mm, curr_cpu)) {
+ if (READ_ONCE(mm->sc_stat.cpu) != -1)
+ WRITE_ONCE(mm->sc_stat.cpu, -1);
+
+ return;
+ }
+
+ if (!zalloc_cpumask_var(&cpus, GFP_KERNEL))
+ return;
+
+ scoped_guard (cpus_read_lock) {
+ guard(rcu)();
+
+ get_scan_cpumasks(cpus, p);
+
+ for_each_cpu(cpu, cpus) {
+ /* XXX sched_cluster_active */
+ struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
+ unsigned long occ, m_occ = 0, a_occ = 0;
+ int m_cpu = -1, i;
+
+ if (!sd)
+ continue;
+
+ for_each_cpu(i, sched_domain_span(sd)) {
+ occ = fraction_mm_sched(cpu_rq(i),
+ per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+ a_occ += occ;
+ if (occ > m_occ) {
+ m_occ = occ;
+ m_cpu = i;
+ }
+
+ cur = rcu_dereference_all(cpu_rq(i)->curr);
+ if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
+ cur->mm == mm)
+ nr_running++;
+ }
+
+ /*
+ * Compare the accumulated occupancy of each LLC. The
+ * reason for using accumulated occupancy rather than average
+ * per CPU occupancy is that it works better in asymmetric LLC
+ * scenarios.
+ * For example, if there are 2 threads in a 4CPU LLC and 3
+ * threads in an 8CPU LLC, it might be better to choose the one
+ * with 3 threads. However, this would not be the case if the
+ * occupancy is divided by the number of CPUs in an LLC (i.e.,
+ * if average per CPU occupancy is used).
+ * Besides, NUMA balancing fault statistics behave similarly:
+ * the total number of faults per node is compared rather than
+ * the average number of faults per CPU. This strategy is also
+ * followed here.
+ */
+ if (a_occ > m_a_occ) {
+ m_a_occ = a_occ;
+ m_a_cpu = m_cpu;
+ }
+
+ if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu)))
+ curr_m_a_occ = a_occ;
+
+ cpumask_andnot(cpus, cpus, sched_domain_span(sd));
+ }
+ }
+
+ if (m_a_occ > (2 * curr_m_a_occ)) {
+ /*
+ * Avoid switching sc_stat.cpu too fast.
+ * The reason to choose 2X is because:
+ * 1. It is better to keep the preferred LLC stable,
+ * rather than changing it frequently and cause migrations
+ * 2. 2X means the new preferred LLC has at least 1 more
+ * busy CPU than the old one(200% vs 100%, eg)
+ * 3. 2X is chosen based on test results, as it delivers
+ * the optimal performance gain so far.
+ */
+ WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu);
+ }
+
+ update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running);
+ free_cpumask_var(cpus);
+}
+
+void init_sched_mm(struct task_struct *p)
+{
+ struct callback_head *work = &p->cache_work;
+
+ init_task_work(work, task_cache_work);
+ work->next = work;
+ /*
+ * Reset new task's preference to avoid
+ * polluting account_llc_enqueue().
+ */
+ p->preferred_llc = -1;
+}
+
+#else /* CONFIG_SCHED_CACHE */
+
+static inline void account_mm_sched(struct rq *rq, struct task_struct *p,
+ s64 delta_exec) { }
+
+void init_sched_mm(struct task_struct *p) { }
+
+static void task_tick_cache(struct rq *rq, struct task_struct *p) { }
+
+static inline int get_pref_llc(struct task_struct *p,
+ struct mm_struct *mm)
+{
+ return -1;
+}
+
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
+
+#endif /* CONFIG_SCHED_CACHE */
+
/*
* Used by other classes to account runtime.
*/
@@ -1549,13 +2156,9 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
se->exec_start = rq_clock_task(rq_of(cfs_rq));
}
-/**************************************************
- * Scheduling class queueing methods:
- */
-
+/* Check sched_smt_active before calling this to avoid overheads in fastpaths */
static inline bool is_core_idle(int cpu)
{
-#ifdef CONFIG_SCHED_SMT
int sibling;
for_each_cpu(sibling, cpu_smt_mask(cpu)) {
@@ -1565,7 +2168,6 @@ static inline bool is_core_idle(int cpu)
if (!idle_cpu(sibling))
return false;
}
-#endif
return true;
}
@@ -2248,12 +2850,11 @@ numa_type numa_classify(unsigned int imbalance_pct,
return node_fully_busy;
}
-#ifdef CONFIG_SCHED_SMT
/* Forward declarations of select_idle_sibling helpers */
static inline bool test_idle_cores(int cpu);
static inline int numa_idle_core(int idle_core, int cpu)
{
- if (!static_branch_likely(&sched_smt_present) ||
+ if (!sched_smt_active() ||
idle_core >= 0 || !test_idle_cores(cpu))
return idle_core;
@@ -2266,12 +2867,6 @@ static inline int numa_idle_core(int idle_core, int cpu)
return idle_core;
}
-#else /* !CONFIG_SCHED_SMT: */
-static inline int numa_idle_core(int idle_core, int cpu)
-{
- return idle_core;
-}
-#endif /* !CONFIG_SCHED_SMT */
/*
* Gather all necessary information to make NUMA balancing placement
@@ -3050,6 +3645,7 @@ static void task_numa_placement(struct task_struct *p)
unsigned long total_faults;
u64 runtime, period;
spinlock_t *group_lock = NULL;
+ long __maybe_unused new_fp;
struct numa_group *ng;
/*
@@ -3124,6 +3720,31 @@ static void task_numa_placement(struct task_struct *p)
ng->total_faults += diff;
group_faults += ng->faults[mem_idx];
}
+#ifdef CONFIG_SCHED_CACHE
+ /*
+ * Per task p->numa_faults[mem_idx] converges,
+ * so the accumulation of each task's faults
+ * converges too - Given the number of threads,
+ * it cannot overflow an unsigned long.
+ * Racy with concurrent updates from other threads
+ * sharing this mm. Acceptable since footprint is a
+ * heuristic and occasional lost updates are tolerable.
+ *
+ * If a task exits, its corresponding footprint must
+ * be subtracted from the mm->sc_stat.footprint, otherwise
+ * the mm->sc_stat.footprint will not converge:
+ * the exiting thread's footprint remains unchanged/undecayed
+ * in mm->sc_stat.footprint. See exit_mm().
+ *
+ * Lost updates and unsynchronized subtraction
+ * in exit_mm() can cause footprint + diff to
+ * go negative. Clamp to zero to prevent the
+ * unsigned footprint from wrapping.
+ */
+ new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff;
+ WRITE_ONCE(p->mm->sc_stat.footprint,
+ max(new_fp, 0L));
+#endif
}
if (!ng) {
@@ -3848,9 +4469,11 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
update_load_add(&cfs_rq->load, se->load.weight);
if (entity_is_task(se)) {
+ struct task_struct *p = task_of(se);
struct rq *rq = rq_of(cfs_rq);
- account_numa_enqueue(rq, task_of(se));
+ account_numa_enqueue(rq, p);
+ account_llc_enqueue(rq, p);
list_add(&se->group_node, &rq->cfs_tasks);
}
cfs_rq->nr_queued++;
@@ -3861,7 +4484,11 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
{
update_load_sub(&cfs_rq->load, se->load.weight);
if (entity_is_task(se)) {
- account_numa_dequeue(rq_of(cfs_rq), task_of(se));
+ struct task_struct *p = task_of(se);
+ struct rq *rq = rq_of(cfs_rq);
+
+ account_numa_dequeue(rq, p);
+ account_llc_dequeue(rq, p);
list_del_init(&se->group_node);
}
cfs_rq->nr_queued--;
@@ -4364,7 +4991,7 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
* For migration heavy workloads, access to tg->load_avg can be
* unbound. Limit the update rate to at most once per ms.
*/
- now = sched_clock_cpu(cpu_of(rq_of(cfs_rq)));
+ now = rq_clock(rq_of(cfs_rq));
if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC)
return;
@@ -4387,7 +5014,7 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
if (cfs_rq->tg == &root_task_group)
return;
- now = sched_clock_cpu(cpu_of(rq_of(cfs_rq)));
+ now = rq_clock(rq_of(cfs_rq));
delta = 0 - cfs_rq->tg_load_avg_contrib;
atomic_long_add(delta, &cfs_rq->tg->load_avg);
cfs_rq->tg_load_avg_contrib = 0;
@@ -4408,13 +5035,13 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
*/
rq_clock_start_loop_update(rq);
- rcu_read_lock();
+ guard(rcu)();
+
list_for_each_entry_rcu(tg, &task_groups, list) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
clear_tg_load_avg(cfs_rq);
}
- rcu_read_unlock();
rq_clock_stop_loop_update(rq);
}
@@ -4930,13 +5557,86 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
trace_pelt_cfs_tp(cfs_rq);
}
+#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
+
+static inline void util_est_update(struct sched_entity *se)
+{
+ unsigned int ewma, dequeued, last_ewma_diff;
+
+ if (!sched_feat(UTIL_EST))
+ return;
+
+ /* Get current estimate of utilization */
+ ewma = READ_ONCE(se->avg.util_est);
+
+ /*
+ * If the PELT values haven't changed since enqueue time,
+ * skip the util_est update.
+ */
+ if (ewma & UTIL_AVG_UNCHANGED)
+ return;
+
+ /* Get utilization at dequeue */
+ dequeued = READ_ONCE(se->avg.util_avg);
+
+ /*
+ * Reset EWMA on utilization increases, the moving average is used only
+ * to smooth utilization decreases.
+ */
+ if (ewma <= dequeued) {
+ ewma = dequeued;
+ goto done;
+ }
+
+ /*
+ * Skip update of task's estimated utilization when its members are
+ * already ~1% close to its last activation value.
+ */
+ last_ewma_diff = ewma - dequeued;
+ if (last_ewma_diff < UTIL_EST_MARGIN)
+ goto done;
+
+ /*
+ * To avoid underestimate of task utilization, skip updates of EWMA if
+ * we cannot grant that thread got all CPU time it wanted.
+ */
+ if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg))
+ goto done;
+
+ /*
+ * Update Task's estimated utilization
+ *
+ * When *p completes an activation we can consolidate another sample
+ * of the task size. This is done by using this value to update the
+ * Exponential Weighted Moving Average (EWMA):
+ *
+ * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
+ * = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
+ * = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
+ * = w * ( -last_ewma_diff ) + ewma(t-1)
+ * = w * (-last_ewma_diff + ewma(t-1) / w)
+ *
+ * Where 'w' is the weight of new samples, which is configured to be
+ * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
+ */
+ ewma <<= UTIL_EST_WEIGHT_SHIFT;
+ ewma -= last_ewma_diff;
+ ewma >>= UTIL_EST_WEIGHT_SHIFT;
+done:
+ ewma |= UTIL_AVG_UNCHANGED;
+ WRITE_ONCE(se->avg.util_est, ewma);
+
+ trace_sched_util_est_se_tp(se);
+}
+
/*
* Optional action to be done while updating the load average
*/
-#define UPDATE_TG 0x1
-#define SKIP_AGE_LOAD 0x2
-#define DO_ATTACH 0x4
-#define DO_DETACH 0x8
+#define UPDATE_TG 0x01
+#define SKIP_AGE_LOAD 0x02
+#define DO_ATTACH 0x04
+#define DO_DETACH 0x08
+#define UPDATE_UTIL_EST 0x10
/* Update task and its cfs_rq load average */
static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
@@ -4979,6 +5679,9 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
if (flags & UPDATE_TG)
update_tg_load_avg(cfs_rq);
}
+
+ if (flags & UPDATE_UTIL_EST)
+ util_est_update(se);
}
/*
@@ -5037,11 +5740,6 @@ static inline unsigned long task_util(struct task_struct *p)
return READ_ONCE(p->se.avg.util_avg);
}
-static inline unsigned long task_runnable(struct task_struct *p)
-{
- return READ_ONCE(p->se.avg.runnable_avg);
-}
-
static inline unsigned long _task_util_est(struct task_struct *p)
{
return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED;
@@ -5084,88 +5782,6 @@ static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
trace_sched_util_est_cfs_tp(cfs_rq);
}
-#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
-
-static inline void util_est_update(struct cfs_rq *cfs_rq,
- struct task_struct *p,
- bool task_sleep)
-{
- unsigned int ewma, dequeued, last_ewma_diff;
-
- if (!sched_feat(UTIL_EST))
- return;
-
- /*
- * Skip update of task's estimated utilization when the task has not
- * yet completed an activation, e.g. being migrated.
- */
- if (!task_sleep)
- return;
-
- /* Get current estimate of utilization */
- ewma = READ_ONCE(p->se.avg.util_est);
-
- /*
- * If the PELT values haven't changed since enqueue time,
- * skip the util_est update.
- */
- if (ewma & UTIL_AVG_UNCHANGED)
- return;
-
- /* Get utilization at dequeue */
- dequeued = task_util(p);
-
- /*
- * Reset EWMA on utilization increases, the moving average is used only
- * to smooth utilization decreases.
- */
- if (ewma <= dequeued) {
- ewma = dequeued;
- goto done;
- }
-
- /*
- * Skip update of task's estimated utilization when its members are
- * already ~1% close to its last activation value.
- */
- last_ewma_diff = ewma - dequeued;
- if (last_ewma_diff < UTIL_EST_MARGIN)
- goto done;
-
- /*
- * To avoid underestimate of task utilization, skip updates of EWMA if
- * we cannot grant that thread got all CPU time it wanted.
- */
- if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p))
- goto done;
-
-
- /*
- * Update Task's estimated utilization
- *
- * When *p completes an activation we can consolidate another sample
- * of the task size. This is done by using this value to update the
- * Exponential Weighted Moving Average (EWMA):
- *
- * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
- * = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
- * = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
- * = w * ( -last_ewma_diff ) + ewma(t-1)
- * = w * (-last_ewma_diff + ewma(t-1) / w)
- *
- * Where 'w' is the weight of new samples, which is configured to be
- * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
- */
- ewma <<= UTIL_EST_WEIGHT_SHIFT;
- ewma -= last_ewma_diff;
- ewma >>= UTIL_EST_WEIGHT_SHIFT;
-done:
- ewma |= UTIL_AVG_UNCHANGED;
- WRITE_ONCE(p->se.avg.util_est, ewma);
-
- trace_sched_util_est_se_tp(&p->se);
-}
-
static inline unsigned long get_actual_cpu_capacity(int cpu)
{
unsigned long capacity = arch_scale_cpu_capacity(cpu);
@@ -5618,7 +6234,7 @@ static bool
dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
{
bool sleep = flags & DEQUEUE_SLEEP;
- int action = UPDATE_TG;
+ int action = 0;
update_curr(cfs_rq);
clear_buddies(cfs_rq, se);
@@ -5638,15 +6254,23 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
if (sched_feat(DELAY_DEQUEUE) && delay &&
!entity_eligible(cfs_rq, se)) {
- update_load_avg(cfs_rq, se, 0);
+ if (entity_is_task(se))
+ action |= UPDATE_UTIL_EST;
+ update_load_avg(cfs_rq, se, action);
update_entity_lag(cfs_rq, se);
set_delayed(se);
return false;
}
}
- if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
- action |= DO_DETACH;
+ action = UPDATE_TG;
+ if (entity_is_task(se)) {
+ if (task_on_rq_migrating(task_of(se)))
+ action |= DO_DETACH;
+
+ if (sleep && !(flags & DEQUEUE_DELAYED))
+ action |= UPDATE_UTIL_EST;
+ }
/*
* When dequeuing a sched_entity, we must:
@@ -5749,11 +6373,11 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags);
* 4) do not run the "skip" process, if something else is available
*/
static struct sched_entity *
-pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq)
+pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect)
{
struct sched_entity *se;
- se = pick_eevdf(cfs_rq);
+ se = pick_eevdf(cfs_rq, protect);
if (se->sched_delayed) {
dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
/*
@@ -5764,8 +6388,6 @@ pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq)
return se;
}
-static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
-
static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
{
/*
@@ -5775,9 +6397,6 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
if (prev->on_rq)
update_curr(cfs_rq);
- /* throttle cfs_rqs exceeding runtime */
- check_cfs_rq_runtime(cfs_rq);
-
if (prev->on_rq) {
update_stats_wait_start_fair(cfs_rq, prev);
/* Put 'current' back into the tree. */
@@ -5912,44 +6531,32 @@ static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
return cfs_rq->runtime_remaining > 0;
}
-/* returns 0 on failure to allocate runtime */
-static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
-{
- struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
- int ret;
+static bool throttle_cfs_rq(struct cfs_rq *cfs_rq);
- raw_spin_lock(&cfs_b->lock);
- ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
- raw_spin_unlock(&cfs_b->lock);
-
- return ret;
-}
-
-static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
+static bool __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
{
/* dock delta_exec before expiring quota (as it could span periods) */
cfs_rq->runtime_remaining -= delta_exec;
if (likely(cfs_rq->runtime_remaining > 0))
- return;
+ return false;
if (cfs_rq->throttled)
- return;
+ return true;
/*
- * if we're unable to extend our runtime we resched so that the active
- * hierarchy can be throttled
+ * throttle_cfs_rq() will try to extend the runtime first
+ * before throttling the hierarchy.
*/
- if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
- resched_curr(rq_of(cfs_rq));
+ return throttle_cfs_rq(cfs_rq);
}
static __always_inline
-void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
+bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
{
if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
- return;
+ return false;
- __account_cfs_rq_runtime(cfs_rq, delta_exec);
+ return __account_cfs_rq_runtime(cfs_rq, delta_exec);
}
static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
@@ -5970,7 +6577,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
{
- return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]);
+ return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu));
}
static inline bool task_is_throttled(struct task_struct *p)
@@ -6116,8 +6723,18 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags);
static int tg_unthrottle_up(struct task_group *tg, void *data)
{
struct rq *rq = data;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
struct task_struct *p, *tmp;
+ LIST_HEAD(throttled_tasks);
+
+ /*
+ * If cfs_rq->curr is set, the cfs_rq might not have caught up
+ * since the last clock update. Do it now before we begin
+ * queueing task onto it to save the need for unnecessarily
+ * unthrottle the hierarchy for this cfs_rq to be throttled
+ * right back again.
+ */
+ update_curr(cfs_rq);
if (--cfs_rq->throttle_count)
return 0;
@@ -6139,13 +6756,31 @@ static int tg_unthrottle_up(struct task_group *tg, void *data)
cfs_rq->throttled_clock_self_time += delta;
}
+ /*
+ * Move the tasks to a local list since an update_curr() during
+ * enqueue_task_fair() can throttle a higher cfs_rq, and it can
+ * see the "throttled_limbo_list" being non-empty in
+ * tg_throttle_down() if throttle_count turned 0 above.
+ */
+ list_splice_init(&cfs_rq->throttled_limbo_list, &throttled_tasks);
+
/* Re-enqueue the tasks that have been throttled at this level. */
- list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) {
+ list_for_each_entry_safe(p, tmp, &throttled_tasks, throttle_node) {
+ /*
+ * Back to being throttled! Break out and put the remaining
+ * tasks back onto the limbo_list to prevent running them
+ * unnecessarily.
+ */
+ if (cfs_rq->throttle_count)
+ break;
+
list_del_init(&p->throttle_node);
p->throttled = false;
- enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP);
+ enqueue_task_fair(rq, p, ENQUEUE_WAKEUP);
}
+ list_splice(&throttled_tasks, &cfs_rq->throttled_limbo_list);
+
/* Add cfs_rq with load or one or more already running entities to the list */
if (!cfs_rq_is_decayed(cfs_rq))
list_add_leaf_cfs_rq(cfs_rq);
@@ -6187,7 +6822,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq)
static int tg_throttle_down(struct task_group *tg, void *data)
{
struct rq *rq = data;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
if (cfs_rq->throttle_count++)
return 0;
@@ -6209,35 +6844,48 @@ static int tg_throttle_down(struct task_group *tg, void *data)
static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
{
- struct rq *rq = rq_of(cfs_rq);
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
- int dequeue = 1;
+ struct sched_entity *curr = cfs_rq->curr;
+ struct rq *rq = rq_of(cfs_rq);
- raw_spin_lock(&cfs_b->lock);
- /* This will start the period timer if necessary */
- if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
+ scoped_guard(raw_spinlock, &cfs_b->lock) {
+ u64 target_runtime = 1;
+
/*
- * We have raced with bandwidth becoming available, and if we
- * actually throttled the timer might not unthrottle us for an
- * entire period. We additionally needed to make sure that any
- * subsequent check_cfs_rq_runtime calls agree not to throttle
- * us, as we may commit to do cfs put_prev+pick_next, so we ask
- * for 1ns of runtime rather than just check cfs_b.
+ * If cfs_rq->curr is still runnable, we are here from an
+ * update_curr(). Request sysctl_sched_cfs_bandwidth_slice
+ * worth of bandwidth to continue running.
+ *
+ * If the curr is not runnable, just request enough bandwidth
+ * to be runnable next time the pick selects this cfs_rq.
*/
- dequeue = 0;
- } else {
+ if (curr && curr->on_rq)
+ target_runtime = sched_cfs_bandwidth_slice();
+
+ /*
+ * Check if We have raced with bandwidth becoming available. If
+ * we actually throttled the timer might not unthrottle us for
+ * an entire period. We additionally needed to make sure that
+ * any subsequent check_cfs_rq_runtime calls agree not to
+ * throttle us, as we may commit to do cfs put_prev+pick_next,
+ * so we ask for 1ns of runtime rather than just check cfs_b.
+ *
+ * This will start the period timer if necessary.
+ */
+ if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime))
+ return false;
+
+ /*
+ * No bandwidth available; Add ourselves on the list to be
+ * unthrottled later.
+ */
list_add_tail_rcu(&cfs_rq->throttled_list,
&cfs_b->throttled_cfs_rq);
}
- raw_spin_unlock(&cfs_b->lock);
-
- if (!dequeue)
- return false; /* Throttle no longer required. */
/* freeze hierarchy runnable averages while throttled */
- rcu_read_lock();
- walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
- rcu_read_unlock();
+ scoped_guard(rcu)
+ walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
/*
* Note: distribution will already see us throttled via the
@@ -6245,6 +6893,17 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
*/
cfs_rq->throttled = 1;
WARN_ON_ONCE(cfs_rq->throttled_clock);
+
+ /*
+ * If current hierarchy was throttled, add throttle work to the
+ * current donor. In case of proxy-execution, the execution
+ * context cannot exit to the userspace while holding a mutex
+ * and the rule of throttle deferral to only throttle the
+ * throttled context at exit to userspace is still preserved.
+ */
+ if (curr && curr->on_rq)
+ task_throttle_setup_work(rq->donor);
+
return true;
}
@@ -6252,7 +6911,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
{
struct rq *rq = rq_of(cfs_rq);
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
- struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+ struct sched_entity *se = cfs_rq_se(cfs_rq);
/*
* It's possible we are called with runtime_remaining < 0 due to things
@@ -6262,21 +6921,25 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
* We can't unthrottle this cfs_rq without any runtime remaining because
* any enqueue in tg_unthrottle_up() will immediately trigger a throttle,
* which is not supposed to happen on unthrottle path.
+ *
+ * Catch up on the remaining runtime since last clock update before
+ * checking runtime remaining.
*/
+ update_curr(cfs_rq);
if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0)
return;
cfs_rq->throttled = 0;
- update_rq_clock(rq);
+ scoped_guard(raw_spinlock, &cfs_b->lock) {
+ list_del_rcu(&cfs_rq->throttled_list);
- raw_spin_lock(&cfs_b->lock);
- if (cfs_rq->throttled_clock) {
+ if (!cfs_rq->throttled_clock)
+ break;
+
cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
cfs_rq->throttled_clock = 0;
}
- list_del_rcu(&cfs_rq->throttled_list);
- raw_spin_unlock(&cfs_b->lock);
/* update hierarchical throttle state */
walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
@@ -6305,9 +6968,8 @@ static void __cfsb_csd_unthrottle(void *arg)
{
struct cfs_rq *cursor, *tmp;
struct rq *rq = arg;
- struct rq_flags rf;
- rq_lock(rq, &rf);
+ guard(rq_lock)(rq);
/*
* Iterating over the list can trigger several call to
@@ -6324,7 +6986,7 @@ static void __cfsb_csd_unthrottle(void *arg)
* race with group being freed in the window between removing it
* from the list and advancing to the next entry in the list.
*/
- rcu_read_lock();
+ guard(rcu)();
list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list,
throttled_csd_list) {
@@ -6334,10 +6996,7 @@ static void __cfsb_csd_unthrottle(void *arg)
unthrottle_cfs_rq(cursor);
}
- rcu_read_unlock();
-
rq_clock_stop_loop_update(rq);
- rq_unlock(rq, &rf);
}
static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
@@ -6346,6 +7005,7 @@ static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
bool first;
if (rq == this_rq()) {
+ update_rq_clock(rq);
unthrottle_cfs_rq(cfs_rq);
return;
}
@@ -6373,15 +7033,14 @@ static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
{
+ bool throttled = false, unthrottle_local = false;
int this_cpu = smp_processor_id();
u64 runtime, remaining = 1;
- bool throttled = false;
- struct cfs_rq *cfs_rq, *tmp;
- struct rq_flags rf;
+ struct cfs_rq *cfs_rq;
struct rq *rq;
- LIST_HEAD(local_unthrottle);
- rcu_read_lock();
+ guard(rcu)();
+
list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
throttled_list) {
rq = rq_of(cfs_rq);
@@ -6391,64 +7050,66 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
break;
}
- rq_lock_irqsave(rq, &rf);
+ guard(rq_lock_irqsave)(rq);
+
if (!cfs_rq_throttled(cfs_rq))
- goto next;
+ continue;
/* Already queued for async unthrottle */
if (!list_empty(&cfs_rq->throttled_csd_list))
- goto next;
+ continue;
+
+ if (cfs_rq->curr) {
+ update_rq_clock(rq);
+ update_curr(cfs_rq);
+ }
/* By the above checks, this should never be true */
WARN_ON_ONCE(cfs_rq->runtime_remaining > 0);
- raw_spin_lock(&cfs_b->lock);
- runtime = -cfs_rq->runtime_remaining + 1;
- if (runtime > cfs_b->runtime)
- runtime = cfs_b->runtime;
- cfs_b->runtime -= runtime;
- remaining = cfs_b->runtime;
- raw_spin_unlock(&cfs_b->lock);
+ scoped_guard(raw_spinlock, &cfs_b->lock) {
+ runtime = -cfs_rq->runtime_remaining + 1;
+ if (runtime > cfs_b->runtime)
+ runtime = cfs_b->runtime;
+ cfs_b->runtime -= runtime;
+ remaining = cfs_b->runtime;
+ }
cfs_rq->runtime_remaining += runtime;
- /* we check whether we're throttled above */
- if (cfs_rq->runtime_remaining > 0) {
- if (cpu_of(rq) != this_cpu) {
- unthrottle_cfs_rq_async(cfs_rq);
- } else {
- /*
- * We currently only expect to be unthrottling
- * a single cfs_rq locally.
- */
- WARN_ON_ONCE(!list_empty(&local_unthrottle));
- list_add_tail(&cfs_rq->throttled_csd_list,
- &local_unthrottle);
- }
- } else {
+ /*
+ * Ran out of bandwidth during distribution!
+ * Indicate throttled entities and break early.
+ */
+ if (cfs_rq->runtime_remaining <= 0) {
throttled = true;
+ break;
}
-next:
- rq_unlock_irqrestore(rq, &rf);
+ /* we check whether we're throttled above */
+ if (cpu_of(rq) != this_cpu) {
+ unthrottle_cfs_rq_async(cfs_rq);
+ continue;
+ }
+
+ /*
+ * Allow a parallel async unthrottle to unthrottle
+ * this cfs_rq too via __cfsb_csd_unthrottle().
+ * If we are first, do it ourselves at the end and
+ * save on an IPI from remote CPUs.
+ */
+ unthrottle_local = list_empty(&rq->cfsb_csd_list);
+ list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list);
}
- list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle,
- throttled_csd_list) {
- struct rq *rq = rq_of(cfs_rq);
-
- rq_lock_irqsave(rq, &rf);
-
- list_del_init(&cfs_rq->throttled_csd_list);
-
- if (cfs_rq_throttled(cfs_rq))
- unthrottle_cfs_rq(cfs_rq);
-
- rq_unlock_irqrestore(rq, &rf);
+ if (unthrottle_local) {
+ /*
+ * Protect against an IPI that is also trying to flush
+ * the unthrottled cfs_rq(s) from this CPU's csd_list.
+ */
+ scoped_guard(irqsave)
+ __cfsb_csd_unthrottle(cpu_rq(this_cpu));
}
- WARN_ON_ONCE(!list_empty(&local_unthrottle));
-
- rcu_read_unlock();
return throttled;
}
@@ -6572,7 +7233,8 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
if (slack_runtime <= 0)
return;
- raw_spin_lock(&cfs_b->lock);
+ guard(raw_spinlock)(&cfs_b->lock);
+
if (cfs_b->quota != RUNTIME_INF) {
cfs_b->runtime += slack_runtime;
@@ -6581,7 +7243,6 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
!list_empty(&cfs_b->throttled_cfs_rq))
start_cfs_slack_bandwidth(cfs_b);
}
- raw_spin_unlock(&cfs_b->lock);
/* even if it's not valid for return we don't want to try again */
cfs_rq->runtime_remaining -= slack_runtime;
@@ -6604,26 +7265,22 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
*/
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{
- u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
- unsigned long flags;
-
/* confirm we're still not at a refresh boundary */
- raw_spin_lock_irqsave(&cfs_b->lock, flags);
- cfs_b->slack_started = false;
+ scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
+ u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
- if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
- return;
+ cfs_b->slack_started = false;
+
+ if (runtime_refresh_within(cfs_b, min_bandwidth_expiration))
+ return;
+
+ if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
+ runtime = cfs_b->runtime;
+
+ if (!runtime)
+ return;
}
- if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
- runtime = cfs_b->runtime;
-
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
-
- if (!runtime)
- return;
-
distribute_cfs_runtime(cfs_b);
}
@@ -6637,7 +7294,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
if (!cfs_bandwidth_used())
return;
- /* an active group must be handled by the update_curr()->put() path */
+ /* an active group must be handled by the update_curr() path */
if (!cfs_rq->runtime_enabled || cfs_rq->curr)
return;
@@ -6647,8 +7304,6 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
/* update runtime allocation */
account_cfs_rq_runtime(cfs_rq, 0);
- if (cfs_rq->runtime_remaining <= 0)
- throttle_cfs_rq(cfs_rq);
}
static void sync_throttle(struct task_group *tg, int cpu)
@@ -6661,8 +7316,8 @@ static void sync_throttle(struct task_group *tg, int cpu)
if (!tg->parent)
return;
- cfs_rq = tg->cfs_rq[cpu];
- pcfs_rq = tg->parent->cfs_rq[cpu];
+ cfs_rq = tg_cfs_rq(tg, cpu);
+ pcfs_rq = tg_cfs_rq(tg->parent, cpu);
cfs_rq->throttle_count = pcfs_rq->throttle_count;
cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
@@ -6678,25 +7333,6 @@ static void sync_throttle(struct task_group *tg, int cpu)
cfs_rq->pelt_clock_throttled = 1;
}
-/* conditionally throttle active cfs_rq's from put_prev_entity() */
-static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
-{
- if (!cfs_bandwidth_used())
- return false;
-
- if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
- return false;
-
- /*
- * it's possible for a throttled entity to be forced into a running
- * state (e.g. set_curr_task), in this case we're finished.
- */
- if (cfs_rq_throttled(cfs_rq))
- return true;
-
- return throttle_cfs_rq(cfs_rq);
-}
-
static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
{
struct cfs_bandwidth *cfs_b =
@@ -6711,18 +7347,18 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
{
struct cfs_bandwidth *cfs_b =
container_of(timer, struct cfs_bandwidth, period_timer);
- unsigned long flags;
int overrun;
int idle = 0;
int count = 0;
- raw_spin_lock_irqsave(&cfs_b->lock, flags);
+ CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
+
for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun)
break;
- idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
+ idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
if (++count > 3) {
u64 new, old = ktime_to_ns(cfs_b->period);
@@ -6755,11 +7391,13 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
count = 0;
}
}
- if (idle)
- cfs_b->period_active = 0;
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
- return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
+ if (idle) {
+ cfs_b->period_active = 0;
+ return HRTIMER_NORESTART;
+ }
+
+ return HRTIMER_RESTART;
}
void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent)
@@ -6826,14 +7464,12 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
*/
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
- unsigned long flags;
if (list_empty(&rq->cfsb_csd_list))
continue;
- local_irq_save(flags);
- __cfsb_csd_unthrottle(rq);
- local_irq_restore(flags);
+ scoped_guard(irqsave)
+ __cfsb_csd_unthrottle(rq);
}
}
@@ -6851,16 +7487,15 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq)
lockdep_assert_rq_held(rq);
- rcu_read_lock();
+ guard(rcu)();
+
list_for_each_entry_rcu(tg, &task_groups, list) {
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
- raw_spin_lock(&cfs_b->lock);
- cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
- raw_spin_unlock(&cfs_b->lock);
+ scoped_guard(raw_spinlock, &cfs_b->lock)
+ cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
}
- rcu_read_unlock();
}
/* cpu offline callback */
@@ -6881,9 +7516,10 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
*/
rq_clock_start_loop_update(rq);
- rcu_read_lock();
+ guard(rcu)();
+
list_for_each_entry_rcu(tg, &task_groups, list) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
if (!cfs_rq->runtime_enabled)
continue;
@@ -6904,7 +7540,6 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
cfs_rq->runtime_remaining = 1;
unthrottle_cfs_rq(cfs_rq);
}
- rcu_read_unlock();
rq_clock_stop_loop_update(rq);
}
@@ -6951,8 +7586,7 @@ static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p)
#else /* !CONFIG_CFS_BANDWIDTH: */
-static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
-static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
+static bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { return false; }
static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
static inline void sync_throttle(struct task_group *tg, int cpu) {}
static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
@@ -7409,7 +8043,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
if (!p->se.sched_delayed)
util_est_dequeue(&rq->cfs, p);
- util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP);
if (dequeue_entities(rq, &p->se, flags) < 0)
return false;
@@ -7782,7 +8415,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p)
return -1;
}
-#ifdef CONFIG_SCHED_SMT
DEFINE_STATIC_KEY_FALSE(sched_smt_present);
EXPORT_SYMBOL_GPL(sched_smt_present);
@@ -7790,7 +8422,7 @@ static inline void set_idle_cores(int cpu, int val)
{
struct sched_domain_shared *sds;
- sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+ sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
if (sds)
WRITE_ONCE(sds->has_idle_cores, val);
}
@@ -7799,7 +8431,7 @@ static inline bool test_idle_cores(int cpu)
{
struct sched_domain_shared *sds;
- sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+ sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
if (sds)
return READ_ONCE(sds->has_idle_cores);
@@ -7808,7 +8440,7 @@ static inline bool test_idle_cores(int cpu)
/*
* Scans the local SMT mask to see if the entire core is idle, and records this
- * information in sd_llc_shared->has_idle_cores.
+ * information in sd_balance_shared->has_idle_cores.
*
* Since SMT siblings share all cache levels, inspecting this limited remote
* state should be fairly cheap.
@@ -7838,7 +8470,8 @@ void __update_idle_core(struct rq *rq)
/*
* Scan the entire LLC domain for idle cores; this dynamically switches off if
* there are no idle cores left in the system; tracked through
- * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
+ * sd_balance_shared->has_idle_cores and enabled through update_idle_core()
+ * above.
*/
static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
{
@@ -7892,29 +8525,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t
return -1;
}
-#else /* !CONFIG_SCHED_SMT: */
-
-static inline void set_idle_cores(int cpu, int val)
-{
-}
-
-static inline bool test_idle_cores(int cpu)
-{
- return false;
-}
-
-static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
-{
- return __select_idle_cpu(core, p);
-}
-
-static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
-{
- return -1;
-}
-
-#endif /* !CONFIG_SCHED_SMT */
-
/*
* Scan the LLC domain for idle CPUs; this is dynamically regulated by
* comparing the average scan cost (tracked in sd->avg_scan_cost) against the
@@ -7925,7 +8535,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
int i, cpu, idle_cpu = -1, nr = INT_MAX;
- if (sched_feat(SIS_UTIL)) {
+ if (sched_feat(SIS_UTIL) && sd->shared) {
/*
* Increment because !--nr is the condition to stop scan.
*
@@ -7990,6 +8600,54 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
}
/*
+ * Idle-capacity scan converts util_fits_cpu() outcomes into preference ranks,
+ * where lower values indicate a better fit - see select_idle_capacity().
+ *
+ * A CPU that both fits the task and sits on a fully-idle SMT core is returned
+ * immediately and is never assigned one of these ranks. On !SMT every CPU is
+ * its own "core", so the early return covers all fits-and-idle cases and the
+ * core-tier ranks below become unreachable.
+ *
+ * Rank Val Tier Meaning
+ * ------------------------------ --- ------ ---------------------------
+ * ASYM_IDLE_UCLAMP_MISFIT -4 core Idle core; capacity fits
+ * util but uclamp_min misses.
+ * ASYM_IDLE_COMPLETE_MISFIT -3 core Idle core; capacity does
+ * not fit. Still beats every
+ * thread-tier rank: a busy
+ * sibling cuts effective
+ * capacity more than a
+ * misfit hurts a quiet core.
+ * ASYM_IDLE_THREAD_FITS -2 thread Busy SMT sibling; capacity
+ * fits util + uclamp.
+ * ASYM_IDLE_THREAD_UCLAMP_MISFIT -1 thread Busy SMT sibling; capacity
+ * fits but uclamp_min misses
+ * (native util_fits_cpu()
+ * return value).
+ * ASYM_IDLE_THREAD_MISFIT 0 thread Busy SMT sibling; capacity
+ * does not fit.
+ *
+ * ASYM_IDLE_CORE_BIAS (-3) is an offset, not a state. On an idle core,
+ * fits += ASYM_IDLE_CORE_BIAS rebases thread-tier ranks into the core tier:
+ *
+ * ASYM_IDLE_THREAD_UCLAMP_MISFIT (-1) + BIAS -> ASYM_IDLE_UCLAMP_MISFIT (-4)
+ * ASYM_IDLE_THREAD_MISFIT (0) + BIAS -> ASYM_IDLE_COMPLETE_MISFIT (-3)
+ *
+ * ASYM_IDLE_THREAD_FITS (-2) is never rebased because a fully-fitting idle-core
+ * candidate early-returns from select_idle_capacity().
+ */
+enum asym_fits_state {
+ ASYM_IDLE_UCLAMP_MISFIT = -4,
+ ASYM_IDLE_COMPLETE_MISFIT,
+ ASYM_IDLE_THREAD_FITS,
+ ASYM_IDLE_THREAD_UCLAMP_MISFIT,
+ ASYM_IDLE_THREAD_MISFIT,
+
+ /* util_fits_cpu() bias for idle core */
+ ASYM_IDLE_CORE_BIAS = -3,
+};
+
+/*
* Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
* the task fits. If no CPU is big enough, but there are idle ones, try to
* maximize capacity.
@@ -7997,10 +8655,17 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
static int
select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
{
+ /*
+ * On !SMT systems, has_idle_core is always false and preferred_core
+ * is always true (CPU == core), so the SMT preference logic below
+ * collapses to the plain capacity scan.
+ */
+ bool has_idle_core = sched_smt_active() && test_idle_cores(target);
unsigned long task_util, util_min, util_max, best_cap = 0;
- int fits, best_fits = 0;
+ int fits, best_fits = ASYM_IDLE_THREAD_MISFIT;
int cpu, best_cpu = -1;
struct cpumask *cpus;
+ int nr = INT_MAX;
cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
@@ -8009,16 +8674,41 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
util_min = uclamp_eff_value(p, UCLAMP_MIN);
util_max = uclamp_eff_value(p, UCLAMP_MAX);
+ if (sched_feat(SIS_UTIL) && sd->shared) {
+ /*
+ * Same nr_idle_scan hint as select_idle_cpu(), nr only limits
+ * the scan when not preferring an idle core.
+ */
+ nr = READ_ONCE(sd->shared->nr_idle_scan) + 1;
+ /* overloaded domain is unlikely to have idle cpu/core */
+ if (nr == 1)
+ return -1;
+ }
+
for_each_cpu_wrap(cpu, cpus, target) {
+ bool preferred_core = !has_idle_core || is_core_idle(cpu);
unsigned long cpu_cap = capacity_of(cpu);
+ /*
+ * Stop when the nr_idle_scan is exhausted (mirrors
+ * select_idle_cpu() logic).
+ */
+ if (!has_idle_core && --nr <= 0)
+ return best_cpu;
+
if (!choose_idle_cpu(cpu, p))
continue;
fits = util_fits_cpu(task_util, util_min, util_max, cpu);
- /* This CPU fits with all requirements */
- if (fits > 0)
+ /*
+ * Perfect fit: capacity satisfies util + uclamp and the CPU
+ * sits on a fully-idle SMT core, this is a !SMT system, or
+ * there is no idle core to find.
+ * Short-circuit the rank-based selection and return
+ * immediately.
+ */
+ if (fits > 0 && preferred_core)
return cpu;
/*
* Only the min performance hint (i.e. uclamp_min) doesn't fit.
@@ -8026,9 +8716,33 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
*/
else if (fits < 0)
cpu_cap = get_actual_cpu_capacity(cpu);
+ /*
+ * fits > 0 implies we are not on a preferred core, but the util
+ * fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS
+ * so the effective range becomes
+ * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_THREAD_MISFIT], where:
+ * ASYM_IDLE_THREAD_MISFIT - does not fit
+ * ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN
+ * ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core
+ */
+ else if (fits > 0)
+ fits = ASYM_IDLE_THREAD_FITS;
/*
- * First, select CPU which fits better (-1 being better than 0).
+ * If we are on a preferred core, translate the range of fits
+ * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_THREAD_MISFIT] to
+ * [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT].
+ * This ensures that an idle core is always given priority over
+ * (partially) busy core.
+ *
+ * A fully fitting idle core would have returned early and hence
+ * fits > 0 for preferred_core need not be dealt with.
+ */
+ if (preferred_core)
+ fits += ASYM_IDLE_CORE_BIAS;
+
+ /*
+ * First, select CPU which fits better (lower is more preferred).
* Then, select the one with best capacity at same level.
*/
if ((fits < best_fits) ||
@@ -8039,6 +8753,19 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
}
}
+ /*
+ * A value in the [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT]
+ * range means the chosen CPU is in a fully idle SMT core. Values above
+ * ASYM_IDLE_COMPLETE_MISFIT mean we never ranked such a CPU best.
+ *
+ * The asym-capacity wakeup path returns from select_idle_sibling()
+ * after this function and never runs select_idle_cpu(), so the usual
+ * select_idle_cpu() tail that clears idle cores must live here when the
+ * idle-core preference did not win.
+ */
+ if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT)
+ set_idle_cores(target, false);
+
return best_cpu;
}
@@ -8047,12 +8774,22 @@ static inline bool asym_fits_cpu(unsigned long util,
unsigned long util_max,
int cpu)
{
- if (sched_asym_cpucap_active())
+ if (sched_asym_cpucap_active()) {
/*
* Return true only if the cpu fully fits the task requirements
* which include the utilization and the performance hints.
+ *
+ * When SMT is active, also require that the core has no busy
+ * siblings.
+ *
+ * Note: gating on is_core_idle() also makes the early-bailout
+ * candidates in select_idle_sibling() (target, prev,
+ * recent_used_cpu) idle-core-aware on ASYM+SMT, which the
+ * NO_ASYM path does not do.
*/
- return (util_fits_cpu(util, util_min, util_max, cpu) > 0);
+ return (!sched_smt_active() || is_core_idle(cpu)) &&
+ (util_fits_cpu(util, util_min, util_max, cpu) > 0);
+ }
return true;
}
@@ -9027,7 +9764,7 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
{
enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK;
struct task_struct *donor = rq->donor;
- struct sched_entity *se = &donor->se, *pse = &p->se;
+ struct sched_entity *nse, *se = &donor->se, *pse = &p->se;
struct cfs_rq *cfs_rq = task_cfs_rq(donor);
int cse_is_idle, pse_is_idle;
@@ -9138,12 +9875,19 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
}
pick:
- /*
- * If @p has become the most eligible task, force preemption.
- */
- if (__pick_eevdf(cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT) == pse)
+ nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT);
+ /* If @p has become the most eligible task, force preemption */
+ if (nse == pse)
goto preempt;
+ /*
+ * Because p is enqueued, nse being null can only mean that we
+ * dequeued a delayed task. If there are still entities queued in
+ * cfs, check if the next one will be p.
+ */
+ if (!nse && cfs_rq->nr_queued)
+ goto pick;
+
if (sched_feat(RUN_TO_PARITY))
update_protect_slice(cfs_rq, se);
@@ -9158,17 +9902,19 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
resched_curr_lazy(rq);
}
-static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
+struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
+ __must_hold(__rq_lockp(rq))
{
struct sched_entity *se;
struct cfs_rq *cfs_rq;
struct task_struct *p;
bool throttled;
+ int new_tasks;
again:
cfs_rq = &rq->cfs;
if (!cfs_rq->nr_queued)
- return NULL;
+ goto idle;
throttled = false;
@@ -9177,9 +9923,7 @@ static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
if (cfs_rq->curr && cfs_rq->curr->on_rq)
update_curr(cfs_rq);
- throttled |= check_cfs_rq_runtime(cfs_rq);
-
- se = pick_next_entity(rq, cfs_rq);
+ se = pick_next_entity(rq, cfs_rq, true);
if (!se)
goto again;
cfs_rq = group_cfs_rq(se);
@@ -9189,95 +9933,19 @@ static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
if (unlikely(throttled))
task_throttle_setup_work(p);
return p;
-}
-
-static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);
-static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first);
-
-struct task_struct *
-pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
- __must_hold(__rq_lockp(rq))
-{
- struct sched_entity *se;
- struct task_struct *p;
- int new_tasks;
-
-again:
- p = pick_task_fair(rq, rf);
- if (!p)
- goto idle;
- se = &p->se;
-
-#ifdef CONFIG_FAIR_GROUP_SCHED
- if (prev->sched_class != &fair_sched_class)
- goto simple;
-
- __put_prev_set_next_dl_server(rq, prev, p);
-
- /*
- * Because of the set_next_buddy() in dequeue_task_fair() it is rather
- * likely that a next task is from the same cgroup as the current.
- *
- * Therefore attempt to avoid putting and setting the entire cgroup
- * hierarchy, only change the part that actually changes.
- *
- * Since we haven't yet done put_prev_entity and if the selected task
- * is a different task than we started out with, try and touch the
- * least amount of cfs_rqs.
- */
- if (prev != p) {
- struct sched_entity *pse = &prev->se;
- struct cfs_rq *cfs_rq;
-
- while (!(cfs_rq = is_same_group(se, pse))) {
- int se_depth = se->depth;
- int pse_depth = pse->depth;
-
- if (se_depth <= pse_depth) {
- put_prev_entity(cfs_rq_of(pse), pse);
- pse = parent_entity(pse);
- }
- if (se_depth >= pse_depth) {
- set_next_entity(cfs_rq_of(se), se, true);
- se = parent_entity(se);
- }
- }
-
- put_prev_entity(cfs_rq, pse);
- set_next_entity(cfs_rq, se, true);
-
- __set_next_task_fair(rq, p, true);
- }
-
- return p;
-
-simple:
-#endif /* CONFIG_FAIR_GROUP_SCHED */
- put_prev_set_next_task(rq, prev, p);
- return p;
idle:
- if (rf) {
- new_tasks = sched_balance_newidle(rq, rf);
-
- /*
- * Because sched_balance_newidle() releases (and re-acquires)
- * rq->lock, it is possible for any higher priority task to
- * appear. In that case we must re-start the pick_next_entity()
- * loop.
- */
- if (new_tasks < 0)
- return RETRY_TASK;
-
- if (new_tasks > 0)
- goto again;
- }
-
+ new_tasks = sched_balance_newidle(rq, rf);
+ if (new_tasks < 0)
+ return RETRY_TASK;
+ if (new_tasks > 0)
+ goto again;
return NULL;
}
static struct task_struct *
fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf)
+ __must_hold(__rq_lockp(dl_se->rq))
{
return pick_task_fair(dl_se->rq, rf);
}
@@ -9298,10 +9966,33 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
{
struct sched_entity *se = &prev->se;
struct cfs_rq *cfs_rq;
+ struct sched_entity *nse = NULL;
- for_each_sched_entity(se) {
+#ifdef CONFIG_FAIR_GROUP_SCHED
+ if (next && next->sched_class == &fair_sched_class)
+ nse = &next->se;
+#endif
+
+ while (se) {
cfs_rq = cfs_rq_of(se);
- put_prev_entity(cfs_rq, se);
+ if (!nse || cfs_rq->curr)
+ put_prev_entity(cfs_rq, se);
+#ifdef CONFIG_FAIR_GROUP_SCHED
+ if (nse) {
+ if (is_same_group(se, nse))
+ break;
+
+ int d = nse->depth - se->depth;
+ if (d >= 0) {
+ /* nse has equal or greater depth, ascend */
+ nse = parent_entity(nse);
+ /* if nse is the deeper, do not ascend se */
+ if (d > 0)
+ continue;
+ }
+ }
+#endif
+ se = parent_entity(se);
}
}
@@ -9523,6 +10214,16 @@ enum group_type {
*/
group_imbalanced,
/*
+ * There are tasks running on non-preferred LLC, possible to move
+ * them to their preferred LLC without creating too much imbalance.
+ * The priority of group_llc_balance is lower than that of
+ * group_overloaded and higher than that of all other group types.
+ * This is because group_llc_balance may exacerbate load imbalance.
+ * If the LLC balancing attempt fails, the nr_balance_failed
+ * mechanism will trigger other group types to rebalance the load.
+ */
+ group_llc_balance,
+ /*
* The CPU is overloaded and can't provide expected CPU cycles to all
* tasks.
*/
@@ -9533,7 +10234,8 @@ enum migration_type {
migrate_load = 0,
migrate_util,
migrate_task,
- migrate_misfit
+ migrate_misfit,
+ migrate_llc_task
};
#define LBF_ALL_PINNED 0x01
@@ -9541,6 +10243,7 @@ enum migration_type {
#define LBF_DST_PINNED 0x04
#define LBF_SOME_PINNED 0x08
#define LBF_ACTIVE_LB 0x10
+#define LBF_LLC_PINNED 0x20
struct lb_env {
struct sched_domain *sd;
@@ -9550,6 +10253,7 @@ struct lb_env {
int dst_cpu;
struct rq *dst_rq;
+ bool dst_core_idle;
struct cpumask *dst_grpmask;
int new_dst_cpu;
@@ -9686,7 +10390,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
struct cfs_rq *dst_cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
- dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu];
+ dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu);
#else
dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
#endif
@@ -9697,6 +10401,298 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
return 0;
}
+#ifdef CONFIG_SCHED_CACHE
+/*
+ * The margin used when comparing LLC utilization with CPU capacity.
+ * It determines the LLC load level where active LLC aggregation is
+ * done.
+ * Derived from fits_capacity().
+ *
+ * (default: ~50%, tunable via debugfs)
+ */
+static bool fits_llc_capacity(unsigned long util, unsigned long max)
+{
+ u32 aggr_pct = llc_overaggr_pct;
+
+ /*
+ * For single core systems, raise the aggregation
+ * threshold to accommodate more tasks.
+ */
+ if (cpu_smt_num_threads == 1)
+ aggr_pct = (aggr_pct * 3 / 2);
+
+ return util * 100 < max * aggr_pct;
+}
+
+/*
+ * The margin used when comparing utilization.
+ * is 'util1' noticeably greater than 'util2'
+ * Derived from capacity_greater().
+ * Bias is in perentage.
+ */
+/* Allows dst util to be bigger than src util by up to bias percent */
+#define util_greater(util1, util2) \
+ ((util1) * 100 > (util2) * (100 + llc_imb_pct))
+
+static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util,
+ unsigned long *cap)
+{
+ struct sched_domain_shared *sd_share;
+
+ sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+ if (!sd_share)
+ return false;
+
+ *util = READ_ONCE(sd_share->util_avg);
+ *cap = READ_ONCE(sd_share->capacity);
+
+ return true;
+}
+
+/*
+ * Decision matrix according to the LLC utilization. To
+ * decide whether we can do task aggregation across LLC.
+ *
+ * By default, 50% is the threshold for treating the LLC
+ * as busy. The reason for choosing 50% is to avoid saturation
+ * of SMT-2, and it is also a safe cutoff for other SMT-n
+ * platforms. SMT-1 has higher threshold because it is
+ * supposed to accommodate more tasks, see fits_llc_capacity().
+ *
+ * 20% is the utilization imbalance percentage to decide
+ * if the preferred LLC is busier than the non-preferred LLC.
+ * 20 is a little higher than the LLC domain's imbalance_pct
+ * 17. The hysteresis is used to avoid task bouncing between the
+ * preferred LLC and the non-preferred LLC, and it will
+ * be turned into tunable debugfs.
+ *
+ * 1. moving towards the preferred LLC, dst is the preferred
+ * LLC, src is not.
+ *
+ * src \ dst 30% 40% 50% 60%
+ * 30% Y Y Y N
+ * 40% Y Y Y Y
+ * 50% Y Y G G
+ * 60% Y Y G G
+ *
+ * 2. moving out of the preferred LLC, src is the preferred
+ * LLC, dst is not:
+ *
+ * src \ dst 30% 40% 50% 60%
+ * 30% N N N N
+ * 40% N N N N
+ * 50% N N G G
+ * 60% Y N G G
+ *
+ * src : src_util
+ * dst : dst_util
+ * Y : Yes, migrate
+ * N : No, do not migrate
+ * G : let the Generic load balance to even the load.
+ *
+ * The intention is that if both LLCs are quite busy, cache aware
+ * load balance should not be performed, and generic load balance
+ * should take effect. However, if one is busy and the other is not,
+ * the preferred LLC capacity(50%) and imbalance criteria(20%) should
+ * be considered to determine whether LLC aggregation should be
+ * performed to bias the load towards the preferred LLC.
+ */
+
+/* migration decision, 3 states are orthogonal. */
+enum llc_mig {
+ mig_forbid = 0, /* N: Don't migrate task, respect LLC preference */
+ mig_llc, /* Y: Do LLC preference based migration */
+ mig_unrestricted /* G: Don't restrict generic load balance migration */
+};
+
+/*
+ * Check if task can be moved from the source LLC to the
+ * destination LLC without breaking cache aware preferrence.
+ * src_cpu and dst_cpu are arbitrary CPUs within the source
+ * and destination LLCs, respectively.
+ */
+static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu,
+ unsigned long tsk_util,
+ bool to_pref)
+{
+ unsigned long src_util, dst_util, src_cap, dst_cap;
+
+ if (!get_llc_stats(src_cpu, &src_util, &src_cap) ||
+ !get_llc_stats(dst_cpu, &dst_util, &dst_cap))
+ return mig_unrestricted;
+
+ src_util = src_util < tsk_util ? 0 : src_util - tsk_util;
+ dst_util = dst_util + tsk_util;
+
+ if (!fits_llc_capacity(dst_util, dst_cap) &&
+ !fits_llc_capacity(src_util, src_cap))
+ return mig_unrestricted;
+
+ if (to_pref) {
+ /*
+ * Don't migrate if we will get preferred LLC too
+ * heavily loaded and if the dest is much busier
+ * than the src, in which case migration will
+ * increase the imbalance too much.
+ */
+ if (!fits_llc_capacity(dst_util, dst_cap) &&
+ util_greater(dst_util, src_util))
+ return mig_forbid;
+ } else {
+ /*
+ * Don't migrate if we will leave preferred LLC
+ * too idle, or if this migration leads to the
+ * non-preferred LLC falls within sysctl_aggr_imb percent
+ * of preferred LLC, leading to migration again
+ * back to preferred LLC.
+ */
+ if (fits_llc_capacity(src_util, src_cap) ||
+ !util_greater(src_util, dst_util))
+ return mig_forbid;
+ }
+ return mig_llc;
+}
+
+/*
+ * Check if task p can migrate from source LLC to
+ * destination LLC in terms of cache aware load balance.
+ */
+static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu,
+ struct task_struct *p)
+{
+ struct mm_struct *mm;
+ bool to_pref;
+ int cpu;
+
+ mm = p->mm;
+ if (!mm)
+ return mig_unrestricted;
+
+ cpu = READ_ONCE(mm->sc_stat.cpu);
+ if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu))
+ return mig_unrestricted;
+
+ /* skip cache aware load balance for too many threads */
+ if (invalid_llc_nr(mm, p, dst_cpu) ||
+ exceed_llc_capacity(mm, dst_cpu)) {
+ if (READ_ONCE(mm->sc_stat.cpu) != -1)
+ WRITE_ONCE(mm->sc_stat.cpu, -1);
+ return mig_unrestricted;
+ }
+
+ if (cpus_share_cache(dst_cpu, cpu))
+ to_pref = true;
+ else if (cpus_share_cache(src_cpu, cpu))
+ to_pref = false;
+ else
+ return mig_unrestricted;
+
+ return can_migrate_llc(src_cpu, dst_cpu,
+ task_util(p), to_pref);
+}
+
+/*
+ * Check if active load balance breaks LLC locality in
+ * terms of cache aware load balance. The load level and
+ * imbalance do not warrant breaking LLC preference per
+ * the can_migrate_llc() policy. Here, the benefit of
+ * LLC locality outweighs the power efficiency gained from
+ * migrating the only runnable task away.
+ */
+static inline bool
+alb_break_llc(struct lb_env *env)
+{
+ if (!sched_cache_enabled())
+ return false;
+
+ if (cpus_share_cache(env->src_cpu, env->dst_cpu))
+ return false;
+ /*
+ * All tasks prefer to stay on their current CPU.
+ * Do not pull a task from its preferred CPU if:
+ * 1. It is the only task running and does not exceed
+ * imbalance allowance; OR
+ * 2. Migrating it away from its preferred LLC would violate
+ * the cache-aware scheduling policy.
+ */
+ if (env->src_rq->nr_pref_llc_running &&
+ env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) {
+ unsigned long util = 0;
+ struct task_struct *cur;
+
+ if (env->src_rq->nr_running <= 1)
+ return true;
+
+ cur = rcu_dereference_all(env->src_rq->curr);
+ if (cur && cur->sched_class == &fair_sched_class)
+ util = task_util(cur);
+
+ if (can_migrate_llc(env->src_cpu, env->dst_cpu,
+ util, false) == mig_forbid)
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Check if migrating task p from env->src_cpu to
+ * env->dst_cpu breaks LLC localiy.
+ */
+static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
+{
+ if (!sched_cache_enabled())
+ return false;
+
+ if (task_has_sched_core(p))
+ return false;
+ /*
+ * Skip over tasks that would degrade LLC locality;
+ * only when nr_balanced_failed is sufficiently high do we
+ * ignore this constraint.
+ *
+ * Threshold of cache_nice_tries is set to 1 higher
+ * than nr_balance_failed to avoid excessive task
+ * migration at the same time.
+ */
+ if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
+ return false;
+
+ /*
+ * We know the env->src_cpu has some tasks prefer to
+ * run on env->dst_cpu, skip the tasks do not prefer
+ * env->dst_cpu, and find the one that prefers.
+ */
+ if (env->migration_type == migrate_llc_task &&
+ READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))
+ return true;
+
+ if (can_migrate_llc_task(env->src_cpu,
+ env->dst_cpu, p) != mig_forbid)
+ return false;
+
+ return true;
+}
+
+#else
+static inline bool get_llc_stats(int cpu, unsigned long *util,
+ unsigned long *cap)
+{
+ return false;
+}
+
+static inline bool
+alb_break_llc(struct lb_env *env)
+{
+ return false;
+}
+
+static inline bool
+migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
+{
+ return false;
+}
+#endif
/*
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
*/
@@ -9793,10 +10789,29 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
return 1;
degrades = migrate_degrades_locality(p, env);
- if (!degrades)
+ if (!degrades) {
+ /*
+ * If the NUMA locality is not broken,
+ * further check if migration would hurt
+ * LLC locality.
+ */
+ if (migrate_degrades_llc(p, env)) {
+ /*
+ * If regular load balancing fails to pull a task
+ * due to LLC locality, this is expected behavior
+ * and we set LBF_LLC_PINNED so we don't increase
+ * nr_balance_failed unecessarily.
+ */
+ if (env->migration_type != migrate_llc_task)
+ env->flags |= LBF_LLC_PINNED;
+
+ return 0;
+ }
+
hot = task_hot(p, env);
- else
+ } else {
hot = degrades > 0;
+ }
if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
if (hot)
@@ -9958,6 +10973,10 @@ static int detach_tasks(struct lb_env *env)
env->imbalance = 0;
break;
+
+ case migrate_llc_task:
+ env->imbalance--;
+ break;
}
detach_task(p, env);
@@ -10091,7 +11110,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
{
struct cfs_rq *cfs_rq, *pos;
bool decayed = false;
- int cpu = cpu_of(rq);
/*
* Iterates the task_group tree in a bottom up fashion, see
@@ -10111,7 +11129,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
}
/* Propagate pending load changes to the parent, if any: */
- se = cfs_rq->tg->se[cpu];
+ se = cfs_rq_se(cfs_rq);
if (se && !skip_blocked_update(se))
update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
@@ -10137,8 +11155,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
*/
static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
{
- struct rq *rq = rq_of(cfs_rq);
- struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
+ struct sched_entity *se = cfs_rq_se(cfs_rq);
unsigned long now = jiffies;
unsigned long load;
@@ -10236,12 +11253,16 @@ struct sg_lb_stats {
enum group_type group_type;
unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
unsigned int group_smt_balance; /* Task on busy SMT be moved */
+ unsigned int group_llc_balance; /* Tasks should be moved to preferred LLC */
unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
unsigned int group_overutilized; /* At least one CPU is overutilized in the group */
#ifdef CONFIG_NUMA_BALANCING
unsigned int nr_numa_running;
unsigned int nr_preferred_running;
#endif
+#ifdef CONFIG_SCHED_CACHE
+ unsigned int nr_pref_dst_llc;
+#endif
};
/*
@@ -10499,6 +11520,9 @@ group_type group_classify(unsigned int imbalance_pct,
if (group_is_overloaded(imbalance_pct, sgs))
return group_overloaded;
+ if (sgs->group_llc_balance)
+ return group_llc_balance;
+
if (sg_imbalanced(group))
return group_imbalanced;
@@ -10653,6 +11677,105 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)
return check_cpu_capacity(rq, sd);
}
+#ifdef CONFIG_SCHED_CACHE
+/*
+ * Record the statistics for this scheduler group for later
+ * use. These values guide load balancing on aggregating tasks
+ * to a LLC.
+ */
+static void record_sg_llc_stats(struct lb_env *env,
+ struct sg_lb_stats *sgs,
+ struct sched_group *group)
+{
+ struct sched_domain_shared *sd_share;
+ int cpu;
+
+ if (!sched_cache_enabled() || env->idle == CPU_NEWLY_IDLE)
+ return;
+
+ /* Only care about sched domain spanning multiple LLCs */
+ if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu)))
+ return;
+
+ /*
+ * At this point we know this group spans a LLC domain.
+ * Record the statistic of this group in its corresponding
+ * shared LLC domain.
+ * Note: sd_share cannot be obtained via sd->child->shared,
+ * because the latter refers to the domain that covers the
+ * local group. Instead, sd_share should be located using
+ * the first CPU of the LLC group.
+ */
+ cpu = cpumask_first(sched_group_span(group));
+ sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+ if (!sd_share)
+ return;
+
+ if (READ_ONCE(sd_share->util_avg) != sgs->group_util)
+ WRITE_ONCE(sd_share->util_avg, sgs->group_util);
+
+ if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity))
+ WRITE_ONCE(sd_share->capacity, sgs->group_capacity);
+}
+
+/*
+ * Do LLC balance on sched group that contains LLC, and have tasks preferring
+ * to run on LLC in idle dst_cpu.
+ */
+static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,
+ struct sched_group *group)
+{
+ if (!sched_cache_enabled())
+ return false;
+
+ if (env->sd->flags & SD_SHARE_LLC)
+ return false;
+
+ /*
+ * Skip cache aware tagging if nr_balanced_failed is sufficiently high.
+ * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed
+ * to avoid excessive task migration at the same time.
+ */
+ if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
+ return false;
+
+ if (sgs->nr_pref_dst_llc &&
+ can_migrate_llc(cpumask_first(sched_group_span(group)),
+ env->dst_cpu, 0, true) == mig_llc)
+ return true;
+
+ return false;
+}
+
+static bool update_llc_busiest(struct lb_env *env,
+ struct sg_lb_stats *busiest,
+ struct sg_lb_stats *sgs)
+{
+ /*
+ * There are more tasks that want to run on dst_cpu's LLC.
+ */
+ return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc;
+}
+#else
+static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs,
+ struct sched_group *group)
+{
+}
+
+static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,
+ struct sched_group *group)
+{
+ return false;
+}
+
+static bool update_llc_busiest(struct lb_env *env,
+ struct sg_lb_stats *busiest,
+ struct sg_lb_stats *sgs)
+{
+ return false;
+}
+#endif
+
/**
* update_sg_lb_stats - Update sched_group's statistics for load balancing.
* @env: The load balancing environment.
@@ -10689,6 +11812,20 @@ static inline void update_sg_lb_stats(struct lb_env *env,
if (cpu_overutilized(i))
sgs->group_overutilized = 1;
+#ifdef CONFIG_SCHED_CACHE
+ if (sched_cache_enabled()) {
+ struct sched_domain *sd_tmp;
+ int dst_llc;
+
+ dst_llc = llc_id(env->dst_cpu);
+ if (llc_id(i) != dst_llc) {
+ sd_tmp = rcu_dereference_all(rq->sd);
+ if (sd_tmp && (unsigned int)dst_llc < sd_tmp->llc_max)
+ sgs->nr_pref_dst_llc += sd_tmp->llc_counts[dst_llc];
+ }
+ }
+#endif
+
/*
* No need to call idle_cpu() if nr_running is not 0
*/
@@ -10729,17 +11866,24 @@ static inline void update_sg_lb_stats(struct lb_env *env,
sgs->group_weight = group->group_weight;
- /* Check if dst CPU is idle and preferred to this group */
- if (!local_group && env->idle && sgs->sum_h_nr_running &&
- sched_group_asym(env, sgs, group))
- sgs->group_asym_packing = 1;
+ if (!local_group) {
+ /* Check if dst CPU is idle and preferred to this group */
+ if (env->idle && sgs->sum_h_nr_running &&
+ sched_group_asym(env, sgs, group))
+ sgs->group_asym_packing = 1;
- /* Check for loaded SMT group to be balanced to dst CPU */
- if (!local_group && smt_balance(env, sgs, group))
- sgs->group_smt_balance = 1;
+ /* Check for loaded SMT group to be balanced to dst CPU */
+ if (smt_balance(env, sgs, group))
+ sgs->group_smt_balance = 1;
+
+ /* Check for tasks in this group can be moved to their preferred LLC */
+ if (llc_balance(env, sgs, group))
+ sgs->group_llc_balance = 1;
+ }
sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
+ record_sg_llc_stats(env, sgs, group);
/* Computing avg_load makes sense only when group is overloaded */
if (sgs->group_type == group_overloaded)
sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
@@ -10775,10 +11919,16 @@ static bool update_sd_pick_busiest(struct lb_env *env,
* We can use max_capacity here as reduction in capacity on some
* CPUs in the group should either be possible to resolve
* internally or be covered by avg_load imbalance (eventually).
+ *
+ * When SMT is active, only pull a misfit to dst_cpu if it is on a
+ * fully idle core; otherwise the effective capacity of the core is
+ * reduced and we may not actually provide more capacity than the
+ * source.
*/
if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
(sgs->group_type == group_misfit_task) &&
- (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
+ (!env->dst_core_idle ||
+ !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
sds->local_stat.group_type != group_has_spare))
return false;
@@ -10798,6 +11948,10 @@ static bool update_sd_pick_busiest(struct lb_env *env,
/* Select the overloaded group with highest avg_load. */
return sgs->avg_load > busiest->avg_load;
+ case group_llc_balance:
+ /* Select the group with most tasks preferring dst LLC */
+ return update_llc_busiest(env, busiest, sgs);
+
case group_imbalanced:
/*
* Select the 1st imbalanced group as we don't have any way to
@@ -11060,6 +12214,7 @@ static bool update_pick_idlest(struct sched_group *idlest,
return false;
break;
+ case group_llc_balance:
case group_imbalanced:
case group_asym_packing:
case group_smt_balance:
@@ -11192,6 +12347,7 @@ sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int
return NULL;
break;
+ case group_llc_balance:
case group_imbalanced:
case group_asym_packing:
case group_smt_balance:
@@ -11342,6 +12498,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
unsigned long sum_util = 0;
bool sg_overloaded = 0, sg_overutilized = 0;
+ env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu);
+
do {
struct sg_lb_stats *sgs = &tmp_sgs;
int local_group;
@@ -11444,6 +12602,15 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
return;
}
+#ifdef CONFIG_SCHED_CACHE
+ if (busiest->group_type == group_llc_balance) {
+ /* Move a task that prefer local LLC */
+ env->migration_type = migrate_llc_task;
+ env->imbalance = 1;
+ return;
+ }
+#endif
+
if (busiest->group_type == group_imbalanced) {
/*
* In the group_imb case we cannot rely on group-wide averages
@@ -11690,7 +12857,8 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env)
* group's child domain.
*/
if (sds.prefer_sibling && local->group_type == group_has_spare &&
- sibling_imbalance(env, &sds, busiest, local) > 1)
+ (busiest->group_type == group_llc_balance ||
+ sibling_imbalance(env, &sds, busiest, local) > 1))
goto force_balance;
if (busiest->group_type != group_overloaded) {
@@ -11749,7 +12917,10 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env,
{
struct rq *busiest = NULL, *rq;
unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
+ unsigned int __maybe_unused busiest_pref_llc = 0;
+ struct sched_domain __maybe_unused *sd_tmp;
unsigned int busiest_nr = 0;
+ int __maybe_unused dst_llc;
int i;
for_each_cpu_and(i, sched_group_span(group), env->cpus) {
@@ -11877,6 +13048,23 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env,
break;
+ case migrate_llc_task:
+#ifdef CONFIG_SCHED_CACHE
+ sd_tmp = rcu_dereference_all(rq->sd);
+ dst_llc = llc_id(env->dst_cpu);
+
+ if (sd_tmp && (unsigned)dst_llc < sd_tmp->llc_max) {
+ unsigned int this_pref_llc =
+ sd_tmp->llc_counts[dst_llc];
+
+ if (busiest_pref_llc < this_pref_llc) {
+ busiest_pref_llc = this_pref_llc;
+ busiest = rq;
+ }
+ }
+#endif
+ break;
+
}
}
@@ -11928,6 +13116,9 @@ static int need_active_balance(struct lb_env *env)
{
struct sched_domain *sd = env->sd;
+ if (alb_break_llc(env))
+ return 0;
+
if (asym_active_balance(env))
return 1;
@@ -11947,7 +13138,8 @@ static int need_active_balance(struct lb_env *env)
return 1;
}
- if (env->migration_type == migrate_misfit)
+ if (env->migration_type == migrate_misfit ||
+ env->migration_type == migrate_llc_task)
return 1;
return 0;
@@ -11992,7 +13184,9 @@ static int should_we_balance(struct lb_env *env)
* balancing cores, but remember the first idle SMT CPU for
* later consideration. Find CPU on an idle core first.
*/
- if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {
+ if (sched_smt_active() &&
+ !(env->sd->flags & SD_SHARE_CPUCAPACITY) &&
+ !is_core_idle(cpu)) {
if (idle_smt == -1)
idle_smt = cpu;
/*
@@ -12000,9 +13194,7 @@ static int should_we_balance(struct lb_env *env)
* idle has been found, then its not needed to check other
* SMT siblings for idleness:
*/
-#ifdef CONFIG_SCHED_SMT
cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));
-#endif
continue;
}
@@ -12040,6 +13232,8 @@ static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd
case migrate_misfit:
__schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);
break;
+ case migrate_llc_task:
+ break;
}
}
@@ -12243,9 +13437,16 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
*
* Similarly for migration_misfit which is not related to
* load/util migration, don't pollute nr_balance_failed.
+ *
+ * The same for cache aware scheduling's allowance for
+ * load imbalance. If regular load balance does not
+ * migrate task due to LLC locality, it is a expected
+ * behavior and don't pollute nr_balance_failed.
+ * See can_migrate_task().
*/
if (idle != CPU_NEWLY_IDLE &&
- env.migration_type != migrate_misfit)
+ env.migration_type != migrate_misfit &&
+ !(env.flags & LBF_LLC_PINNED))
sd->nr_balance_failed++;
if (need_active_balance(&env)) {
@@ -12749,8 +13950,6 @@ static void nohz_balancer_kick(struct rq *rq)
goto out;
}
- rcu_read_lock();
-
sd = rcu_dereference_all(rq->sd);
if (sd) {
/*
@@ -12758,8 +13957,8 @@ static void nohz_balancer_kick(struct rq *rq)
* capacity, kick the ILB to see if there's a better CPU to run on:
*/
if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {
- flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
- goto unlock;
+ flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+ goto out;
}
}
@@ -12775,8 +13974,8 @@ static void nohz_balancer_kick(struct rq *rq)
*/
for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
if (sched_asym(sd, i, cpu)) {
- flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
- goto unlock;
+ flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+ goto out;
}
}
}
@@ -12787,10 +13986,8 @@ static void nohz_balancer_kick(struct rq *rq)
* When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
* to run the misfit task on.
*/
- if (check_misfit_status(rq)) {
- flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
- goto unlock;
- }
+ if (check_misfit_status(rq))
+ flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
/*
* For asymmetric systems, we do not want to nicely balance
@@ -12799,10 +13996,10 @@ static void nohz_balancer_kick(struct rq *rq)
*
* Skip the LLC logic because it's not relevant in that case.
*/
- goto unlock;
+ goto out;
}
- sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+ sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
if (sds) {
/*
* If there is an imbalance between LLC domains (IOW we could
@@ -12814,13 +14011,9 @@ static void nohz_balancer_kick(struct rq *rq)
* like this LLC domain has tasks we could move.
*/
nr_busy = atomic_read(&sds->nr_busy_cpus);
- if (nr_busy > 1) {
- flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
- goto unlock;
- }
+ if (nr_busy > 1)
+ flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
}
-unlock:
- rcu_read_unlock();
out:
if (READ_ONCE(nohz.needs_update))
flags |= NOHZ_NEXT_KICK;
@@ -12832,17 +14025,17 @@ static void nohz_balancer_kick(struct rq *rq)
static void set_cpu_sd_state_busy(int cpu)
{
struct sched_domain *sd;
-
- rcu_read_lock();
sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
- if (!sd || !sd->nohz_idle)
- goto unlock;
+ /*
+ * sd->nohz_idle only pairs with nr_busy_cpus on sd->shared; if this
+ * domain has no shared object there is nothing to clear or account.
+ */
+ if (!sd || !sd->shared || !sd->nohz_idle)
+ return;
sd->nohz_idle = 0;
atomic_inc(&sd->shared->nr_busy_cpus);
-unlock:
- rcu_read_unlock();
}
void nohz_balance_exit_idle(struct rq *rq)
@@ -12861,17 +14054,14 @@ void nohz_balance_exit_idle(struct rq *rq)
static void set_cpu_sd_state_idle(int cpu)
{
struct sched_domain *sd;
-
- rcu_read_lock();
sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
- if (!sd || sd->nohz_idle)
- goto unlock;
+ /* See set_cpu_sd_state_busy(): nohz_idle is only used with sd->shared. */
+ if (!sd || !sd->shared || sd->nohz_idle)
+ return;
sd->nohz_idle = 1;
atomic_dec(&sd->shared->nr_busy_cpus);
-unlock:
- rcu_read_unlock();
}
/*
@@ -13630,7 +14820,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu)
struct cfs_rq *cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
- cfs_rq = task_group(p)->cfs_rq[cpu];
+ cfs_rq = tg_cfs_rq(task_group(p), cpu);
#else
cfs_rq = &cpu_rq(cpu)->cfs;
#endif
@@ -13650,8 +14840,8 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
*/
static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
{
- struct cfs_rq *cfs_rq;
struct sched_entity *se = &curr->se;
+ struct cfs_rq *cfs_rq;
for_each_sched_entity(se) {
cfs_rq = cfs_rq_of(se);
@@ -13664,6 +14854,8 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
if (static_branch_unlikely(&sched_numa_balancing))
task_tick_numa(rq, curr);
+ task_tick_cache(rq, curr);
+
update_misfit_status(curr, rq);
check_update_overutilized_status(task_rq(curr));
@@ -13822,9 +15014,33 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p)
}
}
-static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
+/*
+ * Account for a task changing its policy or group.
+ *
+ * This routine is mostly called to set cfs_rq->curr field when a task
+ * migrates between groups/classes.
+ */
+static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
{
struct sched_entity *se = &p->se;
+ bool throttled = false;
+
+ for_each_sched_entity(se) {
+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+ if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) &&
+ first && cfs_rq->curr)
+ break;
+
+ set_next_entity(cfs_rq, se, first);
+ /* ensure bandwidth has been allocated on our new cfs_rq */
+ throttled |= account_cfs_rq_runtime(cfs_rq, 0);
+ }
+
+ if (throttled)
+ task_throttle_setup_work(p);
+
+ se = &p->se;
if (task_on_rq_queued(p)) {
/*
@@ -13845,27 +15061,6 @@ static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool firs
sched_fair_update_stop_tick(rq, p);
}
-/*
- * Account for a task changing its policy or group.
- *
- * This routine is mostly called to set cfs_rq->curr field when a task
- * migrates between groups/classes.
- */
-static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
-{
- struct sched_entity *se = &p->se;
-
- for_each_sched_entity(se) {
- struct cfs_rq *cfs_rq = cfs_rq_of(se);
-
- set_next_entity(cfs_rq, se, first);
- /* ensure bandwidth has been allocated on our new cfs_rq */
- account_cfs_rq_runtime(cfs_rq, 0);
- }
-
- __set_next_task_fair(rq, p, first);
-}
-
void init_cfs_rq(struct cfs_rq *cfs_rq)
{
cfs_rq->tasks_timeline = RB_ROOT_CACHED;
@@ -13893,56 +15088,38 @@ static void task_change_group_fair(struct task_struct *p)
void free_fair_sched_group(struct task_group *tg)
{
- int i;
-
- for_each_possible_cpu(i) {
- if (tg->cfs_rq)
- kfree(tg->cfs_rq[i]);
- if (tg->se)
- kfree(tg->se[i]);
- }
-
- kfree(tg->cfs_rq);
- kfree(tg->se);
+ free_percpu(tg->cfs_rq);
}
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
{
+ struct cfs_tg_state __percpu *state;
struct sched_entity *se;
struct cfs_rq *cfs_rq;
int i;
- tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
- if (!tg->cfs_rq)
- goto err;
- tg->se = kzalloc_objs(se, nr_cpu_ids);
- if (!tg->se)
+ state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL);
+ if (!state)
goto err;
+ tg->cfs_rq = &state->cfs_rq;
tg->shares = NICE_0_LOAD;
init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
for_each_possible_cpu(i) {
- cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
- GFP_KERNEL, cpu_to_node(i));
+ cfs_rq = tg_cfs_rq(tg, i);
if (!cfs_rq)
goto err;
- se = kzalloc_node(sizeof(struct sched_entity_stats),
- GFP_KERNEL, cpu_to_node(i));
- if (!se)
- goto err_free_rq;
-
+ se = tg_se(tg, i);
init_cfs_rq(cfs_rq);
- init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
+ init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
init_entity_runnable_average(se);
}
return 1;
-err_free_rq:
- kfree(cfs_rq);
err:
return 0;
}
@@ -13956,7 +15133,7 @@ void online_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(i) {
rq = cpu_rq(i);
- se = tg->se[i];
+ se = tg_se(tg, i);
rq_lock_irq(rq, &rf);
update_rq_clock(rq);
attach_entity_cfs_rq(se);
@@ -13972,8 +15149,8 @@ void unregister_fair_sched_group(struct task_group *tg)
destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
for_each_possible_cpu(cpu) {
- struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
- struct sched_entity *se = tg->se[cpu];
+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu);
+ struct sched_entity *se = tg_se(tg, cpu);
struct rq *rq = cpu_rq(cpu);
if (se) {
@@ -14009,9 +15186,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
cfs_rq->rq = rq;
init_cfs_rq_runtime(cfs_rq);
- tg->cfs_rq[cpu] = cfs_rq;
- tg->se[cpu] = se;
-
/* se could be NULL for root_task_group */
if (!se)
return;
@@ -14041,7 +15215,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
/*
* We can't change the weight of the root cgroup.
*/
- if (!tg->se[0])
+ if (is_root_task_group(tg))
return -EINVAL;
shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
@@ -14052,7 +15226,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
tg->shares = shares;
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
- struct sched_entity *se = tg->se[i];
+ struct sched_entity *se = tg_se(tg, i);
struct rq_flags rf;
/* Propagate contribution to hierarchy */
@@ -14103,8 +15277,8 @@ int sched_group_set_idle(struct task_group *tg, long idle)
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
- struct sched_entity *se = tg->se[i];
- struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
+ struct sched_entity *se = tg_se(tg, i);
+ struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i);
bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
long idle_task_delta;
struct rq_flags rf;
@@ -14177,7 +15351,6 @@ DEFINE_SCHED_CLASS(fair) = {
.wakeup_preempt = wakeup_preempt_fair,
.pick_task = pick_task_fair,
- .pick_next_task = pick_next_task_fair,
.put_prev_task = put_prev_task_fair,
.set_next_task = set_next_task_fair,
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 84c4fe3..8f0dee8 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -110,8 +110,16 @@ SCHED_FEAT(WARN_DOUBLE_CLOCK, false)
* rq lock and possibly create a large contention, sending an
* IPI to that CPU and let that CPU push the RT task to where
* it should go may be a better scenario.
+ *
+ * This is best for PREEMPT_RT, but for non-RT it can cause issues
+ * when preemption is disabled for long periods of time. Have
+ * it only default enabled for PREEMPT_RT.
*/
+# ifdef CONFIG_PREEMPT_RT
SCHED_FEAT(RT_PUSH_IPI, true)
+# else
+SCHED_FEAT(RT_PUSH_IPI, false)
+# endif
#endif
SCHED_FEAT(RT_RUNTIME_SHARE, false)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index a83be0c..052435f 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -280,6 +280,14 @@ static void do_idle(void)
int cpu = smp_processor_id();
bool got_tick = false;
+ if (cpu_is_offline(cpu)) {
+ local_irq_disable();
+ /* All per-CPU kernel threads should be done by now. */
+ WARN_ON_ONCE(need_resched());
+ cpuhp_report_idle_dead();
+ arch_cpu_idle_dead();
+ }
+
/*
* Check if we need to update blocked load
*/
@@ -331,11 +339,6 @@ static void do_idle(void)
*/
local_irq_disable();
- if (cpu_is_offline(cpu)) {
- cpuhp_report_idle_dead();
- arch_cpu_idle_dead();
- }
-
arch_cpu_idle_enter();
rcu_nocb_flush_deferred_wakeup();
@@ -462,7 +465,7 @@ select_task_rq_idle(struct task_struct *p, int cpu, int flags)
}
static int
-balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+balance_idle(struct rq *rq, struct rq_flags *rf)
{
return WARN_ON_ONCE(1);
}
diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c
index 6234456..cb957b8 100644
--- a/kernel/sched/membarrier.c
+++ b/kernel/sched/membarrier.c
@@ -164,8 +164,26 @@
| MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK \
| MEMBARRIER_CMD_GET_REGISTRATIONS)
+/*
+ * Scoped guard for memory barriers on entry and exit.
+ * Matches memory barriers before & after rq->curr modification in scheduler.
+ */
+DEFINE_LOCK_GUARD_0(mb, smp_mb(), smp_mb())
static DEFINE_MUTEX(membarrier_ipi_mutex);
+static DEFINE_PER_CPU(struct mutex, membarrier_cpu_mutexes);
+
#define SERIALIZE_IPI() guard(mutex)(&membarrier_ipi_mutex)
+#define SERIALIZE_IPI_CPU(cpu_id) guard(mutex)(&per_cpu(membarrier_cpu_mutexes, cpu_id))
+
+static int __init membarrier_init(void)
+{
+ int i;
+
+ for_each_possible_cpu(i)
+ mutex_init(&per_cpu(membarrier_cpu_mutexes, i));
+ return 0;
+}
+core_initcall(membarrier_init);
static void ipi_mb(void *info)
{
@@ -199,7 +217,16 @@ static void ipi_rseq(void *info)
* is negligible.
*/
smp_mb();
- rseq_sched_switch_event(current);
+ /*
+ * Legacy mode requires that IDs are written and the critical section is
+ * evaluated. V2 optimized mode handles the critical section and IDs are
+ * only updated if they change as a consequence of preemption after
+ * return from this IPI.
+ */
+ if (rseq_v2(current))
+ rseq_sched_switch_event(current);
+ else
+ rseq_force_update();
}
static void ipi_sync_rq_state(void *info)
@@ -249,23 +276,19 @@ void membarrier_update_current_mm(struct mm_struct *next_mm)
static int membarrier_global_expedited(void)
{
+ cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL;
int cpu;
- cpumask_var_t tmpmask;
if (num_online_cpus() == 1)
return 0;
- /*
- * Matches memory barriers after rq->curr modification in
- * scheduler.
- */
- smp_mb(); /* system call entry is not a mb. */
-
if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
return -ENOMEM;
+ guard(mb)();
SERIALIZE_IPI();
- cpus_read_lock();
+ guard(cpus_read_lock)();
+
rcu_read_lock();
for_each_online_cpu(cpu) {
struct task_struct *p;
@@ -301,21 +324,11 @@ static int membarrier_global_expedited(void)
smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
preempt_enable();
- free_cpumask_var(tmpmask);
- cpus_read_unlock();
-
- /*
- * Memory barrier on the caller thread _after_ we finished
- * waiting for the last IPI. Matches memory barriers before
- * rq->curr modification in scheduler.
- */
- smp_mb(); /* exit from system call is not a mb */
return 0;
}
static int membarrier_private_expedited(int flags, int cpu_id)
{
- cpumask_var_t tmpmask;
struct mm_struct *mm = current->mm;
smp_call_func_t ipi_func = ipi_mb;
@@ -352,30 +365,45 @@ static int membarrier_private_expedited(int flags, int cpu_id)
* On RISC-V, this barrier pairing is also needed for the
* SYNC_CORE command when switching between processes, cf.
* the inline comments in membarrier_arch_switch_mm().
+ *
+ * Memory barrier on the caller thread _after_ we finished
+ * waiting for the last IPI. Matches memory barriers before
+ * rq->curr modification in scheduler.
*/
- smp_mb(); /* system call entry is not a mb. */
-
- if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
- return -ENOMEM;
-
- SERIALIZE_IPI();
- cpus_read_lock();
-
+ guard(mb)();
if (cpu_id >= 0) {
+ if (cpu_id >= nr_cpu_ids || !cpu_possible(cpu_id))
+ return 0;
+
+ SERIALIZE_IPI_CPU(cpu_id);
+ guard(cpus_read_lock)();
struct task_struct *p;
- if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id))
- goto out;
+ if (!cpu_online(cpu_id))
+ return 0;
+
rcu_read_lock();
p = rcu_dereference(cpu_rq(cpu_id)->curr);
if (!p || p->mm != mm) {
rcu_read_unlock();
- goto out;
+ return 0;
}
rcu_read_unlock();
+ /*
+ * smp_call_function_single() will call ipi_func() if cpu_id
+ * is the calling CPU.
+ */
+ smp_call_function_single(cpu_id, ipi_func, NULL, 1);
} else {
+ cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL;
int cpu;
+ if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
+ return -ENOMEM;
+
+ SERIALIZE_IPI();
+ guard(cpus_read_lock)();
+
rcu_read_lock();
for_each_online_cpu(cpu) {
struct task_struct *p;
@@ -385,15 +413,6 @@ static int membarrier_private_expedited(int flags, int cpu_id)
__cpumask_set_cpu(cpu, tmpmask);
}
rcu_read_unlock();
- }
-
- if (cpu_id >= 0) {
- /*
- * smp_call_function_single() will call ipi_func() if cpu_id
- * is the calling CPU.
- */
- smp_call_function_single(cpu_id, ipi_func, NULL, 1);
- } else {
/*
* For regular membarrier, we can save a few cycles by
* skipping the current cpu -- we're about to do smp_mb()
@@ -420,18 +439,6 @@ static int membarrier_private_expedited(int flags, int cpu_id)
}
}
-out:
- if (cpu_id < 0)
- free_cpumask_var(tmpmask);
- cpus_read_unlock();
-
- /*
- * Memory barrier on the caller thread _after_ we finished
- * waiting for the last IPI. Matches memory barriers before
- * rq->curr modification in scheduler.
- */
- smp_mb(); /* exit from system call is not a mb */
-
return 0;
}
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 4ee8faf..e474c31 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -19,9 +19,9 @@ int sysctl_sched_rt_period = 1000000;
/*
* part of the period that we allow rt tasks to run in us.
- * default: 0.95s
+ * default: 1s
*/
-int sysctl_sched_rt_runtime = 950000;
+int sysctl_sched_rt_runtime = 1000000;
#ifdef CONFIG_SYSCTL
static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
@@ -1596,8 +1596,14 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
resched_curr(rq);
}
-static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
+static int balance_rt(struct rq *rq, struct rq_flags *rf)
{
+ /*
+ * Note, rq->donor may change during rq lock drops,
+ * so don't re-use p across lock drops
+ */
+ struct task_struct *p = rq->donor;
+
if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
/*
* This is OK, because current is on_cpu, which avoids it being
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9f63b15..c7c2dea 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -421,6 +421,10 @@ extern void ext_server_init(struct rq *rq);
extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq);
extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
u64 runtime, u64 period, bool init);
+extern int dl_server_attach_bw(struct sched_dl_entity *dl_se);
+extern void dl_server_detach_bw(struct sched_dl_entity *dl_se);
+extern int dl_server_swap_bw(struct sched_dl_entity *detach_se,
+ struct sched_dl_entity *attach_se);
static inline bool dl_server_active(struct sched_dl_entity *dl_se)
{
@@ -480,10 +484,8 @@ struct task_group {
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
- /* schedulable entities of this group on each CPU */
- struct sched_entity **se;
/* runqueue "owned" by this group on each CPU */
- struct cfs_rq **cfs_rq;
+ struct cfs_rq __percpu *cfs_rq;
unsigned long shares;
/*
* load_avg can be heavily contended at clock tick time, so put
@@ -889,6 +891,7 @@ struct dl_rq {
bool overloaded;
+ struct sched_dl_entity *curr;
/*
* Tasks on this rq that can be pushed away. They are kept in
* an rb-tree, ordered by tasks' deadlines, with caching
@@ -929,7 +932,8 @@ struct dl_rq {
};
#ifdef CONFIG_FAIR_GROUP_SCHED
-
+/* Check whether a task group is root tg */
+#define is_root_task_group(tg) ((tg) == &root_task_group)
/* An entity is a task if it doesn't "own" a runqueue */
#define entity_is_task(se) (!se->my_q)
@@ -1187,6 +1191,12 @@ struct rq {
struct scx_rq scx;
struct sched_dl_entity ext_server;
#endif
+#ifdef CONFIG_SCHED_CACHE
+ raw_spinlock_t cpu_epoch_lock ____cacheline_aligned;
+ u64 cpu_runtime;
+ unsigned long cpu_epoch;
+ unsigned long cpu_epoch_next;
+#endif
struct sched_dl_entity fair_server;
@@ -1199,6 +1209,12 @@ struct rq {
#ifdef CONFIG_NUMA_BALANCING
unsigned int numa_migrate_on;
#endif
+
+#ifdef CONFIG_SCHED_CACHE
+ unsigned int nr_pref_llc_running;
+ unsigned int nr_llc_running;
+#endif
+
/*
* This is part of a global counter where only the total sum
* over all CPUs matters. A task can increase this counter on
@@ -1546,6 +1562,14 @@ extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags);
extern void sched_core_get(void);
extern void sched_core_put(void);
+static inline bool task_has_sched_core(struct task_struct *p)
+{
+ if (sched_core_disabled())
+ return false;
+
+ return !!p->core_cookie;
+}
+
#else /* !CONFIG_SCHED_CORE: */
static inline bool sched_core_enabled(struct rq *rq)
@@ -1586,6 +1610,11 @@ static inline bool sched_group_cookie_match(struct rq *rq,
return true;
}
+static inline bool task_has_sched_core(struct task_struct *p)
+{
+ return false;
+}
+
#endif /* !CONFIG_SCHED_CORE */
#ifdef CONFIG_RT_GROUP_SCHED
@@ -1667,21 +1696,15 @@ do { \
flags = _raw_spin_rq_lock_irqsave(rq); \
} while (0)
-#ifdef CONFIG_SCHED_SMT
extern void __update_idle_core(struct rq *rq);
static inline void update_idle_core(struct rq *rq)
{
- if (static_branch_unlikely(&sched_smt_present))
+ if (sched_smt_active())
__update_idle_core(rq);
}
-#else /* !CONFIG_SCHED_SMT: */
-static inline void update_idle_core(struct rq *rq) { }
-#endif /* !CONFIG_SCHED_SMT */
-
#ifdef CONFIG_FAIR_GROUP_SCHED
-
static inline struct task_struct *task_of(struct sched_entity *se)
{
WARN_ON_ONCE(!entity_is_task(se));
@@ -2082,6 +2105,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p)
#endif /* !CONFIG_NUMA_BALANCING */
+int task_llc(const struct task_struct *p);
+
static inline void
queue_balance_callback(struct rq *rq,
struct balance_callback *head,
@@ -2171,6 +2196,7 @@ DECLARE_PER_CPU(int, sd_llc_size);
DECLARE_PER_CPU(int, sd_llc_id);
DECLARE_PER_CPU(int, sd_share_id);
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -2267,6 +2293,46 @@ static inline struct task_group *task_group(struct task_struct *p)
return p->sched_task_group;
}
+#ifdef CONFIG_FAIR_GROUP_SCHED
+/*
+ * Defined here to be available before stats.h is included, since
+ * stats.h has dependencies on things defined later in this file.
+ */
+struct cfs_tg_state {
+ struct cfs_rq cfs_rq;
+ struct sched_entity se;
+ struct sched_statistics stats;
+} __no_randomize_layout;
+
+/* Access a specific CPU's cfs_rq from a task group */
+static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu)
+{
+ return per_cpu_ptr(tg->cfs_rq, cpu);
+}
+
+static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
+{
+ struct cfs_tg_state *state;
+
+ if (is_root_task_group(tg))
+ return NULL;
+
+ state = container_of(tg_cfs_rq(tg, cpu), struct cfs_tg_state, cfs_rq);
+ return &state->se;
+}
+
+static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
+{
+ struct cfs_tg_state *state;
+
+ if (is_root_task_group(cfs_rq->tg))
+ return NULL;
+
+ state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq);
+ return &state->se;
+}
+#endif
+
/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
{
@@ -2275,10 +2341,10 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
- set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
- p->se.cfs_rq = tg->cfs_rq[cpu];
- p->se.parent = tg->se[cpu];
- p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0;
+ set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu));
+ p->se.cfs_rq = tg_cfs_rq(tg, cpu);
+ p->se.parent = tg_se(tg, cpu);
+ p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
#endif
#ifdef CONFIG_RT_GROUP_SCHED
@@ -2561,23 +2627,12 @@ struct sched_class {
/*
* schedule/pick_next_task/prev_balance: rq->lock
*/
- int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
+ int (*balance)(struct rq *rq, struct rq_flags *rf);
/*
* schedule/pick_next_task: rq->lock
*/
struct task_struct *(*pick_task)(struct rq *rq, struct rq_flags *rf);
- /*
- * Optional! When implemented pick_next_task() should be equivalent to:
- *
- * next = pick_task();
- * if (next) {
- * put_prev_task(prev);
- * set_next_task_first(next);
- * }
- */
- struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev,
- struct rq_flags *rf);
/*
* sched_change:
@@ -2801,8 +2856,7 @@ static inline bool sched_fair_runnable(struct rq *rq)
return rq->cfs.nr_queued > 0;
}
-extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev,
- struct rq_flags *rf);
+extern struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf);
extern struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf);
#define SCA_CHECK 0x01
@@ -4037,6 +4091,29 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
+#ifdef CONFIG_SCHED_CACHE
+DECLARE_STATIC_KEY_FALSE(sched_cache_present);
+DECLARE_STATIC_KEY_FALSE(sched_cache_active);
+extern int sysctl_sched_cache_user;
+extern unsigned int llc_aggr_tolerance;
+extern unsigned int llc_epoch_period;
+extern unsigned int llc_epoch_affinity_timeout;
+extern unsigned int llc_imb_pct;
+extern unsigned int llc_overaggr_pct;
+
+static inline bool sched_cache_enabled(void)
+{
+ return static_branch_unlikely(&sched_cache_active);
+}
+
+extern void sched_cache_active_set(void);
+
+#endif
+
+void sched_domains_free_llc_id(int cpu);
+
+extern void init_sched_mm(struct task_struct *p);
+
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
static inline
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index a612cf2..ebe0a77 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt
#endif /* CONFIG_SCHEDSTATS */
-#ifdef CONFIG_FAIR_GROUP_SCHED
-struct sched_entity_stats {
- struct sched_entity se;
- struct sched_statistics stats;
-} __no_randomize_layout;
-#endif
-
static inline struct sched_statistics *
__schedstats_from_se(struct sched_entity *se)
{
#ifdef CONFIG_FAIR_GROUP_SCHED
if (!entity_is_task(se))
- return &container_of(se, struct sched_entity_stats, se)->stats;
+ return &container_of(se, struct cfs_tg_state, se)->stats;
#endif
return &task_of(se)->stats;
}
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index f95798b..c909ca0 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -16,7 +16,7 @@ select_task_rq_stop(struct task_struct *p, int cpu, int flags)
}
static int
-balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+balance_stop(struct rq *rq, struct rq_flags *rf)
{
return sched_stop_runnable(rq);
}
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 5847b83..622e2e0 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -19,8 +19,10 @@ void sched_domains_mutex_unlock(void)
}
/* Protected by sched_domains_mutex: */
+static cpumask_var_t sched_domains_llc_id_allocmask;
static cpumask_var_t sched_domains_tmpmask;
static cpumask_var_t sched_domains_tmpmask2;
+int max_lid;
static int __init sched_debug_setup(char *str)
{
@@ -621,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc)
} while (sg != first);
}
+static void free_sched_domain_shared(struct sched_domain_shared *sds)
+{
+ if (sds && atomic_dec_and_test(&sds->ref))
+ kfree(sds);
+}
+
static void destroy_sched_domain(struct sched_domain *sd)
{
/*
@@ -629,9 +637,12 @@ static void destroy_sched_domain(struct sched_domain *sd)
* dropping group/capacity references, freeing where none remain.
*/
free_sched_groups(sd->groups, 1);
+ free_sched_domain_shared(sd->shared);
- if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
- kfree(sd->shared);
+#ifdef CONFIG_SCHED_CACHE
+ /* only the bottom sd has llc_counts array */
+ kfree(sd->llc_counts);
+#endif
kfree(sd);
}
@@ -663,9 +674,10 @@ static void destroy_sched_domains(struct sched_domain *sd)
*/
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DEFINE_PER_CPU(int, sd_llc_size);
-DEFINE_PER_CPU(int, sd_llc_id);
+DEFINE_PER_CPU(int, sd_llc_id) = -1;
DEFINE_PER_CPU(int, sd_share_id);
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -692,7 +704,6 @@ static void update_top_cache_domain(int cpu)
rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
per_cpu(sd_llc_size, cpu) = size;
- per_cpu(sd_llc_id, cpu) = id;
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
sd = lowest_flag_domain(cpu, SD_CLUSTER);
@@ -713,7 +724,18 @@ static void update_top_cache_domain(int cpu)
rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
+ /*
+ * The shared object is attached to sd_asym_cpucapacity only when the
+ * asym domain is non-overlapping (i.e., not built from SD_NUMA).
+ * On overlapping (NUMA) asym domains we fall back to letting the
+ * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
+ * here.
+ */
+ if (sd && sd->shared)
+ sds = sd->shared;
+
rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
+ rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
}
/*
@@ -737,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
/* Pick reference to parent->shared. */
if (parent->shared) {
- WARN_ON_ONCE(tmp->shared);
+ /*
+ * It is safe to free a sd->shared that
+ * has not been published yet. If a
+ * sd->shared was published, the refcount
+ * will end up being non-zero and it will
+ * not be freed here.
+ */
+ free_sched_domain_shared(tmp->shared);
tmp->shared = parent->shared;
parent->shared = NULL;
}
@@ -762,10 +791,20 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
if (sd && sd_degenerate(sd)) {
tmp = sd;
sd = sd->parent;
- destroy_sched_domain(tmp);
+
if (sd) {
struct sched_group *sg = sd->groups;
+#ifdef CONFIG_SCHED_CACHE
+ /* move buffer to parent as child is being destroyed */
+ sd->llc_counts = tmp->llc_counts;
+ sd->llc_max = tmp->llc_max;
+ sd->llc_bytes = tmp->llc_bytes;
+ /* make sure destroy_sched_domain() does not free it */
+ tmp->llc_counts = NULL;
+ tmp->llc_max = 0;
+ tmp->llc_bytes = 0;
+#endif
/*
* sched groups hold the flags of the child sched
* domain for convenience. Clear such flags since
@@ -777,6 +816,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
sd->child = NULL;
}
+
+ destroy_sched_domain(tmp);
}
sched_domain_debug(sd, cpu);
@@ -804,6 +845,239 @@ enum s_alloc {
sa_none,
};
+#ifdef CONFIG_SCHED_CACHE
+/* hardware support for cache aware scheduling */
+DEFINE_STATIC_KEY_FALSE(sched_cache_present);
+/*
+ * Indicator of whether cache aware scheduling
+ * is active, used by the scheduler.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_cache_active);
+/* user wants cache aware scheduling [0 or 1] */
+int sysctl_sched_cache_user = 1;
+
+/*
+ * Get the effective LLC size in bytes that @cpu's bottom sched_domain
+ * can use. A CPU within a cpuset partition can only use a proportion
+ * of the physical LLC, scaled by the ratio of the partition's span
+ * weight to the hardware LLC sharing weight. @sd should be the
+ * topmost domain with SD_SHARE_LLC.
+ *
+ * Returns 0 if cacheinfo is not yet populated. This happens during
+ * early boot when build_sched_domains() runs before the generic
+ * cacheinfo framework has been initialized (cacheinfo_cpu_online()
+ * is a device_initcall cpuhp callback). In that case,
+ * cacheinfo_cpu_online() will later call sched_update_llc_bytes()
+ * to fill in the bottom domain's llc_bytes once the cache attributes
+ * are available.
+ */
+static unsigned long get_effective_llc_bytes(int cpu,
+ struct sched_domain *sd)
+{
+ struct cacheinfo *ci;
+ unsigned int hw_weight;
+
+ ci = get_cpu_cacheinfo_llc(cpu);
+ if (!ci)
+ return 0;
+
+ hw_weight = cpumask_weight(&ci->shared_cpu_map);
+ if (!hw_weight)
+ return 0;
+
+ return div_u64((u64)ci->size * sd->span_weight, hw_weight);
+}
+
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+ struct s_data *d)
+{
+ struct sched_domain *sd, *top_llc, *parent;
+ unsigned int *p;
+ int i;
+
+ for_each_cpu(i, cpu_map) {
+ sd = *per_cpu_ptr(d->sd, i);
+ if (!sd)
+ goto err;
+
+ p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
+ GFP_KERNEL, cpu_to_node(i));
+ if (!p)
+ goto err;
+
+ top_llc = sd;
+ /*
+ * Find the topmost SD_SHARE_LLC domain.
+ * Not yet attached to the CPU, so per_cpu(sd_llc, i)
+ * can not be used.
+ */
+ while ((parent = rcu_dereference_protected(top_llc->parent, true)) &&
+ (parent->flags & SD_SHARE_LLC))
+ top_llc = parent;
+
+ if (top_llc->flags & SD_SHARE_LLC) {
+ sd->llc_max = max_lid + 1;
+ sd->llc_counts = p;
+ sd->llc_bytes = get_effective_llc_bytes(i, top_llc);
+ } else {
+ /* avoid memory leak */
+ kfree(p);
+ }
+ }
+
+ return true;
+err:
+ for_each_cpu(i, cpu_map) {
+ sd = *per_cpu_ptr(d->sd, i);
+ if (sd) {
+ kfree(sd->llc_counts);
+ sd->llc_counts = NULL;
+ sd->llc_max = 0;
+ sd->llc_bytes = 0;
+ }
+ }
+
+ return false;
+}
+
+/*
+ * Enable/disable cache aware scheduling according to
+ * user input and the presence of hardware support.
+ */
+static void _sched_cache_active_set(void)
+{
+ lockdep_assert_cpus_held();
+ lockdep_assert_held(&sched_domains_mutex);
+
+ /* hardware does not support */
+ if (!static_branch_likely(&sched_cache_present)) {
+ static_branch_disable_cpuslocked(&sched_cache_active);
+ if (sched_debug())
+ pr_info("%s: cache aware scheduling not supported on this platform\n", __func__);
+ return;
+ }
+
+ /*
+ * user wants it or not ?
+ * TBD: read before writing the static key.
+ * It is not in the critical path, leave as-is
+ * for now.
+ */
+ if (sysctl_sched_cache_user) {
+ static_branch_enable_cpuslocked(&sched_cache_active);
+ if (sched_debug())
+ pr_info("%s: enabling cache aware scheduling\n", __func__);
+ } else {
+ static_branch_disable_cpuslocked(&sched_cache_active);
+ if (sched_debug())
+ pr_info("%s: disabling cache aware scheduling\n", __func__);
+ }
+}
+
+/* used by debugfs */
+void sched_cache_active_set(void)
+{
+ cpus_read_lock();
+ sched_domains_mutex_lock();
+ _sched_cache_active_set();
+ sched_domains_mutex_unlock();
+ cpus_read_unlock();
+}
+
+/*
+ * Update the bottom sched_domain's llc_bytes for @cpu and all its
+ * LLC siblings. Called from cacheinfo_cpu_online() or
+ * cacheinfo_cpu_pre_down() with cpu hotplug lock held.
+ *
+ * Note: get_effective_llc_bytes() returns 0 on PowerPC.
+ * thus cache aware scheduling is disabled on PowerPC for
+ * now. PowerPC does not use the generic cacheinfo framework --
+ * it has its own cacheinfo with a separate struct cache hierarchy
+ * and does not populates the per-CPU struct cpu_cacheinfo array
+ * that get_cpu_cacheinfo_llc() reads.
+ */
+void sched_update_llc_bytes(unsigned int cpu)
+{
+ struct sched_domain *sd, *sdp;
+ unsigned int i;
+
+ sched_domains_mutex_lock();
+
+ sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
+ if (!sdp)
+ goto unlock;
+
+ /*
+ * ci->shared_cpu_map is built incrementally as CPUs come
+ * online, so the first CPU in an LLC initially sees
+ * hw_weight == 1 and computes an inflated llc_bytes in
+ * get_effective_llc_bytes(). Re-evaluating every LLC
+ * sibling on each online event corrects this once the full
+ * shared_cpu_map is known.
+ */
+ for_each_cpu(i, sched_domain_span(sdp)) {
+ sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
+ if (sd)
+ sd->llc_bytes = get_effective_llc_bytes(i, sdp);
+ }
+
+unlock:
+ sched_domains_mutex_unlock();
+}
+
+static void sched_cache_set(bool has_multi_llcs)
+{
+ /*
+ * TBD: check before writing to it. sched domain rebuild
+ * is not in the critical path, leave as-is for now.
+ */
+ if (has_multi_llcs)
+ static_branch_enable_cpuslocked(&sched_cache_present);
+ else
+ static_branch_disable_cpuslocked(&sched_cache_present);
+
+ _sched_cache_active_set();
+}
+#else
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+ struct s_data *d)
+{
+ return false;
+}
+static inline void sched_cache_set(bool has_multi_llcs) { }
+#endif
+
+/*
+ * Return true if @sd belongs to an LLC group whose enclosing
+ * partition spans more than one LLC. @sd must be the topmost
+ * SD_SHARE_LLC domain.
+ *
+ * Any duplicated parent domains with the same span as @sd are
+ * skipped: before cpu_attach_domain() degeneration these still
+ * exist, after degeneration the loop is a no-op. This makes the
+ * helper usable both during sched domain build and against an
+ * already-attached domain tree.
+ *
+ * Note: For systems with a single LLC per node, cache-aware
+ * scheduling is still enabled when multiple nodes exist.
+ * However, NUMA balancing decisions take precedence over
+ * cache-aware scheduling. Conversely, if there is only one
+ * LLC per partition, cache-aware scheduling should be disabled.
+ */
+static bool sd_in_multi_llcs(struct sched_domain *sd)
+{
+ struct sched_domain *sdp = sd->parent;
+
+ /* it does not make sense to aggregate to 1 CPU */
+ if (sd->span_weight == 1)
+ return false;
+
+ while (sdp && sdp->span_weight == sd->span_weight)
+ sdp = sdp->parent;
+
+ return !!sdp;
+}
+
/*
* Return the canonical balance CPU for this group, this is the first CPU
* of this group that's also in the balance mask.
@@ -1310,9 +1584,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
cpumask_copy(mask, sched_group_span(sg));
for_each_cpu(cpu, mask) {
cores++;
-#ifdef CONFIG_SCHED_SMT
cpumask_andnot(mask, mask, cpu_smt_mask(cpu));
-#endif
}
sg->cores = cores;
@@ -1790,8 +2062,22 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu
{
return cpu_coregroup_mask(cpu);
}
+
+/*
+ * Majority of architectures have LLC at MC domain level with exception
+ * such as powerpc. Provide a way for arch to specify where its LLC is
+ * if it falls in exception category
+ */
+# ifndef arch_llc_mask
+#define arch_llc_mask(cpu) cpu_coregroup_mask(cpu)
+# endif
+
+#else
+#define arch_llc_mask(cpu) cpumask_of(cpu)
#endif
+#define llc_mask(cpu) arch_llc_mask(cpu)
+
const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu)
{
return cpu_node_mask(cpu);
@@ -2650,14 +2936,153 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc)
}
}
+static void
+init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags)
+{
+ struct sched_domain_shared *sds = NULL;
+ int cpu;
+
+ /*
+ * Multiple domains can try to claim a shared object like
+ * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to
+ * same cpumask_first(sched_domain_span(sd)) CPU and can
+ * cause "nr_idle_scan" to be populated incorrectly during
+ * load balancing.
+ *
+ * Find the first CPU in sched_domain_span(sd) with an
+ * unclaimed domain (!alloc_flags) or where the alloc_flag
+ * matches the requested flag (SD_* flag)
+ *
+ * If the domain only has single CPU, allow temporary overlap
+ * in allocation since the domains will be degenerated later.
+ */
+ for_each_cpu(cpu, sched_domain_span(sd)) {
+ sds = *per_cpu_ptr(d->sds, cpu);
+
+ if (!sds->alloc_flags ||
+ sd->span_weight == 1 ||
+ sds->alloc_flags == flags) {
+ sds->alloc_flags = flags;
+ sd->shared = sds;
+ break;
+ }
+ }
+
+ /*
+ * Use the sd_shared corresponding to the last
+ * CPU in the span if none are avaialable.
+ */
+ if (WARN_ON_ONCE(!sd->shared))
+ sd->shared = sds;
+
+ /*
+ * nr_busy_cpus is consumed only by the NOHZ kick path via
+ * sd_balance_shared; on the asym-capacity path it is initialized but
+ * never read.
+ */
+ atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
+ atomic_inc(&sd->shared->ref);
+}
+
+/*
+ * For asymmetric CPU capacity, attach sched_domain_shared on the innermost
+ * SD_ASYM_CPUCAPACITY_FULL ancestor of @cpu's base domain when that ancestor is
+ * not an overlapping NUMA-built domain (then LLC should claim shared).
+ *
+ * A CPU may lack any FULL ancestor (e.g., exclusive cpuset symmetric island),
+ * then LLC must claim shared instead.
+ *
+ * Note: SD_ASYM_CPUCAPACITY_FULL is only set when all CPU capacity values
+ * are present in the domain span, so the asym domain we attach to cannot
+ * degenerate into a single-capacity group. The relevant edge cases are instead
+ * covered by the caveats above.
+ *
+ * Return true if this CPU's asym path claimed sd->shared, false otherwise.
+ */
+static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu)
+{
+ struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu);
+ struct sched_domain *sd_asym;
+
+ if (!sd)
+ return false;
+
+ sd_asym = sd;
+ while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL))
+ sd_asym = sd_asym->parent;
+
+ if (!sd_asym || (sd_asym->flags & SD_NUMA))
+ return false;
+
+ init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY);
+ return true;
+}
+
+static int __sched_domains_alloc_llc_id(void)
+{
+ int lid, max;
+
+ lockdep_assert_held(&sched_domains_mutex);
+
+ lid = cpumask_first_zero(sched_domains_llc_id_allocmask);
+ /*
+ * llc_id space should never grow larger than the
+ * possible number of CPUs in the system.
+ */
+ if (lid >= nr_cpu_ids)
+ return -1;
+
+ __cpumask_set_cpu(lid, sched_domains_llc_id_allocmask);
+ max = cpumask_last(sched_domains_llc_id_allocmask);
+ if (max > max_lid)
+ max_lid = max;
+
+ return lid;
+}
+
+static void __sched_domains_free_llc_id(int cpu)
+{
+ int i, lid, max;
+
+ lockdep_assert_held(&sched_domains_mutex);
+
+ lid = per_cpu(sd_llc_id, cpu);
+ if (lid == -1 || lid >= nr_cpu_ids)
+ return;
+
+ per_cpu(sd_llc_id, cpu) = -1;
+
+ for_each_cpu(i, llc_mask(cpu)) {
+ /* An online CPU owns the llc_id. */
+ if (per_cpu(sd_llc_id, i) == lid)
+ return;
+ }
+
+ __cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask);
+
+ max = cpumask_last(sched_domains_llc_id_allocmask);
+ /* shrink max lid to save memory */
+ if (max < max_lid)
+ max_lid = max;
+}
+
+void sched_domains_free_llc_id(int cpu)
+{
+ sched_domains_mutex_lock();
+ __sched_domains_free_llc_id(cpu);
+ sched_domains_mutex_unlock();
+}
+
/*
* Build sched domains for a given set of CPUs and attach the sched domains
* to the individual CPUs
*/
static int
-build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
+build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+ bool *multi_llcs)
{
enum s_alloc alloc_state = sa_none;
+ bool has_multi_llcs = false;
struct sched_domain *sd;
struct s_data d;
struct rq *rq = NULL;
@@ -2675,6 +3100,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
/* Set up domains for CPUs specified by the cpu_map: */
for_each_cpu(i, cpu_map) {
struct sched_domain_topology_level *tl;
+ int lid;
sd = NULL;
for_each_sd_topology(tl) {
@@ -2688,6 +3114,29 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
if (cpumask_equal(cpu_map, sched_domain_span(sd)))
break;
}
+
+ lid = per_cpu(sd_llc_id, i);
+ if (lid == -1) {
+ /* try to reuse the llc_id of its siblings */
+ for (int j = cpumask_first(llc_mask(i));
+ j < nr_cpu_ids;
+ j = cpumask_next(j, llc_mask(i))) {
+ if (i == j)
+ continue;
+
+ lid = per_cpu(sd_llc_id, j);
+
+ if (lid != -1) {
+ per_cpu(sd_llc_id, i) = lid;
+
+ break;
+ }
+ }
+
+ /* a new LLC is detected */
+ if (lid == -1)
+ per_cpu(sd_llc_id, i) = __sched_domains_alloc_llc_id();
+ }
}
if (WARN_ON(!topology_span_sane(cpu_map)))
@@ -2712,23 +3161,27 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
if (!sd)
continue;
+ if (has_asym)
+ claim_asym_sched_domain_shared(&d, i);
+
/* First, find the topmost SD_SHARE_LLC domain */
while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
sd = sd->parent;
if (sd->flags & SD_SHARE_LLC) {
- int sd_id = cpumask_first(sched_domain_span(sd));
-
- sd->shared = *per_cpu_ptr(d.sds, sd_id);
- atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
- atomic_inc(&sd->shared->ref);
+ init_sched_domain_shared(&d, sd, SD_SHARE_LLC);
/*
* In presence of higher domains, adjust the
* NUMA imbalance stats for the hierarchy.
*/
- if (IS_ENABLED(CONFIG_NUMA) && sd->parent)
- adjust_numa_imbalance(sd);
+ if (sd->parent) {
+ if (IS_ENABLED(CONFIG_NUMA))
+ adjust_numa_imbalance(sd);
+
+ if (sd_in_multi_llcs(sd))
+ has_multi_llcs = true;
+ }
}
}
@@ -2743,6 +3196,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
init_sched_groups_capacity(i, sd);
}
+ alloc_sd_llc(cpu_map, &d);
+
/* Attach the domains */
rcu_read_lock();
for_each_cpu(i, cpu_map) {
@@ -2767,6 +3222,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
ret = 0;
error:
+ *multi_llcs = has_multi_llcs;
__free_domain_allocs(&d, alloc_state, cpu_map);
return ret;
@@ -2829,8 +3285,10 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
*/
int __init sched_init_domains(const struct cpumask *cpu_map)
{
+ bool multi_llcs;
int err;
+ zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL);
zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
@@ -2842,7 +3300,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
if (!doms_cur)
doms_cur = &fallback_doms;
cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN));
- err = build_sched_domains(doms_cur[0], NULL);
+ err = build_sched_domains(doms_cur[0], NULL, &multi_llcs);
+ if (!err)
+ sched_cache_set(multi_llcs);
return err;
}
@@ -2915,6 +3375,7 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
struct sched_domain_attr *dattr_new)
{
bool __maybe_unused has_eas = false;
+ bool has_multi_llcs = false, multi_llcs;
int i, j, n;
int new_topology;
@@ -2964,14 +3425,41 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
for (i = 0; i < ndoms_new; i++) {
for (j = 0; j < n && !new_topology; j++) {
if (cpumask_equal(doms_new[i], doms_cur[j]) &&
- dattrs_equal(dattr_new, i, dattr_cur, j))
+ dattrs_equal(dattr_new, i, dattr_cur, j)) {
+ /*
+ * Reused partition has to be taken care
+ * of here, because there could be a corner
+ * case that if the reused partition is skipped
+ * and only new partition is considered, an
+ * incorrect has_multi_llcs would be set. For
+ * example:
+ * If the only multi-LLC partition is reused
+ * and a new single-LLC partition is built,
+ * sched_cache_set(false) disables cache-aware
+ * scheduling globally despite the reused
+ * multi-LLC partition still being active.
+ */
+ struct sched_domain *sd;
+ int cpu = cpumask_first(doms_cur[j]);
+
+ guard(rcu)();
+ sd = rcu_dereference(cpu_rq(cpu)->sd);
+ while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC))
+ sd = sd->parent;
+ if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent &&
+ sd_in_multi_llcs(sd))
+ has_multi_llcs = true;
goto match2;
+ }
}
/* No match - add a new doms_new */
- build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
+ build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL,
+ &multi_llcs);
+ has_multi_llcs |= multi_llcs;
match2:
;
}
+ sched_cache_set(has_multi_llcs);
#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
/* Build perf domains: */
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 3fe6b0c..773d8e9 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
EXPORT_SYMBOL_GPL(stop_machine);
#ifdef CONFIG_SCHED_SMT
+/*
+ * INTEL_IFS is the only user of this API. That selftest can
+ * only be compiled if SMP=y. On x86 it selects SCHED_SMT.
+ * Keep the ifdefs for now.
+ */
int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
{
const struct cpumask *smt_mask = cpu_smt_mask(cpu);
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 02aac7c..d098ac3 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -16,10 +16,6 @@
config ARCH_WANTS_CLOCKSOURCE_READ_INLINE
bool
-# Timekeeping vsyscall support
-config GENERIC_TIME_VSYSCALL
- bool
-
# The generic clock events infrastructure
config GENERIC_CLOCKEVENTS
def_bool !LEGACY_TIMER_TICK
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index 6e173d7..ea5be58 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -337,48 +337,32 @@ void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
EXPORT_SYMBOL_GPL(alarm_init);
/**
- * alarm_start - Sets an absolute alarm to fire
- * @alarm: ptr to alarm to set
- * @start: time to run the alarm
+ * alarm_start_timer - Sets an alarm to fire
+ * @alarm: Pointer to alarm to set
+ * @expires: Expiry time
+ * @relative: True if @expires is relative
+ *
+ * Returns: True if the alarm was queued. False if it already expired
*/
-void alarm_start(struct alarm *alarm, ktime_t start)
+bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative)
{
struct alarm_base *base = &alarm_bases[alarm->type];
- scoped_guard(spinlock_irqsave, &base->lock) {
- alarm->node.expires = start;
- alarmtimer_enqueue(base, alarm);
- hrtimer_start(&alarm->timer, alarm->node.expires, HRTIMER_MODE_ABS);
- }
+ if (relative)
+ expires = ktime_add_safe(expires, base->get_ktime());
trace_alarmtimer_start(alarm, base->get_ktime());
-}
-EXPORT_SYMBOL_GPL(alarm_start);
-
-/**
- * alarm_start_relative - Sets a relative alarm to fire
- * @alarm: ptr to alarm to set
- * @start: time relative to now to run the alarm
- */
-void alarm_start_relative(struct alarm *alarm, ktime_t start)
-{
- struct alarm_base *base = &alarm_bases[alarm->type];
-
- start = ktime_add_safe(start, base->get_ktime());
- alarm_start(alarm, start);
-}
-EXPORT_SYMBOL_GPL(alarm_start_relative);
-
-void alarm_restart(struct alarm *alarm)
-{
- struct alarm_base *base = &alarm_bases[alarm->type];
guard(spinlock_irqsave)(&base->lock);
- hrtimer_set_expires(&alarm->timer, alarm->node.expires);
- hrtimer_restart(&alarm->timer);
+ alarm->node.expires = expires;
alarmtimer_enqueue(base, alarm);
+ if (!hrtimer_start_range_ns_user(&alarm->timer, expires, 0, HRTIMER_MODE_ABS)) {
+ alarmtimer_dequeue(base, alarm);
+ return false;
+ }
+ return true;
}
-EXPORT_SYMBOL_GPL(alarm_restart);
+EXPORT_SYMBOL_GPL(alarm_start_timer);
/**
* alarm_try_to_cancel - Tries to cancel an alarm timer
@@ -512,8 +496,6 @@ static enum alarmtimer_type clock2alarm(clockid_t clockid)
* @now: time at the timer expiration
*
* Posix timer callback for expired alarm timers.
- *
- * Return: whether the timer is to be restarted
*/
static void alarm_handle_timer(struct alarm *alarm, ktime_t now)
{
@@ -527,12 +509,12 @@ static void alarm_handle_timer(struct alarm *alarm, ktime_t now)
* alarm_timer_rearm - Posix timer callback for rearming timer
* @timr: Pointer to the posixtimer data struct
*/
-static void alarm_timer_rearm(struct k_itimer *timr)
+static bool alarm_timer_rearm(struct k_itimer *timr)
{
struct alarm *alarm = &timr->it.alarm.alarmtimer;
timr->it_overrun += alarm_forward_now(alarm, timr->it_interval);
- alarm_start(alarm, alarm->node.expires);
+ return alarm_start_timer(alarm, alarm->node.expires, false);
}
/**
@@ -588,7 +570,7 @@ static void alarm_timer_wait_running(struct k_itimer *timr)
* @absolute: Expiry value is absolute time
* @sigev_none: Posix timer does not deliver signals
*/
-static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
+static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
bool absolute, bool sigev_none)
{
struct alarm *alarm = &timr->it.alarm.alarmtimer;
@@ -596,10 +578,16 @@ static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
if (!absolute)
expires = ktime_add_safe(expires, base->get_ktime());
- if (sigev_none)
+
+ /*
+ * sigev_none needs to update the expires value and pretend
+ * that the timer is queued
+ */
+ if (sigev_none) {
alarm->node.expires = expires;
- else
- alarm_start(&timr->it.alarm.alarmtimer, expires);
+ return true;
+ }
+ return alarm_start_timer(&timr->it.alarm.alarmtimer, expires, false);
}
/**
@@ -706,7 +694,9 @@ static int alarmtimer_do_nsleep(struct alarm *alarm, ktime_t absexp,
alarm->data = (void *)current;
do {
set_current_state(TASK_INTERRUPTIBLE);
- alarm_start(alarm, absexp);
+ if (!alarm_start_timer(alarm, absexp, false))
+ alarm->data = NULL;
+
if (likely(alarm->data))
schedule();
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 5e22697..0014d16 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -301,7 +301,7 @@ static int clockevents_program_min_delta(struct clock_event_device *dev)
#include <asm/clock_inlined.h>
#else
static __always_inline void
-arch_inlined_clockevent_set_next_coupled(u64 u64 cycles, struct clock_event_device *dev) { }
+arch_inlined_clockevent_set_next_coupled(u64 cycles, struct clock_event_device *dev) { }
#endif
static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires)
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index baee13a..e48c4d3 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -1222,14 +1222,8 @@ static void clocksource_enqueue(struct clocksource *cs)
* @cs: clocksource to be registered
* @scale: Scale factor multiplied against freq to get clocksource hz
* @freq: clocksource frequency (cycles per second) divided by scale
- *
- * This should only be called from the clocksource->enable() method.
- *
- * This *SHOULD NOT* be called directly! Please use the
- * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper
- * functions.
*/
-void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
+static void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
{
u64 sec;
@@ -1287,7 +1281,6 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq
pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
}
-EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
/**
* __clocksource_register_scale - Used to install new clocksources
@@ -1338,6 +1331,26 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
}
EXPORT_SYMBOL_GPL(__clocksource_register_scale);
+static void __devm_clocksource_unregister(void *data)
+{
+ struct clocksource *cs = data;
+
+ clocksource_unregister(cs);
+}
+
+int __devm_clocksource_register_scale(struct device *dev, struct clocksource *cs,
+ u32 scale, u32 freq)
+{
+ int ret;
+
+ ret = __clocksource_register_scale(cs, scale, freq);
+ if (ret)
+ return ret;
+
+ return devm_add_action_or_reset(dev, __devm_clocksource_unregister, cs);
+}
+EXPORT_SYMBOL_GPL(__devm_clocksource_register_scale);
+
/*
* Unbind clocksource @cs. Called with clocksource_mutex held
*/
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 5bd6efe..638ce623 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1352,8 +1352,14 @@ static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool
return hrtimer_prefer_local(is_local, is_first, is_pinned);
}
-static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
- const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
+enum {
+ HRTIMER_REPROGRAM_NONE,
+ HRTIMER_REPROGRAM,
+ HRTIMER_REPROGRAM_FORCE,
+};
+
+static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
+ const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
{
struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
bool is_pinned, first, was_first, keep_base = false;
@@ -1410,7 +1416,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
/* If a deferred rearm is pending skip reprogramming the device */
if (cpu_base->deferred_rearm) {
cpu_base->deferred_needs_update = true;
- return false;
+ return HRTIMER_REPROGRAM_NONE;
}
if (!was_first || cpu_base != this_cpu_base) {
@@ -1423,7 +1429,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
* callbacks.
*/
if (likely(hrtimer_base_is_online(this_cpu_base)))
- return first;
+ return first ? HRTIMER_REPROGRAM : HRTIMER_REPROGRAM_NONE;
/*
* Timer was enqueued remote because the current base is
@@ -1432,7 +1438,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
*/
if (first)
smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
- return false;
+ return HRTIMER_REPROGRAM_NONE;
}
/*
@@ -1446,7 +1452,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
*/
if (timer->is_lazy) {
if (cpu_base->expires_next <= hrtimer_get_expires(timer))
- return false;
+ return HRTIMER_REPROGRAM_NONE;
}
/*
@@ -1455,8 +1461,24 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
* reprogram the hardware by evaluating the new first expiring
* timer.
*/
- hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
- return false;
+ return HRTIMER_REPROGRAM_FORCE;
+}
+
+static int hrtimer_start_range_ns_common(struct hrtimer *timer, ktime_t tim,
+ u64 delta_ns, const enum hrtimer_mode mode,
+ struct hrtimer_clock_base *base)
+{
+ /*
+ * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
+ * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
+ * expiry mode because unmarked timers are moved to softirq expiry.
+ */
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
+ else
+ WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
+
+ return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, base);
}
/**
@@ -1476,25 +1498,105 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
debug_hrtimer_assert_init(timer);
- /*
- * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
- * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
- * expiry mode because unmarked timers are moved to softirq expiry.
- */
- if (!IS_ENABLED(CONFIG_PREEMPT_RT))
- WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
- else
- WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
-
base = lock_hrtimer_base(timer, &flags);
- if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
+ switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
+ case HRTIMER_REPROGRAM:
hrtimer_reprogram(timer, true);
+ break;
+ case HRTIMER_REPROGRAM_FORCE:
+ hrtimer_force_reprogram(timer->base->cpu_base, 1);
+ break;
+ case HRTIMER_REPROGRAM_NONE:
+ break;
+ }
unlock_hrtimer_base(timer, &flags);
}
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
+static inline bool hrtimer_check_user_timer(struct hrtimer *timer)
+{
+ struct hrtimer_cpu_base *cpu_base = timer->base->cpu_base;
+ ktime_t expires;
+
+ /*
+ * This uses soft expires because that's the user provided
+ * expiry time, while expires can be further in the past
+ * due to a slack value added to the user expiry time.
+ */
+ expires = hrtimer_get_softexpires(timer);
+
+ /* Convert to monotonic */
+ expires = ktime_sub(expires, timer->base->offset);
+
+ /*
+ * Check whether this timer will end up as the first expiring timer in
+ * the CPU base. If not, no further checks required as it's then
+ * guaranteed to expire in the future.
+ */
+ if (expires >= cpu_base->expires_next)
+ return true;
+
+ /* Validate that the expiry time is in the future. */
+ if (expires > ktime_get())
+ return true;
+
+ debug_hrtimer_deactivate(timer);
+ __remove_hrtimer(timer, timer->base, HRTIMER_STATE_INACTIVE, false);
+ trace_hrtimer_start_expired(timer);
+ return false;
+}
+
+/**
+ * hrtimer_start_range_ns_user - (re)start an user controlled hrtimer
+ * @timer: the timer to be added
+ * @tim: expiry time
+ * @delta_ns: "slack" range for the timer
+ * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
+ * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
+ * softirq based mode is considered for debug purpose only!
+ *
+ * Returns: True when the timer was queued, false if it was already expired
+ *
+ * This function cannot invoke the timer callback for expired timers as it might
+ * be called under a lock which the timer callback needs to acquire. So the
+ * caller has to handle that case.
+ */
+bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim,
+ u64 delta_ns, const enum hrtimer_mode mode)
+{
+ struct hrtimer_clock_base *base;
+ unsigned long flags;
+ bool ret = true;
+
+ debug_hrtimer_assert_init(timer);
+
+ base = lock_hrtimer_base(timer, &flags);
+
+ switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
+ case HRTIMER_REPROGRAM:
+ ret = hrtimer_check_user_timer(timer);
+ if (ret)
+ hrtimer_reprogram(timer, true);
+ break;
+ case HRTIMER_REPROGRAM_FORCE:
+ ret = hrtimer_check_user_timer(timer);
+ /*
+ * The base must always be reevaluated, independent of the
+ * result above because the timer was the first pending timer.
+ */
+ hrtimer_force_reprogram(timer->base->cpu_base, 1);
+ break;
+ case HRTIMER_REPROGRAM_NONE:
+ break;
+ }
+
+ unlock_hrtimer_base(timer, &flags);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(hrtimer_start_range_ns_user);
+
/**
* hrtimer_try_to_cancel - try to deactivate a timer
* @timer: hrtimer to stop
@@ -1681,10 +1783,10 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
*
* Returns the next expiry time or KTIME_MAX if no timer is pending.
*/
-u64 hrtimer_get_next_event(void)
+ktime_t hrtimer_get_next_event(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
- u64 expires = KTIME_MAX;
+ ktime_t expires = KTIME_MAX;
guard(raw_spinlock_irqsave)(&cpu_base->lock);
if (!hrtimer_hres_active(cpu_base))
@@ -1700,10 +1802,10 @@ u64 hrtimer_get_next_event(void)
* Returns the next expiry time over all timers except for the @exclude one or
* KTIME_MAX if none of them is pending.
*/
-u64 hrtimer_next_event_without(const struct hrtimer *exclude)
+ktime_t hrtimer_next_event_without(const struct hrtimer *exclude)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
- u64 expires = KTIME_MAX;
+ ktime_t expires = KTIME_MAX;
unsigned int active;
guard(raw_spinlock_irqsave)(&cpu_base->lock);
@@ -2213,7 +2315,11 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode
if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
mode |= HRTIMER_MODE_HARD;
- hrtimer_start_expires(&sl->timer, mode);
+ /* If already expired, clear the task pointer and set current state to running */
+ if (!hrtimer_start_expires_user(&sl->timer, mode)) {
+ sl->task = NULL;
+ __set_current_state(TASK_RUNNING);
+ }
}
EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c
index 4bca3f7..5fa0af6 100644
--- a/kernel/time/namespace.c
+++ b/kernel/time/namespace.c
@@ -57,6 +57,7 @@ ktime_t do_timens_ktime_to_host(clockid_t clockid, ktime_t tim,
return tim;
}
+EXPORT_SYMBOL_GPL(do_timens_ktime_to_host);
static struct ucounts *inc_time_namespaces(struct user_namespace *ns)
{
@@ -351,6 +352,7 @@ struct time_namespace init_time_ns = {
.user_ns = &init_user_ns,
.frozen_offsets = true,
};
+EXPORT_SYMBOL_GPL(init_time_ns);
void __init time_ns_init(void)
{
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 0de2bb7..395e297 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -19,7 +19,7 @@
#include "posix-timers.h"
-static void posix_cpu_timer_rearm(struct k_itimer *timer);
+static bool posix_cpu_timer_rearm(struct k_itimer *timer);
void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit)
{
@@ -1011,24 +1011,27 @@ static void check_process_timers(struct task_struct *tsk,
/*
* This is called from the signal code (via posixtimer_rearm)
* when the last timer signal was delivered and we have to reload the timer.
+ *
+ * Return true unconditionally so the core code assumes the timer to be
+ * armed. Otherwise it would requeue the signal.
*/
-static void posix_cpu_timer_rearm(struct k_itimer *timer)
+static bool posix_cpu_timer_rearm(struct k_itimer *timer)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
- struct task_struct *p;
struct sighand_struct *sighand;
+ struct task_struct *p;
unsigned long flags;
u64 now;
- rcu_read_lock();
+ guard(rcu)();
p = cpu_timer_task_rcu(timer);
if (!p)
- goto out;
+ return true;
/* Protect timer list r/w in arm_timer() */
sighand = lock_task_sighand(p, &flags);
if (unlikely(sighand == NULL))
- goto out;
+ return true;
/*
* Fetch the current sample and update the timer's expiry time.
@@ -1045,8 +1048,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer)
*/
arm_timer(timer, p);
unlock_task_sighand(p, &flags);
-out:
- rcu_read_unlock();
+ return true;
}
/**
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index 9331e16..436ba79 100644
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -288,16 +288,18 @@ static inline int timer_overrun_to_int(struct k_itimer *timr)
return (int)timr->it_overrun_last;
}
-static void common_hrtimer_rearm(struct k_itimer *timr)
+static bool common_hrtimer_rearm(struct k_itimer *timr)
{
struct hrtimer *timer = &timr->it.real.timer;
timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval);
- hrtimer_restart(timer);
+ return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
}
static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr)
{
+ bool queued;
+
guard(spinlock)(&timr->it_lock);
/*
@@ -311,12 +313,18 @@ static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_it
if (!timr->it_interval || WARN_ON_ONCE(timr->it_status != POSIX_TIMER_REQUEUE_PENDING))
return true;
- timr->kclock->timer_rearm(timr);
- timr->it_status = POSIX_TIMER_ARMED;
+ /* timer_rearm() updates timr::it_overrun */
+ queued = timr->kclock->timer_rearm(timr);
+
timr->it_overrun_last = timr->it_overrun;
timr->it_overrun = -1LL;
++timr->it_signal_seq;
info->si_overrun = timer_overrun_to_int(timr);
+
+ if (queued)
+ timr->it_status = POSIX_TIMER_ARMED;
+ else
+ posix_timer_queue_signal(timr);
return true;
}
@@ -795,7 +803,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id)
return timer_overrun_to_int(scoped_timer);
}
-static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
+static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
bool absolute, bool sigev_none)
{
struct hrtimer *timer = &timr->it.real.timer;
@@ -820,8 +828,11 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
expires = ktime_add_safe(expires, hrtimer_cb_get_time(timer));
hrtimer_set_expires(timer, expires);
- if (!sigev_none)
- hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
+ /* For sigev_none pretend that the timer is queued */
+ if (sigev_none)
+ return true;
+
+ return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
}
static int common_hrtimer_try_to_cancel(struct k_itimer *timr)
@@ -903,9 +914,13 @@ int common_timer_set(struct k_itimer *timr, int flags,
expires = timens_ktime_to_host(timr->it_clock, expires);
sigev_none = timr->it_sigev_notify == SIGEV_NONE;
- kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none);
- if (!sigev_none)
- timr->it_status = POSIX_TIMER_ARMED;
+ if (kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none)) {
+ if (!sigev_none)
+ timr->it_status = POSIX_TIMER_ARMED;
+ } else {
+ /* Timer was already expired, queue the signal */
+ posix_timer_queue_signal(timr);
+ }
return 0;
}
diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h
index 7f259e8..4ea9611 100644
--- a/kernel/time/posix-timers.h
+++ b/kernel/time/posix-timers.h
@@ -27,11 +27,11 @@ struct k_clock {
int (*timer_del)(struct k_itimer *timr);
void (*timer_get)(struct k_itimer *timr,
struct itimerspec64 *cur_setting);
- void (*timer_rearm)(struct k_itimer *timr);
+ bool (*timer_rearm)(struct k_itimer *timr);
s64 (*timer_forward)(struct k_itimer *timr, ktime_t now);
ktime_t (*timer_remaining)(struct k_itimer *timr, ktime_t now);
int (*timer_try_to_cancel)(struct k_itimer *timr);
- void (*timer_arm)(struct k_itimer *timr, ktime_t expires,
+ bool (*timer_arm)(struct k_itimer *timr, ktime_t expires,
bool absolute, bool sigev_none);
void (*timer_wait_running)(struct k_itimer *timr);
};
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index cbbb87a..98a9cae 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -285,8 +285,6 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) &&
tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
touch_softlockup_watchdog_sched();
- if (is_idle_task(current))
- ts->idle_jiffies++;
/*
* In case the current tick fired too early past its expected
* expiration, make sure we don't bypass the next clock reprogramming
@@ -751,119 +749,6 @@ static void tick_nohz_update_jiffies(ktime_t now)
touch_softlockup_watchdog_sched();
}
-static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
-{
- ktime_t delta;
-
- if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)))
- return;
-
- delta = ktime_sub(now, ts->idle_entrytime);
-
- write_seqcount_begin(&ts->idle_sleeptime_seq);
- if (nr_iowait_cpu(smp_processor_id()) > 0)
- ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
- else
- ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
-
- ts->idle_entrytime = now;
- tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);
- write_seqcount_end(&ts->idle_sleeptime_seq);
-
- sched_clock_idle_wakeup_event();
-}
-
-static void tick_nohz_start_idle(struct tick_sched *ts)
-{
- write_seqcount_begin(&ts->idle_sleeptime_seq);
- ts->idle_entrytime = ktime_get();
- tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);
- write_seqcount_end(&ts->idle_sleeptime_seq);
-
- sched_clock_idle_sleep_event();
-}
-
-static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,
- bool compute_delta, u64 *last_update_time)
-{
- ktime_t now, idle;
- unsigned int seq;
-
- if (!tick_nohz_active)
- return -1;
-
- now = ktime_get();
- if (last_update_time)
- *last_update_time = ktime_to_us(now);
-
- do {
- seq = read_seqcount_begin(&ts->idle_sleeptime_seq);
-
- if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) {
- ktime_t delta = ktime_sub(now, ts->idle_entrytime);
-
- idle = ktime_add(*sleeptime, delta);
- } else {
- idle = *sleeptime;
- }
- } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));
-
- return ktime_to_us(idle);
-
-}
-
-/**
- * get_cpu_idle_time_us - get the total idle time of a CPU
- * @cpu: CPU number to query
- * @last_update_time: variable to store update time in. Do not update
- * counters if NULL.
- *
- * Return the cumulative idle time (since boot) for a given
- * CPU, in microseconds. Note that this is partially broken due to
- * the counter of iowait tasks that can be remotely updated without
- * any synchronization. Therefore it is possible to observe backward
- * values within two consecutive reads.
- *
- * This time is measured via accounting rather than sampling,
- * and is as accurate as ktime_get() is.
- *
- * Return: -1 if NOHZ is not enabled, else total idle time of the @cpu
- */
-u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
-{
- struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-
- return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,
- !nr_iowait_cpu(cpu), last_update_time);
-}
-EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
-
-/**
- * get_cpu_iowait_time_us - get the total iowait time of a CPU
- * @cpu: CPU number to query
- * @last_update_time: variable to store update time in. Do not update
- * counters if NULL.
- *
- * Return the cumulative iowait time (since boot) for a given
- * CPU, in microseconds. Note this is partially broken due to
- * the counter of iowait tasks that can be remotely updated without
- * any synchronization. Therefore it is possible to observe backward
- * values within two consecutive reads.
- *
- * This time is measured via accounting rather than sampling,
- * and is as accurate as ktime_get() is.
- *
- * Return: -1 if NOHZ is not enabled, else total iowait time of @cpu
- */
-u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
-{
- struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
-
- return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,
- nr_iowait_cpu(cpu), last_update_time);
-}
-EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
-
/* Simplified variant of hrtimer_forward_now() */
static ktime_t tick_forward_now(ktime_t expires, ktime_t now)
{
@@ -1273,7 +1158,7 @@ void tick_nohz_idle_stop_tick(void)
ts->idle_expires = expires;
if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
- ts->idle_jiffies = ts->last_jiffies;
+ kcpustat_dyntick_start(ts->idle_entrytime);
nohz_balance_enter_idle(cpu);
}
} else {
@@ -1286,6 +1171,20 @@ void tick_nohz_idle_retain_tick(void)
tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
}
+static void tick_nohz_clock_sleep(struct tick_sched *ts)
+{
+ tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);
+ sched_clock_idle_sleep_event();
+}
+
+static void tick_nohz_clock_wakeup(struct tick_sched *ts)
+{
+ if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) {
+ tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);
+ sched_clock_idle_wakeup_event();
+ }
+}
+
/**
* tick_nohz_idle_enter - prepare for entering idle on the current CPU
*
@@ -1300,11 +1199,10 @@ void tick_nohz_idle_enter(void)
local_irq_disable();
ts = this_cpu_ptr(&tick_cpu_sched);
-
WARN_ON_ONCE(ts->timer_expires_base);
-
tick_sched_flag_set(ts, TS_FLAG_INIDLE);
- tick_nohz_start_idle(ts);
+ ts->idle_entrytime = ktime_get();
+ tick_nohz_clock_sleep(ts);
local_irq_enable();
}
@@ -1332,10 +1230,14 @@ void tick_nohz_irq_exit(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
- if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
- tick_nohz_start_idle(ts);
- else
+ if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) {
+ tick_nohz_clock_sleep(ts);
+ ts->idle_entrytime = ktime_get();
+ if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
+ kcpustat_irq_exit(ts->idle_entrytime);
+ } else {
tick_nohz_full_update_tick(ts);
+ }
}
/**
@@ -1407,8 +1309,7 @@ ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
* If the next highres timer to expire is earlier than 'next_event', the
* idle governor needs to know that.
*/
- next_event = min_t(u64, next_event,
- hrtimer_next_event_without(&ts->sched_timer));
+ next_event = min(next_event, hrtimer_next_event_without(&ts->sched_timer));
return ktime_sub(next_event, now);
}
@@ -1429,36 +1330,20 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
return ts->idle_calls;
}
-static void tick_nohz_account_idle_time(struct tick_sched *ts,
- ktime_t now)
-{
- unsigned long ticks;
-
- ts->idle_exittime = now;
-
- if (vtime_accounting_enabled_this_cpu())
- return;
- /*
- * We stopped the tick in idle. update_process_times() would miss the
- * time we slept, as it does only a 1 tick accounting.
- * Enforce that this is accounted to idle !
- */
- ticks = jiffies - ts->idle_jiffies;
- /*
- * We might be one off. Do not randomly account a huge number of ticks!
- */
- if (ticks && ticks < LONG_MAX)
- account_idle_ticks(ticks);
-}
-
void tick_nohz_idle_restart_tick(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
- ktime_t now = ktime_get();
- tick_nohz_restart_sched_tick(ts, now);
- tick_nohz_account_idle_time(ts, now);
+ /*
+ * Update entrytime here in case the tick restart is due to temporary
+ * polling on forced broadcast. The tick may be stopped again later within
+ * the same idle trip. The idle_entrytime was updated recently but make sure
+ * no tiny amount of idle time is accounted twice.
+ */
+ ts->idle_entrytime = ktime_get();
+ kcpustat_dyntick_stop(ts->idle_entrytime);
+ tick_nohz_restart_sched_tick(ts, ts->idle_entrytime);
}
}
@@ -1468,8 +1353,6 @@ static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
__tick_nohz_full_update_tick(ts, now);
else
tick_nohz_restart_sched_tick(ts, now);
-
- tick_nohz_account_idle_time(ts, now);
}
/**
@@ -1491,7 +1374,6 @@ static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
void tick_nohz_idle_exit(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
- bool idle_active, tick_stopped;
ktime_t now;
local_irq_disable();
@@ -1500,17 +1382,13 @@ void tick_nohz_idle_exit(void)
WARN_ON_ONCE(ts->timer_expires_base);
tick_sched_flag_clear(ts, TS_FLAG_INIDLE);
- idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE);
- tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
+ tick_nohz_clock_wakeup(ts);
- if (idle_active || tick_stopped)
+ if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
now = ktime_get();
-
- if (idle_active)
- tick_nohz_stop_idle(ts, now);
-
- if (tick_stopped)
+ kcpustat_dyntick_stop(now);
tick_nohz_idle_update_tick(ts, now);
+ }
local_irq_enable();
}
@@ -1565,11 +1443,14 @@ static inline void tick_nohz_irq_enter(void)
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
ktime_t now;
- if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE))
+ tick_nohz_clock_wakeup(ts);
+
+ if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED))
return;
+
now = ktime_get();
- if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))
- tick_nohz_stop_idle(ts, now);
+ kcpustat_irq_enter(now);
+
/*
* If all CPUs are idle we may need to update a stale jiffies value.
* Note nohz_full is a special case: a timekeeper is guaranteed to stay
@@ -1577,8 +1458,7 @@ static inline void tick_nohz_irq_enter(void)
* rare case (typically stop machine). So we must make sure we have a
* last resort.
*/
- if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
- tick_nohz_update_jiffies(now);
+ tick_nohz_update_jiffies(now);
}
#else
@@ -1648,20 +1528,15 @@ void tick_setup_sched_timer(bool hrtimer)
void tick_sched_timer_dying(int cpu)
{
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
- ktime_t idle_sleeptime, iowait_sleeptime;
unsigned long idle_calls, idle_sleeps;
/* This must happen before hrtimers are migrated! */
if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
hrtimer_cancel(&ts->sched_timer);
- idle_sleeptime = ts->idle_sleeptime;
- iowait_sleeptime = ts->iowait_sleeptime;
idle_calls = ts->idle_calls;
idle_sleeps = ts->idle_sleeps;
memset(ts, 0, sizeof(*ts));
- ts->idle_sleeptime = idle_sleeptime;
- ts->iowait_sleeptime = iowait_sleeptime;
ts->idle_calls = idle_calls;
ts->idle_sleeps = idle_sleeps;
}
diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h
index b4a7822..79b9252 100644
--- a/kernel/time/tick-sched.h
+++ b/kernel/time/tick-sched.h
@@ -44,9 +44,7 @@ struct tick_device {
* to resume the tick timer operation in the timeline
* when the CPU returns from nohz sleep.
* @next_tick: Next tick to be fired when in dynticks mode.
- * @idle_jiffies: jiffies at the entry to idle for idle time accounting
* @idle_waketime: Time when the idle was interrupted
- * @idle_sleeptime_seq: sequence counter for data consistency
* @idle_entrytime: Time when the idle call was entered
* @last_jiffies: Base jiffies snapshot when next event was last computed
* @timer_expires_base: Base time clock monotonic for @timer_expires
@@ -55,9 +53,6 @@ struct tick_device {
* @idle_expires: Next tick in idle, for debugging purpose only
* @idle_calls: Total number of idle calls
* @idle_sleeps: Number of idle calls, where the sched tick was stopped
- * @idle_exittime: Time when the idle state was left
- * @idle_sleeptime: Sum of the time slept in idle with sched tick stopped
- * @iowait_sleeptime: Sum of the time slept in idle with sched tick stopped, with IO outstanding
* @tick_dep_mask: Tick dependency mask - is set, if someone needs the tick
* @check_clocks: Notification mechanism about clocksource changes
*/
@@ -73,12 +68,10 @@ struct tick_sched {
struct hrtimer sched_timer;
ktime_t last_tick;
ktime_t next_tick;
- unsigned long idle_jiffies;
ktime_t idle_waketime;
unsigned int got_idle_tick;
/* Idle entry */
- seqcount_t idle_sleeptime_seq;
ktime_t idle_entrytime;
/* Tick stop */
@@ -90,11 +83,6 @@ struct tick_sched {
unsigned long idle_calls;
unsigned long idle_sleeps;
- /* Idle exit */
- ktime_t idle_exittime;
- ktime_t idle_sleeptime;
- ktime_t iowait_sleeptime;
-
/* Full dynticks handling */
atomic_t tick_dep_mask;
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 0d83231..771cef8 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -207,7 +207,7 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
get_user(new_ts.tv_nsec, &tv->tv_usec))
return -EFAULT;
- if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0)
+ if (new_ts.tv_nsec >= USEC_PER_SEC || new_ts.tv_nsec < 0)
return -EINVAL;
new_ts.tv_nsec *= NSEC_PER_USEC;
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index c493a40..0d5b67f 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -67,6 +67,7 @@ static inline bool tk_is_aux(const struct timekeeper *tk)
{
return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;
}
+static inline struct tk_data *aux_get_tk_data(clockid_t id);
#else
static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
{
@@ -77,6 +78,10 @@ static inline bool tk_is_aux(const struct timekeeper *tk)
{
return false;
}
+static inline struct tk_data *aux_get_tk_data(clockid_t id)
+{
+ return NULL;
+}
#endif
static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs)
@@ -315,6 +320,7 @@ static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr)
return clock->read(clock);
}
+
static inline void clocksource_disable_inline_read(void) { }
static inline void clocksource_enable_inline_read(void) { }
#endif
@@ -1182,44 +1188,107 @@ noinstr time64_t __ktime_get_real_seconds(void)
return tk->xtime_sec;
}
-/**
- * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
- * @systime_snapshot: pointer to struct receiving the system time snapshot
- */
-void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
+static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
+ struct clocksource_hw_snapshot *chs)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- unsigned int seq;
- ktime_t base_raw;
- ktime_t base_real;
- ktime_t base_boot;
- u64 nsec_raw;
- u64 nsec_real;
- u64 now;
+ struct clocksource *clock = READ_ONCE(tkr->clock);
- WARN_ON_ONCE(timekeeping_suspended);
+ if (unlikely(clock->read_snapshot))
+ return clock->read_snapshot(clock, chs);
+
+ return clock->read(clock);
+}
+
+
+/**
+ * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with
+ * CLOCK_MONOTONIC_RAW and the underlying
+ * clocksource counter value.
+ * @clock_id: The clock ID to snapshot
+ * @systime_snapshot: Pointer to struct receiving the system time snapshot
+ */
+void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot)
+{
+ ktime_t base_raw, base_sys, offs_sys, *offs, offs_zero = 0;
+ u64 nsec_raw, nsec_sys, now;
+ struct timekeeper *tk;
+ struct tk_data *tkd;
+ unsigned int seq;
+
+ /* Invalidate the snapshot for all failure cases */
+ systime_snapshot->valid = false;
+
+ if (WARN_ON_ONCE(timekeeping_suspended))
+ return;
+
+ switch (clock_id) {
+ case CLOCK_REALTIME:
+ tkd = &tk_core;
+ offs = &tk_core.timekeeper.offs_real;
+ break;
+ /* Map RAW to MONOTONIC so the loop below is trivial */
+ case CLOCK_MONOTONIC_RAW:
+ case CLOCK_MONOTONIC:
+ tkd = &tk_core;
+ offs = &offs_zero;
+ break;
+ case CLOCK_BOOTTIME:
+ tkd = &tk_core;
+ offs = &tk_core.timekeeper.offs_boot;
+ break;
+ case CLOCK_AUX ... CLOCK_AUX_LAST:
+ tkd = aux_get_tk_data(clock_id);
+ if (!tkd)
+ return;
+ offs = &tkd->timekeeper.offs_aux;
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ return;
+ }
+
+ tk = &tkd->timekeeper;
do {
- seq = read_seqcount_begin(&tk_core.seq);
- now = tk_clock_read(&tk->tkr_mono);
+ struct clocksource_hw_snapshot chs = { };
+
+ seq = read_seqcount_begin(&tkd->seq);
+
+ /* Aux clocks can be invalid */
+ if (!tk->clock_valid)
+ return;
+
+ now = tk_clock_read_snapshot(&tk->tkr_mono, &chs);
systime_snapshot->cs_id = tk->tkr_mono.clock->id;
+
+ systime_snapshot->hw_cycles = chs.hw_cycles;
+ systime_snapshot->hw_csid = chs.hw_csid;
+
systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
- base_real = ktime_add(tk->tkr_mono.base,
- tk_core.timekeeper.offs_real);
- base_boot = ktime_add(tk->tkr_mono.base,
- tk_core.timekeeper.offs_boot);
+
+ base_sys = tk->tkr_mono.base;
+ offs_sys = *offs;
base_raw = tk->tkr_raw.base;
- nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
- nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
- } while (read_seqcount_retry(&tk_core.seq, seq));
+
+ nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
+ nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
+ } while (read_seqcount_retry(&tkd->seq, seq));
systime_snapshot->cycles = now;
- systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
- systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);
- systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
+ systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys);
+ systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw);
+
+ /*
+ * Special case for PTP. Just transfer the raw time into sys,
+ * so the call sites can consistently use snap::systime.
+ */
+ if (clock_id == CLOCK_MONOTONIC_RAW)
+ systime_snapshot->systime = systime_snapshot->monoraw;
+ /* Tell the consumer that this snapshot is valid */
+ systime_snapshot->valid = true;
}
-EXPORT_SYMBOL_GPL(ktime_get_snapshot);
+EXPORT_SYMBOL_GPL(ktime_get_snapshot_id);
/* Scale base by mult/div checking for overflow */
static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
@@ -1262,7 +1331,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history,
struct system_device_crosststamp *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
- u64 corr_raw, corr_real;
+ u64 corr_raw, corr_sys;
bool interp_forward;
int ret;
@@ -1279,8 +1348,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history,
* Scale the monotonic raw time delta by:
* partial_history_cycles / total_history_cycles
*/
- corr_raw = (u64)ktime_to_ns(
- ktime_sub(ts->sys_monoraw, history->raw));
+ corr_raw = (u64)ktime_to_ns(ktime_sub(ts->sys_monoraw, history->monoraw));
ret = scale64_check_overflow(partial_history_cycles,
total_history_cycles, &corr_raw);
if (ret)
@@ -1288,30 +1356,29 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history,
/*
* If there is a discontinuity in the history, scale monotonic raw
- * correction by:
- * mult(real)/mult(raw) yielding the realtime correction
- * Otherwise, calculate the realtime correction similar to monotonic
- * raw calculation
+ * correction by:
+ * mult(sys)/mult(raw) yielding the system time correction
+ *
+ * Otherwise, calculate the system time correction similar to monotonic
+ * raw calculation
*/
if (discontinuity) {
- corr_real = mul_u64_u32_div
- (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
+ corr_sys = mul_u64_u32_div(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
} else {
- corr_real = (u64)ktime_to_ns(
- ktime_sub(ts->sys_realtime, history->real));
- ret = scale64_check_overflow(partial_history_cycles,
- total_history_cycles, &corr_real);
+ corr_sys = (u64)ktime_to_ns(ktime_sub(ts->sys_systime, history->systime));
+ ret = scale64_check_overflow(partial_history_cycles, total_history_cycles,
+ &corr_sys);
if (ret)
return ret;
}
- /* Fixup monotonic raw and real time time values */
+ /* Fixup monotonic raw and system time time values */
if (interp_forward) {
- ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
- ts->sys_realtime = ktime_add_ns(history->real, corr_real);
+ ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw);
+ ts->sys_systime = ktime_add_ns(history->systime, corr_sys);
} else {
ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
- ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
+ ts->sys_systime = ktime_sub_ns(ts->sys_systime, corr_sys);
}
return 0;
@@ -1368,6 +1435,8 @@ static bool convert_base_to_cs(struct system_counterval_t *scv)
return false;
scv->cycles += base->offset;
+ /* Set the clocksource ID as scv::cycles is now clocksource based */
+ scv->cs_id = cs->id;
return true;
}
@@ -1435,11 +1504,11 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
/**
* get_device_system_crosststamp - Synchronously capture system/device timestamp
- * @get_time_fn: Callback to get simultaneous device time and
- * system counter from the device driver
+ * @get_time_fn: Callback to get simultaneous device time and system counter
+ * from the device driver
* @ctx: Context passed to get_time_fn()
- * @history_begin: Historical reference point used to interpolate system
- * time when counter provided by the driver is before the current interval
+ * @history_begin: Historical reference point used to interpolate system time when
+ * the counter value provided by the driver is before the current interval
* @xtstamp: Receives simultaneously captured system and device time
*
* Reads a timestamp from a device and correlates it to system time
@@ -1452,36 +1521,54 @@ int get_device_system_crosststamp(int (*get_time_fn)
struct system_time_snapshot *history_begin,
struct system_device_crosststamp *xtstamp)
{
- struct system_counterval_t system_counterval = {};
- struct timekeeper *tk = &tk_core.timekeeper;
- u64 cycles, now, interval_start;
- unsigned int clock_was_set_seq = 0;
- ktime_t base_real, base_raw;
- u64 nsec_real, nsec_raw;
+ u64 syscnt_cycles, cycles, now, interval_start;
+ unsigned int seq, clock_was_set_seq = 0;
+ ktime_t base_sys, base_raw, *offs;
+ u64 nsec_sys, nsec_raw;
u8 cs_was_changed_seq;
- unsigned int seq;
bool do_interp;
+ struct timekeeper *tk;
+ struct tk_data *tkd;
int ret;
+ switch (xtstamp->clock_id) {
+ case CLOCK_REALTIME:
+ tkd = &tk_core;
+ offs = &tk_core.timekeeper.offs_real;
+ break;
+ case CLOCK_AUX ... CLOCK_AUX_LAST:
+ tkd = aux_get_tk_data(xtstamp->clock_id);
+ if (!tkd)
+ return -ENODEV;
+ offs = &tkd->timekeeper.offs_aux;
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ return -ENODEV;
+ }
+
+ tk = &tkd->timekeeper;
+
do {
- seq = read_seqcount_begin(&tk_core.seq);
+ seq = read_seqcount_begin(&tkd->seq);
/*
* Try to synchronously capture device time and a system
* counter value calling back into the device driver
*/
- ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
+ ret = get_time_fn(&xtstamp->device, &xtstamp->sys_counter, ctx);
if (ret)
return ret;
/*
* Verify that the clocksource ID associated with the captured
* system counter value is the same as for the currently
- * installed timekeeper clocksource
+ * installed timekeeper clocksource and convert to it.
*/
- if (system_counterval.cs_id == CSID_GENERIC ||
- !convert_base_to_cs(&system_counterval))
+ if (xtstamp->sys_counter.cs_id == CSID_GENERIC ||
+ !convert_base_to_cs(&xtstamp->sys_counter))
return -ENODEV;
- cycles = system_counterval.cycles;
+
+ cycles = syscnt_cycles = xtstamp->sys_counter.cycles;
/*
* Check whether the system counter value provided by the
@@ -1498,15 +1585,14 @@ int get_device_system_crosststamp(int (*get_time_fn)
do_interp = false;
}
- base_real = ktime_add(tk->tkr_mono.base,
- tk_core.timekeeper.offs_real);
+ base_sys = ktime_add(tk->tkr_mono.base, *offs);
base_raw = tk->tkr_raw.base;
- nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);
+ nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);
- } while (read_seqcount_retry(&tk_core.seq, seq));
+ } while (read_seqcount_retry(&tkd->seq, seq));
- xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
+ xtstamp->sys_systime = ktime_add_ns(base_sys, nsec_sys);
xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
/*
@@ -1523,24 +1609,19 @@ int get_device_system_crosststamp(int (*get_time_fn)
* clocksource change
*/
if (!history_begin ||
- !timestamp_in_interval(history_begin->cycles,
- cycles, system_counterval.cycles) ||
+ !timestamp_in_interval(history_begin->cycles, cycles, syscnt_cycles) ||
history_begin->cs_was_changed_seq != cs_was_changed_seq)
return -EINVAL;
- partial_history_cycles = cycles - system_counterval.cycles;
- total_history_cycles = cycles - history_begin->cycles;
- discontinuity =
- history_begin->clock_was_set_seq != clock_was_set_seq;
- ret = adjust_historical_crosststamp(history_begin,
- partial_history_cycles,
- total_history_cycles,
- discontinuity, xtstamp);
- if (ret)
- return ret;
+ partial_history_cycles = cycles - syscnt_cycles;
+ total_history_cycles = cycles - history_begin->cycles;
+ discontinuity = history_begin->clock_was_set_seq != clock_was_set_seq;
+
+ ret = adjust_historical_crosststamp(history_begin, partial_history_cycles,
+ total_history_cycles, discontinuity, xtstamp);
}
- return 0;
+ return ret;
}
EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 04d928c..655a8c6 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1932,7 +1932,7 @@ static void timer_recalc_next_expiry(struct timer_base *base)
*/
static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
{
- u64 nextevt = hrtimer_get_next_event();
+ u64 nextevt = ktime_to_ns(hrtimer_get_next_event());
/*
* If high resolution timers are enabled
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 427d7dd..514802d 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -152,14 +152,10 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
P_flag(highres, TS_FLAG_HIGHRES);
P_ns(last_tick);
P_flag(tick_stopped, TS_FLAG_STOPPED);
- P(idle_jiffies);
P(idle_calls);
P(idle_sleeps);
P_ns(idle_entrytime);
P_ns(idle_waketime);
- P_ns(idle_exittime);
- P_ns(idle_sleeptime);
- P_ns(iowait_sleeptime);
P(last_jiffies);
P(next_timer);
P_ns(idle_expires);
@@ -256,7 +252,7 @@ static void timer_list_show_tickdevices_header(struct seq_file *m)
static inline void timer_list_header(struct seq_file *m, u64 now)
{
- SEQ_printf(m, "Timer List Version: v0.10\n");
+ SEQ_printf(m, "Timer List Version: v0.11\n");
SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES);
SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now);
SEQ_printf(m, "\n");
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 155eeae..fda4214 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -102,7 +102,7 @@
* active CPU/group information atomic_try_cmpxchg() is used instead and only
* the per CPU tmigr_cpu->lock is held.
*
- * During the setup of groups tmigr_level_list is required. It is protected by
+ * During the setup of groups, hier->level_list is required. It is protected by
* @tmigr_mutex.
*
* When @timer_base->lock as well as tmigr related locks are required, the lock
@@ -416,13 +416,12 @@
*/
static DEFINE_MUTEX(tmigr_mutex);
-static struct list_head *tmigr_level_list __read_mostly;
+
+static LIST_HEAD(tmigr_hierarchy_list);
static unsigned int tmigr_hierarchy_levels __read_mostly;
static unsigned int tmigr_crossnode_level __read_mostly;
-static struct tmigr_group *tmigr_root;
-
static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu);
/*
@@ -978,8 +977,12 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now,
/* Drop the lock to allow the remote CPU to exit idle */
raw_spin_unlock_irq(&tmc->lock);
- if (cpu != smp_processor_id())
- timer_expire_remote(cpu);
+ /*
+ * This can't exclude the local CPU because jiffies might have advanced
+ * after the timer softirq invoked run_timer_base(BASE_GLOBAL) and the
+ * point where the jiffies snapshot @jif was taken in tmigr_handle_remote().
+ */
+ timer_expire_remote(cpu);
/*
* Lock ordering needs to be preserved - timer_base locks before tmigr
@@ -1465,6 +1468,34 @@ static long tmigr_trigger_active(void *unused)
return 0;
}
+static unsigned int tmigr_get_capacity(int cpu)
+{
+ /*
+ * nohz_full CPUs need to make sure there is always an available (online)
+ * and never idle migrator to handle all their global timers. That duty
+ * is served by the timekeeper which then never stops its tick. But the
+ * timekeeper must then belong to the same hierarchy as all the nohz_full
+ * CPUs. Simply turn off capacity awareness when nohz_full is running.
+ */
+ if (tick_nohz_full_enabled())
+ return SCHED_CAPACITY_SCALE;
+ else
+ return arch_scale_cpu_capacity(cpu);
+}
+
+static struct tmigr_hierarchy *__tmigr_get_hierarchy(int cpu)
+{
+ unsigned int capacity = tmigr_get_capacity(cpu);
+ struct tmigr_hierarchy *iter;
+
+ list_for_each_entry(iter, &tmigr_hierarchy_list, node) {
+ if (iter->capacity == capacity)
+ return iter;
+ }
+
+ return NULL;
+}
+
static int tmigr_clear_cpu_available(unsigned int cpu)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
@@ -1489,8 +1520,21 @@ static int tmigr_clear_cpu_available(unsigned int cpu)
}
if (firstexp != KTIME_MAX) {
- migrator = cpumask_any(tmigr_available_cpumask);
- work_on_cpu(migrator, tmigr_trigger_active, NULL);
+ struct tmigr_hierarchy *hier = __tmigr_get_hierarchy(cpu);
+
+ if (WARN_ON_ONCE(!hier))
+ return -EINVAL;
+
+ migrator = cpumask_any_and(tmigr_available_cpumask, hier->cpumask);
+ if (migrator < nr_cpu_ids) {
+ work_on_cpu(migrator, tmigr_trigger_active, NULL);
+ } else {
+ /*
+ * If deactivation returned an expiration, it belongs to an available
+ * nohz CPU in the hierarchy.
+ */
+ WARN_ONCE(1, "Expected available CPU in the hierarchy\n");
+ }
}
return 0;
@@ -1653,14 +1697,14 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
group->groupevt.ignore = true;
}
-static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
+static struct tmigr_group *tmigr_get_group(struct tmigr_hierarchy *hier, int node, unsigned int lvl)
{
struct tmigr_group *tmp, *group = NULL;
lockdep_assert_held(&tmigr_mutex);
/* Try to attach to an existing group first */
- list_for_each_entry(tmp, &tmigr_level_list[lvl], list) {
+ list_for_each_entry(tmp, &hier->level_list[lvl], list) {
/*
* If @lvl is below the cross NUMA node level, check whether
* this group belongs to the same NUMA node.
@@ -1694,14 +1738,14 @@ static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
tmigr_init_group(group, lvl, node);
/* Setup successful. Add it to the hierarchy */
- list_add(&group->list, &tmigr_level_list[lvl]);
+ list_add(&group->list, &hier->level_list[lvl]);
trace_tmigr_group_set(group);
return group;
}
-static bool tmigr_init_root(struct tmigr_group *group, bool activate)
+static bool tmigr_init_root(struct tmigr_hierarchy *hier, struct tmigr_group *group, bool activate)
{
- if (!group->parent && group != tmigr_root) {
+ if (!group->parent && group != hier->root) {
/*
* This is the new top-level, prepare its groupmask in advance
* to avoid accidents where yet another new top-level is
@@ -1717,11 +1761,10 @@ static bool tmigr_init_root(struct tmigr_group *group, bool activate)
}
-static void tmigr_connect_child_parent(struct tmigr_group *child,
- struct tmigr_group *parent,
- bool activate)
+static void tmigr_connect_child_parent(struct tmigr_hierarchy *hier, struct tmigr_group *child,
+ struct tmigr_group *parent, bool activate)
{
- if (tmigr_init_root(parent, activate)) {
+ if (tmigr_init_root(hier, parent, activate)) {
/*
* The previous top level had prepared its groupmask already,
* simply account it in advance as the first child. If some groups
@@ -1754,13 +1797,13 @@ static void tmigr_connect_child_parent(struct tmigr_group *child,
*/
smp_store_release(&child->parent, parent);
- trace_tmigr_connect_child_parent(child);
+ trace_tmigr_connect_child_parent(hier, child);
}
-static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
- struct tmigr_group *start, bool activate)
+static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu,
+ unsigned int node, struct tmigr_group *start, bool activate)
{
- struct tmigr_group *group, *child, **stack;
+ struct tmigr_group *root = hier->root, *group, *child, **stack;
int i, top = 0, err = 0, start_lvl = 0;
bool root_mismatch = false;
@@ -1773,11 +1816,11 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
start_lvl = start->level + 1;
}
- if (tmigr_root)
- root_mismatch = tmigr_root->numa_node != node;
+ if (root)
+ root_mismatch = root->numa_node != node;
for (i = start_lvl; i < tmigr_hierarchy_levels; i++) {
- group = tmigr_get_group(node, i);
+ group = tmigr_get_group(hier, node, i);
if (IS_ERR(group)) {
err = PTR_ERR(group);
i--;
@@ -1799,7 +1842,7 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
if (group->parent)
break;
if ((!root_mismatch || i >= tmigr_crossnode_level) &&
- list_is_singular(&tmigr_level_list[i]))
+ list_is_singular(&hier->level_list[i]))
break;
}
@@ -1827,15 +1870,15 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
tmc->tmgroup = group;
tmc->groupmask = BIT(group->num_children++);
- tmigr_init_root(group, activate);
+ tmigr_init_root(hier, group, activate);
- trace_tmigr_connect_cpu_parent(tmc);
+ trace_tmigr_connect_cpu_parent(hier, tmc);
/* There are no children that need to be connected */
continue;
} else {
child = stack[i - 1];
- tmigr_connect_child_parent(child, group, activate);
+ tmigr_connect_child_parent(hier, child, group, activate);
}
}
@@ -1860,31 +1903,54 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
* child to the new parents. So tmigr_active_up() activates the
* new parents while walking up from the old root to the new.
*
- * * It is ensured that @start is active, as this setup path is
- * executed in hotplug prepare callback. This is executed by an
- * already connected and !idle CPU. Even if all other CPUs go idle,
- * the CPU executing the setup will be responsible up to current top
- * level group. And the next time it goes inactive, it will release
- * the new childmask and parent to subsequent walkers through this
- * @child. Therefore propagate active state unconditionally.
+ * * It is ensured that @start is active, (or on the way to be activated
+ * by another CPU that woke up before the current one) as this setup path
+ * is executed in hotplug prepare callback. This is executed by an already
+ * connected and !idle CPU in the hierarchy.
+ *
+ * * The below RmW atomic operation ensures that:
+ *
+ * 1) If the old root has been completely activated, the latest state is
+ * acquired (the below implicit acquire pairs with the implicit release
+ * from cmpxchg() in tmigr_active_up()).
+ *
+ * 2) If the old root is still on the way to be activated, the lagging behind
+ * CPU performing the activation will acquire the links up to the new root.
+ * (The below implicit release pairs with the implicit acquire from cmpxchg()
+ * in tmigr_active_up()).
+ *
+ * 3) Every subsequent CPU below the old root will acquire the new links while
+ * walking through the old root (The below implicit release pairs with the
+ * implicit acquire from cmpxchg() in either tmigr_active_up()) or
+ * tmigr_inactive_up().
*/
- state.state = atomic_read(&start->migr_state);
- WARN_ON_ONCE(!state.active);
+ state.state = atomic_fetch_or(0, &start->migr_state);
WARN_ON_ONCE(!start->parent);
- data.childmask = start->groupmask;
- __walk_groups_from(tmigr_active_up, &data, start, start->parent);
+ /*
+ * If the state of the old root is inactive, another CPU is on its way to activate
+ * it and propagate to the new root.
+ */
+ if (state.active) {
+ data.childmask = start->groupmask;
+ __walk_groups_from(tmigr_active_up, &data, start, start->parent);
+ }
+ } else if (start) {
+ union tmigr_state state;
+
+ /* Remote activation assumes the whole target's hierarchy is inactive */
+ state.state = atomic_read(&start->migr_state);
+ WARN_ON_ONCE(state.active);
}
/* Root update */
- if (list_is_singular(&tmigr_level_list[top])) {
- group = list_first_entry(&tmigr_level_list[top],
- typeof(*group), list);
+ if (list_is_singular(&hier->level_list[top])) {
+ group = list_first_entry(&hier->level_list[top], typeof(*group), list);
WARN_ON_ONCE(group->parent);
- if (tmigr_root) {
+ if (root) {
/* Old root should be the same or below */
- WARN_ON_ONCE(tmigr_root->level > top);
+ WARN_ON_ONCE(root->level > top);
}
- tmigr_root = group;
+ hier->root = group;
}
out:
kfree(stack);
@@ -1892,34 +1958,123 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
return err;
}
+static struct tmigr_hierarchy *tmigr_get_hierarchy(int cpu)
+{
+ struct tmigr_hierarchy *hier;
+
+ hier = __tmigr_get_hierarchy(cpu);
+
+ if (hier)
+ return hier;
+
+ hier = kzalloc_flex(*hier, level_list, tmigr_hierarchy_levels);
+ if (!hier)
+ return ERR_PTR(-ENOMEM);
+
+ hier->cpumask = kzalloc(cpumask_size(), GFP_KERNEL);
+ if (!hier->cpumask) {
+ kfree(hier);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ for (int i = 0; i < tmigr_hierarchy_levels; i++)
+ INIT_LIST_HEAD(&hier->level_list[i]);
+
+ hier->capacity = tmigr_get_capacity(cpu);
+ list_add_tail(&hier->node, &tmigr_hierarchy_list);
+
+ return hier;
+}
+
+static int tmigr_connect_old_root(struct tmigr_hierarchy *hier, int cpu,
+ struct tmigr_group *old_root, bool activate)
+{
+ /*
+ * The target CPU must never do the prepare work, except
+ * on early boot when the boot CPU is the target. Otherwise
+ * it may spuriously activate the old top level group inside
+ * the new one (nevertheless whether old top level group is
+ * active or not) and/or release an uninitialized childmask.
+ */
+ WARN_ON_ONCE(cpu == smp_processor_id());
+ if (activate) {
+ /*
+ * The current CPU is expected to be online in the hierarchy,
+ * otherwise the old root may not be active as expected.
+ */
+ WARN_ON_ONCE(!__this_cpu_read(tmigr_cpu.available));
+ }
+
+ return tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, activate);
+}
+
+static long connect_old_root_work(void *arg)
+{
+ struct tmigr_group *old_root = arg;
+ struct tmigr_hierarchy *hier;
+ int cpu = smp_processor_id();
+
+ hier = __tmigr_get_hierarchy(cpu);
+ if (WARN_ON_ONCE(!hier))
+ return -EINVAL;
+
+ return tmigr_connect_old_root(hier, cpu, old_root, true);
+}
+
static int tmigr_add_cpu(unsigned int cpu)
{
- struct tmigr_group *old_root = tmigr_root;
+ struct tmigr_hierarchy *hier;
+ struct tmigr_group *old_root;
int node = cpu_to_node(cpu);
int ret;
guard(mutex)(&tmigr_mutex);
- ret = tmigr_setup_groups(cpu, node, NULL, false);
+ hier = tmigr_get_hierarchy(cpu);
+ if (IS_ERR(hier))
+ return PTR_ERR(hier);
+
+ old_root = hier->root;
+
+ ret = tmigr_setup_groups(hier, cpu, node, NULL, false);
+
+ if (ret < 0)
+ return ret;
/* Root has changed? Connect the old one to the new */
- if (ret >= 0 && old_root && old_root != tmigr_root) {
- /*
- * The target CPU must never do the prepare work, except
- * on early boot when the boot CPU is the target. Otherwise
- * it may spuriously activate the old top level group inside
- * the new one (nevertheless whether old top level group is
- * active or not) and/or release an uninitialized childmask.
- */
- WARN_ON_ONCE(cpu == raw_smp_processor_id());
- /*
- * The (likely) current CPU is expected to be online in the hierarchy,
- * otherwise the old root may not be active as expected.
- */
- WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available);
- ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true);
+ if (old_root && old_root != hier->root) {
+ guard(migrate)();
+
+ if (cpumask_test_cpu(smp_processor_id(), hier->cpumask)) {
+ /*
+ * If the target belong to the same hierarchy, the old root is expected
+ * to be active. Link and propagate to the new root.
+ */
+ ret = tmigr_connect_old_root(hier, cpu, old_root, true);
+ } else {
+ int target = cpumask_first_and(hier->cpumask, tmigr_available_cpumask);
+
+ if (target < nr_cpu_ids) {
+ /*
+ * If the target doesn't belong to the same hierarchy as the current
+ * CPU, activate from a relevant one to make sure the old root is
+ * active.
+ */
+ ret = work_on_cpu(target, connect_old_root_work, old_root);
+ } else {
+ /*
+ * No other available CPUs in the remote hierarchy. Link the
+ * old root remotely but don't propagate activation since the
+ * old root is not expected to be active.
+ */
+ ret = tmigr_connect_old_root(hier, cpu, old_root, false);
+ }
+ }
}
+ if (ret >= 0)
+ cpumask_set_cpu(cpu, hier->cpumask);
+
return ret;
}
@@ -1952,7 +2107,7 @@ static int tmigr_cpu_prepare(unsigned int cpu)
static int __init tmigr_init(void)
{
- unsigned int cpulvl, nodelvl, cpus_per_node, i;
+ unsigned int cpulvl, nodelvl, cpus_per_node;
unsigned int nnodes = num_possible_nodes();
unsigned int ncpus = num_possible_cpus();
int ret = -ENOMEM;
@@ -1999,14 +2154,6 @@ static int __init tmigr_init(void)
*/
tmigr_crossnode_level = cpulvl;
- tmigr_level_list = kzalloc_objs(struct list_head,
- tmigr_hierarchy_levels);
- if (!tmigr_level_list)
- goto err;
-
- for (i = 0; i < tmigr_hierarchy_levels; i++)
- INIT_LIST_HEAD(&tmigr_level_list[i]);
-
pr_info("Timer migration: %d hierarchy levels; %d children per group;"
" %d crossnode level\n",
tmigr_hierarchy_levels, TMIGR_CHILDREN_PER_GROUP,
diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h
index 70879cd..31735dd 100644
--- a/kernel/time/timer_migration.h
+++ b/kernel/time/timer_migration.h
@@ -6,6 +6,24 @@
#define TMIGR_CHILDREN_PER_GROUP 8
/**
+ * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity.
+ * Homogeneous systems have only one hierarchy.
+ * Heterogenous have one hierarchy per CPU capacity.
+ * @cpumask: CPUs belonging to this hierarchy
+ * @root: The current root of the hierarchy
+ * @capacity: CPU capacity associated to this hierarchy
+ * @node: Node in the global hierarchy list
+ * @level_list: Per level lists of tmigr groups
+ */
+struct tmigr_hierarchy {
+ struct cpumask *cpumask;
+ struct tmigr_group *root;
+ unsigned long capacity;
+ struct list_head node;
+ struct list_head level_list[];
+};
+
+/**
* struct tmigr_event - a timer event associated to a CPU
* @nextevt: The node to enqueue an event in the parent group queue
* @cpu: The CPU to which this event belongs
@@ -75,15 +93,17 @@ struct tmigr_group {
/**
* struct tmigr_cpu - timer migration per CPU group
* @lock: Lock protecting the tmigr_cpu group information
- * @online: Indicates whether the CPU is online; In deactivate path
- * it is required to know whether the migrator in the top
- * level group is to be set offline, while a timer is
- * pending. Then another online CPU needs to be notified to
- * take over the migrator role. Furthermore the information
- * is required in CPU hotplug path as the CPU is able to go
- * idle before the timer migration hierarchy hotplug AP is
- * reached. During this phase, the CPU has to handle the
+ * @available: Indicates whether the CPU is available for handling
+ * global timers. In the deactivate path it is required to
+ * know whether the migrator in the top level group is to
+ * be set offline, while a timer is pending. Then another
+ * available CPU needs to be notified to take over the
+ * migrator role. Furthermore the information is required
+ * in the CPU hotplug path as the CPU is able to go idle
+ * before the timer migration hierarchy hotplug callback is
+ * reached. During this phase, the CPU has to handle the
* global timers on its own and must not act as a migrator.
+
* @idle: Indicates whether the CPU is idle in the timer migration
* hierarchy
* @remote: Is set when timers of the CPU are expired remotely
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 1decdce8..8d3d96e 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -143,8 +143,8 @@
targets += undefsyms_base.o
KASAN_SANITIZE_undefsyms_base.o := y
-UNDEFINED_ALLOWLIST = __asan __gcov __kasan __kcsan __hwasan __sancov __sanitizer __tsan __ubsan __x86_indirect_thunk \
- __msan simple_ring_buffer \
+UNDEFINED_ALLOWLIST = __asan __gcov __kasan __kcsan __hwasan __sancov __sanitizer __tsan __ubsan __msan \
+ __aeabi_unwind_cpp __s390_indirect_jump __x86_indirect_thunk simple_ring_buffer \
$(shell $(NM) -u $(obj)/undefsyms_base.o 2>/dev/null | awk '{print $$2}')
quiet_cmd_check_undefined = NM $<
@@ -154,7 +154,8 @@
echo "Unexpected symbols in $<:" >&2; \
echo "$$undefsyms" >&2; \
false; \
- fi
+ fi; \
+ touch $@
$(obj)/%.o.checked: $(obj)/%.o $(obj)/undefsyms_base.o FORCE
$(call if_changed,check_undefined)
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index af7079a..a02bd25 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -2384,7 +2384,8 @@ static void bpf_kprobe_multi_link_release(struct bpf_link *link)
struct bpf_kprobe_multi_link *kmulti_link;
kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
- unregister_fprobe(&kmulti_link->fp);
+ /* Don't wait for RCU GP here. */
+ unregister_fprobe_async(&kmulti_link->fp);
kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt);
}
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index cc49ebd..f378613 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -1093,14 +1093,15 @@ static int unregister_fprobe_nolock(struct fprobe *fp)
}
/**
- * unregister_fprobe() - Unregister fprobe.
+ * unregister_fprobe_async() - Unregister fprobe without RCU GP wait
* @fp: A fprobe data structure to be unregistered.
*
* Unregister fprobe (and remove ftrace hooks from the function entries).
+ * This function will NOT wait until the fprobe is no longer used.
*
* Return 0 if @fp is unregistered successfully, -errno if not.
*/
-int unregister_fprobe(struct fprobe *fp)
+int unregister_fprobe_async(struct fprobe *fp)
{
guard(mutex)(&fprobe_mutex);
if (!fp || !fprobe_registered(fp))
@@ -1108,6 +1109,24 @@ int unregister_fprobe(struct fprobe *fp)
return unregister_fprobe_nolock(fp);
}
+
+/**
+ * unregister_fprobe() - Unregister fprobe with RCU GP wait
+ * @fp: A fprobe data structure to be unregistered.
+ *
+ * Unregister fprobe (and remove ftrace hooks from the function entries).
+ * This function will block until the fprobe is no longer used.
+ *
+ * Return 0 if @fp is unregistered successfully, -errno if not.
+ */
+int unregister_fprobe(struct fprobe *fp)
+{
+ int ret = unregister_fprobe_async(fp);
+
+ if (!ret)
+ synchronize_rcu();
+ return ret;
+}
EXPORT_SYMBOL_GPL(unregister_fprobe);
static int __init fprobe_initcall(void)
diff --git a/kernel/trace/remote_test.c b/kernel/trace/remote_test.c
index 6c1b770..a3e2c9b 100644
--- a/kernel/trace/remote_test.c
+++ b/kernel/trace/remote_test.c
@@ -110,9 +110,9 @@ static struct trace_buffer_desc *remote_test_load(unsigned long size, void *unus
return remote_test_buffer_desc;
err_unload:
- for_each_ring_buffer_desc(rb_desc, cpu, remote_test_buffer_desc)
+ for_each_ring_buffer_desc(rb_desc, cpu, desc)
remote_test_unload_simple_rb(rb_desc->cpu);
- trace_remote_free_buffer(remote_test_buffer_desc);
+ trace_remote_free_buffer(desc);
err_free_desc:
kfree(desc);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 5326924..ebae64e 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -7,6 +7,7 @@
#include <linux/ring_buffer_types.h>
#include <linux/sched/isolation.h>
#include <linux/trace_recursion.h>
+#include <linux/panic_notifier.h>
#include <linux/trace_events.h>
#include <linux/ring_buffer.h>
#include <linux/trace_clock.h>
@@ -31,6 +32,7 @@
#include <linux/oom.h>
#include <linux/mm.h>
+#include <asm/ring_buffer.h>
#include <asm/local64.h>
#include <asm/local.h>
#include <asm/setup.h>
@@ -559,6 +561,7 @@ struct trace_buffer {
unsigned long range_addr_start;
unsigned long range_addr_end;
+ struct notifier_block flush_nb;
struct ring_buffer_meta *meta;
@@ -2521,6 +2524,16 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
kfree(cpu_buffer);
}
+/* Stop recording on a persistent buffer and flush cache if needed. */
+static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data)
+{
+ struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb);
+
+ ring_buffer_record_off(buffer);
+ arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end);
+ return NOTIFY_DONE;
+}
+
static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
int order, unsigned long start,
unsigned long end,
@@ -2651,6 +2664,12 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags,
mutex_init(&buffer->mutex);
+ /* Persistent ring buffer needs to flush cache before reboot. */
+ if (start && end) {
+ buffer->flush_nb.notifier_call = rb_flush_buffer_cb;
+ atomic_notifier_chain_register(&panic_notifier_list, &buffer->flush_nb);
+ }
+
return_ptr(buffer);
fail_free_buffers:
@@ -2749,6 +2768,9 @@ ring_buffer_free(struct trace_buffer *buffer)
{
int cpu;
+ if (buffer->range_addr_start && buffer->range_addr_end)
+ atomic_notifier_chain_unregister(&panic_notifier_list, &buffer->flush_nb);
+
cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);
irq_work_sync(&buffer->irq_work.work);
@@ -3769,13 +3791,6 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
return skip_time_extend(event);
}
-#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
-static inline bool sched_clock_stable(void)
-{
- return true;
-}
-#endif
-
static void
rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
struct rb_event_info *info)
@@ -5407,6 +5422,7 @@ static void rb_iter_reset(struct ring_buffer_iter *iter)
iter->head_page = cpu_buffer->reader_page;
iter->head = cpu_buffer->reader_page->read;
iter->next_event = iter->head;
+ iter->missed_events = 0;
iter->cache_reader_page = iter->head_page;
iter->cache_read = cpu_buffer->read;
@@ -6086,10 +6102,7 @@ ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts,
*/
bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter)
{
- bool ret = iter->missed_events != 0;
-
- iter->missed_events = 0;
- return ret;
+ return iter->missed_events != 0;
}
EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped);
@@ -6251,7 +6264,7 @@ void ring_buffer_iter_advance(struct ring_buffer_iter *iter)
unsigned long flags;
raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
-
+ iter->missed_events = 0;
rb_advance_iter(iter);
raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c
index 02af229..f4642f5 100644
--- a/kernel/trace/simple_ring_buffer.c
+++ b/kernel/trace/simple_ring_buffer.c
@@ -395,7 +395,6 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer,
memset(cpu_buffer->meta, 0, sizeof(*cpu_buffer->meta));
cpu_buffer->meta->meta_page_size = PAGE_SIZE;
- cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages;
/* The reader page is not part of the ring initially */
page = load_page(desc->page_va[0]);
@@ -431,12 +430,13 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer,
if (ret) {
for (i--; i >= 0; i--)
- unload_page((void *)desc->page_va[i]);
+ unload_page(bpages[i].page);
unload_page(cpu_buffer->meta);
return ret;
}
+ cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages;
/* Close the ring */
bpage->link.next = &cpu_buffer->tail_page->link;
cpu_buffer->tail_page->link.prev = &bpage->link;
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 6809b37..d1564db 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -373,10 +373,10 @@ __init static int init_annotated_branch_stats(void)
int ret;
ret = register_stat_tracer(&annotated_branch_stats);
- if (!ret) {
+ if (ret) {
printk(KERN_WARNING "Warning: could not register "
"annotated branches stats\n");
- return 1;
+ return ret;
}
return 0;
}
@@ -438,10 +438,10 @@ __init static int all_annotated_branch_stats(void)
int ret;
ret = register_stat_tracer(&all_branch_stats);
- if (!ret) {
+ if (ret) {
printk(KERN_WARNING "Warning: could not register "
"all branches stats\n");
- return 1;
+ return ret;
}
return 0;
}
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index 0dbbf6c..eb2c2bc 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -1369,10 +1369,8 @@ static const char *hist_field_name(struct hist_field *field,
len = snprintf(full_name, sizeof(full_name), fmt,
field->system, field->event_name,
field->name);
- if (len >= sizeof(full_name))
- return NULL;
-
- field_name = full_name;
+ if (len < sizeof(full_name))
+ field_name = full_name;
} else
field_name = field->name;
} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index e1c7306..e0d3a0d 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -1523,6 +1523,12 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
parg->offset = *size;
*size += parg->type->size * (parg->count ?: 1);
+ if (*size > MAX_PROBE_EVENT_SIZE) {
+ ret = -E2BIG;
+ trace_probe_log_err(ctx->offset, EVENT_TOO_BIG);
+ goto fail;
+ }
+
if (parg->count) {
len = strlen(parg->type->fmttype) + 6;
parg->fmt = kmalloc(len, GFP_KERNEL);
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 9fc56c9..262d870 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -38,6 +38,7 @@
#define MAX_BTF_ARGS_LEN 128
#define MAX_DENTRY_ARGS_LEN 256
#define MAX_STRING_SIZE PATH_MAX
+#define MAX_PROBE_EVENT_SIZE 3072
/* Reserved field names */
#define FIELD_STRING_IP "__probe_ip"
@@ -561,7 +562,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
C(BAD_TYPE4STR, "This type does not fit for string."),\
C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\
C(TOO_MANY_ARGS, "Too many arguments are specified"), \
- C(TOO_MANY_EARGS, "Too many entry arguments specified"),
+ C(TOO_MANY_EARGS, "Too many entry arguments specified"), \
+ C(EVENT_TOO_BIG, "Event too big (too many fields?)"),
#undef C
#define C(a, b) TP_ERR_##a
diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c
index bf1a507..0dd7927 100644
--- a/kernel/trace/tracing_map.c
+++ b/kernel/trace/tracing_map.c
@@ -386,13 +386,11 @@ static void tracing_map_elt_init_fields(struct tracing_map_elt *elt)
}
}
-static void tracing_map_elt_free(struct tracing_map_elt *elt)
+static void __tracing_map_elt_free(struct tracing_map_elt *elt)
{
if (!elt)
return;
- if (elt->map->ops && elt->map->ops->elt_free)
- elt->map->ops->elt_free(elt);
kfree(elt->fields);
kfree(elt->vars);
kfree(elt->var_set);
@@ -400,6 +398,17 @@ static void tracing_map_elt_free(struct tracing_map_elt *elt)
kfree(elt);
}
+static void tracing_map_elt_free(struct tracing_map_elt *elt)
+{
+ if (!elt)
+ return;
+
+ /* Only objects initialized with alloc_elt() should be passed to free_elt().*/
+ if (elt->map->ops && elt->map->ops->elt_free)
+ elt->map->ops->elt_free(elt);
+ __tracing_map_elt_free(elt);
+}
+
static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map)
{
struct tracing_map_elt *elt;
@@ -444,7 +453,7 @@ static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map)
}
return elt;
free:
- tracing_map_elt_free(elt);
+ __tracing_map_elt_free(elt);
return ERR_PTR(err);
}
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 5f747f2..0c265ea 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -2296,6 +2296,18 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",
work->func, wq->name))) {
+ struct work_offq_data offqd;
+
+ /*
+ * State on entry: PENDING is set, work is off-queue (no
+ * insert_work() has run).
+ *
+ * Returning without clearing PENDING would leave the work
+ * in a weird state (PENDING=1, PWQ=0, entry empty)
+ */
+ work_offqd_unpack(&offqd, *work_data_bits(work));
+ set_work_pool_and_clear_pending(work, offqd.pool_id,
+ work_offqd_pack_flags(&offqd));
return;
}
rcu_read_lock();
@@ -5642,7 +5654,9 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
}
- return ret;
+ if (ret)
+ goto enomem;
+ return 0;
enomem:
if (wq->cpu_pwq) {
@@ -5906,6 +5920,21 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
return NULL;
}
+__printf(1, 0)
+static struct workqueue_struct *alloc_workqueue_va(const char *fmt,
+ unsigned int flags,
+ int max_active,
+ va_list args)
+{
+ struct workqueue_struct *wq;
+
+ wq = __alloc_workqueue(fmt, flags, max_active, args);
+ if (wq)
+ wq_init_lockdep(wq);
+
+ return wq;
+}
+
__printf(1, 4)
struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,
unsigned int flags,
@@ -5915,12 +5944,8 @@ struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,
va_list args;
va_start(args, max_active);
- wq = __alloc_workqueue(fmt, flags, max_active, args);
+ wq = alloc_workqueue_va(fmt, flags, max_active, args);
va_end(args);
- if (!wq)
- return NULL;
-
- wq_init_lockdep(wq);
return wq;
}
@@ -5932,15 +5957,15 @@ static void devm_workqueue_release(void *res)
}
__printf(2, 5) struct workqueue_struct *
-devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags,
- int max_active, ...)
+devm_alloc_workqueue_noprof(struct device *dev, const char *fmt,
+ unsigned int flags, int max_active, ...)
{
struct workqueue_struct *wq;
va_list args;
int ret;
va_start(args, max_active);
- wq = alloc_workqueue(fmt, flags, max_active, args);
+ wq = alloc_workqueue_va(fmt, flags, max_active, args);
va_end(args);
if (!wq)
return NULL;
@@ -5951,7 +5976,7 @@ devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags,
return wq;
}
-EXPORT_SYMBOL_GPL(devm_alloc_workqueue);
+EXPORT_SYMBOL_GPL(devm_alloc_workqueue_noprof);
#ifdef CONFIG_LOCKDEP
__printf(1, 5)
@@ -8187,11 +8212,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1)
static bool __init cpus_share_smt(int cpu0, int cpu1)
{
-#ifdef CONFIG_SCHED_SMT
return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
-#else
- return false;
-#endif
}
static bool __init cpus_share_numa(int cpu0, int cpu1)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 8ff5adc..d9d7556 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -630,7 +630,7 @@
config WARN_CONTEXT_ANALYSIS
bool "Compiler context-analysis warnings"
- depends on CC_IS_CLANG && CLANG_VERSION >= 220100
+ depends on CC_IS_CLANG && CLANG_VERSION >= 230000
# Branch profiling re-defines "if", which messes with the compiler's
# ability to analyze __cond_acquires(..), resulting in false positives.
depends on !TRACE_BRANCH_PROFILING
@@ -641,7 +641,7 @@
and releasing user-definable "context locks".
Clang's name of the feature is "Thread Safety Analysis". Requires
- Clang 22.1.0 or later.
+ Clang 23 or later.
Produces warnings by default. Select CONFIG_WERROR if you wish to
turn these warnings into errors.
@@ -3070,7 +3070,7 @@
config LONGEST_SYM_KUNIT_TEST
tristate "Test the longest symbol possible" if !KUNIT_ALL_TESTS
depends on KUNIT && KPROBES
- depends on !PREFIX_SYMBOLS && !CFI && !GCOV_KERNEL
+ depends on !CALL_PADDING && !CFI && !GCOV_KERNEL
default KUNIT_ALL_TESTS
help
Tests the longest symbol possible
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 12e2e42..b18a682 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -711,6 +711,15 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket
return NULL;
}
+static inline bool debug_objects_is_pi_blocked_on(void)
+{
+#ifdef CONFIG_RT_MUTEXES
+ return current->pi_blocked_on != NULL;
+#else
+ return false;
+#endif
+}
+
static void debug_objects_fill_pool(void)
{
if (!static_branch_likely(&obj_cache_enabled))
@@ -727,11 +736,12 @@ static void debug_objects_fill_pool(void)
/*
* On RT enabled kernels the pool refill must happen in preemptible
- * context -- for !RT kernels we rely on the fact that spinlock_t and
- * raw_spinlock_t are basically the same type and this lock-type
- * inversion works just fine.
+ * context and not enqueued on an rt_mutex -- for !RT kernels we rely
+ * on the fact that spinlock_t and raw_spinlock_t are basically the
+ * same type and this lock-type inversion works just fine.
*/
- if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible() || system_state < SYSTEM_SCHEDULING) {
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT) || system_state < SYSTEM_SCHEDULING ||
+ (preemptible() && !debug_objects_is_pi_blocked_on())) {
/*
* Annotate away the spinlock_t inside raw_spinlock_t warning
* by temporarily raising the wait-type to LD_WAIT_CONFIG, matching
@@ -1212,7 +1222,7 @@ struct self_test {
static __initconst const struct debug_obj_descr descr_type_test;
-static bool __init is_static_object(void *addr)
+static __noipa bool __init is_static_object(void *addr)
{
struct self_test *obj = addr;
diff --git a/lib/fonts/font_rotate.c b/lib/fonts/font_rotate.c
index 065e0fc..27540600 100644
--- a/lib/fonts/font_rotate.c
+++ b/lib/fonts/font_rotate.c
@@ -106,7 +106,7 @@ static void __font_glyph_rotate_180(const unsigned char *glyph,
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) {
if (font_glyph_test_bit(glyph, x, y, bit_pitch)) {
- font_glyph_set_bit(out, width - (1 + x + shift), height - (1 + y),
+ font_glyph_set_bit(out, bit_pitch - 1 - x - shift, height - 1 - y,
bit_pitch);
}
}
diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig
index 498cc51..94ff8e4 100644
--- a/lib/kunit/Kconfig
+++ b/lib/kunit/Kconfig
@@ -16,8 +16,9 @@
if KUNIT
config KUNIT_DEBUGFS
- bool "KUnit - Enable /sys/kernel/debug/kunit debugfs representation" if !KUNIT_ALL_TESTS
- default KUNIT_ALL_TESTS
+ bool "KUnit - Enable /sys/kernel/debug/kunit debugfs representation"
+ depends on DEBUG_FS
+ default y
help
Enable debugfs representation for kunit. Currently this consists
of /sys/kernel/debug/kunit/<test_suite>/results files for each
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index 1fef217..b0f8a41 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -15,6 +15,16 @@ extern struct kunit_suite * const __kunit_suites_end[];
extern struct kunit_suite * const __kunit_init_suites_start[];
extern struct kunit_suite * const __kunit_init_suites_end[];
+static struct kunit_suite_set kunit_boot_suites;
+
+void kunit_free_boot_suites(void)
+{
+ if (kunit_boot_suites.start) {
+ kunit_free_suite_set(kunit_boot_suites);
+ kunit_boot_suites = (struct kunit_suite_set){ NULL, NULL };
+ }
+}
+
static char *action_param;
module_param_named(action, action_param, charp, 0400);
@@ -411,9 +421,12 @@ int kunit_run_all_tests(void)
pr_err("kunit executor: unknown action '%s'\n", action_param);
free_out:
- if (filter_glob_param || filter_param)
- kunit_free_suite_set(suite_set);
- else if (init_num_suites > 0)
+ if (filter_glob_param || filter_param) {
+ if (err)
+ kunit_free_suite_set(suite_set);
+ else
+ kunit_boot_suites = suite_set;
+ } else if (init_num_suites > 0)
/* Don't use kunit_free_suite_set because suites aren't individually allocated */
kfree(suite_set.start);
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 41e1c89..99773e0 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -1075,6 +1075,7 @@ static void __exit kunit_exit(void)
kunit_bus_shutdown();
kunit_debugfs_cleanup();
+ kunit_free_boot_suites();
}
module_exit(kunit_exit);
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index d939403..bfafe12 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -1431,10 +1431,9 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
{
int saved_preempt_count = preempt_count();
#ifdef CONFIG_PREEMPT_RT
-#ifdef CONFIG_SMP
int saved_mgd_count = current->migration_disabled;
-#endif
int saved_rcu_count = current->rcu_read_lock_nesting;
+ int saved_sched_rt_mutex = current->sched_rt_mutex;
#endif
WARN_ON(irqs_disabled());
@@ -1471,10 +1470,10 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
preempt_count_set(saved_preempt_count);
#ifdef CONFIG_PREEMPT_RT
-#ifdef CONFIG_SMP
+ current->sched_rt_mutex = saved_sched_rt_mutex;
+
while (current->migration_disabled > saved_mgd_count)
migrate_enable();
-#endif
while (current->rcu_read_lock_nesting > saved_rcu_count)
rcu_read_unlock();
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index d18d7ed..60ae5e6 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2,7 +2,7 @@
/*
* Maple Tree implementation
* Copyright (c) 2018-2022 Oracle Corporation
- * Authors: Liam R. Howlett <Liam.Howlett@oracle.com>
+ * Authors: Liam R. Howlett <liam@infradead.org>
* Matthew Wilcox <willy@infradead.org>
* Copyright (c) 2023 ByteDance
* Author: Peng Zhang <zhangpeng.00@bytedance.com>
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 6074ed5..04b3a80 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -114,6 +114,14 @@ static void bucket_table_free(const struct bucket_table *tbl)
kvfree(tbl);
}
+static void bucket_table_free_atomic(const struct bucket_table *tbl)
+{
+ if (tbl->nest)
+ nested_bucket_table_free(tbl);
+
+ kvfree_atomic(tbl);
+}
+
static void bucket_table_free_rcu(struct rcu_head *head)
{
bucket_table_free(container_of(head, struct bucket_table, rcu));
@@ -441,10 +449,33 @@ static void rht_deferred_worker(struct work_struct *work)
mutex_unlock(&ht->mutex);
+ /*
+ * Re-arm via @run_work, not @run_irq_work.
+ * rhashtable_free_and_destroy() drains async work as irq_work_sync()
+ * followed by cancel_work_sync(). If this site queued irq_work while
+ * cancel_work_sync() was waiting for us, irq_work_sync() would already
+ * have returned and the stale irq_work could fire post-teardown.
+ * cancel_work_sync() natively handles self-requeue on @run_work.
+ */
if (err)
schedule_work(&ht->run_work);
}
+/*
+ * Insert-path callers can run under a raw spinlock (e.g. an insecure_elasticity
+ * user). Calling schedule_work() under that lock records caller_lock ->
+ * pool->lock -> pi_lock -> rq->__lock, closing a locking cycle if any of
+ * these is acquired in the reverse direction elsewhere. Bounce through
+ * irq_work so the schedule_work() runs with the caller's lock no longer held.
+ */
+static void rht_deferred_irq_work(struct irq_work *irq_work)
+{
+ struct rhashtable *ht = container_of(irq_work, struct rhashtable,
+ run_irq_work);
+
+ schedule_work(&ht->run_work);
+}
+
static int rhashtable_insert_rehash(struct rhashtable *ht,
struct bucket_table *tbl)
{
@@ -473,11 +504,11 @@ static int rhashtable_insert_rehash(struct rhashtable *ht,
err = rhashtable_rehash_attach(ht, tbl, new_tbl);
if (err) {
- bucket_table_free(new_tbl);
+ bucket_table_free_atomic(new_tbl);
if (err == -EEXIST)
err = 0;
} else
- schedule_work(&ht->run_work);
+ irq_work_queue(&ht->run_irq_work);
return err;
@@ -488,7 +519,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht,
/* Schedule async rehash to retry allocation in process context. */
if (err == -ENOMEM)
- schedule_work(&ht->run_work);
+ irq_work_queue(&ht->run_irq_work);
return err;
}
@@ -538,7 +569,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht,
return NULL;
}
- if (elasticity <= 0)
+ if (elasticity <= 0 && !ht->p.insecure_elasticity)
return ERR_PTR(-EAGAIN);
return ERR_PTR(-ENOENT);
@@ -568,7 +599,8 @@ static struct bucket_table *rhashtable_insert_one(
if (unlikely(rht_grow_above_max(ht, tbl)))
return ERR_PTR(-E2BIG);
- if (unlikely(rht_grow_above_100(ht, tbl)))
+ if (unlikely(rht_grow_above_100(ht, tbl)) &&
+ !ht->p.insecure_elasticity)
return ERR_PTR(-EAGAIN);
head = rht_ptr(bkt, tbl, hash);
@@ -629,7 +661,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
rht_unlock(tbl, bkt, flags);
if (inserted && rht_grow_above_75(ht, tbl))
- schedule_work(&ht->run_work);
+ irq_work_queue(&ht->run_irq_work);
}
} while (!IS_ERR_OR_NULL(new_tbl));
@@ -1084,6 +1116,7 @@ int rhashtable_init_noprof(struct rhashtable *ht,
RCU_INIT_POINTER(ht->tbl, tbl);
INIT_WORK(&ht->run_work, rht_deferred_worker);
+ init_irq_work(&ht->run_irq_work, rht_deferred_irq_work);
return 0;
}
@@ -1141,6 +1174,11 @@ static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj,
* This function will eventually sleep to wait for an async resize
* to complete. The caller is responsible that no further write operations
* occurs in parallel.
+ *
+ * After cancel_work_sync() has returned, the deferred rehash worker is
+ * quiesced and, per the contract above, no other concurrent access to the
+ * rhashtable is possible. The tables are therefore owned exclusively by
+ * this function and can be walked without ht->mutex held.
*/
void rhashtable_free_and_destroy(struct rhashtable *ht,
void (*free_fn)(void *ptr, void *arg),
@@ -1149,10 +1187,18 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
struct bucket_table *tbl, *next_tbl;
unsigned int i;
+ irq_work_sync(&ht->run_irq_work);
cancel_work_sync(&ht->run_work);
- mutex_lock(&ht->mutex);
- tbl = rht_dereference(ht->tbl, ht);
+ /*
+ * Do NOT take ht->mutex here. The rehash worker establishes
+ * ht->mutex -> fs_reclaim via GFP_KERNEL bucket allocation under
+ * the mutex; callers on the reclaim path (e.g. simple_xattr_ht_free()
+ * from evict() under the dcache shrinker for shmem/kernfs/pidfs
+ * inodes) would otherwise close a circular dependency
+ * fs_reclaim -> ht->mutex.
+ */
+ tbl = rcu_dereference_raw(ht->tbl);
restart:
if (free_fn) {
for (i = 0; i < tbl->size; i++) {
@@ -1161,22 +1207,21 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
cond_resched();
for (pos = rht_ptr_exclusive(rht_bucket(tbl, i)),
next = !rht_is_a_nulls(pos) ?
- rht_dereference(pos->next, ht) : NULL;
+ rcu_dereference_raw(pos->next) : NULL;
!rht_is_a_nulls(pos);
pos = next,
next = !rht_is_a_nulls(pos) ?
- rht_dereference(pos->next, ht) : NULL)
+ rcu_dereference_raw(pos->next) : NULL)
rhashtable_free_one(ht, pos, free_fn, arg);
}
}
- next_tbl = rht_dereference(tbl->future_tbl, ht);
+ next_tbl = rcu_dereference_raw(tbl->future_tbl);
bucket_table_free(tbl);
if (next_tbl) {
tbl = next_tbl;
goto restart;
}
- mutex_unlock(&ht->mutex);
}
EXPORT_SYMBOL_GPL(rhashtable_free_and_destroy);
diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c
index 06b4a6a..316f4df 100644
--- a/lib/test_context-analysis.c
+++ b/lib/test_context-analysis.c
@@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock,
struct test_mutex_data {
struct mutex mtx;
int counter __guarded_by(&mtx);
+
+ struct mutex mtx2;
+ int anyread __guarded_by(&mtx, &mtx2);
+ int *anyptr __pt_guarded_by(&mtx, &mtx2);
};
static void __used test_mutex_init(struct test_mutex_data *d)
@@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d)
}
}
+static void __used test_mutex_multiguard(struct test_mutex_data *d)
+{
+ mutex_lock(&d->mtx);
+ (void)d->anyread;
+ (void)*d->anyptr;
+ mutex_unlock(&d->mtx);
+
+ mutex_lock(&d->mtx2);
+ (void)d->anyread;
+ (void)*d->anyptr;
+ mutex_unlock(&d->mtx2);
+
+ mutex_lock(&d->mtx);
+ mutex_lock(&d->mtx2);
+ d->anyread++;
+ (*d->anyptr)++;
+ mutex_unlock(&d->mtx2);
+ mutex_unlock(&d->mtx);
+}
+
struct test_seqlock_data {
seqlock_t sl;
int counter __guarded_by(&sl);
diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index 434d8a2..b9367c6 100644
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -2,7 +2,7 @@
/*
* test_maple_tree.c: Test the maple tree API
* Copyright (c) 2018-2022 Oracle Corporation
- * Author: Liam R. Howlett <Liam.Howlett@Oracle.com>
+ * Author: Liam R. Howlett <liam@infradead.org>
*
* Any tests that only require the interface of the tree.
*/
@@ -4021,6 +4021,6 @@ static void __exit maple_tree_harvest(void)
module_init(maple_tree_seed);
module_exit(maple_tree_harvest);
-MODULE_AUTHOR("Liam R. Howlett <Liam.Howlett@Oracle.com>");
+MODULE_AUTHOR("Liam R. Howlett <liam@infradead.org>");
MODULE_DESCRIPTION("maple tree API test module");
MODULE_LICENSE("GPL");
diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c
index 37bd6eb..f02f7b7 100644
--- a/lib/tests/kunit_iov_iter.c
+++ b/lib/tests/kunit_iov_iter.c
@@ -1128,7 +1128,7 @@ static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test)
struct kvec kvec;
size_t bufsize;
- bufsize = 0x100000;
+ bufsize = 0x200000;
iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
kvec.iov_base = data.buffer;
@@ -1146,7 +1146,7 @@ static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test)
struct bio_vec *bvec;
struct iov_iter iter;
- bufsize = 0x100000;
+ bufsize = 0x200000;
iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec),
@@ -1173,7 +1173,7 @@ static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test)
struct iov_iter iter;
size_t bufsize;
- bufsize = 0x100000;
+ bufsize = 0x200000;
iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
folioq = iov_kunit_create_folioq(test);
@@ -1190,7 +1190,7 @@ static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test)
struct iov_iter iter;
size_t bufsize;
- bufsize = 0x100000;
+ bufsize = 0x200000;
iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
xarray = iov_kunit_create_xarray(test);
@@ -1206,7 +1206,7 @@ static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test)
struct iov_iter iter;
size_t bufsize;
- bufsize = 0x100000;
+ bufsize = 0x200000;
iov_kunit_iter_to_sg_init(test, bufsize, true, &data);
iov_iter_ubuf(&iter, READ, data.ubuf, bufsize);
diff --git a/lib/tests/test_kprobes.c b/lib/tests/test_kprobes.c
index b7582010..06e729e 100644
--- a/lib/tests/test_kprobes.c
+++ b/lib/tests/test_kprobes.c
@@ -12,6 +12,12 @@
#define div_factor 3
+#define KP_CLEAR(_kp) \
+do { \
+ (_kp).addr = NULL; \
+ (_kp).flags = 0; \
+} while (0)
+
static u32 rand1, preh_val, posth_val;
static u32 (*target)(u32 value);
static u32 (*recursed_target)(u32 value);
@@ -125,10 +131,6 @@ static void test_kprobes(struct kunit *test)
current_test = test;
- /* addr and flags should be cleard for reusing kprobe. */
- kp.addr = NULL;
- kp.flags = 0;
-
KUNIT_EXPECT_EQ(test, 0, register_kprobes(kps, 2));
preh_val = 0;
posth_val = 0;
@@ -226,9 +228,6 @@ static void test_kretprobes(struct kunit *test)
struct kretprobe *rps[2] = {&rp, &rp2};
current_test = test;
- /* addr and flags should be cleard for reusing kprobe. */
- rp.kp.addr = NULL;
- rp.kp.flags = 0;
KUNIT_EXPECT_EQ(test, 0, register_kretprobes(rps, 2));
krph_val = 0;
@@ -290,8 +289,6 @@ static void test_stacktrace_on_kretprobe(struct kunit *test)
unsigned long myretaddr = (unsigned long)__builtin_return_address(0);
current_test = test;
- rp3.kp.addr = NULL;
- rp3.kp.flags = 0;
/*
* Run the stacktrace_driver() to record correct return address in
@@ -352,8 +349,6 @@ static void test_stacktrace_on_nested_kretprobe(struct kunit *test)
struct kretprobe *rps[2] = {&rp3, &rp4};
current_test = test;
- rp3.kp.addr = NULL;
- rp3.kp.flags = 0;
//KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver());
@@ -367,6 +362,18 @@ static void test_stacktrace_on_nested_kretprobe(struct kunit *test)
static int kprobes_test_init(struct kunit *test)
{
+ KP_CLEAR(kp);
+ KP_CLEAR(kp2);
+ KP_CLEAR(kp_missed);
+#ifdef CONFIG_KRETPROBES
+ KP_CLEAR(rp.kp);
+ KP_CLEAR(rp2.kp);
+#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE
+ KP_CLEAR(rp3.kp);
+ KP_CLEAR(rp4.kp);
+#endif
+#endif
+
target = kprobe_target;
target2 = kprobe_target2;
recursed_target = kprobe_recursed_target;
diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
index cf5d784..17d37b8 100644
--- a/lib/vdso/datastore.c
+++ b/lib/vdso/datastore.c
@@ -11,21 +11,21 @@
static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
#ifdef CONFIG_GENERIC_GETTIMEOFDAY
-struct vdso_time_data *vdso_k_time_data __refdata =
+struct vdso_time_data *vdso_k_time_data __ro_after_init =
(void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
#ifdef CONFIG_VDSO_GETRANDOM
-struct vdso_rng_data *vdso_k_rng_data __refdata =
+struct vdso_rng_data *vdso_k_rng_data __ro_after_init =
(void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
#endif /* CONFIG_VDSO_GETRANDOM */
#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
-struct vdso_arch_data *vdso_k_arch_data __refdata =
+struct vdso_arch_data *vdso_k_arch_data __ro_after_init =
(void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
index a5798bd..e0f289d 100644
--- a/lib/vdso/gettimeofday.c
+++ b/lib/vdso/gettimeofday.c
@@ -126,7 +126,7 @@ bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock
}
static __always_inline
-const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
+const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd)
{
return (void *)vd + PAGE_SIZE;
}
@@ -135,7 +135,7 @@ static __always_inline
bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
clockid_t clk, struct __kernel_timespec *ts)
{
- const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
+ const struct vdso_time_data *vd = vdso_timens_data(vdns);
const struct timens_offset *offs = &vcns->offset[clk];
const struct vdso_clock *vc = vd->clock_data;
u32 seq;
@@ -191,7 +191,7 @@ static __always_inline
bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
clockid_t clk, struct __kernel_timespec *ts)
{
- const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
+ const struct vdso_time_data *vd = vdso_timens_data(vdns);
const struct timens_offset *offs = &vcns->offset[clk];
const struct vdso_clock *vc = vd->clock_data;
const struct vdso_timestamp *vdso_ts;
@@ -248,11 +248,10 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti
vc = &vd->aux_clock_data[idx];
do {
- if (vdso_read_begin_timens(vc, &seq)) {
- vd = __arch_get_vdso_u_timens_data(vd);
+ while (vdso_read_begin_timens(vc, &seq)) {
+ /* Re-read from the real time data page, reload seq by looping */
+ vd = vdso_timens_data(vd);
vc = &vd->aux_clock_data[idx];
- /* Re-read from the real time data page */
- continue;
}
/* Auxclock disabled? */
@@ -361,7 +360,7 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
if (unlikely(tz != NULL)) {
if (vdso_is_timens_clock(vc))
- vd = __arch_get_vdso_u_timens_data(vd);
+ vd = vdso_timens_data(vd);
tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
@@ -384,7 +383,7 @@ __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
__kernel_old_time_t t;
if (vdso_is_timens_clock(vc)) {
- vd = __arch_get_vdso_u_timens_data(vd);
+ vd = vdso_timens_data(vd);
vc = vd->clock_data;
}
@@ -415,7 +414,7 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc
return false;
if (vdso_is_timens_clock(vc))
- vd = __arch_get_vdso_u_timens_data(vd);
+ vd = vdso_timens_data(vd);
/*
* Convert the clockid to a bitmask and use it to check which
diff --git a/mm/cma.c b/mm/cma.c
index c7ca567..a13ce49 100644
--- a/mm/cma.c
+++ b/mm/cma.c
@@ -188,10 +188,13 @@ static void __init cma_activate_area(struct cma *cma)
/* Expose all pages to the buddy, they are useless for CMA. */
if (!test_bit(CMA_RESERVE_PAGES_ON_ERROR, &cma->flags)) {
- for (r = 0; r < allocrange; r++) {
+ for (r = 0; r < cma->nranges; r++) {
+ unsigned long start_pfn;
+
cmr = &cma->ranges[r];
+ start_pfn = r <= allocrange ? early_pfn[r] : cmr->early_pfn;
end_pfn = cmr->base_pfn + cmr->count;
- for (pfn = early_pfn[r]; pfn < end_pfn; pfn++)
+ for (pfn = start_pfn; pfn < end_pfn; pfn++)
free_reserved_page(pfn_to_page(pfn));
}
}
diff --git a/mm/cma_debug.c b/mm/cma_debug.c
index 5ae38f5..523ba4a 100644
--- a/mm/cma_debug.c
+++ b/mm/cma_debug.c
@@ -205,7 +205,8 @@ static int __init cma_debugfs_init(void)
cma_debugfs_root = debugfs_create_dir("cma", NULL);
for (i = 0; i < cma_area_count; i++)
- cma_debugfs_add_one(&cma_areas[i], cma_debugfs_root);
+ if (test_bit(CMA_ACTIVATED, &cma_areas[i].flags))
+ cma_debugfs_add_one(&cma_areas[i], cma_debugfs_root);
return 0;
}
diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c
index 554559d..8494040 100644
--- a/mm/damon/lru_sort.c
+++ b/mm/damon/lru_sort.c
@@ -161,15 +161,6 @@ module_param(monitor_region_end, ulong, 0600);
*/
static unsigned long addr_unit __read_mostly = 1;
-/*
- * PID of the DAMON thread
- *
- * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread.
- * Else, -1.
- */
-static int kdamond_pid __read_mostly = -1;
-module_param(kdamond_pid, int, 0400);
-
static struct damos_stat damon_lru_sort_hot_stat;
DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat,
lru_sort_tried_hot_regions, lru_sorted_hot_regions,
@@ -386,12 +377,8 @@ static int damon_lru_sort_turn(bool on)
{
int err;
- if (!on) {
- err = damon_stop(&ctx, 1);
- if (!err)
- kdamond_pid = -1;
- return err;
- }
+ if (!on)
+ return damon_stop(&ctx, 1);
err = damon_lru_sort_apply_parameters();
if (err)
@@ -400,9 +387,6 @@ static int damon_lru_sort_turn(bool on)
err = damon_start(&ctx, 1, true);
if (err)
return err;
- kdamond_pid = damon_kdamond_pid(ctx);
- if (kdamond_pid < 0)
- return kdamond_pid;
return damon_call(ctx, &call_control);
}
@@ -430,42 +414,83 @@ module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600);
MODULE_PARM_DESC(addr_unit,
"Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)");
+static bool damon_lru_sort_enabled(void)
+{
+ if (!ctx)
+ return false;
+ return damon_is_running(ctx);
+}
+
static int damon_lru_sort_enabled_store(const char *val,
const struct kernel_param *kp)
{
- bool is_enabled = enabled;
- bool enable;
int err;
- err = kstrtobool(val, &enable);
+ err = kstrtobool(val, &enabled);
if (err)
return err;
- if (is_enabled == enable)
+ if (damon_lru_sort_enabled() == enabled)
return 0;
/* Called before init function. The function will handle this. */
if (!damon_initialized())
- goto set_param_out;
+ return 0;
- err = damon_lru_sort_turn(enable);
- if (err)
- return err;
+ return damon_lru_sort_turn(enabled);
+}
-set_param_out:
- enabled = enable;
- return err;
+static int damon_lru_sort_enabled_load(char *buffer,
+ const struct kernel_param *kp)
+{
+ return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N');
}
static const struct kernel_param_ops enabled_param_ops = {
.set = damon_lru_sort_enabled_store,
- .get = param_get_bool,
+ .get = damon_lru_sort_enabled_load,
};
module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
MODULE_PARM_DESC(enabled,
"Enable or disable DAMON_LRU_SORT (default: disabled)");
+static int damon_lru_sort_kdamond_pid_store(const char *val,
+ const struct kernel_param *kp)
+{
+ /*
+ * kdamond_pid is read-only, but kernel command line could write it.
+ * Do nothing here.
+ */
+ return 0;
+}
+
+static int damon_lru_sort_kdamond_pid_load(char *buffer,
+ const struct kernel_param *kp)
+{
+ int kdamond_pid = -1;
+
+ if (ctx) {
+ kdamond_pid = damon_kdamond_pid(ctx);
+ if (kdamond_pid < 0)
+ kdamond_pid = -1;
+ }
+ return sprintf(buffer, "%d\n", kdamond_pid);
+}
+
+static const struct kernel_param_ops kdamond_pid_param_ops = {
+ .set = damon_lru_sort_kdamond_pid_store,
+ .get = damon_lru_sort_kdamond_pid_load,
+};
+
+/*
+ * PID of the DAMON thread
+ *
+ * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread.
+ * Else, -1.
+ */
+module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400);
+
static int __init damon_lru_sort_init(void)
{
int err;
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index 8c6d613..c3e4c87 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -32,9 +32,9 @@ struct folio *damon_get_folio(unsigned long pfn)
return NULL;
folio = page_folio(page);
- if (!folio_test_lru(folio) || !folio_try_get(folio))
+ if (!folio_try_get(folio))
return NULL;
- if (unlikely(page_folio(page) != folio || !folio_test_lru(folio))) {
+ if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) {
folio_put(folio);
folio = NULL;
}
diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c
index 86da147..fe7fce2 100644
--- a/mm/damon/reclaim.c
+++ b/mm/damon/reclaim.c
@@ -144,15 +144,6 @@ static unsigned long addr_unit __read_mostly = 1;
static bool skip_anon __read_mostly;
module_param(skip_anon, bool, 0600);
-/*
- * PID of the DAMON thread
- *
- * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
- * Else, -1.
- */
-static int kdamond_pid __read_mostly = -1;
-module_param(kdamond_pid, int, 0400);
-
static struct damos_stat damon_reclaim_stat;
DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
reclaim_tried_regions, reclaimed_regions, quota_exceeds);
@@ -288,12 +279,8 @@ static int damon_reclaim_turn(bool on)
{
int err;
- if (!on) {
- err = damon_stop(&ctx, 1);
- if (!err)
- kdamond_pid = -1;
- return err;
- }
+ if (!on)
+ return damon_stop(&ctx, 1);
err = damon_reclaim_apply_parameters();
if (err)
@@ -302,9 +289,6 @@ static int damon_reclaim_turn(bool on)
err = damon_start(&ctx, 1, true);
if (err)
return err;
- kdamond_pid = damon_kdamond_pid(ctx);
- if (kdamond_pid < 0)
- return kdamond_pid;
return damon_call(ctx, &call_control);
}
@@ -332,42 +316,83 @@ module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600);
MODULE_PARM_DESC(addr_unit,
"Scale factor for DAMON_RECLAIM to ops address conversion (default: 1)");
+static bool damon_reclaim_enabled(void)
+{
+ if (!ctx)
+ return false;
+ return damon_is_running(ctx);
+}
+
static int damon_reclaim_enabled_store(const char *val,
const struct kernel_param *kp)
{
- bool is_enabled = enabled;
- bool enable;
int err;
- err = kstrtobool(val, &enable);
+ err = kstrtobool(val, &enabled);
if (err)
return err;
- if (is_enabled == enable)
+ if (damon_reclaim_enabled() == enabled)
return 0;
/* Called before init function. The function will handle this. */
if (!damon_initialized())
- goto set_param_out;
+ return 0;
- err = damon_reclaim_turn(enable);
- if (err)
- return err;
+ return damon_reclaim_turn(enabled);
+}
-set_param_out:
- enabled = enable;
- return err;
+static int damon_reclaim_enabled_load(char *buffer,
+ const struct kernel_param *kp)
+{
+ return sprintf(buffer, "%c\n", damon_reclaim_enabled() ? 'Y' : 'N');
}
static const struct kernel_param_ops enabled_param_ops = {
.set = damon_reclaim_enabled_store,
- .get = param_get_bool,
+ .get = damon_reclaim_enabled_load,
};
module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
MODULE_PARM_DESC(enabled,
"Enable or disable DAMON_RECLAIM (default: disabled)");
+static int damon_reclaim_kdamond_pid_store(const char *val,
+ const struct kernel_param *kp)
+{
+ /*
+ * kdamond_pid is read-only, but kernel command line could write it.
+ * Do nothing here.
+ */
+ return 0;
+}
+
+static int damon_reclaim_kdamond_pid_load(char *buffer,
+ const struct kernel_param *kp)
+{
+ int kdamond_pid = -1;
+
+ if (ctx) {
+ kdamond_pid = damon_kdamond_pid(ctx);
+ if (kdamond_pid < 0)
+ kdamond_pid = -1;
+ }
+ return sprintf(buffer, "%d\n", kdamond_pid);
+}
+
+static const struct kernel_param_ops kdamond_pid_param_ops = {
+ .set = damon_reclaim_kdamond_pid_store,
+ .get = damon_reclaim_kdamond_pid_load,
+};
+
+/*
+ * PID of the DAMON thread
+ *
+ * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
+ * Else, -1.
+ */
+module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400);
+
static int __init damon_reclaim_init(void)
{
int err;
diff --git a/mm/damon/stat.c b/mm/damon/stat.c
index 99ba346..3951b76 100644
--- a/mm/damon/stat.c
+++ b/mm/damon/stat.c
@@ -19,14 +19,17 @@
static int damon_stat_enabled_store(
const char *val, const struct kernel_param *kp);
+static int damon_stat_enabled_load(char *buffer,
+ const struct kernel_param *kp);
+
static const struct kernel_param_ops enabled_param_ops = {
.set = damon_stat_enabled_store,
- .get = param_get_bool,
+ .get = damon_stat_enabled_load,
};
static bool enabled __read_mostly = IS_ENABLED(
CONFIG_DAMON_STAT_ENABLED_DEFAULT);
-module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
+module_param_cb(enabled, &enabled_param_ops, NULL, 0600);
MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT");
static unsigned long estimated_memory_bandwidth __read_mostly;
@@ -273,17 +276,23 @@ static void damon_stat_stop(void)
damon_stat_context = NULL;
}
+static bool damon_stat_enabled(void)
+{
+ if (!damon_stat_context)
+ return false;
+ return damon_is_running(damon_stat_context);
+}
+
static int damon_stat_enabled_store(
const char *val, const struct kernel_param *kp)
{
- bool is_enabled = enabled;
int err;
err = kstrtobool(val, &enabled);
if (err)
return err;
- if (is_enabled == enabled)
+ if (damon_stat_enabled() == enabled)
return 0;
if (!damon_initialized())
@@ -293,16 +302,17 @@ static int damon_stat_enabled_store(
*/
return 0;
- if (enabled) {
- err = damon_stat_start();
- if (err)
- enabled = false;
- return err;
- }
+ if (enabled)
+ return damon_stat_start();
damon_stat_stop();
return 0;
}
+static int damon_stat_enabled_load(char *buffer, const struct kernel_param *kp)
+{
+ return sprintf(buffer, "%c\n", damon_stat_enabled() ? 'Y' : 'N');
+}
+
static int __init damon_stat_init(void)
{
int err = 0;
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 5186966..a801478 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -88,7 +88,6 @@ static void damon_sysfs_scheme_region_release(struct kobject *kobj)
struct damon_sysfs_scheme_region *region = container_of(kobj,
struct damon_sysfs_scheme_region, kobj);
- list_del(®ion->list);
kfree(region);
}
@@ -164,7 +163,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
struct damon_sysfs_scheme_region *r, *next;
list_for_each_entry_safe(r, next, ®ions->regions_list, list) {
- /* release function deletes it from the list */
+ list_del(&r->list);
kobject_put(&r->kobj);
regions->nr_regions--;
}
@@ -533,9 +532,14 @@ static ssize_t memcg_path_show(struct kobject *kobj,
{
struct damon_sysfs_scheme_filter *filter = container_of(kobj,
struct damon_sysfs_scheme_filter, kobj);
+ int len;
- return sysfs_emit(buf, "%s\n",
+ if (!mutex_trylock(&damon_sysfs_lock))
+ return -EBUSY;
+ len = sysfs_emit(buf, "%s\n",
filter->memcg_path ? filter->memcg_path : "");
+ mutex_unlock(&damon_sysfs_lock);
+ return len;
}
static ssize_t memcg_path_store(struct kobject *kobj,
@@ -550,8 +554,13 @@ static ssize_t memcg_path_store(struct kobject *kobj,
return -ENOMEM;
strscpy(path, buf, count + 1);
+ if (!mutex_trylock(&damon_sysfs_lock)) {
+ kfree(path);
+ return -EBUSY;
+ }
kfree(filter->memcg_path);
filter->memcg_path = path;
+ mutex_unlock(&damon_sysfs_lock);
return count;
}
@@ -1187,8 +1196,13 @@ static ssize_t path_show(struct kobject *kobj,
{
struct damos_sysfs_quota_goal *goal = container_of(kobj,
struct damos_sysfs_quota_goal, kobj);
+ int len;
- return sysfs_emit(buf, "%s\n", goal->path ? goal->path : "");
+ if (!mutex_trylock(&damon_sysfs_lock))
+ return -EBUSY;
+ len = sysfs_emit(buf, "%s\n", goal->path ? goal->path : "");
+ mutex_unlock(&damon_sysfs_lock);
+ return len;
}
static ssize_t path_store(struct kobject *kobj,
@@ -1203,8 +1217,13 @@ static ssize_t path_store(struct kobject *kobj,
return -ENOMEM;
strscpy(path, buf, count + 1);
+ if (!mutex_trylock(&damon_sysfs_lock)) {
+ kfree(path);
+ return -EBUSY;
+ }
kfree(goal->path);
goal->path = path;
+ mutex_unlock(&damon_sysfs_lock);
return count;
}
@@ -2574,6 +2593,7 @@ static int damon_sysfs_memcg_path_to_id(char *memcg_path, u64 *id)
if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
*id = mem_cgroup_id(memcg);
found = true;
+ mem_cgroup_iter_break(NULL, memcg);
break;
}
}
@@ -2907,14 +2927,15 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes,
if (!region)
return;
region->sz_filter_passed = sz_filter_passed;
- list_add_tail(®ion->list, &sysfs_regions->regions_list);
- sysfs_regions->nr_regions++;
if (kobject_init_and_add(®ion->kobj,
&damon_sysfs_scheme_region_ktype,
&sysfs_regions->kobj, "%d",
sysfs_regions->nr_regions++)) {
kobject_put(®ion->kobj);
+ return;
}
+ list_add_tail(®ion->list, &sysfs_regions->regions_list);
+ sysfs_regions->nr_regions++;
}
int damon_sysfs_schemes_clear_regions(
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 970e077..653f2dc 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3015,9 +3015,9 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
if (!folio_test_referenced(folio) && pud_young(old_pud))
folio_set_referenced(folio);
folio_remove_rmap_pud(folio, page, vma);
- folio_put(folio);
add_mm_counter(vma->vm_mm, mm_counter_file(folio),
-HPAGE_PUD_NR);
+ folio_put(folio);
}
void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
@@ -3133,7 +3133,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
if (!folio_test_referenced(folio) && pmd_young(old_pmd))
folio_set_referenced(folio);
folio_remove_rmap_pmd(folio, page, vma);
+ add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
folio_put(folio);
+ return;
}
add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
return;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index f24bf49b..c921287 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -116,7 +116,11 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init;
/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+ struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+ bool check_locks);
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
unsigned long start, unsigned long end, bool take_locks);
static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
@@ -413,21 +417,17 @@ static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
}
}
-/*
- * vma specific semaphore used for pmd sharing and fault/truncation
- * synchronization
- */
-int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
{
struct hugetlb_vma_lock *vma_lock;
/* Only establish in (flags) sharable vmas */
if (!vma || !(vma->vm_flags & VM_MAYSHARE))
- return 0;
+ return;
/* Should never get here with non-NULL vm_private_data */
if (vma->vm_private_data)
- return -EINVAL;
+ return;
vma_lock = kmalloc_obj(*vma_lock);
if (!vma_lock) {
@@ -442,15 +442,13 @@ int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
* allocation failure.
*/
pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
- return -EINVAL;
+ return;
}
kref_init(&vma_lock->refs);
init_rwsem(&vma_lock->rw_sema);
vma_lock->vma = vma;
vma->vm_private_data = vma_lock;
-
- return 0;
}
/* Helper that removes a struct file_region from the resv_map cache and returns
@@ -1147,30 +1145,22 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
}
}
+static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
+{
+ VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
+ VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
+
+ set_vma_private_data(vma, (unsigned long)map);
+}
+
static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
{
VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
- VM_WARN_ON_ONCE_VMA(vma->vm_flags & VM_MAYSHARE, vma);
+ VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
set_vma_private_data(vma, get_vma_private_data(vma) | flags);
}
-static void set_vma_desc_resv_map(struct vm_area_desc *desc, struct resv_map *map)
-{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
- VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
-
- desc->private_data = map;
-}
-
-static void set_vma_desc_resv_flags(struct vm_area_desc *desc, unsigned long flags)
-{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
- VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
-
- desc->private_data = (void *)((unsigned long)desc->private_data | flags);
-}
-
static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
{
VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
@@ -1178,13 +1168,6 @@ static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
return (get_vma_private_data(vma) & flag) != 0;
}
-static bool is_vma_desc_resv_set(struct vm_area_desc *desc, unsigned long flag)
-{
- VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
-
- return ((unsigned long)desc->private_data) & flag;
-}
-
bool __vma_private_lock(struct vm_area_struct *vma)
{
return !(vma->vm_flags & VM_MAYSHARE) &&
@@ -4994,6 +4977,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
addr, dst_vma);
folio_put(pte_folio);
if (ret) {
+ restore_reserve_on_error(h, dst_vma, addr, new_folio);
folio_put(new_folio);
break;
}
@@ -6290,6 +6274,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
folio_put(*foliop);
*foliop = NULL;
if (ret) {
+ restore_reserve_on_error(h, dst_vma, dst_addr, folio);
folio_put(folio);
goto out;
}
@@ -6553,7 +6538,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
long hugetlb_reserve_pages(struct inode *inode,
long from, long to,
- struct vm_area_desc *desc,
+ struct vm_area_struct *vma,
vma_flags_t vma_flags)
{
long chg = -1, add = -1, spool_resv, gbl_resv;
@@ -6571,6 +6556,12 @@ long hugetlb_reserve_pages(struct inode *inode,
}
/*
+ * vma specific semaphore used for pmd sharing and fault/truncation
+ * synchronization
+ */
+ hugetlb_vma_lock_alloc(vma);
+
+ /*
* Only apply hugepage reservation if asked. At fault time, an
* attempt will be made for VM_NORESERVE to allocate a page
* without using reserves
@@ -6582,9 +6573,9 @@ long hugetlb_reserve_pages(struct inode *inode,
* Shared mappings base their reservation on the number of pages that
* are already allocated on behalf of the file. Private mappings need
* to reserve the full area even if read-only as mprotect() may be
- * called to make the mapping read-write. Assume !desc is a shm mapping
+ * called to make the mapping read-write. Assume !vma is a shm mapping
*/
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
/*
* resv_map can not be NULL as hugetlb_reserve_pages is only
* called for inodes for which resv_maps were created (see
@@ -6603,8 +6594,8 @@ long hugetlb_reserve_pages(struct inode *inode,
chg = to - from;
- set_vma_desc_resv_map(desc, resv_map);
- set_vma_desc_resv_flags(desc, HPAGE_RESV_OWNER);
+ set_vma_resv_map(vma, resv_map);
+ set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
}
if (chg < 0) {
@@ -6618,7 +6609,7 @@ long hugetlb_reserve_pages(struct inode *inode,
if (err < 0)
goto out_err;
- if (desc && !vma_desc_test(desc, VMA_MAYSHARE_BIT) && h_cg) {
+ if (vma && !vma_test(vma, VMA_MAYSHARE_BIT) && h_cg) {
/* For private mappings, the hugetlb_cgroup uncharge info hangs
* of the resv_map.
*/
@@ -6655,7 +6646,7 @@ long hugetlb_reserve_pages(struct inode *inode,
* consumed reservations are stored in the map. Hence, nothing
* else has to be done for private mappings here
*/
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
add = region_add(resv_map, from, to, regions_needed, h, h_cg);
if (unlikely(add < 0)) {
@@ -6719,15 +6710,16 @@ long hugetlb_reserve_pages(struct inode *inode,
hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
chg * pages_per_huge_page(h), h_cg);
out_err:
- if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT))
+ hugetlb_vma_lock_free(vma);
+ if (!vma || vma_test(vma, VMA_MAYSHARE_BIT))
/* Only call region_abort if the region_chg succeeded but the
* region_add failed or didn't run.
*/
if (chg >= 0 && add < 0)
region_abort(resv_map, from, to, regions_needed);
- if (desc && is_vma_desc_resv_set(desc, HPAGE_RESV_OWNER)) {
+ if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
kref_put(&resv_map->refs, resv_map_release);
- set_vma_desc_resv_map(desc, NULL);
+ set_vma_resv_map(vma, NULL);
}
return err;
}
@@ -6904,6 +6896,31 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
return pte;
}
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+ struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+ bool check_locks)
+{
+ unsigned long sz = huge_page_size(hstate_vma(vma));
+ struct mm_struct *mm = vma->vm_mm;
+ pgd_t *pgd = pgd_offset(mm, addr);
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
+
+ if (sz != PMD_SIZE)
+ return 0;
+ if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
+ return 0;
+ i_mmap_assert_write_locked(vma->vm_file->f_mapping);
+ if (check_locks)
+ hugetlb_vma_assert_locked(vma);
+ pud_clear(pud);
+
+ tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
+
+ mm_dec_nr_pmds(mm);
+ return 1;
+}
+
/**
* huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users
* @tlb: the current mmu_gather.
@@ -6923,24 +6940,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
- unsigned long sz = huge_page_size(hstate_vma(vma));
- struct mm_struct *mm = vma->vm_mm;
- pgd_t *pgd = pgd_offset(mm, addr);
- p4d_t *p4d = p4d_offset(pgd, addr);
- pud_t *pud = pud_offset(p4d, addr);
-
- if (sz != PMD_SIZE)
- return 0;
- if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
- return 0;
- i_mmap_assert_write_locked(vma->vm_file->f_mapping);
- hugetlb_vma_assert_locked(vma);
- pud_clear(pud);
-
- tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
-
- mm_dec_nr_pmds(mm);
- return 1;
+ return __huge_pmd_unshare(tlb, vma, addr, ptep, /*check_locks=*/true);
}
/*
@@ -6974,6 +6974,13 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
return NULL;
}
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+ struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+ bool check_locks)
+{
+ return 0;
+}
+
int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
@@ -7154,17 +7161,6 @@ int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison
return ret;
}
-int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
- bool *migratable_cleared)
-{
- int ret;
-
- spin_lock_irq(&hugetlb_lock);
- ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
- spin_unlock_irq(&hugetlb_lock);
- return ret;
-}
-
/**
* folio_putback_hugetlb - unisolate a hugetlb folio
* @folio: the isolated hugetlb folio
@@ -7282,7 +7278,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
if (!ptep)
continue;
ptl = huge_pte_lock(h, mm, ptep);
- huge_pmd_unshare(&tlb, vma, address, ptep);
+ __huge_pmd_unshare(&tlb, vma, address, ptep, take_locks);
spin_unlock(ptl);
}
huge_pmd_unshare_flush(&tlb, vma);
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index f83ae49..7693cce 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -204,6 +204,7 @@ void __init hugetlb_cma_reserve(void)
*/
per_node = DIV_ROUND_UP(hugetlb_cma_size,
nodes_weight(hugetlb_bootmem_nodes));
+ per_node = round_up(per_node, PAGE_SIZE << order);
pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
}
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4a077d2..133b46d 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -207,6 +207,8 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
/* Remapping the head page requires r/w */
if (unlikely(walk->nr_walked == 0 && walk->vmemmap_head)) {
+ VM_WARN_ON_ONCE(!PageHead((const struct page *)addr));
+
list_del(&walk->vmemmap_head->lru);
/*
@@ -218,6 +220,8 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
entry = mk_pte(walk->vmemmap_head, PAGE_KERNEL);
} else {
+ VM_WARN_ON_ONCE(!PageTail((const struct page *)addr));
+
/*
* Remap the tail pages as read-only to catch illegal write
* operation to the tail pages.
@@ -232,33 +236,28 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk)
{
- struct page *page;
- struct page *from, *to;
-
- page = list_first_entry(walk->vmemmap_pages, struct page, lru);
- list_del(&page->lru);
+ struct page *src = pte_page(ptep_get(pte)), *dst;
/*
- * Initialize tail pages in the newly allocated vmemmap page.
- *
- * There is folio-scope metadata that is encoded in the first few
- * tail pages.
- *
- * Use the value last tail page in the page with the head page
- * to initialize the rest of tail pages.
+ * When rolling back vmemmap_remap_free(), keep the copied head page
+ * mapping and restore only PTEs currently pointing at the shared tail
+ * page.
*/
- from = compound_head((struct page *)addr) +
- PAGE_SIZE / sizeof(struct page) - 1;
- to = page_to_virt(page);
- for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++, to++)
- *to = *from;
+ if (walk->vmemmap_tail && walk->vmemmap_tail != src)
+ return;
+
+ VM_WARN_ON_ONCE(PageHead((const struct page *)addr));
+
+ dst = list_first_entry(walk->vmemmap_pages, struct page, lru);
+ list_del(&dst->lru);
+ copy_page(page_to_virt(dst), page_to_virt(src));
/*
* Makes sure that preceding stores to the page contents become visible
* before the set_pte_at() write.
*/
smp_wmb();
- set_pte_at(&init_mm, addr, pte, mk_pte(page, PAGE_KERNEL));
+ set_pte_at(&init_mm, addr, pte, mk_pte(dst, PAGE_KERNEL));
}
/**
@@ -324,6 +323,7 @@ static int vmemmap_remap_free(unsigned long start, unsigned long end,
*/
walk = (struct vmemmap_remap_walk) {
.remap_pte = vmemmap_restore_pte,
+ .vmemmap_tail = vmemmap_tail,
.vmemmap_pages = vmemmap_pages,
.flags = 0,
};
diff --git a/mm/memblock.c b/mm/memblock.c
index a6a1c91..ccd43f3 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -989,13 +989,15 @@ void __init_memblock memblock_free(void *ptr, size_t size)
int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size)
{
phys_addr_t end = base + size - 1;
- int ret;
+ int ret = 0;
memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
&base, &end, (void *)_RET_IP_);
kmemleak_free_part_phys(base, size);
- ret = memblock_remove_range(&memblock.reserved, base, size);
+
+ if (!slab_is_available() || IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
+ ret = memblock_remove_range(&memblock.reserved, base, size);
if (slab_is_available())
__free_reserved_area(base, base + size, -1);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c3d98ab..1a4fd25 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -679,7 +679,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, long val,
if (!val)
return;
- css_rstat_updated(&memcg->css, cpu);
+ __css_rstat_updated(&memcg->css, cpu);
statc_pcpu = memcg->vmstats_percpu;
for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
statc = this_cpu_ptr(statc_pcpu);
@@ -805,12 +805,17 @@ static long memcg_state_val_in_pages(int idx, long val)
* Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with
* reparenting of non-hierarchical state_locals.
*/
-static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg)
+static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
+ bool *rcu_locked)
{
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ /* Rebinding can cause this value to be changed at runtime */
+ if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
+ *rcu_locked = false;
return memcg;
+ }
rcu_read_lock();
+ *rcu_locked = true;
while (memcg_is_dying(memcg))
memcg = parent_mem_cgroup(memcg);
@@ -818,20 +823,21 @@ static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *me
return memcg;
}
-static inline void get_non_dying_memcg_end(void)
+static inline void get_non_dying_memcg_end(bool rcu_locked)
{
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ if (!rcu_locked)
return;
rcu_read_unlock();
}
#else
-static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg)
+static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
+ bool *rcu_locked)
{
return memcg;
}
-static inline void get_non_dying_memcg_end(void)
+static inline void get_non_dying_memcg_end(bool rcu_locked)
{
}
#endif
@@ -865,12 +871,14 @@ static void __mod_memcg_state(struct mem_cgroup *memcg,
void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
int val)
{
+ bool rcu_locked = false;
+
if (mem_cgroup_disabled())
return;
- memcg = get_non_dying_memcg_start(memcg);
+ memcg = get_non_dying_memcg_start(memcg, &rcu_locked);
__mod_memcg_state(memcg, idx, val);
- get_non_dying_memcg_end();
+ get_non_dying_memcg_end(rcu_locked);
}
#ifdef CONFIG_MEMCG_V1
@@ -933,14 +941,15 @@ static void mod_memcg_lruvec_state(struct lruvec *lruvec,
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup_per_node *pn;
struct mem_cgroup *memcg;
+ bool rcu_locked = false;
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
- memcg = get_non_dying_memcg_start(pn->memcg);
+ memcg = get_non_dying_memcg_start(pn->memcg, &rcu_locked);
pn = memcg->nodeinfo[pgdat->node_id];
__mod_memcg_lruvec_state(pn, idx, val);
- get_non_dying_memcg_end();
+ get_non_dying_memcg_end(rcu_locked);
}
/**
@@ -2002,6 +2011,7 @@ struct memcg_stock_pcp {
struct work_struct work;
unsigned long flags;
+ uint8_t drain_idx;
};
static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pcp, memcg_stock) = {
@@ -2185,7 +2195,9 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
if (!success) {
i = empty_slot;
if (i == -1) {
- i = get_random_u32_below(NR_MEMCG_STOCK);
+ i = stock->drain_idx++;
+ if (stock->drain_idx == NR_MEMCG_STOCK)
+ stock->drain_idx = 0;
drain_stock(stock, i);
}
css_get(&memcg->css);
@@ -2787,7 +2799,7 @@ static inline void account_slab_nmi_safe(struct mem_cgroup *memcg,
struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
/* preemption is disabled in_nmi(). */
- css_rstat_updated(&memcg->css, smp_processor_id());
+ __css_rstat_updated(&memcg->css, smp_processor_id());
if (idx == NR_SLAB_RECLAIMABLE_B)
atomic_add(nr, &pn->slab_reclaimable);
else
@@ -3010,7 +3022,7 @@ static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val)
mod_memcg_state(memcg, MEMCG_KMEM, val);
} else {
/* preemption is disabled in_nmi(). */
- css_rstat_updated(&memcg->css, smp_processor_id());
+ __css_rstat_updated(&memcg->css, smp_processor_id());
atomic_add(val, &memcg->kmem_stat);
}
}
@@ -4343,6 +4355,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
lstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ memcg->vmstats->state[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
if (atomic_read(&pn->slab_unreclaimable)) {
int slab = atomic_xchg(&pn->slab_unreclaimable, 0);
@@ -4351,6 +4366,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
lstats->state[index] += slab;
if (plstats)
plstats->state_pending[index] += slab;
+ memcg->vmstats->state[index] += slab;
+ if (parent)
+ parent->vmstats->state_pending[index] += slab;
}
}
}
diff --git a/mm/memfd.c b/mm/memfd.c
index fb425f4..abe13b2 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -283,6 +283,12 @@ int memfd_add_seals(struct file *file, unsigned int seals)
goto unlock;
}
+ /*
+ * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
+ */
+ if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
+ seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
+
if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
error = mapping_deny_writable(file->f_mapping);
if (error)
@@ -295,12 +301,6 @@ int memfd_add_seals(struct file *file, unsigned int seals)
}
}
- /*
- * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
- */
- if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
- seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
-
*file_seals |= seals;
error = 0;
diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c
index b02b503..59de210 100644
--- a/mm/memfd_luo.c
+++ b/mm/memfd_luo.c
@@ -50,6 +50,11 @@
* memfds are always opened with ``O_RDWR`` and ``O_LARGEFILE``. This property
* is maintained.
*
+ * Seals
+ * File seals set on the memfd are preserved and re-applied on restore.
+ * Only seals known to this LUO version (see ``MEMFD_LUO_ALL_SEALS``) may
+ * be present; preservation fails with ``-EOPNOTSUPP`` otherwise.
+ *
* Non-Preserved Properties
* ========================
*
@@ -61,10 +66,6 @@
* A memfd can be created with the ``MFD_CLOEXEC`` flag that sets the
* ``FD_CLOEXEC`` on the file. This flag is not preserved and must be set
* again after restore via ``fcntl()``.
- *
- * Seals
- * File seals are not preserved. The file is unsealed on restore and if
- * needed, must be sealed again via ``fcntl()``.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -259,7 +260,7 @@ static int memfd_luo_preserve(struct liveupdate_file_op_args *args)
struct inode *inode = file_inode(args->file);
struct memfd_luo_folio_ser *folios_ser;
struct memfd_luo_ser *ser;
- u64 nr_folios;
+ u64 nr_folios, inode_size;
int err = 0, seals;
inode_lock(inode);
@@ -285,7 +286,18 @@ static int memfd_luo_preserve(struct liveupdate_file_op_args *args)
}
ser->pos = args->file->f_pos;
- ser->size = i_size_read(inode);
+ inode_size = i_size_read(inode);
+
+ /*
+ * memfd_pin_folios() caps at UINT_MAX folios; refuse larger
+ * files to avoid silently preserving only a prefix.
+ */
+ if (DIV_ROUND_UP_ULL(inode_size, PAGE_SIZE) > UINT_MAX) {
+ err = -EFBIG;
+ goto err_free_ser;
+ }
+
+ ser->size = inode_size;
ser->seals = seals;
err = memfd_luo_preserve_folios(args->file, &ser->folios,
@@ -427,6 +439,7 @@ static int memfd_luo_retrieve_folios(struct file *file,
if (!folio) {
pr_err("Unable to restore folio at physical address: %llx\n",
phys);
+ err = -EIO;
goto put_folios;
}
index = pfolio->index;
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index ee42d43..d47aef2 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -1966,20 +1966,19 @@ void folio_clear_hugetlb_hwpoison(struct folio *folio)
folio_free_raw_hwp(folio, true);
}
-/*
- * Called from hugetlb code with hugetlb_lock held.
- */
-int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
+static int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
bool *migratable_cleared)
{
struct page *page = pfn_to_page(pfn);
- struct folio *folio = page_folio(page);
+ struct folio *folio;
bool count_increased = false;
int ret, rc;
+ spin_lock_irq(&hugetlb_lock);
+ folio = page_folio(page);
if (!folio_test_hugetlb(folio)) {
ret = MF_HUGETLB_NON_HUGEPAGE;
- goto out;
+ goto out_unlock;
} else if (flags & MF_COUNT_INCREASED) {
ret = MF_HUGETLB_IN_USED;
count_increased = true;
@@ -1995,13 +1994,13 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
} else {
ret = MF_HUGETLB_RETRY;
if (!(flags & MF_NO_RETRY))
- goto out;
+ goto out_unlock;
}
rc = hugetlb_update_hwpoison(folio, page);
if (rc >= MF_HUGETLB_FOLIO_PRE_POISONED) {
ret = rc;
- goto out;
+ goto out_unlock;
}
/*
@@ -2013,8 +2012,10 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
*migratable_cleared = true;
}
+ spin_unlock_irq(&hugetlb_lock);
return ret;
-out:
+out_unlock:
+ spin_unlock_irq(&hugetlb_lock);
if (count_increased)
folio_put(folio);
return ret;
diff --git a/mm/memory.c b/mm/memory.c
index ea65685..86a9731 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -612,6 +612,21 @@ static void print_bad_page_map(struct vm_area_struct *vma,
dump_stack();
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}
+
+static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
+{
+ switch (level) {
+ case PGTABLE_LEVEL_PTE:
+ return IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL);
+ case PGTABLE_LEVEL_PMD:
+ return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP);
+ case PGTABLE_LEVEL_PUD:
+ return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP);
+ default:
+ return false;
+ }
+}
+
#define print_bad_pte(vma, addr, pte, page) \
print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
@@ -684,7 +699,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn, bool special,
unsigned long long entry, enum pgtable_level level)
{
- if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
+ if (pgtable_level_has_pxx_special(level)) {
if (unlikely(special)) {
#ifdef CONFIG_FIND_NORMAL_PAGE
if (vma->vm_ops && vma->vm_ops->find_normal_page)
@@ -699,8 +714,9 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
return NULL;
}
/*
- * With CONFIG_ARCH_HAS_PTE_SPECIAL, any special page table
- * mappings (incl. shared zero folios) are marked accordingly.
+ * With working pte_special()/pmd_special()..., any special page
+ * table mappings (incl. shared zero folios) are marked
+ * accordingly.
*/
} else {
if (unlikely(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) {
@@ -1739,7 +1755,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
* consider uffd-wp bit when zap. For more information,
* see zap_install_uffd_wp_if_needed().
*/
- WARN_ON_ONCE(!vma_is_anonymous(vma));
+ WARN_ON_ONCE(!folio_test_anon(folio));
rss[mm_counter(folio)]--;
folio_remove_rmap_pte(folio, page, vma);
folio_put(folio);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2a943ec..40c7915 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1422,6 +1422,8 @@ static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
altmap = mem->altmap;
mem->altmap = NULL;
+ /* drop the ref. we got via find_memory_block() */
+ put_device(&mem->dev);
remove_memory_block_devices(cur_start, memblock_size);
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index fbfe571..19cd14b 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -840,7 +840,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
} else {
if (folio_is_zone_device(folio) &&
!folio_is_device_coherent(folio)) {
- goto abort;
+ goto free_abort;
}
entry = folio_mk_pmd(folio, vma->vm_page_prot);
if (vma->vm_flags & VM_WRITE)
@@ -850,7 +850,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
ptl = pmd_lock(vma->vm_mm, pmdp);
csa_ret = check_stable_address_space(vma->vm_mm);
if (csa_ret)
- goto abort;
+ goto unlock_abort;
/*
* Check for userfaultfd but do not deliver the fault. Instead,
@@ -893,6 +893,8 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
unlock_abort:
spin_unlock(ptl);
+free_abort:
+ pte_free(vma->vm_mm, pgtable);
abort:
for (i = 0; i < HPAGE_PMD_NR; i++)
src[i] &= ~MIGRATE_PFN_MIGRATE;
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 88cd53d..833f743 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -1835,7 +1835,9 @@ static int balance_dirty_pages(struct bdi_writeback *wb,
balance_domain_limits(mdtc, strictlimit);
}
- if (nr_dirty > gdtc->bg_thresh && !writeback_in_progress(wb))
+ if (!writeback_in_progress(wb) &&
+ (nr_dirty > gdtc->bg_thresh ||
+ (strictlimit && gdtc->wb_dirty > gdtc->wb_bg_thresh)))
wb_start_background_writeback(wb);
/*
@@ -1862,15 +1864,9 @@ static int balance_dirty_pages(struct bdi_writeback *wb,
* Unconditionally start background writeback if it's not
* already in progress. We need to do this because the global
* dirty threshold check above (nr_dirty > gdtc->bg_thresh)
- * doesn't account for these cases:
- *
- * a) strictlimit BDIs: throttling is calculated using per-wb
- * thresholds. The per-wb threshold can be exceeded even when
- * nr_dirty < gdtc->bg_thresh
- *
- * b) memcg-based throttling: memcg uses its own dirty count and
- * thresholds and can trigger throttling even when global
- * nr_dirty < gdtc->bg_thresh
+ * doesn't account for the memcg-based throttling case. memcg
+ * uses its own dirty count and thresholds and can trigger
+ * throttling even when global nr_dirty < gdtc->bg_thresh
*
* Writeback needs to be started else the writer stalls in the
* throttle loop waiting for dirty pages to be written back
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 65e2051..d49c254 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1035,6 +1035,7 @@ static inline bool page_expected_state(struct page *page,
#ifdef CONFIG_MEMCG
page->memcg_data |
#endif
+ page_pool_page_is_pp(page) |
(page->flags.f & check_flags)))
return false;
@@ -1061,6 +1062,8 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
if (unlikely(page->memcg_data))
bad_reason = "page still charged to cgroup";
#endif
+ if (unlikely(page_pool_page_is_pp(page)))
+ bad_reason = "page_pool leak";
return bad_reason;
}
@@ -1377,17 +1380,9 @@ __always_inline bool __free_pages_prepare(struct page *page,
mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
folio->mapping = NULL;
}
- if (unlikely(page_has_type(page))) {
- /* networking expects to clear its page type before releasing */
- if (is_check_pages_enabled()) {
- if (unlikely(PageNetpp(page))) {
- bad_page(page, "page_pool leak");
- return false;
- }
- }
+ if (unlikely(page_has_type(page)))
/* Reset the page_type (which overlays _mapcount) */
page->page_type = UINT_MAX;
- }
if (is_check_pages_enabled()) {
if (free_page_is_bad(page))
@@ -1808,9 +1803,9 @@ static inline bool should_skip_init(gfp_t flags)
inline void post_alloc_hook(struct page *page, unsigned int order,
gfp_t gfp_flags)
{
+ const bool zero_tags = gfp_flags & __GFP_ZEROTAGS;
bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
!should_skip_init(gfp_flags);
- bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);
int i;
set_page_private(page, 0);
@@ -1832,11 +1827,11 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
*/
/*
- * If memory tags should be zeroed
- * (which happens only when memory should be initialized as well).
+ * Clearing tags can efficiently clear the memory for us as well, if
+ * required.
*/
if (zero_tags)
- init = !tag_clear_highpages(page, 1 << order);
+ init = tag_clear_highpages(page, 1 << order, /* clear_pages= */init);
if (!should_skip_kasan_unpoison(gfp_flags) &&
kasan_unpoison_pages(page, order, init)) {
@@ -7737,6 +7732,11 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
return NULL;
+
+ /* On UP, spin_trylock() always succeeds even when it is locked */
+ if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+ return NULL;
+
if (!pcp_allowed_order(order))
return NULL;
diff --git a/mm/rmap.c b/mm/rmap.c
index 78b7fb5..99e1b3d 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2030,6 +2030,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
+ nr_pages = 1;
+
/*
* If the folio is in an mlock()d vma, we must not swap it out.
*/
diff --git a/mm/slab_common.c b/mm/slab_common.c
index d5a70a8..8b661ff 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -2110,7 +2110,9 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
{
if (cache_has_sheaves(s)) {
+ cpus_read_lock();
flush_rcu_sheaves_on_cache(s);
+ cpus_read_unlock();
rcu_barrier();
}
diff --git a/mm/slub.c b/mm/slub.c
index 161079a..a2bf375 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4024,6 +4024,7 @@ void flush_rcu_sheaves_on_cache(struct kmem_cache *s)
struct slub_flush_work *sfw;
unsigned int cpu;
+ lockdep_assert_cpus_held();
mutex_lock(&flush_lock);
for_each_online_cpu(cpu) {
@@ -5339,6 +5340,10 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node)
if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
return NULL;
+ /* On UP, spin_trylock() always succeeds even when it is locked */
+ if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+ return NULL;
+
retry:
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return NULL;
@@ -6879,6 +6884,22 @@ void kvfree(const void *addr)
EXPORT_SYMBOL(kvfree);
/**
+ * kvfree_atomic() - Free memory.
+ * @addr: Pointer to allocated memory.
+ *
+ * Same as kvfree(), but uses vfree_atomic() for vmalloc
+ * backed memory. Must not be called from NMI context.
+ */
+void kvfree_atomic(const void *addr)
+{
+ if (is_vmalloc_addr(addr))
+ vfree_atomic(addr);
+ else
+ kfree(addr);
+}
+EXPORT_SYMBOL(kvfree_atomic);
+
+/**
* kvfree_sensitive - Free a data object containing sensitive information.
* @addr: address of the data object to be freed.
* @len: length of the data object.
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 885da1e..80cc8be 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -14,6 +14,8 @@
#include <linux/userfaultfd_k.h>
#include <linux/mmu_notifier.h>
#include <linux/hugetlb.h>
+#include <linux/file.h>
+#include <linux/cleanup.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include "internal.h"
@@ -66,7 +68,7 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)
{
if (vma_is_anonymous(vma))
return &anon_uffd_ops;
- return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL;
+ return vma->vm_ops->uffd_ops;
}
static __always_inline
@@ -443,14 +445,80 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
return ret;
}
-static int mfill_copy_folio_retry(struct mfill_state *state, struct folio *folio)
+#define MFILL_RETRY_STATE_VMA_FLAGS \
+ append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)
+
+/*
+ * VMA state saved before dropping the locks in mfill_copy_folio_retry().
+ * Used to detect VMA replacement or incompatible changes after reacquiring the
+ * locks.
+ */
+struct mfill_retry_state {
+ const struct vm_uffd_ops *ops;
+ struct file *file;
+ vma_flags_t flags;
+ pgoff_t pgoff;
+};
+
+static void mfill_retry_state_save(struct mfill_retry_state *s,
+ struct vm_area_struct *vma)
{
- unsigned long src_addr = state->src_addr;
+ s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS);
+ s->ops = vma_uffd_ops(vma);
+ s->pgoff = vma->vm_pgoff;
+
+ if (vma->vm_file)
+ s->file = get_file(vma->vm_file);
+}
+
+static bool mfill_retry_state_changed(struct mfill_retry_state *state,
+ struct vm_area_struct *vma)
+{
+ vma_flags_t flags = vma_flags_and_mask(&vma->flags,
+ MFILL_RETRY_STATE_VMA_FLAGS);
+
+ /* Have any UFFD flags (missing, WP, minor) changed? */
+ if (!vma_flags_same_pair(&state->flags, &flags))
+ return true;
+
+ /* VMA type or effective uffd_ops changed while the lock was dropped */
+ if (state->ops != vma_uffd_ops(vma))
+ return true;
+
+ /* VMA was anonymous before; changed only if it no longer is */
+ if (!state->file)
+ return !vma_is_anonymous(vma);
+
+ /* VMA was file backed, but file, inode or offset has changed */
+ if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode ||
+ state->file != vma->vm_file || vma->vm_pgoff != state->pgoff)
+ return true;
+
+ return false;
+}
+
+static void mfill_retry_state_put(struct mfill_retry_state *s)
+{
+ if (s->file)
+ fput(s->file);
+}
+
+DEFINE_FREE(retry_put, struct mfill_retry_state *,
+ if (_T) mfill_retry_state_put(_T));
+
+static int mfill_copy_folio_retry(struct mfill_state *mfill_state,
+ struct folio *folio)
+{
+ struct mfill_retry_state retry_state = { 0 };
+ struct mfill_retry_state *for_free __free(retry_put) = &retry_state;
+ unsigned long src_addr = mfill_state->src_addr;
void *kaddr;
int err;
+ mfill_retry_state_save(&retry_state, mfill_state->vma);
+
/* retry copying with mm_lock dropped */
- mfill_put_vma(state);
+ mfill_put_vma(mfill_state);
kaddr = kmap_local_folio(folio, 0);
err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE);
@@ -461,11 +529,14 @@ static int mfill_copy_folio_retry(struct mfill_state *state, struct folio *folio
flush_dcache_folio(folio);
/* reget VMA and PMD, they could change underneath us */
- err = mfill_get_vma(state);
+ err = mfill_get_vma(mfill_state);
if (err)
return err;
- err = mfill_establish_pmd(state);
+ if (mfill_retry_state_changed(&retry_state, mfill_state->vma))
+ return -EAGAIN;
+
+ err = mfill_establish_pmd(mfill_state);
if (err)
return err;
@@ -481,6 +552,11 @@ static int __mfill_atomic_pte(struct mfill_state *state,
struct folio *folio;
int ret;
+ if (!ops) {
+ VM_WARN_ONCE(1, "UFFDIO_COPY for unsupported VMA");
+ return -EOPNOTSUPP;
+ }
+
folio = ops->alloc_folio(state->vma, state->dst_addr);
if (!folio)
return -ENOMEM;
diff --git a/mm/util.c b/mm/util.c
index 232c393..3cc949a 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -1232,7 +1232,7 @@ int __compat_vma_mmap(struct vm_area_desc *desc,
/* Update the VMA from the descriptor. */
compat_set_vma_from_desc(vma, desc);
/* Complete any specified mmap actions. */
- return mmap_action_complete(vma, &desc->action);
+ return mmap_action_complete(vma, &desc->action, /*is_compat=*/true);
}
EXPORT_SYMBOL(__compat_vma_mmap);
@@ -1389,7 +1389,8 @@ static int call_vma_mapped(struct vm_area_struct *vma)
}
static int mmap_action_finish(struct vm_area_struct *vma,
- struct mmap_action *action, int err)
+ struct mmap_action *action, int err,
+ bool is_compat)
{
size_t len;
@@ -1400,8 +1401,12 @@ static int mmap_action_finish(struct vm_area_struct *vma,
/* do_munmap() might take rmap lock, so release if held. */
maybe_rmap_unlock_action(vma, action);
- if (!err)
- return 0;
+ /*
+ * If this is invoked from the compatibility layer, post-mmap() hook
+ * logic will handle cleanup for us.
+ */
+ if (!err || is_compat)
+ return err;
/*
* If an error occurs, unmap the VMA altogether and return an error. We
@@ -1451,13 +1456,15 @@ EXPORT_SYMBOL(mmap_action_prepare);
* mmap_action_complete - Execute VMA descriptor action.
* @vma: The VMA to perform the action upon.
* @action: The action to perform.
+ * @is_compat: Is this being invoked from the compatibility layer?
*
* Similar to mmap_action_prepare().
*
- * Return: 0 on success, or error, at which point the VMA will be unmapped.
+ * Return: 0 on success, or error, at which point the VMA will be unmapped if
+ * !@is_compat.
*/
int mmap_action_complete(struct vm_area_struct *vma,
- struct mmap_action *action)
+ struct mmap_action *action, bool is_compat)
{
int err = 0;
@@ -1478,7 +1485,7 @@ int mmap_action_complete(struct vm_area_struct *vma,
break;
}
- return mmap_action_finish(vma, action, err);
+ return mmap_action_finish(vma, action, err, is_compat);
}
EXPORT_SYMBOL(mmap_action_complete);
#else
@@ -1500,7 +1507,8 @@ int mmap_action_prepare(struct vm_area_desc *desc)
EXPORT_SYMBOL(mmap_action_prepare);
int mmap_action_complete(struct vm_area_struct *vma,
- struct mmap_action *action)
+ struct mmap_action *action,
+ bool is_compat)
{
int err = 0;
@@ -1517,7 +1525,7 @@ int mmap_action_complete(struct vm_area_struct *vma,
break;
}
- return mmap_action_finish(vma, action, err);
+ return mmap_action_finish(vma, action, err, is_compat);
}
EXPORT_SYMBOL(mmap_action_complete);
#endif
diff --git a/mm/vma.c b/mm/vma.c
index 377321b..d90791b 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -2780,7 +2780,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
__mmap_complete(&map, vma);
if (have_mmap_prepare && allocated_new) {
- error = mmap_action_complete(vma, &desc.action);
+ error = mmap_action_complete(vma, &desc.action,
+ /*is_compat=*/false);
if (error)
return error;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index aa08651..bb6ae08 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3203,7 +3203,7 @@ struct vm_struct *__get_vm_area_node(unsigned long size,
struct vm_struct *area;
unsigned long requested_size = size;
- BUG_ON(in_interrupt());
+ BUG_ON(in_nmi() || in_hardirq());
size = ALIGN(size, 1ul << shift);
if (unlikely(!size))
return NULL;
@@ -4361,7 +4361,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align
return NULL;
if (p) {
- memcpy(n, p, old_size);
+ memcpy(n, p, min(size, old_size));
vfree(p);
}
diff --git a/net/6lowpan/iphc.c b/net/6lowpan/iphc.c
index e116d30..37eaff3 100644
--- a/net/6lowpan/iphc.c
+++ b/net/6lowpan/iphc.c
@@ -1086,12 +1086,12 @@ static u8 lowpan_iphc_mcast_ctx_addr_compress(u8 **hc_ptr,
const struct lowpan_iphc_ctx *ctx,
const struct in6_addr *ipaddr)
{
- u8 data[6];
+ u8 data[6] = {};
/* flags/scope, reserved (RIID) */
memcpy(data, &ipaddr->s6_addr[1], 2);
/* group ID */
- memcpy(&data[1], &ipaddr->s6_addr[11], 4);
+ memcpy(&data[2], &ipaddr->s6_addr[12], 4);
lowpan_push_hc_data(hc_ptr, data, 6);
return LOWPAN_IPHC_DAM_00;
diff --git a/net/802/garp.c b/net/802/garp.c
index 6f563b6..c7a39f2 100644
--- a/net/802/garp.c
+++ b/net/802/garp.c
@@ -453,7 +453,7 @@ static int garp_pdu_parse_attr(struct garp_applicant *app, struct sk_buff *skb,
if (!pskb_may_pull(skb, ga->len))
return -1;
skb_pull(skb, ga->len);
- dlen = sizeof(*ga) - ga->len;
+ dlen = ga->len - sizeof(*ga);
if (attrtype > app->app->maxattr)
return 0;
diff --git a/net/802/mrp.c b/net/802/mrp.c
index ff0e805..160a3b1 100644
--- a/net/802/mrp.c
+++ b/net/802/mrp.c
@@ -703,6 +703,12 @@ static int mrp_pdu_parse_vecattr(struct mrp_applicant *app,
valen = be16_to_cpu(get_unaligned(&mrp_cb(skb)->vah->lenflags) &
MRP_VECATTR_HDR_LEN_MASK);
+ /* If valen is 0, only a LeaveAllEvent is present; FirstValue and
+ * Vector fields are absent per IEEE 802.1ak.
+ */
+ if (valen == 0)
+ return 0;
+
/* The VectorAttribute structure in a PDU carries event information
* about one or more attributes having consecutive values. Only the
* value for the first attribute is contained in the structure. So
@@ -753,6 +759,9 @@ static int mrp_pdu_parse_vecattr(struct mrp_applicant *app,
vaevents %= __MRP_VECATTR_EVENT_MAX;
vaevent = vaevents;
mrp_pdu_parse_vecattr_event(app, skb, vaevent);
+ valen--;
+ mrp_attrvalue_inc(mrp_cb(skb)->attrvalue,
+ mrp_cb(skb)->mh->attrlen);
}
return 0;
}
diff --git a/net/appletalk/aarp.c b/net/appletalk/aarp.c
index e7315c0..078fb7a6 100644
--- a/net/appletalk/aarp.c
+++ b/net/appletalk/aarp.c
@@ -393,7 +393,7 @@ static void aarp_purge(void)
*/
static struct aarp_entry *aarp_alloc(void)
{
- struct aarp_entry *a = kmalloc_obj(*a, GFP_ATOMIC);
+ struct aarp_entry *a = kzalloc_obj(*a, GFP_ATOMIC);
if (!a)
return NULL;
@@ -542,6 +542,11 @@ int aarp_send_ddp(struct net_device *dev, struct sk_buff *skb,
struct ddpehdr *ddp = (struct ddpehdr *)skb->data;
int ft = 2;
+ if (!at) {
+ kfree_skb(skb);
+ return NET_XMIT_DROP;
+ }
+
/*
* Compressible ?
*
diff --git a/net/atm/signaling.c b/net/atm/signaling.c
index 358fbe5..b991d93 100644
--- a/net/atm/signaling.c
+++ b/net/atm/signaling.c
@@ -179,6 +179,7 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb)
break;
default:
pr_alert("bad message type %d\n", (int)msg->type);
+ dev_kfree_skb(skb);
/* Paired with find_get_vcc(msg->vcc) above */
sock_put(sk);
return -EINVAL;
diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c
index f28e9cb..b8b1b99 100644
--- a/net/batman-adv/bat_iv_ogm.c
+++ b/net/batman-adv/bat_iv_ogm.c
@@ -173,19 +173,12 @@ batadv_iv_ogm_orig_get(struct batadv_priv *bat_priv, const u8 *addr)
static struct batadv_neigh_node *
batadv_iv_ogm_neigh_new(struct batadv_hard_iface *hard_iface,
const u8 *neigh_addr,
- struct batadv_orig_node *orig_node,
- struct batadv_orig_node *orig_neigh)
+ struct batadv_orig_node *orig_node)
{
struct batadv_neigh_node *neigh_node;
neigh_node = batadv_neigh_node_get_or_create(orig_node,
hard_iface, neigh_addr);
- if (!neigh_node)
- goto out;
-
- neigh_node->orig_node = orig_neigh;
-
-out:
return neigh_node;
}
@@ -231,6 +224,8 @@ static void batadv_iv_ogm_iface_disable(struct batadv_hard_iface *hard_iface)
hard_iface->bat_iv.ogm_buff = NULL;
mutex_unlock(&hard_iface->bat_iv.ogm_buff_mutex);
+
+ cancel_delayed_work_sync(&hard_iface->bat_iv.reschedule_work);
}
static void batadv_iv_ogm_iface_update_mac(struct batadv_hard_iface *hard_iface)
@@ -335,7 +330,7 @@ static void batadv_iv_ogm_send_to_if(struct batadv_forw_packet *forw_packet,
struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
const char *fwd_str;
u8 packet_num;
- s16 buff_pos;
+ int buff_pos;
struct batadv_ogm_packet *batadv_ogm_packet;
struct sk_buff *skb;
u8 *packet_pos;
@@ -543,8 +538,10 @@ batadv_iv_ogm_can_aggregate(const struct batadv_ogm_packet *new_bat_ogm_packet,
* @if_incoming: interface where the packet was received
* @if_outgoing: interface for which the retransmission should be considered
* @own_packet: true if it is a self-generated ogm
+ *
+ * Return: whether forward packet was scheduled
*/
-static void batadv_iv_ogm_aggregate_new(const unsigned char *packet_buff,
+static bool batadv_iv_ogm_aggregate_new(const unsigned char *packet_buff,
int packet_len, unsigned long send_time,
bool direct_link,
struct batadv_hard_iface *if_incoming,
@@ -568,13 +565,13 @@ static void batadv_iv_ogm_aggregate_new(const unsigned char *packet_buff,
skb = netdev_alloc_skb_ip_align(NULL, skb_size);
if (!skb)
- return;
+ return false;
forw_packet_aggr = batadv_forw_packet_alloc(if_incoming, if_outgoing,
queue_left, bat_priv, skb);
if (!forw_packet_aggr) {
kfree_skb(skb);
- return;
+ return false;
}
forw_packet_aggr->skb->priority = TC_PRIO_CONTROL;
@@ -597,6 +594,8 @@ static void batadv_iv_ogm_aggregate_new(const unsigned char *packet_buff,
batadv_iv_send_outstanding_bat_ogm_packet);
batadv_forw_packet_ogmv1_queue(bat_priv, forw_packet_aggr, send_time);
+
+ return true;
}
/* aggregate a new packet into the existing ogm packet */
@@ -624,8 +623,10 @@ static void batadv_iv_ogm_aggregate(struct batadv_forw_packet *forw_packet_aggr,
* @if_outgoing: interface for which the retransmission should be considered
* @own_packet: true if it is a self-generated ogm
* @send_time: timestamp (jiffies) when the packet is to be sent
+ *
+ * Return: whether forward packet was scheduled
*/
-static void batadv_iv_ogm_queue_add(struct batadv_priv *bat_priv,
+static bool batadv_iv_ogm_queue_add(struct batadv_priv *bat_priv,
unsigned char *packet_buff,
int packet_len,
struct batadv_hard_iface *if_incoming,
@@ -677,14 +678,16 @@ static void batadv_iv_ogm_queue_add(struct batadv_priv *bat_priv,
if (!own_packet && atomic_read(&bat_priv->aggregated_ogms))
send_time += max_aggregation_jiffies;
- batadv_iv_ogm_aggregate_new(packet_buff, packet_len,
- send_time, direct_link,
- if_incoming, if_outgoing,
- own_packet);
+ return batadv_iv_ogm_aggregate_new(packet_buff, packet_len,
+ send_time, direct_link,
+ if_incoming, if_outgoing,
+ own_packet);
} else {
batadv_iv_ogm_aggregate(forw_packet_aggr, packet_buff,
packet_len, direct_link);
spin_unlock_bh(&bat_priv->forw_bat_list_lock);
+
+ return true;
}
}
@@ -797,6 +800,9 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface)
u32 seqno;
u16 tvlv_len = 0;
unsigned long send_time;
+ bool reschedule = false;
+ bool scheduled;
+ int ret;
lockdep_assert_held(&hard_iface->bat_iv.ogm_buff_mutex);
@@ -820,9 +826,15 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface)
* appended as it may alter the tt tvlv container
*/
batadv_tt_local_commit_changes(bat_priv);
- tvlv_len = batadv_tvlv_container_ogm_append(bat_priv, ogm_buff,
- ogm_buff_len,
- BATADV_OGM_HLEN);
+ ret = batadv_tvlv_container_ogm_append(bat_priv, ogm_buff,
+ ogm_buff_len,
+ BATADV_OGM_HLEN);
+ if (ret < 0) {
+ reschedule = true;
+ goto out;
+ }
+
+ tvlv_len = ret;
}
batadv_ogm_packet = (struct batadv_ogm_packet *)(*ogm_buff);
@@ -841,8 +853,11 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface)
/* OGMs from secondary interfaces are only scheduled on their
* respective interfaces.
*/
- batadv_iv_ogm_queue_add(bat_priv, *ogm_buff, *ogm_buff_len,
- hard_iface, hard_iface, 1, send_time);
+ scheduled = batadv_iv_ogm_queue_add(bat_priv, *ogm_buff, *ogm_buff_len,
+ hard_iface, hard_iface, 1, send_time);
+ if (!scheduled)
+ reschedule = true;
+
goto out;
}
@@ -854,15 +869,28 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface)
if (!kref_get_unless_zero(&tmp_hard_iface->refcount))
continue;
- batadv_iv_ogm_queue_add(bat_priv, *ogm_buff,
- *ogm_buff_len, hard_iface,
- tmp_hard_iface, 1, send_time);
-
+ scheduled = batadv_iv_ogm_queue_add(bat_priv, *ogm_buff,
+ *ogm_buff_len, hard_iface,
+ tmp_hard_iface, 1, send_time);
batadv_hardif_put(tmp_hard_iface);
+
+ if (!scheduled && tmp_hard_iface == hard_iface)
+ reschedule = true;
}
rcu_read_unlock();
out:
+ if (reschedule) {
+ /* there was a failure scheduling the own forward packet.
+ * as result, the batadv_iv_send_outstanding_bat_ogm_packet()
+ * work item is no longer scheduled. it is therefore necessary
+ * to reschedule it manually
+ */
+ queue_delayed_work(batadv_event_workqueue,
+ &hard_iface->bat_iv.reschedule_work,
+ msecs_to_jiffies(atomic_read(&bat_priv->orig_interval)));
+ }
+
batadv_hardif_put(primary_if);
}
@@ -877,6 +905,17 @@ static void batadv_iv_ogm_schedule(struct batadv_hard_iface *hard_iface)
mutex_unlock(&hard_iface->bat_iv.ogm_buff_mutex);
}
+static void batadv_iv_ogm_reschedule(struct work_struct *work)
+{
+ struct delayed_work *delayed_work = to_delayed_work(work);
+ struct batadv_hard_iface *hard_iface;
+
+ hard_iface = container_of(delayed_work,
+ struct batadv_hard_iface,
+ bat_iv.reschedule_work);
+ batadv_iv_ogm_schedule(hard_iface);
+}
+
/**
* batadv_iv_orig_ifinfo_sum() - Get bcast_own sum for originator over interface
* @orig_node: originator which reproadcasted the OGMs directly
@@ -907,6 +946,31 @@ static u8 batadv_iv_orig_ifinfo_sum(struct batadv_orig_node *orig_node,
}
/**
+ * batadv_iv_ogm_neigh_ifinfo_sum() - Get bcast_own sum for a last-hop neighbor
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @neigh_node: last-hop neighbor of an originator
+ *
+ * Return: Number of replied (rebroadcasted) OGMs for the originator currently
+ * announced by the neighbor. Returns 0 if the neighbor's originator entry is
+ * not available anymore.
+ */
+static u8 batadv_iv_ogm_neigh_ifinfo_sum(struct batadv_priv *bat_priv,
+ const struct batadv_neigh_node *neigh_node)
+{
+ struct batadv_orig_node *orig_neigh;
+ u8 sum;
+
+ orig_neigh = batadv_orig_hash_find(bat_priv, neigh_node->addr);
+ if (!orig_neigh)
+ return 0;
+
+ sum = batadv_iv_orig_ifinfo_sum(orig_neigh, neigh_node->if_incoming);
+ batadv_orig_node_put(orig_neigh);
+
+ return sum;
+}
+
+/**
* batadv_iv_ogm_orig_update() - use OGM to update corresponding data in an
* originator
* @bat_priv: the bat priv with all the mesh interface information
@@ -975,17 +1039,9 @@ batadv_iv_ogm_orig_update(struct batadv_priv *bat_priv,
}
if (!neigh_node) {
- struct batadv_orig_node *orig_tmp;
-
- orig_tmp = batadv_iv_ogm_orig_get(bat_priv, ethhdr->h_source);
- if (!orig_tmp)
- goto unlock;
-
neigh_node = batadv_iv_ogm_neigh_new(if_incoming,
ethhdr->h_source,
- orig_node, orig_tmp);
-
- batadv_orig_node_put(orig_tmp);
+ orig_node);
if (!neigh_node)
goto unlock;
} else {
@@ -1037,10 +1093,9 @@ batadv_iv_ogm_orig_update(struct batadv_priv *bat_priv,
*/
if (router_ifinfo &&
neigh_ifinfo->bat_iv.tq_avg == router_ifinfo->bat_iv.tq_avg) {
- sum_orig = batadv_iv_orig_ifinfo_sum(router->orig_node,
- router->if_incoming);
- sum_neigh = batadv_iv_orig_ifinfo_sum(neigh_node->orig_node,
- neigh_node->if_incoming);
+ sum_orig = batadv_iv_ogm_neigh_ifinfo_sum(bat_priv, router);
+ sum_neigh = batadv_iv_ogm_neigh_ifinfo_sum(bat_priv,
+ neigh_node);
if (sum_orig >= sum_neigh)
goto out;
}
@@ -1106,7 +1161,6 @@ static bool batadv_iv_ogm_calc_tq(struct batadv_orig_node *orig_node,
if (!neigh_node)
neigh_node = batadv_iv_ogm_neigh_new(if_incoming,
orig_neigh_node->orig,
- orig_neigh_node,
orig_neigh_node);
if (!neigh_node)
@@ -1303,6 +1357,32 @@ batadv_iv_ogm_update_seqnos(const struct ethhdr *ethhdr,
}
/**
+ * batadv_orig_to_direct_router() - get direct next hop neighbor to an orig address
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @orig_addr: the originator MAC address to search the best next hop router for
+ * @if_outgoing: the interface where the OGM should be sent to
+ *
+ * Return: A neighbor node which is the best router towards the given originator
+ * address. Bonding candidates are ignored.
+ */
+static struct batadv_neigh_node *
+batadv_orig_to_direct_router(struct batadv_priv *bat_priv, u8 *orig_addr,
+ struct batadv_hard_iface *if_outgoing)
+{
+ struct batadv_neigh_node *neigh_node;
+ struct batadv_orig_node *orig_node;
+
+ orig_node = batadv_orig_hash_find(bat_priv, orig_addr);
+ if (!orig_node)
+ return NULL;
+
+ neigh_node = batadv_orig_router_get(orig_node, if_outgoing);
+ batadv_orig_node_put(orig_node);
+
+ return neigh_node;
+}
+
+/**
* batadv_iv_ogm_process_per_outif() - process a batman iv OGM for an outgoing
* interface
* @skb: the skb containing the OGM
@@ -1372,8 +1452,9 @@ batadv_iv_ogm_process_per_outif(const struct sk_buff *skb, int ogm_offset,
router = batadv_orig_router_get(orig_node, if_outgoing);
if (router) {
- router_router = batadv_orig_router_get(router->orig_node,
- if_outgoing);
+ router_router = batadv_orig_to_direct_router(bat_priv,
+ router->addr,
+ if_outgoing);
router_ifinfo = batadv_neigh_ifinfo_get(router, if_outgoing);
}
@@ -2227,6 +2308,8 @@ batadv_iv_ogm_neigh_is_sob(struct batadv_neigh_node *neigh1,
static void batadv_iv_iface_enabled(struct batadv_hard_iface *hard_iface)
{
+ INIT_DELAYED_WORK(&hard_iface->bat_iv.reschedule_work, batadv_iv_ogm_reschedule);
+
/* begin scheduling originator messages on that interface */
batadv_iv_ogm_schedule(hard_iface);
}
diff --git a/net/batman-adv/bat_v_ogm.c b/net/batman-adv/bat_v_ogm.c
index e387049..d66ca77 100644
--- a/net/batman-adv/bat_v_ogm.c
+++ b/net/batman-adv/bat_v_ogm.c
@@ -113,14 +113,14 @@ static void batadv_v_ogm_start_timer(struct batadv_priv *bat_priv)
/**
* batadv_v_ogm_send_to_if() - send a batman ogm using a given interface
+ * @bat_priv: the bat priv with all the mesh interface information
* @skb: the OGM to send
* @hard_iface: the interface to use to send the OGM
*/
-static void batadv_v_ogm_send_to_if(struct sk_buff *skb,
+static void batadv_v_ogm_send_to_if(struct batadv_priv *bat_priv,
+ struct sk_buff *skb,
struct batadv_hard_iface *hard_iface)
{
- struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
-
if (hard_iface->if_status != BATADV_IF_ACTIVE) {
kfree_skb(skb);
return;
@@ -187,6 +187,7 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface)
/**
* batadv_v_ogm_aggr_send() - flush & send aggregation queue
+ * @bat_priv: the bat priv with all the mesh interface information
* @hard_iface: the interface with the aggregation queue to flush
*
* Aggregates all OGMv2 packets currently in the aggregation queue into a
@@ -196,7 +197,8 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface)
*
* Caller needs to hold the hard_iface->bat_v.aggr_list.lock.
*/
-static void batadv_v_ogm_aggr_send(struct batadv_hard_iface *hard_iface)
+static void batadv_v_ogm_aggr_send(struct batadv_priv *bat_priv,
+ struct batadv_hard_iface *hard_iface)
{
unsigned int aggr_len = hard_iface->bat_v.aggr_len;
struct sk_buff *skb_aggr;
@@ -226,27 +228,32 @@ static void batadv_v_ogm_aggr_send(struct batadv_hard_iface *hard_iface)
consume_skb(skb);
}
- batadv_v_ogm_send_to_if(skb_aggr, hard_iface);
+ batadv_v_ogm_send_to_if(bat_priv, skb_aggr, hard_iface);
}
/**
* batadv_v_ogm_queue_on_if() - queue a batman ogm on a given interface
+ * @bat_priv: the bat priv with all the mesh interface information
* @skb: the OGM to queue
* @hard_iface: the interface to queue the OGM on
*/
-static void batadv_v_ogm_queue_on_if(struct sk_buff *skb,
+static void batadv_v_ogm_queue_on_if(struct batadv_priv *bat_priv,
+ struct sk_buff *skb,
struct batadv_hard_iface *hard_iface)
{
- struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
+ if (hard_iface->mesh_iface != bat_priv->mesh_iface) {
+ kfree_skb(skb);
+ return;
+ }
if (!atomic_read(&bat_priv->aggregated_ogms)) {
- batadv_v_ogm_send_to_if(skb, hard_iface);
+ batadv_v_ogm_send_to_if(bat_priv, skb, hard_iface);
return;
}
spin_lock_bh(&hard_iface->bat_v.aggr_list.lock);
if (!batadv_v_ogm_queue_left(skb, hard_iface))
- batadv_v_ogm_aggr_send(hard_iface);
+ batadv_v_ogm_aggr_send(bat_priv, hard_iface);
hard_iface->bat_v.aggr_len += batadv_v_ogm_len(skb);
__skb_queue_tail(&hard_iface->bat_v.aggr_list, skb);
@@ -262,10 +269,10 @@ static void batadv_v_ogm_send_meshif(struct batadv_priv *bat_priv)
struct batadv_hard_iface *hard_iface;
struct batadv_ogm2_packet *ogm_packet;
struct sk_buff *skb, *skb_tmp;
- unsigned char *ogm_buff;
+ unsigned char **ogm_buff;
struct list_head *iter;
- int ogm_buff_len;
- u16 tvlv_len = 0;
+ int *ogm_buff_len;
+ u16 tvlv_len;
int ret;
lockdep_assert_held(&bat_priv->bat_v.ogm_buff_mutex);
@@ -273,25 +280,27 @@ static void batadv_v_ogm_send_meshif(struct batadv_priv *bat_priv)
if (atomic_read(&bat_priv->mesh_state) == BATADV_MESH_DEACTIVATING)
goto out;
- ogm_buff = bat_priv->bat_v.ogm_buff;
- ogm_buff_len = bat_priv->bat_v.ogm_buff_len;
+ ogm_buff = &bat_priv->bat_v.ogm_buff;
+ ogm_buff_len = &bat_priv->bat_v.ogm_buff_len;
+
/* tt changes have to be committed before the tvlv data is
* appended as it may alter the tt tvlv container
*/
batadv_tt_local_commit_changes(bat_priv);
- tvlv_len = batadv_tvlv_container_ogm_append(bat_priv, &ogm_buff,
- &ogm_buff_len,
- BATADV_OGM2_HLEN);
+ ret = batadv_tvlv_container_ogm_append(bat_priv, ogm_buff,
+ ogm_buff_len,
+ BATADV_OGM2_HLEN);
+ if (ret < 0)
+ goto reschedule;
- bat_priv->bat_v.ogm_buff = ogm_buff;
- bat_priv->bat_v.ogm_buff_len = ogm_buff_len;
+ tvlv_len = ret;
- skb = netdev_alloc_skb_ip_align(NULL, ETH_HLEN + ogm_buff_len);
+ skb = netdev_alloc_skb_ip_align(NULL, ETH_HLEN + *ogm_buff_len);
if (!skb)
goto reschedule;
skb_reserve(skb, ETH_HLEN);
- skb_put_data(skb, ogm_buff, ogm_buff_len);
+ skb_put_data(skb, *ogm_buff, *ogm_buff_len);
ogm_packet = (struct batadv_ogm2_packet *)skb->data;
ogm_packet->seqno = htonl(atomic_read(&bat_priv->bat_v.ogm_seqno));
@@ -343,7 +352,7 @@ static void batadv_v_ogm_send_meshif(struct batadv_priv *bat_priv)
break;
}
- batadv_v_ogm_queue_on_if(skb_tmp, hard_iface);
+ batadv_v_ogm_queue_on_if(bat_priv, skb_tmp, hard_iface);
batadv_hardif_put(hard_iface);
}
rcu_read_unlock();
@@ -383,12 +392,14 @@ void batadv_v_ogm_aggr_work(struct work_struct *work)
{
struct batadv_hard_iface_bat_v *batv;
struct batadv_hard_iface *hard_iface;
+ struct batadv_priv *bat_priv;
batv = container_of(work, struct batadv_hard_iface_bat_v, aggr_wq.work);
hard_iface = container_of(batv, struct batadv_hard_iface, bat_v);
+ bat_priv = netdev_priv(hard_iface->mesh_iface);
spin_lock_bh(&hard_iface->bat_v.aggr_list.lock);
- batadv_v_ogm_aggr_send(hard_iface);
+ batadv_v_ogm_aggr_send(bat_priv, hard_iface);
spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock);
batadv_v_ogm_start_queue_timer(hard_iface);
@@ -578,7 +589,7 @@ static void batadv_v_ogm_forward(struct batadv_priv *bat_priv,
if_outgoing->net_dev->name, ntohl(ogm_forward->throughput),
ogm_forward->ttl, if_incoming->net_dev->name);
- batadv_v_ogm_queue_on_if(skb, if_outgoing);
+ batadv_v_ogm_queue_on_if(bat_priv, skb, if_outgoing);
out:
batadv_orig_ifinfo_put(orig_ifinfo);
diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c
index 51fe028..ffe8540 100644
--- a/net/batman-adv/bridge_loop_avoidance.c
+++ b/net/batman-adv/bridge_loop_avoidance.c
@@ -318,8 +318,8 @@ batadv_bla_del_backbone_claims(struct batadv_bla_backbone_gw *backbone_gw)
if (claim->backbone_gw != backbone_gw)
continue;
- batadv_claim_put(claim);
hlist_del_rcu(&claim->hash_entry);
+ batadv_claim_put(claim);
}
spin_unlock_bh(list_lock);
}
@@ -356,12 +356,14 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac,
sizeof(local_claim_dest));
local_claim_dest.type = claimtype;
- mesh_iface = primary_if->mesh_iface;
+ mesh_iface = READ_ONCE(primary_if->mesh_iface);
+ if (!mesh_iface)
+ goto out;
skb = arp_create(ARPOP_REPLY, ETH_P_ARP,
/* IP DST: 0.0.0.0 */
zeroip,
- primary_if->mesh_iface,
+ mesh_iface,
/* IP SRC: 0.0.0.0 */
zeroip,
/* Ethernet DST: Broadcast */
@@ -514,8 +516,8 @@ batadv_bla_get_backbone_gw(struct batadv_priv *bat_priv, const u8 *orig,
entry->crc = BATADV_BLA_CRC_INIT;
entry->bat_priv = bat_priv;
spin_lock_init(&entry->crc_lock);
- atomic_set(&entry->request_sent, 0);
- atomic_set(&entry->wait_periods, 0);
+ entry->state = BATADV_BLA_BACKBONE_GW_SYNCED;
+ entry->wait_periods = 0;
ether_addr_copy(entry->orig, orig);
INIT_WORK(&entry->report_work, batadv_bla_loopdetect_report);
kref_init(&entry->refcount);
@@ -544,9 +546,13 @@ batadv_bla_get_backbone_gw(struct batadv_priv *bat_priv, const u8 *orig,
batadv_bla_send_announce(bat_priv, entry);
/* this will be decreased in the worker thread */
- atomic_inc(&entry->request_sent);
- atomic_set(&entry->wait_periods, BATADV_BLA_WAIT_PERIODS);
- atomic_inc(&bat_priv->bla.num_requests);
+ spin_lock_bh(&bat_priv->bla.num_requests_lock);
+ if (entry->state == BATADV_BLA_BACKBONE_GW_SYNCED) {
+ entry->state = BATADV_BLA_BACKBONE_GW_UNSYNCED;
+ entry->wait_periods = BATADV_BLA_WAIT_PERIODS;
+ atomic_inc(&bat_priv->bla.num_requests);
+ }
+ spin_unlock_bh(&bat_priv->bla.num_requests_lock);
}
return entry;
@@ -649,10 +655,12 @@ static void batadv_bla_send_request(struct batadv_bla_backbone_gw *backbone_gw)
backbone_gw->vid, BATADV_CLAIM_TYPE_REQUEST);
/* no local broadcasts should be sent or received, for now. */
- if (!atomic_read(&backbone_gw->request_sent)) {
+ spin_lock_bh(&backbone_gw->bat_priv->bla.num_requests_lock);
+ if (backbone_gw->state == BATADV_BLA_BACKBONE_GW_SYNCED) {
+ backbone_gw->state = BATADV_BLA_BACKBONE_GW_UNSYNCED;
atomic_inc(&backbone_gw->bat_priv->bla.num_requests);
- atomic_set(&backbone_gw->request_sent, 1);
}
+ spin_unlock_bh(&backbone_gw->bat_priv->bla.num_requests_lock);
}
/**
@@ -723,6 +731,7 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv,
if (unlikely(hash_added != 0)) {
/* only local changes happened. */
+ batadv_backbone_gw_put(backbone_gw);
kfree(claim);
return;
}
@@ -872,10 +881,12 @@ static bool batadv_handle_announce(struct batadv_priv *bat_priv, u8 *an_addr,
/* if we have sent a request and the crc was OK,
* we can allow traffic again.
*/
- if (atomic_read(&backbone_gw->request_sent)) {
+ spin_lock_bh(&bat_priv->bla.num_requests_lock);
+ if (backbone_gw->state == BATADV_BLA_BACKBONE_GW_UNSYNCED) {
+ backbone_gw->state = BATADV_BLA_BACKBONE_GW_SYNCED;
atomic_dec(&backbone_gw->bat_priv->bla.num_requests);
- atomic_set(&backbone_gw->request_sent, 0);
}
+ spin_unlock_bh(&bat_priv->bla.num_requests_lock);
}
batadv_backbone_gw_put(backbone_gw);
@@ -1223,6 +1234,7 @@ static void batadv_bla_purge_backbone_gw(struct batadv_priv *bat_priv, int now)
struct hlist_head *head;
struct batadv_hashtable *hash;
spinlock_t *list_lock; /* protects write access to the hash lists */
+ bool purged;
int i;
hash = bat_priv->bla.backbone_hash;
@@ -1233,30 +1245,49 @@ static void batadv_bla_purge_backbone_gw(struct batadv_priv *bat_priv, int now)
head = &hash->table[i];
list_lock = &hash->list_locks[i];
- spin_lock_bh(list_lock);
- hlist_for_each_entry_safe(backbone_gw, node_tmp,
- head, hash_entry) {
- if (now)
- goto purge_now;
- if (!batadv_has_timed_out(backbone_gw->lasttime,
- BATADV_BLA_BACKBONE_TIMEOUT))
- continue;
+ do {
+ purged = false;
- batadv_dbg(BATADV_DBG_BLA, backbone_gw->bat_priv,
- "%s(): backbone gw %pM timed out\n",
- __func__, backbone_gw->orig);
+ spin_lock_bh(list_lock);
+ hlist_for_each_entry_safe(backbone_gw, node_tmp,
+ head, hash_entry) {
+ if (now)
+ goto purge_now;
+ if (!batadv_has_timed_out(backbone_gw->lasttime,
+ BATADV_BLA_BACKBONE_TIMEOUT))
+ continue;
+
+ batadv_dbg(BATADV_DBG_BLA, backbone_gw->bat_priv,
+ "%s(): backbone gw %pM timed out\n",
+ __func__, backbone_gw->orig);
purge_now:
- /* don't wait for the pending request anymore */
- if (atomic_read(&backbone_gw->request_sent))
- atomic_dec(&bat_priv->bla.num_requests);
+ purged = true;
- batadv_bla_del_backbone_claims(backbone_gw);
+ /* don't wait for the pending request anymore */
+ spin_lock_bh(&bat_priv->bla.num_requests_lock);
+ if (backbone_gw->state == BATADV_BLA_BACKBONE_GW_UNSYNCED)
+ atomic_dec(&bat_priv->bla.num_requests);
- hlist_del_rcu(&backbone_gw->hash_entry);
- batadv_backbone_gw_put(backbone_gw);
- }
- spin_unlock_bh(list_lock);
+ backbone_gw->state = BATADV_BLA_BACKBONE_GW_STOPPED;
+ spin_unlock_bh(&bat_priv->bla.num_requests_lock);
+
+ batadv_bla_del_backbone_claims(backbone_gw);
+
+ hlist_del_rcu(&backbone_gw->hash_entry);
+ break;
+ }
+ spin_unlock_bh(list_lock);
+
+ if (purged) {
+ /* reference for pending report_work */
+ if (cancel_work_sync(&backbone_gw->report_work))
+ batadv_backbone_gw_put(backbone_gw);
+
+ /* reference for hash_entry */
+ batadv_backbone_gw_put(backbone_gw);
+ }
+ } while (purged);
}
}
@@ -1288,6 +1319,13 @@ static void batadv_bla_purge_claims(struct batadv_priv *bat_priv,
rcu_read_lock();
hlist_for_each_entry_rcu(claim, head, hash_entry) {
+ /* only purge claims not currently in the process of being released.
+ * Such claims could otherwise have a NULL-ptr backbone_gw set because
+ * they already went through batadv_claim_release()
+ */
+ if (!kref_get_unless_zero(&claim->refcount))
+ continue;
+
backbone_gw = batadv_bla_claim_get_backbone_gw(claim);
if (now)
goto purge_now;
@@ -1313,6 +1351,7 @@ static void batadv_bla_purge_claims(struct batadv_priv *bat_priv,
claim->addr, claim->vid);
skip:
batadv_backbone_gw_put(backbone_gw);
+ batadv_claim_put(claim);
}
rcu_read_unlock();
}
@@ -1483,7 +1522,7 @@ static void batadv_bla_periodic_work(struct work_struct *work)
batadv_bla_send_loopdetect(bat_priv,
backbone_gw);
- /* request_sent is only set after creation to avoid
+ /* state is only set to unsynced after creation to avoid
* problems when we are not yet known as backbone gw
* in the backbone.
*
@@ -1492,14 +1531,21 @@ static void batadv_bla_periodic_work(struct work_struct *work)
* some grace time.
*/
- if (atomic_read(&backbone_gw->request_sent) == 0)
- continue;
+ spin_lock_bh(&bat_priv->bla.num_requests_lock);
+ if (backbone_gw->state != BATADV_BLA_BACKBONE_GW_UNSYNCED)
+ goto unlock_next;
- if (!atomic_dec_and_test(&backbone_gw->wait_periods))
- continue;
+ if (backbone_gw->wait_periods > 0)
+ backbone_gw->wait_periods--;
+ if (backbone_gw->wait_periods > 0)
+ goto unlock_next;
+
+ backbone_gw->state = BATADV_BLA_BACKBONE_GW_SYNCED;
atomic_dec(&backbone_gw->bat_priv->bla.num_requests);
- atomic_set(&backbone_gw->request_sent, 0);
+
+unlock_next:
+ spin_unlock_bh(&bat_priv->bla.num_requests_lock);
}
rcu_read_unlock();
}
diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c
index 3efc4cf..0a8bd95e 100644
--- a/net/batman-adv/distributed-arp-table.c
+++ b/net/batman-adv/distributed-arp-table.c
@@ -696,6 +696,9 @@ static bool batadv_dat_forward_data(struct batadv_priv *bat_priv,
goto free_orig;
tmp_skb = pskb_copy_for_clone(skb, GFP_ATOMIC);
+ if (!tmp_skb)
+ goto free_neigh;
+
if (!batadv_send_skb_prepare_unicast_4addr(bat_priv, tmp_skb,
cand[i].orig_node,
packet_subtype)) {
diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c
index f4e45cc..e9553db 100644
--- a/net/batman-adv/fragmentation.c
+++ b/net/batman-adv/fragmentation.c
@@ -17,6 +17,7 @@
#include <linux/lockdep.h>
#include <linux/minmax.h>
#include <linux/netdevice.h>
+#include <linux/overflow.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
@@ -80,9 +81,9 @@ void batadv_frag_purge_orig(struct batadv_orig_node *orig_node,
*
* Return: the maximum size of payload that can be fragmented.
*/
-static int batadv_frag_size_limit(void)
+static size_t batadv_frag_size_limit(void)
{
- int limit = BATADV_FRAG_MAX_FRAG_SIZE;
+ size_t limit = BATADV_FRAG_MAX_FRAG_SIZE;
limit -= sizeof(struct batadv_frag_packet);
limit *= BATADV_FRAG_MAX_FRAGMENTS;
@@ -143,7 +144,9 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
struct batadv_frag_packet *frag_packet;
u8 bucket;
u16 seqno, hdr_size = sizeof(struct batadv_frag_packet);
+ bool overflow = false;
bool ret = false;
+ size_t data_len;
/* Linearize packet to avoid linearizing 16 packets in a row when doing
* the later merge. Non-linear merge should be added to remove this
@@ -153,6 +156,7 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
goto err;
frag_packet = (struct batadv_frag_packet *)skb->data;
+ data_len = skb->len - hdr_size;
seqno = ntohs(frag_packet->seqno);
bucket = seqno % BATADV_FRAG_BUFFER_COUNT;
@@ -171,7 +175,7 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
spin_lock_bh(&chain->lock);
if (batadv_frag_init_chain(chain, seqno)) {
hlist_add_head(&frag_entry_new->list, &chain->fragment_list);
- chain->size = skb->len - hdr_size;
+ chain->size = data_len;
chain->timestamp = jiffies;
chain->total_size = ntohs(frag_packet->total_size);
ret = true;
@@ -188,7 +192,11 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
if (frag_entry_curr->no < frag_entry_new->no) {
hlist_add_before(&frag_entry_new->list,
&frag_entry_curr->list);
- chain->size += skb->len - hdr_size;
+
+ if (check_add_overflow(chain->size, data_len,
+ &chain->size))
+ overflow = true;
+
chain->timestamp = jiffies;
ret = true;
goto out;
@@ -201,13 +209,16 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
/* Reached the end of the list, so insert after 'frag_entry_last'. */
if (likely(frag_entry_last)) {
hlist_add_behind(&frag_entry_new->list, &frag_entry_last->list);
- chain->size += skb->len - hdr_size;
+
+ if (check_add_overflow(chain->size, data_len, &chain->size))
+ overflow = true;
+
chain->timestamp = jiffies;
ret = true;
}
out:
- if (chain->size > batadv_frag_size_limit() ||
+ if (overflow || chain->size > batadv_frag_size_limit() ||
chain->total_size != ntohs(frag_packet->total_size) ||
chain->total_size > batadv_frag_size_limit()) {
/* Clear chain if total size of either the list or the packet
@@ -294,6 +305,31 @@ batadv_frag_merge_packets(struct hlist_head *chain)
}
/**
+ * batadv_skb_is_frag() - check if newly merged skb contains unicast fragment
+ * @skb: newly merged skb
+ *
+ * Return: if newly merged skb is of type BATADV_UNICAST_FRAG
+ */
+static bool batadv_skb_is_frag(struct sk_buff *skb)
+{
+ struct batadv_ogm_packet *batadv_ogm_packet;
+
+ /* packet should hold at least type and version */
+ if (unlikely(!pskb_may_pull(skb, 2)))
+ return false;
+
+ batadv_ogm_packet = (struct batadv_ogm_packet *)skb->data;
+
+ if (batadv_ogm_packet->version != BATADV_COMPAT_VERSION)
+ return false;
+
+ if (batadv_ogm_packet->packet_type != BATADV_UNICAST_FRAG)
+ return false;
+
+ return true;
+}
+
+/**
* batadv_frag_skb_buffer() - buffer fragment for later merge
* @skb: skb to buffer
* @orig_node_src: originator that the skb is received from
@@ -326,6 +362,16 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb,
if (!skb_out)
goto out_err;
+ /* fragment in fragment is not allowed. otherwise it is possible
+ * to exhaust the stack when receiving a matryoshka-style
+ * "fragments in a fragment packet"
+ */
+ if (batadv_skb_is_frag(skb_out)) {
+ kfree_skb(skb_out);
+ skb_out = NULL;
+ goto out_err;
+ }
+
out:
ret = true;
out_err:
diff --git a/net/batman-adv/gateway_client.c b/net/batman-adv/gateway_client.c
index 51e9c081..a9d0346 100644
--- a/net/batman-adv/gateway_client.c
+++ b/net/batman-adv/gateway_client.c
@@ -478,10 +478,14 @@ void batadv_gw_node_delete(struct batadv_priv *bat_priv,
*/
void batadv_gw_node_free(struct batadv_priv *bat_priv)
{
+ struct batadv_gw_node *curr_gw;
struct batadv_gw_node *gw_node;
struct hlist_node *node_tmp;
spin_lock_bh(&bat_priv->gw.list_lock);
+ curr_gw = rcu_replace_pointer(bat_priv->gw.curr_gw, NULL, true);
+ batadv_gw_node_put(curr_gw);
+
hlist_for_each_entry_safe(gw_node, node_tmp,
&bat_priv->gw.gateway_list, list) {
hlist_del_init_rcu(&gw_node->list);
diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c
index 3a35aad..a4d33ee 100644
--- a/net/batman-adv/main.c
+++ b/net/batman-adv/main.c
@@ -249,6 +249,7 @@ void batadv_mesh_free(struct net_device *mesh_iface)
atomic_set(&bat_priv->mesh_state, BATADV_MESH_DEACTIVATING);
batadv_purge_outstanding_packets(bat_priv, NULL);
+ batadv_tp_stop_all(bat_priv);
batadv_gw_node_free(bat_priv);
diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c
index 56ca1c1..e7aa45b 100644
--- a/net/batman-adv/mesh-interface.c
+++ b/net/batman-adv/mesh-interface.c
@@ -787,6 +787,7 @@ static int batadv_meshif_init_late(struct net_device *dev)
atomic_set(&bat_priv->tt.ogm_append_cnt, 0);
#ifdef CONFIG_BATMAN_ADV_BLA
atomic_set(&bat_priv->bla.num_requests, 0);
+ spin_lock_init(&bat_priv->bla.num_requests_lock);
#endif
atomic_set(&bat_priv->tp_num, 0);
diff --git a/net/batman-adv/originator.c b/net/batman-adv/originator.c
index b3468cc..ad4921b 100644
--- a/net/batman-adv/originator.c
+++ b/net/batman-adv/originator.c
@@ -835,8 +835,6 @@ static void batadv_orig_node_free_rcu(struct rcu_head *rcu)
orig_node = container_of(rcu, struct batadv_orig_node, rcu);
- batadv_mcast_purge_orig(orig_node);
-
batadv_frag_purge_orig(orig_node, NULL);
kfree(orig_node->tt_buff);
@@ -887,6 +885,8 @@ void batadv_orig_node_release(struct kref *ref)
}
spin_unlock_bh(&orig_node->vlan_list_lock);
+ batadv_mcast_purge_orig(orig_node);
+
call_rcu(&orig_node->rcu, batadv_orig_node_free_rcu);
}
diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c
index 2e42f6b..0fc4ca7 100644
--- a/net/batman-adv/tp_meter.c
+++ b/net/batman-adv/tp_meter.c
@@ -8,10 +8,12 @@
#include "main.h"
#include <linux/atomic.h>
+#include <linux/bug.h>
#include <linux/build_bug.h>
#include <linux/byteorder/generic.h>
#include <linux/cache.h>
#include <linux/compiler.h>
+#include <linux/completion.h>
#include <linux/container_of.h>
#include <linux/err.h>
#include <linux/etherdevice.h>
@@ -253,6 +255,7 @@ static void batadv_tp_batctl_error_notify(enum batadv_tp_meter_reason reason,
* batadv_tp_list_find() - find a tp_vars object in the global list
* @bat_priv: the bat priv with all the mesh interface information
* @dst: the other endpoint MAC address to look for
+ * @role: role of the session
*
* Look for a tp_vars object matching dst as end_point and return it after
* having increment the refcounter. Return NULL is not found
@@ -260,7 +263,8 @@ static void batadv_tp_batctl_error_notify(enum batadv_tp_meter_reason reason,
* Return: matching tp_vars or NULL when no tp_vars with @dst was found
*/
static struct batadv_tp_vars *batadv_tp_list_find(struct batadv_priv *bat_priv,
- const u8 *dst)
+ const u8 *dst,
+ enum batadv_tp_meter_role role)
{
struct batadv_tp_vars *pos, *tp_vars = NULL;
@@ -269,6 +273,9 @@ static struct batadv_tp_vars *batadv_tp_list_find(struct batadv_priv *bat_priv,
if (!batadv_compare_eth(pos->other_end, dst))
continue;
+ if (pos->role != role)
+ continue;
+
/* most of the time this function is invoked during the normal
* process..it makes sens to pay more when the session is
* finished and to speed the process up during the measurement
@@ -285,11 +292,32 @@ static struct batadv_tp_vars *batadv_tp_list_find(struct batadv_priv *bat_priv,
}
/**
+ * batadv_tp_list_active() - check if session from/to destination is ongoing
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @dst: the other endpoint MAC address to look for
+ *
+ * Return: if matching session with @dst was found
+ */
+static bool batadv_tp_list_active(struct batadv_priv *bat_priv, const u8 *dst)
+ __must_hold(&bat_priv->tp_list_lock)
+{
+ struct batadv_tp_vars *tp_vars;
+
+ hlist_for_each_entry_rcu(tp_vars, &bat_priv->tp_list, list) {
+ if (batadv_compare_eth(tp_vars->other_end, dst))
+ return true;
+ }
+
+ return false;
+}
+
+/**
* batadv_tp_list_find_session() - find tp_vars session object in the global
* list
* @bat_priv: the bat priv with all the mesh interface information
* @dst: the other endpoint MAC address to look for
* @session: session identifier
+ * @role: role of the session
*
* Look for a tp_vars object matching dst as end_point, session as tp meter
* session and return it after having increment the refcounter. Return NULL
@@ -299,7 +327,7 @@ static struct batadv_tp_vars *batadv_tp_list_find(struct batadv_priv *bat_priv,
*/
static struct batadv_tp_vars *
batadv_tp_list_find_session(struct batadv_priv *bat_priv, const u8 *dst,
- const u8 *session)
+ const u8 *session, enum batadv_tp_meter_role role)
{
struct batadv_tp_vars *pos, *tp_vars = NULL;
@@ -311,6 +339,9 @@ batadv_tp_list_find_session(struct batadv_priv *bat_priv, const u8 *dst,
if (memcmp(pos->session, session, sizeof(pos->session)) != 0)
continue;
+ if (pos->role != role)
+ continue;
+
/* most of the time this function is invoked during the normal
* process..it makes sense to pay more when the session is
* finished and to speed the process up during the measurement
@@ -365,32 +396,41 @@ static void batadv_tp_vars_put(struct batadv_tp_vars *tp_vars)
}
/**
- * batadv_tp_sender_cleanup() - cleanup sender data and drop and timer
- * @bat_priv: the bat priv with all the mesh interface information
- * @tp_vars: the private data of the current TP meter session to cleanup
+ * batadv_tp_list_detach() - remove tp session from mesh session list once
+ * @tp_vars: the private data of the current TP meter session
*/
-static void batadv_tp_sender_cleanup(struct batadv_priv *bat_priv,
- struct batadv_tp_vars *tp_vars)
+static void batadv_tp_list_detach(struct batadv_tp_vars *tp_vars)
{
- cancel_delayed_work(&tp_vars->finish_work);
+ bool detached = false;
spin_lock_bh(&tp_vars->bat_priv->tp_list_lock);
- hlist_del_rcu(&tp_vars->list);
+ if (!hlist_unhashed(&tp_vars->list)) {
+ hlist_del_init_rcu(&tp_vars->list);
+ detached = true;
+ }
spin_unlock_bh(&tp_vars->bat_priv->tp_list_lock);
- /* drop list reference */
- batadv_tp_vars_put(tp_vars);
+ if (!detached)
+ return;
atomic_dec(&tp_vars->bat_priv->tp_num);
+ /* drop list reference */
+ batadv_tp_vars_put(tp_vars);
+}
+
+/**
+ * batadv_tp_sender_cleanup() - cleanup sender data and drop and timer
+ * @tp_vars: the private data of the current TP meter session to cleanup
+ */
+static void batadv_tp_sender_cleanup(struct batadv_tp_vars *tp_vars)
+{
+ cancel_delayed_work_sync(&tp_vars->finish_work);
+
+ batadv_tp_list_detach(tp_vars);
+
/* kill the timer and remove its reference */
- timer_delete_sync(&tp_vars->timer);
- /* the worker might have rearmed itself therefore we kill it again. Note
- * that if the worker should run again before invoking the following
- * timer_delete(), it would not re-arm itself once again because the status
- * is OFF now
- */
- timer_delete(&tp_vars->timer);
+ timer_shutdown_sync(&tp_vars->timer);
batadv_tp_vars_put(tp_vars);
}
@@ -402,11 +442,14 @@ static void batadv_tp_sender_cleanup(struct batadv_priv *bat_priv,
static void batadv_tp_sender_end(struct batadv_priv *bat_priv,
struct batadv_tp_vars *tp_vars)
{
+ enum batadv_tp_meter_reason reason;
u32 session_cookie;
+ reason = atomic_read(&tp_vars->send_result);
+
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Test towards %pM finished..shutting down (reason=%d)\n",
- tp_vars->other_end, tp_vars->reason);
+ tp_vars->other_end, reason);
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Last timing stats: SRTT=%ums RTTVAR=%ums RTO=%ums\n",
@@ -419,7 +462,7 @@ static void batadv_tp_sender_end(struct batadv_priv *bat_priv,
session_cookie = batadv_tp_session_cookie(tp_vars->session,
tp_vars->icmp_uid);
- batadv_tp_batctl_notify(tp_vars->reason,
+ batadv_tp_batctl_notify(reason,
tp_vars->other_end,
bat_priv,
tp_vars->start_time,
@@ -435,10 +478,18 @@ static void batadv_tp_sender_end(struct batadv_priv *bat_priv,
static void batadv_tp_sender_shutdown(struct batadv_tp_vars *tp_vars,
enum batadv_tp_meter_reason reason)
{
- if (!atomic_dec_and_test(&tp_vars->sending))
- return;
+ atomic_cmpxchg(&tp_vars->send_result, 0, reason);
+}
- tp_vars->reason = reason;
+/**
+ * batadv_tp_sender_stopped() - check if tp session was stopped with reason
+ * @tp_vars: the private data of the current TP meter session
+ *
+ * Return: whether stop reason was found
+ */
+static bool batadv_tp_sender_stopped(struct batadv_tp_vars *tp_vars)
+{
+ return atomic_read(&tp_vars->send_result) != 0;
}
/**
@@ -468,7 +519,7 @@ static void batadv_tp_reset_sender_timer(struct batadv_tp_vars *tp_vars)
/* most of the time this function is invoked while normal packet
* reception...
*/
- if (unlikely(atomic_read(&tp_vars->sending) == 0))
+ if (unlikely(batadv_tp_sender_stopped(tp_vars)))
/* timer ref will be dropped in batadv_tp_sender_cleanup */
return;
@@ -488,7 +539,7 @@ static void batadv_tp_sender_timeout(struct timer_list *t)
struct batadv_tp_vars *tp_vars = timer_container_of(tp_vars, t, timer);
struct batadv_priv *bat_priv = tp_vars->bat_priv;
- if (atomic_read(&tp_vars->sending) == 0)
+ if (batadv_tp_sender_stopped(tp_vars))
return;
/* if the user waited long enough...shutdown the test */
@@ -643,11 +694,11 @@ static void batadv_tp_recv_ack(struct batadv_priv *bat_priv,
/* find the tp_vars */
tp_vars = batadv_tp_list_find_session(bat_priv, icmp->orig,
- icmp->session);
+ icmp->session, BATADV_TP_SENDER);
if (unlikely(!tp_vars))
return;
- if (unlikely(atomic_read(&tp_vars->sending) == 0))
+ if (unlikely(batadv_tp_sender_stopped(tp_vars)))
goto out;
/* old ACK? silently drop it.. */
@@ -813,21 +864,21 @@ static int batadv_tp_send(void *arg)
if (unlikely(tp_vars->role != BATADV_TP_SENDER)) {
err = BATADV_TP_REASON_DST_UNREACHABLE;
- tp_vars->reason = err;
+ batadv_tp_sender_shutdown(tp_vars, err);
goto out;
}
orig_node = batadv_orig_hash_find(bat_priv, tp_vars->other_end);
if (unlikely(!orig_node)) {
err = BATADV_TP_REASON_DST_UNREACHABLE;
- tp_vars->reason = err;
+ batadv_tp_sender_shutdown(tp_vars, err);
goto out;
}
primary_if = batadv_primary_if_get_selected(bat_priv);
if (unlikely(!primary_if)) {
err = BATADV_TP_REASON_DST_UNREACHABLE;
- tp_vars->reason = err;
+ batadv_tp_sender_shutdown(tp_vars, err);
goto out;
}
@@ -846,7 +897,7 @@ static int batadv_tp_send(void *arg)
queue_delayed_work(batadv_event_workqueue, &tp_vars->finish_work,
msecs_to_jiffies(tp_vars->test_length));
- while (atomic_read(&tp_vars->sending) != 0) {
+ while (!batadv_tp_sender_stopped(tp_vars)) {
if (unlikely(!batadv_tp_avail(tp_vars, payload_len))) {
batadv_tp_wait_available(tp_vars, payload_len);
continue;
@@ -869,8 +920,7 @@ static int batadv_tp_send(void *arg)
"Meter: %s() cannot send packets (%d)\n",
__func__, err);
/* ensure nobody else tries to stop the thread now */
- if (atomic_dec_and_test(&tp_vars->sending))
- tp_vars->reason = err;
+ batadv_tp_sender_shutdown(tp_vars, err);
break;
}
@@ -886,7 +936,8 @@ static int batadv_tp_send(void *arg)
batadv_orig_node_put(orig_node);
batadv_tp_sender_end(bat_priv, tp_vars);
- batadv_tp_sender_cleanup(bat_priv, tp_vars);
+ batadv_tp_sender_cleanup(tp_vars);
+ complete(&tp_vars->finished);
batadv_tp_vars_put(tp_vars);
@@ -918,7 +969,8 @@ static void batadv_tp_start_kthread(struct batadv_tp_vars *tp_vars)
batadv_tp_vars_put(tp_vars);
/* cleanup of failed tp meter variables */
- batadv_tp_sender_cleanup(bat_priv, tp_vars);
+ batadv_tp_sender_cleanup(tp_vars);
+ complete(&tp_vars->finished);
return;
}
@@ -947,10 +999,15 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
/* look for an already existing test towards this node */
spin_lock_bh(&bat_priv->tp_list_lock);
- tp_vars = batadv_tp_list_find(bat_priv, dst);
- if (tp_vars) {
+ if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE) {
spin_unlock_bh(&bat_priv->tp_list_lock);
- batadv_tp_vars_put(tp_vars);
+ batadv_tp_batctl_error_notify(BATADV_TP_REASON_DST_UNREACHABLE,
+ dst, bat_priv, session_cookie);
+ return;
+ }
+
+ if (batadv_tp_list_active(bat_priv, dst)) {
+ spin_unlock_bh(&bat_priv->tp_list_lock);
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Meter: test to or from the same node already ongoing, aborting\n");
batadv_tp_batctl_error_notify(BATADV_TP_REASON_ALREADY_ONGOING,
@@ -969,6 +1026,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
tp_vars = kmalloc_obj(*tp_vars, GFP_ATOMIC);
if (!tp_vars) {
+ atomic_dec(&bat_priv->tp_num);
spin_unlock_bh(&bat_priv->tp_list_lock);
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Meter: %s cannot allocate list elements\n",
@@ -982,7 +1040,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
ether_addr_copy(tp_vars->other_end, dst);
kref_init(&tp_vars->refcount);
tp_vars->role = BATADV_TP_SENDER;
- atomic_set(&tp_vars->sending, 1);
+ atomic_set(&tp_vars->send_result, 0);
memcpy(tp_vars->session, session_id, sizeof(session_id));
tp_vars->icmp_uid = icmp_uid;
@@ -1017,6 +1075,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
tp_vars->start_time = jiffies;
init_waitqueue_head(&tp_vars->more_bytes);
+ init_completion(&tp_vars->finished);
spin_lock_init(&tp_vars->unacked_lock);
INIT_LIST_HEAD(&tp_vars->unacked_list);
@@ -1069,16 +1128,16 @@ void batadv_tp_stop(struct batadv_priv *bat_priv, const u8 *dst,
if (!orig_node)
return;
- tp_vars = batadv_tp_list_find(bat_priv, orig_node->orig);
+ tp_vars = batadv_tp_list_find(bat_priv, orig_node->orig, BATADV_TP_SENDER);
if (!tp_vars) {
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Meter: trying to interrupt an already over connection\n");
- goto out;
+ goto out_put_orig_node;
}
batadv_tp_sender_shutdown(tp_vars, return_value);
batadv_tp_vars_put(tp_vars);
-out:
+out_put_orig_node:
batadv_orig_node_put(orig_node);
}
@@ -1119,14 +1178,7 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t)
"Shutting down for inactivity (more than %dms) from %pM\n",
BATADV_TP_RECV_TIMEOUT, tp_vars->other_end);
- spin_lock_bh(&tp_vars->bat_priv->tp_list_lock);
- hlist_del_rcu(&tp_vars->list);
- spin_unlock_bh(&tp_vars->bat_priv->tp_list_lock);
-
- /* drop list reference */
- batadv_tp_vars_put(tp_vars);
-
- atomic_dec(&bat_priv->tp_num);
+ batadv_tp_list_detach(tp_vars);
spin_lock_bh(&tp_vars->unacked_lock);
list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) {
@@ -1136,6 +1188,9 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t)
spin_unlock_bh(&tp_vars->unacked_lock);
/* drop reference of timer */
+ if (WARN_ON(atomic_xchg(&tp_vars->receiving, 0) != 1))
+ return;
+
batadv_tp_vars_put(tp_vars);
}
@@ -1329,11 +1384,14 @@ static struct batadv_tp_vars *
batadv_tp_init_recv(struct batadv_priv *bat_priv,
const struct batadv_icmp_tp_packet *icmp)
{
- struct batadv_tp_vars *tp_vars;
+ struct batadv_tp_vars *tp_vars = NULL;
spin_lock_bh(&bat_priv->tp_list_lock);
+ if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE)
+ goto out_unlock;
+
tp_vars = batadv_tp_list_find_session(bat_priv, icmp->orig,
- icmp->session);
+ icmp->session, BATADV_TP_RECEIVER);
if (tp_vars)
goto out_unlock;
@@ -1344,11 +1402,14 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv,
}
tp_vars = kmalloc_obj(*tp_vars, GFP_ATOMIC);
- if (!tp_vars)
+ if (!tp_vars) {
+ atomic_dec(&bat_priv->tp_num);
goto out_unlock;
+ }
ether_addr_copy(tp_vars->other_end, icmp->orig);
tp_vars->role = BATADV_TP_RECEIVER;
+ atomic_set(&tp_vars->receiving, 1);
memcpy(tp_vars->session, icmp->session, sizeof(tp_vars->session));
tp_vars->last_recv = BATADV_TP_FIRST_SEQ;
tp_vars->bat_priv = bat_priv;
@@ -1401,7 +1462,7 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv,
}
} else {
tp_vars = batadv_tp_list_find_session(bat_priv, icmp->orig,
- icmp->session);
+ icmp->session, BATADV_TP_RECEIVER);
if (!tp_vars) {
batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
"Unexpected packet from %pM!\n",
@@ -1410,13 +1471,6 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv,
}
}
- if (unlikely(tp_vars->role != BATADV_TP_RECEIVER)) {
- batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
- "Meter: dropping packet: not expected (role=%u)\n",
- tp_vars->role);
- goto out;
- }
-
tp_vars->last_recv_time = jiffies;
/* if the packet is a duplicate, it may be the case that an ACK has been
@@ -1464,6 +1518,9 @@ void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb)
{
struct batadv_icmp_tp_packet *icmp;
+ if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE)
+ goto out;
+
icmp = (struct batadv_icmp_tp_packet *)skb->data;
switch (icmp->subtype) {
@@ -1478,10 +1535,62 @@ void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb)
"Received unknown TP Metric packet type %u\n",
icmp->subtype);
}
+
+out:
consume_skb(skb);
}
/**
+ * batadv_tp_stop_all() - stop all currently running tp meter sessions
+ * @bat_priv: the bat priv with all the mesh interface information
+ */
+void batadv_tp_stop_all(struct batadv_priv *bat_priv)
+{
+ struct batadv_tp_vars *tp_vars[BATADV_TP_MAX_NUM];
+ struct batadv_tp_vars *tp_var;
+ size_t count = 0;
+ size_t i;
+
+ spin_lock_bh(&bat_priv->tp_list_lock);
+ hlist_for_each_entry(tp_var, &bat_priv->tp_list, list) {
+ if (WARN_ON_ONCE(count >= BATADV_TP_MAX_NUM))
+ break;
+
+ if (!kref_get_unless_zero(&tp_var->refcount))
+ continue;
+
+ tp_vars[count++] = tp_var;
+ }
+ spin_unlock_bh(&bat_priv->tp_list_lock);
+
+ for (i = 0; i < count; i++) {
+ tp_var = tp_vars[i];
+
+ switch (tp_var->role) {
+ case BATADV_TP_SENDER:
+ batadv_tp_sender_shutdown(tp_var,
+ BATADV_TP_REASON_CANCEL);
+ wake_up(&tp_var->more_bytes);
+ wait_for_completion(&tp_var->finished);
+ break;
+ case BATADV_TP_RECEIVER:
+ batadv_tp_list_detach(tp_var);
+ timer_shutdown_sync(&tp_var->timer);
+
+ if (atomic_xchg(&tp_var->receiving, 0) != 1)
+ break;
+
+ batadv_tp_vars_put(tp_var);
+ break;
+ }
+
+ batadv_tp_vars_put(tp_var);
+ }
+
+ synchronize_net();
+}
+
+/**
* batadv_tp_meter_init() - initialize global tp_meter structures
*/
void __init batadv_tp_meter_init(void)
diff --git a/net/batman-adv/tp_meter.h b/net/batman-adv/tp_meter.h
index f0046d3..4e97cd1 100644
--- a/net/batman-adv/tp_meter.h
+++ b/net/batman-adv/tp_meter.h
@@ -17,6 +17,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
u32 test_length, u32 *cookie);
void batadv_tp_stop(struct batadv_priv *bat_priv, const u8 *dst,
u8 return_value);
+void batadv_tp_stop_all(struct batadv_priv *bat_priv);
void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb);
#endif /* _NET_BATMAN_ADV_TP_METER_H_ */
diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c
index 05cddcf..9f6e677 100644
--- a/net/batman-adv/translation-table.c
+++ b/net/batman-adv/translation-table.c
@@ -797,24 +797,33 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node,
s32 *tt_len)
{
u16 num_vlan = 0;
- u16 num_entries = 0;
u16 tvlv_len = 0;
unsigned int change_offset;
struct batadv_tvlv_tt_vlan_data *tt_vlan;
struct batadv_orig_node_vlan *vlan;
+ u16 total_entries = 0;
u8 *tt_change_ptr;
+ int vlan_entries;
+ u16 sum_entries;
spin_lock_bh(&orig_node->vlan_list_lock);
hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
+ vlan_entries = atomic_read(&vlan->tt.num_entries);
+
+ if (check_add_overflow(vlan_entries, total_entries, &sum_entries)) {
+ *tt_len = 0;
+ goto out;
+ }
+
+ total_entries = sum_entries;
num_vlan++;
- num_entries += atomic_read(&vlan->tt.num_entries);
}
change_offset = struct_size(*tt_data, vlan_data, num_vlan);
/* if tt_len is negative, allocate the space needed by the full table */
if (*tt_len < 0)
- *tt_len = batadv_tt_len(num_entries);
+ *tt_len = batadv_tt_len(total_entries);
if (change_offset > U16_MAX || *tt_len > U16_MAX - change_offset) {
*tt_len = 0;
@@ -835,14 +844,26 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node,
(*tt_data)->num_vlan = htons(num_vlan);
tt_vlan = (*tt_data)->vlan_data;
+ num_vlan = 0;
hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
+ vlan_entries = atomic_read(&vlan->tt.num_entries);
+ if (vlan_entries < 1)
+ continue;
+
tt_vlan->vid = htons(vlan->vid);
tt_vlan->crc = htonl(vlan->tt.crc);
tt_vlan->reserved = 0;
tt_vlan++;
+ num_vlan++;
}
+ /* recalculate in case number of VLANs reduced */
+ change_offset = struct_size(*tt_data, vlan_data, num_vlan);
+ tvlv_len = *tt_len + change_offset;
+
+ (*tt_data)->num_vlan = htons(num_vlan);
+
tt_change_ptr = (u8 *)*tt_data + change_offset;
*tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;
@@ -877,21 +898,25 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
{
struct batadv_tvlv_tt_vlan_data *tt_vlan;
struct batadv_meshif_vlan *vlan;
+ size_t change_offset;
u16 num_vlan = 0;
- u16 vlan_entries = 0;
u16 total_entries = 0;
u16 tvlv_len;
u8 *tt_change_ptr;
- int change_offset;
+ int vlan_entries;
+ u16 sum_entries;
spin_lock_bh(&bat_priv->meshif_vlan_list_lock);
hlist_for_each_entry(vlan, &bat_priv->meshif_vlan_list, list) {
vlan_entries = atomic_read(&vlan->tt.num_entries);
- if (vlan_entries < 1)
- continue;
+ if (check_add_overflow(vlan_entries, total_entries, &sum_entries)) {
+ tvlv_len = 0;
+ goto out;
+ }
+
+ total_entries = sum_entries;
num_vlan++;
- total_entries += vlan_entries;
}
change_offset = struct_size(*tt_data, vlan_data, num_vlan);
@@ -900,8 +925,10 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
if (*tt_len < 0)
*tt_len = batadv_tt_len(total_entries);
- tvlv_len = *tt_len;
- tvlv_len += change_offset;
+ if (check_add_overflow(*tt_len, change_offset, &tvlv_len)) {
+ tvlv_len = 0;
+ goto out;
+ }
*tt_data = kmalloc(tvlv_len, GFP_ATOMIC);
if (!*tt_data) {
@@ -914,6 +941,7 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
(*tt_data)->num_vlan = htons(num_vlan);
tt_vlan = (*tt_data)->vlan_data;
+ num_vlan = 0;
hlist_for_each_entry(vlan, &bat_priv->meshif_vlan_list, list) {
vlan_entries = atomic_read(&vlan->tt.num_entries);
if (vlan_entries < 1)
@@ -924,8 +952,15 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
tt_vlan->reserved = 0;
tt_vlan++;
+ num_vlan++;
}
+ /* recalculate in case number of VLANs reduced */
+ change_offset = struct_size(*tt_data, vlan_data, num_vlan);
+ tvlv_len = *tt_len + change_offset;
+
+ (*tt_data)->num_vlan = htons(num_vlan);
+
tt_change_ptr = (u8 *)*tt_data + change_offset;
*tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;
diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c
index 8129a3f9..cc6ac58 100644
--- a/net/batman-adv/tvlv.c
+++ b/net/batman-adv/tvlv.c
@@ -8,10 +8,12 @@
#include <linux/byteorder/generic.h>
#include <linux/container_of.h>
+#include <linux/errno.h>
#include <linux/etherdevice.h>
#include <linux/gfp.h>
#include <linux/if_ether.h>
#include <linux/kref.h>
+#include <linux/limits.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/netdevice.h>
@@ -159,10 +161,10 @@ batadv_tvlv_container_get(struct batadv_priv *bat_priv, u8 type, u8 version)
*
* Return: size of all currently registered tvlv containers in bytes.
*/
-static u16 batadv_tvlv_container_list_size(struct batadv_priv *bat_priv)
+static size_t batadv_tvlv_container_list_size(struct batadv_priv *bat_priv)
{
struct batadv_tvlv_container *tvlv;
- u16 tvlv_len = 0;
+ size_t tvlv_len = 0;
lockdep_assert_held(&bat_priv->tvlv.container_list_lock);
@@ -306,26 +308,35 @@ static bool batadv_tvlv_realloc_packet_buff(unsigned char **packet_buff,
* The ogm packet might be enlarged or shrunk depending on the current size
* and the size of the to-be-appended tvlv containers.
*
- * Return: size of all appended tvlv containers in bytes.
+ * Return: size of all appended tvlv containers in bytes (max U16_MAX), negative
+ * if operation failed
*/
-u16 batadv_tvlv_container_ogm_append(struct batadv_priv *bat_priv,
+int batadv_tvlv_container_ogm_append(struct batadv_priv *bat_priv,
unsigned char **packet_buff,
int *packet_buff_len, int packet_min_len)
{
struct batadv_tvlv_container *tvlv;
struct batadv_tvlv_hdr *tvlv_hdr;
- u16 tvlv_value_len;
+ size_t tvlv_value_len;
void *tvlv_value;
+ int tvlv_len_ret;
bool ret;
spin_lock_bh(&bat_priv->tvlv.container_list_lock);
tvlv_value_len = batadv_tvlv_container_list_size(bat_priv);
+ if (tvlv_value_len > U16_MAX) {
+ tvlv_len_ret = -E2BIG;
+ goto end;
+ }
ret = batadv_tvlv_realloc_packet_buff(packet_buff, packet_buff_len,
packet_min_len, tvlv_value_len);
-
- if (!ret)
+ if (!ret) {
+ tvlv_len_ret = -ENOMEM;
goto end;
+ }
+
+ tvlv_len_ret = tvlv_value_len;
if (!tvlv_value_len)
goto end;
@@ -344,7 +355,8 @@ u16 batadv_tvlv_container_ogm_append(struct batadv_priv *bat_priv,
end:
spin_unlock_bh(&bat_priv->tvlv.container_list_lock);
- return tvlv_value_len;
+
+ return tvlv_len_ret;
}
/**
diff --git a/net/batman-adv/tvlv.h b/net/batman-adv/tvlv.h
index e569723..f96f6b3 100644
--- a/net/batman-adv/tvlv.h
+++ b/net/batman-adv/tvlv.h
@@ -16,7 +16,7 @@
void batadv_tvlv_container_register(struct batadv_priv *bat_priv,
u8 type, u8 version,
void *tvlv_value, u16 tvlv_value_len);
-u16 batadv_tvlv_container_ogm_append(struct batadv_priv *bat_priv,
+int batadv_tvlv_container_ogm_append(struct batadv_priv *bat_priv,
unsigned char **packet_buff,
int *packet_buff_len, int packet_min_len);
void batadv_tvlv_ogm_receive(struct batadv_priv *bat_priv,
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 8fc5fe0..a01ee46 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -14,6 +14,7 @@
#include <linux/average.h>
#include <linux/bitops.h>
#include <linux/compiler.h>
+#include <linux/completion.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/kref.h>
@@ -82,6 +83,9 @@ struct batadv_hard_iface_bat_iv {
/** @ogm_seqno: OGM sequence number - used to identify each OGM */
atomic_t ogm_seqno;
+ /** @reschedule_work: recover OGM schedule after schedule error */
+ struct delayed_work reschedule_work;
+
/** @ogm_buff_mutex: lock protecting ogm_buff and ogm_buff_len */
struct mutex ogm_buff_mutex;
};
@@ -300,7 +304,7 @@ struct batadv_frag_table_entry {
u16 seqno;
/** @size: accumulated size of packets in list */
- u16 size;
+ size_t size;
/** @total_size: expected size of the assembled packet */
u16 total_size;
@@ -451,7 +455,7 @@ struct batadv_orig_node {
* @tt_buff_len: length of the last tt changeset this node received
* from the orig node
*/
- s16 tt_buff_len;
+ u16 tt_buff_len;
/** @tt_buff_lock: lock that protects tt_buff and tt_buff_len */
spinlock_t tt_buff_lock;
@@ -992,7 +996,7 @@ struct batadv_priv_tt {
* @last_changeset_len: length of last tt changeset this host has
* generated
*/
- s16 last_changeset_len;
+ u16 last_changeset_len;
/**
* @last_changeset_lock: lock protecting last_changeset &
@@ -1023,6 +1027,12 @@ struct batadv_priv_bla {
atomic_t num_requests;
/**
+ * @num_requests_lock: locks update num_requests +
+ * batadv_backbone_gw::state + batadv_backbone_gw::wait_periods update
+ */
+ spinlock_t num_requests_lock;
+
+ /**
* @claim_hash: hash table containing mesh nodes this host has claimed
*/
struct batadv_hashtable *claim_hash;
@@ -1319,15 +1329,21 @@ struct batadv_tp_vars {
/** @role: receiver/sender modi */
enum batadv_tp_meter_role role;
- /** @sending: sending binary semaphore: 1 if sending, 0 is not */
- atomic_t sending;
+ /**
+ * @send_result: 0 when sending is ongoing and otherwise
+ * enum batadv_tp_meter_reason
+ */
+ atomic_t send_result;
- /** @reason: reason for a stopped session */
- enum batadv_tp_meter_reason reason;
+ /** @receiving: receiving binary semaphore: 1 if receiving, 0 is not */
+ atomic_t receiving;
/** @finish_work: work item for the finishing procedure */
struct delayed_work finish_work;
+ /** @finished: completion signaled when a sender thread exits */
+ struct completion finished;
+
/** @test_length: test length in milliseconds */
u32 test_length;
@@ -1662,6 +1678,27 @@ struct batadv_priv {
#ifdef CONFIG_BATMAN_ADV_BLA
+enum batadv_bla_backbone_gw_state {
+ /**
+ * @BATADV_BLA_BACKBONE_GW_STOPPED: backbone gw is being removed
+ * and it must not longer work on requests
+ */
+ BATADV_BLA_BACKBONE_GW_STOPPED,
+
+ /**
+ * @BATADV_BLA_BACKBONE_GW_UNSYNCED: backbone was detected out
+ * of sync and a request was send. No traffic is forwarded until the
+ * situation is resolved
+ */
+ BATADV_BLA_BACKBONE_GW_UNSYNCED,
+
+ /**
+ * @BATADV_BLA_BACKBONE_GW_SYNCED: backbone is consider to be in
+ * sync. traffic can be forwarded
+ */
+ BATADV_BLA_BACKBONE_GW_SYNCED,
+};
+
/**
* struct batadv_bla_backbone_gw - batman-adv gateway bridged into the LAN
*/
@@ -1687,16 +1724,12 @@ struct batadv_bla_backbone_gw {
/**
* @wait_periods: grace time for bridge forward delays and bla group
* forming at bootup phase - no bcast traffic is formwared until it has
- * elapsed
+ * elapsed. Must only be access with num_requests_lock.
*/
- atomic_t wait_periods;
+ u8 wait_periods;
- /**
- * @request_sent: if this bool is set to true we are out of sync with
- * this backbone gateway - no bcast traffic is formwared until the
- * situation was resolved
- */
- atomic_t request_sent;
+ /** @state: sync state. Must only be access with num_requests_lock. */
+ enum batadv_bla_backbone_gw_state state;
/** @crc: crc16 checksum over all claims */
u16 crc;
diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c
index 2f03b78..960a19b 100644
--- a/net/bluetooth/6lowpan.c
+++ b/net/bluetooth/6lowpan.c
@@ -486,6 +486,8 @@ static int send_mcast_pkt(struct sk_buff *skb, struct net_device *netdev)
int ret;
local_skb = skb_clone(skb, GFP_ATOMIC);
+ if (!local_skb)
+ continue;
BT_DBG("xmit %s to %pMR type %u IP %pI6c chan %p",
netdev->name,
diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c
index 33d053d..1a6aa3f 100644
--- a/net/bluetooth/af_bluetooth.c
+++ b/net/bluetooth/af_bluetooth.c
@@ -154,6 +154,7 @@ struct sock *bt_sock_alloc(struct net *net, struct socket *sock,
sock_init_data(sock, sk);
INIT_LIST_HEAD(&bt_sk(sk)->accept_q);
+ spin_lock_init(&bt_sk(sk)->accept_q_lock);
sock_reset_flag(sk, SOCK_ZAPPED);
@@ -214,6 +215,7 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
{
const struct cred *old_cred;
struct pid *old_pid;
+ struct bt_sock *par = bt_sk(parent);
BT_DBG("parent %p, sk %p", parent, sk);
@@ -224,9 +226,13 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
else
lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
- list_add_tail(&bt_sk(sk)->accept_q, &bt_sk(parent)->accept_q);
bt_sk(sk)->parent = parent;
+ spin_lock_bh(&par->accept_q_lock);
+ list_add_tail(&bt_sk(sk)->accept_q, &par->accept_q);
+ sk_acceptq_added(parent);
+ spin_unlock_bh(&par->accept_q_lock);
+
/* Copy credentials from parent since for incoming connections the
* socket is allocated by the kernel.
*/
@@ -244,8 +250,6 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
bh_unlock_sock(sk);
else
release_sock(sk);
-
- sk_acceptq_added(parent);
}
EXPORT_SYMBOL(bt_accept_enqueue);
@@ -254,45 +258,72 @@ EXPORT_SYMBOL(bt_accept_enqueue);
*/
void bt_accept_unlink(struct sock *sk)
{
+ struct sock *parent = bt_sk(sk)->parent;
+
BT_DBG("sk %p state %d", sk, sk->sk_state);
+ spin_lock_bh(&bt_sk(parent)->accept_q_lock);
list_del_init(&bt_sk(sk)->accept_q);
- sk_acceptq_removed(bt_sk(sk)->parent);
+ sk_acceptq_removed(parent);
+ spin_unlock_bh(&bt_sk(parent)->accept_q_lock);
bt_sk(sk)->parent = NULL;
sock_put(sk);
}
EXPORT_SYMBOL(bt_accept_unlink);
+static struct sock *bt_accept_get(struct sock *parent, struct sock *sk)
+{
+ struct bt_sock *bt = bt_sk(parent);
+ struct sock *next = NULL;
+
+ /* accept_q is modified from child teardown paths too, so take a
+ * temporary reference before dropping the queue lock.
+ */
+ spin_lock_bh(&bt->accept_q_lock);
+
+ if (sk) {
+ if (bt_sk(sk)->parent != parent)
+ goto out;
+
+ if (!list_is_last(&bt_sk(sk)->accept_q, &bt->accept_q)) {
+ next = &list_next_entry(bt_sk(sk), accept_q)->sk;
+ sock_hold(next);
+ }
+ } else if (!list_empty(&bt->accept_q)) {
+ next = &list_first_entry(&bt->accept_q,
+ struct bt_sock, accept_q)->sk;
+ sock_hold(next);
+ }
+
+out:
+ spin_unlock_bh(&bt->accept_q_lock);
+ return next;
+}
+
struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock)
{
- struct bt_sock *s, *n;
- struct sock *sk;
+ struct sock *sk, *next;
BT_DBG("parent %p", parent);
restart:
- list_for_each_entry_safe(s, n, &bt_sk(parent)->accept_q, accept_q) {
- sk = (struct sock *)s;
-
+ for (sk = bt_accept_get(parent, NULL); sk; sk = next) {
/* Prevent early freeing of sk due to unlink and sock_kill */
- sock_hold(sk);
lock_sock(sk);
/* Check sk has not already been unlinked via
* bt_accept_unlink() due to serialisation caused by sk locking
*/
- if (!bt_sk(sk)->parent) {
+ if (bt_sk(sk)->parent != parent) {
BT_DBG("sk %p, already unlinked", sk);
release_sock(sk);
sock_put(sk);
- /* Restart the loop as sk is no longer in the list
- * and also avoid a potential infinite loop because
- * list_for_each_entry_safe() is not thread safe.
- */
goto restart;
}
+ next = bt_accept_get(parent, sk);
+
/* sk is safely in the parent list so reduce reference count */
sock_put(sk);
@@ -309,7 +340,19 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock)
if (newsock)
sock_graft(sk, newsock);
+ /* Hand the caller a reference taken while sk is
+ * still locked. bt_accept_unlink() just dropped
+ * the accept-queue reference; without this hold a
+ * concurrent teardown (e.g. l2cap_conn_del() ->
+ * l2cap_sock_kill()) could free sk between
+ * release_sock() and the caller using it. Every
+ * caller drops this with sock_put() when done.
+ */
+ sock_hold(sk);
+
release_sock(sk);
+ if (next)
+ sock_put(next);
return sk;
}
@@ -518,18 +561,28 @@ EXPORT_SYMBOL(bt_sock_stream_recvmsg);
static inline __poll_t bt_accept_poll(struct sock *parent)
{
- struct bt_sock *s, *n;
+ struct bt_sock *bt = bt_sk(parent);
+ struct bt_sock *s;
struct sock *sk;
+ __poll_t mask = 0;
- list_for_each_entry_safe(s, n, &bt_sk(parent)->accept_q, accept_q) {
+ spin_lock_bh(&bt->accept_q_lock);
+ list_for_each_entry(s, &bt->accept_q, accept_q) {
+ int state;
+
sk = (struct sock *)s;
- if (sk->sk_state == BT_CONNECTED ||
- (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags) &&
- sk->sk_state == BT_CONNECT2))
- return EPOLLIN | EPOLLRDNORM;
- }
+ state = READ_ONCE(sk->sk_state);
- return 0;
+ if (state == BT_CONNECTED ||
+ (test_bit(BT_SK_DEFER_SETUP, &bt->flags) &&
+ state == BT_CONNECT2)) {
+ mask = EPOLLIN | EPOLLRDNORM;
+ break;
+ }
+ }
+ spin_unlock_bh(&bt->accept_q_lock);
+
+ return mask;
}
__poll_t bt_sock_poll(struct file *file, struct socket *sock,
diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c
index d44987d..5c5f53f 100644
--- a/net/bluetooth/bnep/core.c
+++ b/net/bluetooth/bnep/core.c
@@ -206,14 +206,11 @@ static int bnep_ctrl_set_mcfilter(struct bnep_session *s, u8 *data, int len)
return 0;
}
-static int bnep_rx_control(struct bnep_session *s, void *data, int len)
+static int bnep_rx_control_cmd(struct bnep_session *s, u8 cmd, void *data,
+ int len)
{
- u8 cmd = *(u8 *)data;
int err = 0;
- data++;
- len--;
-
switch (cmd) {
case BNEP_CMD_NOT_UNDERSTOOD:
case BNEP_SETUP_CONN_RSP:
@@ -254,6 +251,14 @@ static int bnep_rx_control(struct bnep_session *s, void *data, int len)
return err;
}
+static int bnep_rx_control(struct bnep_session *s, void *data, int len)
+{
+ if (len < 1)
+ return -EILSEQ;
+
+ return bnep_rx_control_cmd(s, *(u8 *)data, data + 1, len - 1);
+}
+
static int bnep_rx_extension(struct bnep_session *s, struct sk_buff *skb)
{
struct bnep_ext_hdr *h;
@@ -299,19 +304,26 @@ static int bnep_rx_frame(struct bnep_session *s, struct sk_buff *skb)
{
struct net_device *dev = s->dev;
struct sk_buff *nskb;
+ u8 *data;
u8 type, ctrl_type;
dev->stats.rx_bytes += skb->len;
- type = *(u8 *) skb->data;
- skb_pull(skb, 1);
- ctrl_type = *(u8 *)skb->data;
+ data = skb_pull_data(skb, sizeof(type));
+ if (!data)
+ goto badframe;
+ type = *data;
if ((type & BNEP_TYPE_MASK) >= sizeof(__bnep_rx_hlen))
goto badframe;
if ((type & BNEP_TYPE_MASK) == BNEP_CONTROL) {
- if (bnep_rx_control(s, skb->data, skb->len) < 0) {
+ data = skb_pull_data(skb, sizeof(ctrl_type));
+ if (!data)
+ goto badframe;
+ ctrl_type = *data;
+
+ if (bnep_rx_control_cmd(s, ctrl_type, skb->data, skb->len) < 0) {
dev->stats.tx_errors++;
kfree_skb(skb);
return 0;
@@ -324,15 +336,25 @@ static int bnep_rx_frame(struct bnep_session *s, struct sk_buff *skb)
/* Verify and pull ctrl message since it's already processed */
switch (ctrl_type) {
- case BNEP_SETUP_CONN_REQ:
- /* Pull: ctrl type (1 b), len (1 b), data (len bytes) */
- if (!skb_pull(skb, 2 + *(u8 *)(skb->data + 1) * 2))
+ case BNEP_SETUP_CONN_REQ: {
+ u8 uuid_size;
+
+ /* Pull uuid_size and the dst/src service UUIDs. */
+ data = skb_pull_data(skb, sizeof(uuid_size));
+ if (!data)
+ goto badframe;
+ uuid_size = *data;
+ if (!skb_pull(skb, uuid_size + uuid_size))
goto badframe;
break;
+ }
case BNEP_FILTER_MULTI_ADDR_SET:
case BNEP_FILTER_NET_TYPE_SET:
- /* Pull: ctrl type (1 b), len (2 b), data (len bytes) */
- if (!skb_pull(skb, 3 + *(u16 *)(skb->data + 1) * 2))
+ /* Pull: len (2 b), data (len bytes) */
+ data = skb_pull_data(skb, sizeof(u16));
+ if (!data)
+ goto badframe;
+ if (!skb_pull(skb, get_unaligned_be16(data)))
goto badframe;
break;
default:
@@ -638,8 +660,8 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock)
goto failed;
}
- up_write(&bnep_session_sem);
strcpy(req->device, dev->name);
+ up_write(&bnep_session_sem);
return 0;
failed:
diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
index 3a05925..54eabaa 100644
--- a/net/bluetooth/hci_conn.c
+++ b/net/bluetooth/hci_conn.c
@@ -480,40 +480,107 @@ bool hci_setup_sync(struct hci_conn *conn, __u16 handle)
return hci_setup_sync_conn(conn, handle);
}
-u8 hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency,
- u16 to_multiplier)
+struct le_conn_update_data {
+ struct hci_conn *conn;
+ u16 min;
+ u16 max;
+ u16 latency;
+ u16 to_multiplier;
+};
+
+static int le_conn_update_sync(struct hci_dev *hdev, void *data)
{
- struct hci_dev *hdev = conn->hdev;
+ struct le_conn_update_data *d = data;
+ struct hci_conn *conn = d->conn;
struct hci_conn_params *params;
struct hci_cp_le_conn_update cp;
+ u16 timeout;
+ u8 store_hint;
+ int err;
+ /* Verify connection is still alive and read conn fields under
+ * the same lock to prevent a concurrent disconnect from freeing
+ * or reusing the connection while we build the HCI command.
+ */
+ hci_dev_lock(hdev);
+
+ if (!hci_conn_valid(hdev, conn)) {
+ hci_dev_unlock(hdev);
+ return -ECANCELED;
+ }
+
+ memset(&cp, 0, sizeof(cp));
+ cp.handle = cpu_to_le16(conn->handle);
+ cp.conn_interval_min = cpu_to_le16(d->min);
+ cp.conn_interval_max = cpu_to_le16(d->max);
+ cp.conn_latency = cpu_to_le16(d->latency);
+ cp.supervision_timeout = cpu_to_le16(d->to_multiplier);
+ cp.min_ce_len = cpu_to_le16(0x0000);
+ cp.max_ce_len = cpu_to_le16(0x0000);
+ timeout = conn->conn_timeout;
+
+ hci_dev_unlock(hdev);
+
+ err = __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CONN_UPDATE,
+ sizeof(cp), &cp,
+ HCI_EV_LE_CONN_UPDATE_COMPLETE,
+ timeout, NULL);
+ if (err)
+ return err;
+
+ /* Update stored connection parameters after the controller has
+ * confirmed the update via the LE Connection Update Complete event.
+ */
hci_dev_lock(hdev);
params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type);
if (params) {
- params->conn_min_interval = min;
- params->conn_max_interval = max;
- params->conn_latency = latency;
- params->supervision_timeout = to_multiplier;
+ params->conn_min_interval = d->min;
+ params->conn_max_interval = d->max;
+ params->conn_latency = d->latency;
+ params->supervision_timeout = d->to_multiplier;
+ store_hint = 0x01;
+ } else {
+ store_hint = 0x00;
}
hci_dev_unlock(hdev);
- memset(&cp, 0, sizeof(cp));
- cp.handle = cpu_to_le16(conn->handle);
- cp.conn_interval_min = cpu_to_le16(min);
- cp.conn_interval_max = cpu_to_le16(max);
- cp.conn_latency = cpu_to_le16(latency);
- cp.supervision_timeout = cpu_to_le16(to_multiplier);
- cp.min_ce_len = cpu_to_le16(0x0000);
- cp.max_ce_len = cpu_to_le16(0x0000);
+ mgmt_new_conn_param(hdev, &conn->dst, conn->dst_type, store_hint,
+ d->min, d->max, d->latency, d->to_multiplier);
- hci_send_cmd(hdev, HCI_OP_LE_CONN_UPDATE, sizeof(cp), &cp);
+ return 0;
+}
- if (params)
- return 0x01;
+static void le_conn_update_complete(struct hci_dev *hdev, void *data, int err)
+{
+ struct le_conn_update_data *d = data;
- return 0x00;
+ hci_conn_put(d->conn);
+ kfree(d);
+}
+
+void hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency,
+ u16 to_multiplier)
+{
+ struct le_conn_update_data *d;
+
+ d = kzalloc_obj(*d);
+ if (!d)
+ return;
+
+ hci_conn_get(conn);
+ d->conn = conn;
+ d->min = min;
+ d->max = max;
+ d->latency = latency;
+ d->to_multiplier = to_multiplier;
+
+ if (hci_cmd_sync_queue(conn->hdev, le_conn_update_sync, d,
+ le_conn_update_complete) < 0) {
+ hci_conn_put(conn);
+ kfree(d);
+ }
}
void hci_le_start_enc(struct hci_conn *conn, __le16 ediv, __le64 rand,
@@ -803,8 +870,10 @@ static int hci_le_big_terminate(struct hci_dev *hdev, struct hci_conn *conn)
d->big_sync_term = true;
}
- if (!d->pa_sync_term && !d->big_sync_term)
+ if (!d->pa_sync_term && !d->big_sync_term) {
+ kfree(d);
return 0;
+ }
ret = hci_cmd_sync_queue(hdev, big_terminate_sync, d,
terminate_big_destroy);
@@ -2130,6 +2199,9 @@ static int create_big_sync(struct hci_dev *hdev, void *data)
u32 flags = 0;
int err;
+ if (!hci_conn_valid(hdev, conn))
+ return -ECANCELED;
+
if (qos->bcast.out.phys == BIT(1))
flags |= MGMT_ADV_FLAG_SEC_2M;
@@ -2204,11 +2276,24 @@ static void create_big_complete(struct hci_dev *hdev, void *data, int err)
bt_dev_dbg(hdev, "conn %p", conn);
+ if (err == -ECANCELED)
+ goto done;
+
+ hci_dev_lock(hdev);
+
+ if (!hci_conn_valid(hdev, conn))
+ goto unlock;
+
if (err) {
bt_dev_err(hdev, "Unable to create BIG: %d", err);
hci_connect_cfm(conn, err);
hci_conn_del(conn);
}
+
+unlock:
+ hci_dev_unlock(hdev);
+done:
+ hci_conn_put(conn);
}
struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst, __u8 sid,
@@ -2336,10 +2421,11 @@ struct hci_conn *hci_connect_bis(struct hci_dev *hdev, bdaddr_t *dst,
BT_BOUND, &data);
/* Queue start periodic advertising and create BIG */
- err = hci_cmd_sync_queue(hdev, create_big_sync, conn,
+ err = hci_cmd_sync_queue(hdev, create_big_sync, hci_conn_get(conn),
create_big_complete);
if (err < 0) {
hci_conn_drop(conn);
+ hci_conn_put(conn);
return ERR_PTR(err);
}
diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
index c46c123..28d7929 100644
--- a/net/bluetooth/hci_core.c
+++ b/net/bluetooth/hci_core.c
@@ -539,46 +539,9 @@ static int hci_dev_do_reset(struct hci_dev *hdev)
hci_req_sync_lock(hdev);
- /* Drop queues */
- skb_queue_purge(&hdev->rx_q);
- skb_queue_purge(&hdev->cmd_q);
-
- /* Cancel these to avoid queueing non-chained pending work */
- hci_dev_set_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
- /* Wait for
- *
- * if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE))
- * queue_delayed_work(&hdev->{cmd,ncmd}_timer)
- *
- * inside RCU section to see the flag or complete scheduling.
- */
- synchronize_rcu();
- /* Explicitly cancel works in case scheduled after setting the flag. */
- cancel_delayed_work(&hdev->cmd_timer);
- cancel_delayed_work(&hdev->ncmd_timer);
-
- /* Avoid potential lockdep warnings from the *_flush() calls by
- * ensuring the workqueue is empty up front.
- */
- drain_workqueue(hdev->workqueue);
-
- hci_dev_lock(hdev);
- hci_inquiry_cache_flush(hdev);
- hci_conn_hash_flush(hdev);
- hci_dev_unlock(hdev);
-
- if (hdev->flush)
- hdev->flush(hdev);
-
- hci_dev_clear_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
-
- atomic_set(&hdev->cmd_cnt, 1);
- hdev->acl_cnt = 0;
- hdev->sco_cnt = 0;
- hdev->le_cnt = 0;
- hdev->iso_cnt = 0;
-
- ret = hci_reset_sync(hdev);
+ ret = hci_dev_close_sync(hdev);
+ if (!ret)
+ ret = hci_dev_open_sync(hdev);
hci_req_sync_unlock(hdev);
return ret;
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index b2ee6b6..eea2f81 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -7118,9 +7118,29 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data,
continue;
}
- if (hci_conn_set_handle(conn,
- __le16_to_cpu(ev->bis_handle[i++])))
+ if (ev->num_bis <= i) {
+ bt_dev_err(hdev,
+ "Not enough BIS handles for BIG 0x%2.2x",
+ ev->handle);
+ ev->status = HCI_ERROR_UNSPECIFIED;
+ hci_connect_cfm(conn, ev->status);
+ hci_conn_del(conn);
continue;
+ }
+
+ if (hci_conn_set_handle(conn,
+ __le16_to_cpu(ev->bis_handle[i++]))) {
+ bt_dev_err(hdev,
+ "Failed to set BIS handle for BIG 0x%2.2x",
+ ev->handle);
+ /* Force error so BIG gets terminated as not all BIS
+ * could be connected.
+ */
+ ev->status = HCI_ERROR_UNSPECIFIED;
+ hci_connect_cfm(conn, ev->status);
+ hci_conn_del(conn);
+ continue;
+ }
conn->state = BT_CONNECTED;
set_bit(HCI_CONN_BIG_CREATED, &conn->flags);
@@ -7129,7 +7149,10 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data,
hci_iso_setup_path(conn);
}
- if (!ev->status && !i)
+ /* If there is an unexpected error or if no BISes have been connected
+ * for the BIG, terminate it.
+ */
+ if (ev->status == HCI_ERROR_UNSPECIFIED || (!ev->status && !i))
/* If no BISes have been connected for the BIG,
* terminate. This is in case all bound connections
* have been closed before the BIG creation
@@ -7168,7 +7191,7 @@ static void hci_le_big_sync_established_evt(struct hci_dev *hdev, void *data,
clear_bit(HCI_CONN_CREATE_BIG_SYNC, &conn->flags);
conn->num_bis = 0;
- memset(conn->bis, 0, sizeof(conn->num_bis));
+ memset(conn->bis, 0, sizeof(conn->bis));
for (i = 0; i < ev->num_bis; i++) {
u16 handle = le16_to_cpu(ev->bis[i]);
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index fd3aacde..df23245 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -1725,6 +1725,11 @@ static int hci_adv_bcast_annoucement(struct hci_dev *hdev, struct adv_info *adv)
/* Generate Broadcast ID */
get_random_bytes(bid, sizeof(bid));
len = eir_append_service_data(ad, 0, 0x1852, bid, sizeof(bid));
+ if (adv->adv_data_len > sizeof(ad) - len) {
+ bt_dev_err(hdev, "No room for Broadcast Announcement");
+ return -EINVAL;
+ }
+
memcpy(ad + len, adv->adv_data, adv->adv_data_len);
hci_set_adv_instance_data(hdev, adv->instance, len + adv->adv_data_len,
ad, 0, NULL);
@@ -4438,6 +4443,9 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev)
events[4] |= 0x02; /* LE BIG Info Advertising Report */
}
+ if (ll_ext_feature_capable(hdev))
+ events[5] |= BIT(2);
+
if (le_cs_capable(hdev)) {
/* Channel Sounding events */
events[5] |= 0x08; /* LE CS Read Remote Supported Cap Complete event */
@@ -5298,6 +5306,12 @@ int hci_dev_close_sync(struct hci_dev *hdev)
bt_dev_dbg(hdev, "");
+ /* Set HCI_DRAIN_WORKQUEUE flag to prevent queuing work during
+ * reset/close. See hci_cmd_work() and handle_cmd_cnt_and_timer().
+ */
+ hci_dev_set_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
+ synchronize_rcu();
+
if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) {
disable_delayed_work(&hdev->power_off);
disable_delayed_work(&hdev->ncmd_timer);
@@ -5321,6 +5335,7 @@ int hci_dev_close_sync(struct hci_dev *hdev)
if (!test_and_clear_bit(HCI_UP, &hdev->flags)) {
cancel_delayed_work_sync(&hdev->cmd_timer);
+ hci_dev_clear_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
return err;
}
@@ -5383,6 +5398,10 @@ int hci_dev_close_sync(struct hci_dev *hdev)
/* Reset device */
skb_queue_purge(&hdev->cmd_q);
atomic_set(&hdev->cmd_cnt, 1);
+ hdev->acl_cnt = 0;
+ hdev->sco_cnt = 0;
+ hdev->le_cnt = 0;
+ hdev->iso_cnt = 0;
if (hci_test_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE) &&
!auto_off && !hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) {
set_bit(HCI_INIT, &hdev->flags);
@@ -5420,6 +5439,7 @@ int hci_dev_close_sync(struct hci_dev *hdev)
/* Clear flags */
hdev->flags &= BIT(HCI_RAW);
hci_dev_clear_volatile_flags(hdev);
+ hci_dev_clear_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
memset(hdev->eir, 0, sizeof(hdev->eir));
memset(hdev->dev_class, 0, sizeof(hdev->dev_class));
@@ -6696,6 +6716,7 @@ int hci_le_create_cis_sync(struct hci_dev *hdev)
DEFINE_FLEX(struct hci_cp_le_create_cis, cmd, cis, num_cis, 0x1f);
size_t aux_num_cis = 0;
struct hci_conn *conn;
+ u16 timeout = 0;
u8 cig = BT_ISO_QOS_CIG_UNSET;
/* The spec allows only one pending LE Create CIS command at a time. If
@@ -6766,6 +6787,7 @@ int hci_le_create_cis_sync(struct hci_dev *hdev)
set_bit(HCI_CONN_CREATE_CIS, &conn->flags);
cis->acl_handle = cpu_to_le16(conn->parent->handle);
cis->cis_handle = cpu_to_le16(conn->handle);
+ timeout = conn->conn_timeout;
aux_num_cis++;
if (aux_num_cis >= cmd->num_cis)
@@ -6785,7 +6807,7 @@ int hci_le_create_cis_sync(struct hci_dev *hdev)
return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CREATE_CIS,
struct_size(cmd, cis, cmd->num_cis),
cmd, HCI_EVT_LE_CIS_ESTABLISHED,
- conn->conn_timeout, NULL);
+ timeout, NULL);
}
int hci_le_remove_cig_sync(struct hci_dev *hdev, u8 handle)
@@ -7413,9 +7435,6 @@ static int hci_le_read_all_remote_features_sync(struct hci_dev *hdev,
sizeof(cp), &cp,
HCI_EVT_LE_ALL_REMOTE_FEATURES_COMPLETE,
HCI_CMD_TIMEOUT, NULL);
-
- return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_ALL_REMOTE_FEATURES,
- sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_le_read_remote_features_sync(struct hci_dev *hdev, void *data)
diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c
index 041ce9a..8957ce7 100644
--- a/net/bluetooth/hci_sysfs.c
+++ b/net/bluetooth/hci_sysfs.c
@@ -83,10 +83,12 @@ static void bt_host_release(struct device *dev)
{
struct hci_dev *hdev = to_hci_dev(dev);
- if (hci_dev_test_flag(hdev, HCI_UNREGISTER))
+ if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) {
hci_release_dev(hdev);
- else
+ } else {
+ cleanup_srcu_struct(&hdev->srcu);
kfree(hdev);
+ }
module_put(THIS_MODULE);
}
diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c
index 7bcf8c5..70344bd 100644
--- a/net/bluetooth/hidp/core.c
+++ b/net/bluetooth/hidp/core.c
@@ -179,12 +179,21 @@ static void hidp_input_report(struct hidp_session *session, struct sk_buff *skb)
{
struct input_dev *dev = session->input;
unsigned char *keys = session->keys;
- unsigned char *udata = skb->data + 1;
- signed char *sdata = skb->data + 1;
- int i, size = skb->len - 1;
+ unsigned char *udata;
+ signed char *sdata;
+ u8 *hdr;
+ int i;
- switch (skb->data[0]) {
+ hdr = skb_pull_data(skb, 1);
+ if (!hdr)
+ return;
+
+ switch (*hdr) {
case 0x01: /* Keyboard report */
+ udata = skb_pull_data(skb, 8);
+ if (!udata)
+ break;
+
for (i = 0; i < 8; i++)
input_report_key(dev, hidp_keycode[i + 224], (udata[0] >> i) & 1);
@@ -213,6 +222,10 @@ static void hidp_input_report(struct hidp_session *session, struct sk_buff *skb)
break;
case 0x02: /* Mouse report */
+ sdata = skb_pull_data(skb, 3);
+ if (!sdata)
+ break;
+
input_report_key(dev, BTN_LEFT, sdata[0] & 0x01);
input_report_key(dev, BTN_RIGHT, sdata[0] & 0x02);
input_report_key(dev, BTN_MIDDLE, sdata[0] & 0x04);
@@ -222,7 +235,7 @@ static void hidp_input_report(struct hidp_session *session, struct sk_buff *skb)
input_report_rel(dev, REL_X, sdata[1]);
input_report_rel(dev, REL_Y, sdata[2]);
- if (size > 3)
+ if (skb->len > 0)
input_report_rel(dev, REL_WHEEL, sdata[3]);
break;
}
@@ -1036,6 +1049,28 @@ static struct hidp_session *hidp_session_find(const bdaddr_t *bdaddr)
}
/*
+ * Consume session->conn: clear the member under hidp_session_sem, then
+ * l2cap_unregister_user() and l2cap_conn_put() the snapshot outside the
+ * sem. At most one caller wins; later callers see NULL and skip. The
+ * reference is the one hidp_session_new() took via l2cap_conn_get().
+ */
+static void hidp_session_unregister_conn(struct hidp_session *session)
+{
+ struct l2cap_conn *conn;
+
+ down_write(&hidp_session_sem);
+ conn = session->conn;
+ if (conn)
+ session->conn = NULL;
+ up_write(&hidp_session_sem);
+
+ if (conn) {
+ l2cap_unregister_user(conn, &session->user);
+ l2cap_conn_put(conn);
+ }
+}
+
+/*
* Start session synchronously
* This starts a session thread and waits until initialization
* is done or returns an error if it couldn't be started.
@@ -1311,8 +1346,7 @@ static int hidp_session_thread(void *arg)
* Instead, this call has the same semantics as if user-space tried to
* delete the session.
*/
- if (session->conn)
- l2cap_unregister_user(session->conn, &session->user);
+ hidp_session_unregister_conn(session);
hidp_session_put(session);
@@ -1418,7 +1452,7 @@ int hidp_connection_del(struct hidp_conndel_req *req)
HIDP_CTRL_VIRTUAL_CABLE_UNPLUG,
NULL, 0);
else
- l2cap_unregister_user(session->conn, &session->user);
+ hidp_session_unregister_conn(session);
hidp_session_put(session);
diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c
index be145e2..3abd811 100644
--- a/net/bluetooth/iso.c
+++ b/net/bluetooth/iso.c
@@ -337,16 +337,25 @@ static int iso_connect_bis(struct sock *sk)
struct iso_conn *conn;
struct hci_conn *hcon;
struct hci_dev *hdev;
+ bdaddr_t src, dst;
+ u8 src_type, bc_sid;
int err;
- BT_DBG("%pMR (SID 0x%2.2x)", &iso_pi(sk)->src, iso_pi(sk)->bc_sid);
+ lock_sock(sk);
+ bacpy(&src, &iso_pi(sk)->src);
+ bacpy(&dst, &iso_pi(sk)->dst);
+ src_type = iso_pi(sk)->src_type;
+ bc_sid = iso_pi(sk)->bc_sid;
+ release_sock(sk);
- hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src,
- iso_pi(sk)->src_type);
+ BT_DBG("%pMR (SID 0x%2.2x)", &src, bc_sid);
+
+ hdev = hci_get_route(&dst, &src, src_type);
if (!hdev)
return -EHOSTUNREACH;
hci_dev_lock(hdev);
+ lock_sock(sk);
if (!bis_capable(hdev)) {
err = -EOPNOTSUPP;
@@ -399,13 +408,9 @@ static int iso_connect_bis(struct sock *sk)
goto unlock;
}
- lock_sock(sk);
-
err = iso_chan_add(conn, sk, NULL);
- if (err) {
- release_sock(sk);
+ if (err)
goto unlock;
- }
/* Update source addr of the socket */
bacpy(&iso_pi(sk)->src, &hcon->src);
@@ -421,9 +426,8 @@ static int iso_connect_bis(struct sock *sk)
iso_sock_set_timer(sk, READ_ONCE(sk->sk_sndtimeo));
}
- release_sock(sk);
-
unlock:
+ release_sock(sk);
hci_dev_unlock(hdev);
hci_dev_put(hdev);
return err;
@@ -434,16 +438,24 @@ static int iso_connect_cis(struct sock *sk)
struct iso_conn *conn;
struct hci_conn *hcon;
struct hci_dev *hdev;
+ bdaddr_t src, dst;
+ u8 src_type;
int err;
- BT_DBG("%pMR -> %pMR", &iso_pi(sk)->src, &iso_pi(sk)->dst);
+ lock_sock(sk);
+ bacpy(&src, &iso_pi(sk)->src);
+ bacpy(&dst, &iso_pi(sk)->dst);
+ src_type = iso_pi(sk)->src_type;
+ release_sock(sk);
- hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src,
- iso_pi(sk)->src_type);
+ BT_DBG("%pMR -> %pMR", &src, &dst);
+
+ hdev = hci_get_route(&dst, &src, src_type);
if (!hdev)
return -EHOSTUNREACH;
hci_dev_lock(hdev);
+ lock_sock(sk);
if (!cis_central_capable(hdev)) {
err = -EOPNOTSUPP;
@@ -498,13 +510,9 @@ static int iso_connect_cis(struct sock *sk)
goto unlock;
}
- lock_sock(sk);
-
err = iso_chan_add(conn, sk, NULL);
- if (err) {
- release_sock(sk);
+ if (err)
goto unlock;
- }
/* Update source addr of the socket */
bacpy(&iso_pi(sk)->src, &hcon->src);
@@ -520,9 +528,8 @@ static int iso_connect_cis(struct sock *sk)
iso_sock_set_timer(sk, READ_ONCE(sk->sk_sndtimeo));
}
- release_sock(sk);
-
unlock:
+ release_sock(sk);
hci_dev_unlock(hdev);
hci_dev_put(hdev);
return err;
@@ -572,7 +579,7 @@ static void iso_recv_frame(struct iso_conn *conn, struct sk_buff *skb)
struct sock *sk;
iso_conn_lock(conn);
- sk = conn->sk;
+ sk = iso_sock_hold(conn);
iso_conn_unlock(conn);
if (!sk)
@@ -581,11 +588,15 @@ static void iso_recv_frame(struct iso_conn *conn, struct sk_buff *skb)
BT_DBG("sk %p len %d", sk, skb->len);
if (sk->sk_state != BT_CONNECTED)
- goto drop;
+ goto drop_put;
- if (!sock_queue_rcv_skb(sk, skb))
+ if (!sock_queue_rcv_skb(sk, skb)) {
+ sock_put(sk);
return;
+ }
+drop_put:
+ sock_put(sk);
drop:
kfree_skb(skb);
}
@@ -759,6 +770,8 @@ static void iso_sock_cleanup_listen(struct sock *parent)
while ((sk = bt_accept_dequeue(parent, NULL))) {
iso_sock_close(sk);
iso_sock_kill(sk);
+ /* Drop the reference handed back by bt_accept_dequeue(). */
+ sock_put(sk);
}
/* If listening socket has a hcon, properly disconnect it */
@@ -866,8 +879,8 @@ static void __iso_sock_close(struct sock *sk)
/* Must be called on unlocked socket. */
static void iso_sock_close(struct sock *sk)
{
- iso_sock_clear_timer(sk);
lock_sock(sk);
+ iso_sock_clear_timer(sk);
__iso_sock_close(sk);
release_sock(sk);
iso_sock_kill(sk);
@@ -1084,7 +1097,7 @@ static int iso_sock_rebind_bc(struct sock *sk, struct sockaddr_iso *sa,
* ordering.
*/
release_sock(sk);
- hci_dev_lock(bis->hdev);
+ hci_dev_lock(hdev);
lock_sock(sk);
if (!iso_pi(sk)->conn || iso_pi(sk)->conn->hcon != bis) {
@@ -1193,7 +1206,7 @@ static int iso_sock_connect(struct socket *sock, struct sockaddr_unsized *addr,
release_sock(sk);
- if (bacmp(&iso_pi(sk)->dst, BDADDR_ANY))
+ if (bacmp(&sa->iso_bdaddr, BDADDR_ANY))
err = iso_connect_cis(sk);
else
err = iso_connect_bis(sk);
@@ -1214,18 +1227,25 @@ static int iso_sock_connect(struct socket *sock, struct sockaddr_unsized *addr,
static int iso_listen_bis(struct sock *sk)
{
- struct hci_dev *hdev;
- int err = 0;
struct iso_conn *conn;
struct hci_conn *hcon;
+ struct hci_dev *hdev;
+ bdaddr_t src, dst;
+ u8 src_type, bc_sid;
+ int err = 0;
- BT_DBG("%pMR -> %pMR (SID 0x%2.2x)", &iso_pi(sk)->src,
- &iso_pi(sk)->dst, iso_pi(sk)->bc_sid);
+ lock_sock(sk);
+ bacpy(&src, &iso_pi(sk)->src);
+ bacpy(&dst, &iso_pi(sk)->dst);
+ src_type = iso_pi(sk)->src_type;
+ bc_sid = iso_pi(sk)->bc_sid;
+ release_sock(sk);
+
+ BT_DBG("%pMR -> %pMR (SID 0x%2.2x)", &src, &dst, bc_sid);
write_lock(&iso_sk_list.lock);
- if (__iso_get_sock_listen_by_sid(&iso_pi(sk)->src, &iso_pi(sk)->dst,
- iso_pi(sk)->bc_sid))
+ if (__iso_get_sock_listen_by_sid(&src, &dst, bc_sid))
err = -EADDRINUSE;
write_unlock(&iso_sk_list.lock);
@@ -1233,8 +1253,7 @@ static int iso_listen_bis(struct sock *sk)
if (err)
return err;
- hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src,
- iso_pi(sk)->src_type);
+ hdev = hci_get_route(&dst, &src, src_type);
if (!hdev)
return -EHOSTUNREACH;
@@ -1364,8 +1383,13 @@ static int iso_sock_accept(struct socket *sock, struct socket *newsock,
}
ch = bt_accept_dequeue(sk, newsock);
- if (ch)
+ if (ch) {
+ /* Drop the bridging ref from bt_accept_dequeue();
+ * the grafted socket keeps ch alive from here.
+ */
+ sock_put(ch);
break;
+ }
if (!timeo) {
err = -EAGAIN;
@@ -1565,9 +1589,16 @@ static void iso_conn_big_sync(struct sock *sk)
{
int err;
struct hci_dev *hdev;
+ bdaddr_t src, dst;
+ u8 src_type;
- hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src,
- iso_pi(sk)->src_type);
+ lock_sock(sk);
+ bacpy(&src, &iso_pi(sk)->src);
+ bacpy(&dst, &iso_pi(sk)->dst);
+ src_type = iso_pi(sk)->src_type;
+ release_sock(sk);
+
+ hdev = hci_get_route(&dst, &src, src_type);
if (!hdev)
return;
@@ -1592,6 +1623,7 @@ static void iso_conn_big_sync(struct sock *sk)
release_sock(sk);
hci_dev_unlock(hdev);
+ hci_dev_put(hdev);
}
static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg,
@@ -2256,8 +2288,10 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
sk = iso_get_sock(hdev, &hdev->bdaddr, bdaddr, BT_LISTEN,
iso_match_sid, ev1);
if (sk && !ev1->status) {
+ lock_sock(sk);
iso_pi(sk)->sync_handle = le16_to_cpu(ev1->handle);
iso_pi(sk)->bc_sid = ev1->sid;
+ release_sock(sk);
}
goto done;
@@ -2268,8 +2302,10 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
sk = iso_get_sock(hdev, &hdev->bdaddr, bdaddr, BT_LISTEN,
iso_match_sid_past, ev1a);
if (sk && !ev1a->status) {
+ lock_sock(sk);
iso_pi(sk)->sync_handle = le16_to_cpu(ev1a->sync_handle);
iso_pi(sk)->bc_sid = ev1a->sid;
+ release_sock(sk);
}
goto done;
@@ -2296,27 +2332,35 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
ev2);
if (sk) {
- int err;
- struct hci_conn *hcon = iso_pi(sk)->conn->hcon;
+ int err = 0;
+ bool big_sync;
+ struct hci_conn *hcon;
+ lock_sock(sk);
+
+ hcon = iso_pi(sk)->conn->hcon;
iso_pi(sk)->qos.bcast.encryption = ev2->encryption;
if (ev2->num_bis < iso_pi(sk)->bc_num_bis)
iso_pi(sk)->bc_num_bis = ev2->num_bis;
- if (!test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags) &&
- !test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) {
+ big_sync = !test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags) &&
+ !test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags);
+
+ if (big_sync)
err = hci_conn_big_create_sync(hdev, hcon,
&iso_pi(sk)->qos,
iso_pi(sk)->sync_handle,
iso_pi(sk)->bc_num_bis,
iso_pi(sk)->bc_bis);
- if (err) {
- bt_dev_err(hdev, "hci_le_big_create_sync: %d",
- err);
- sock_put(sk);
- sk = NULL;
- }
+
+ release_sock(sk);
+
+ if (big_sync && err) {
+ bt_dev_err(hdev, "hci_le_big_create_sync: %d",
+ err);
+ sock_put(sk);
+ sk = NULL;
}
}
@@ -2370,8 +2414,10 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
if (!base || base_len > BASE_MAX_LENGTH)
goto done;
+ lock_sock(sk);
memcpy(iso_pi(sk)->base, base, base_len);
iso_pi(sk)->base_len = base_len;
+ release_sock(sk);
} else {
/* This is a PA data fragment. Keep pa_data_len set to 0
* until all data has been reassembled.
@@ -2587,6 +2633,11 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags)
break;
case ISO_END:
+ if (!conn->rx_len) {
+ BT_ERR("Unexpected end frame (len %d)", skb->len);
+ goto drop;
+ }
+
skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len),
skb->len);
conn->rx_len -= skb->len;
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 77dec10..c4ccfbd 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -411,8 +411,10 @@ static void l2cap_chan_timeout(struct work_struct *work)
BT_DBG("chan %p state %s", chan, state_to_string(chan->state));
- if (!conn)
+ if (!conn) {
+ l2cap_chan_put(chan);
return;
+ }
mutex_lock(&conn->lock);
/* __set_chan_timer() calls l2cap_chan_hold(chan) while scheduling
@@ -4706,16 +4708,8 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
l2cap_send_cmd(conn, cmd->ident, L2CAP_CONN_PARAM_UPDATE_RSP,
sizeof(rsp), &rsp);
- if (!err) {
- u8 store_hint;
-
- store_hint = hci_le_conn_update(hcon, min, max, latency,
- to_multiplier);
- mgmt_new_conn_param(hcon->hdev, &hcon->dst, hcon->dst_type,
- store_hint, min, max, latency,
- to_multiplier);
-
- }
+ if (!err)
+ hci_le_conn_update(hcon, min, max, latency, to_multiplier);
return 0;
}
@@ -5268,6 +5262,7 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
cmd_len -= sizeof(*rsp);
list_for_each_entry_safe(chan, tmp, &conn->chan_l, list) {
+ struct l2cap_chan *orig;
u16 dcid;
if (chan->ident != cmd->ident ||
@@ -5289,8 +5284,10 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
BT_DBG("dcid[%d] 0x%4.4x", i, dcid);
+ orig = __l2cap_get_chan_by_dcid(conn, dcid);
+
/* Check if dcid is already in use */
- if (dcid && __l2cap_get_chan_by_dcid(conn, dcid)) {
+ if (dcid && orig) {
/* If a device receives a
* L2CAP_CREDIT_BASED_CONNECTION_RSP packet with an
* already-assigned Destination CID, then both the
@@ -5299,10 +5296,24 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
*/
l2cap_chan_del(chan, ECONNREFUSED);
l2cap_chan_unlock(chan);
- chan = __l2cap_get_chan_by_dcid(conn, dcid);
- l2cap_chan_lock(chan);
- l2cap_chan_del(chan, ECONNRESET);
- l2cap_chan_unlock(chan);
+
+ /* Check that the dcid channel mode is
+ * L2CAP_MODE_EXT_FLOWCTL since this procedure is only
+ * valid for that mode and shouldn't disconnect a dcid
+ * in other modes.
+ */
+ if (orig->mode == L2CAP_MODE_EXT_FLOWCTL) {
+ l2cap_chan_lock(orig);
+ /* Disconnect the original channel as it may be
+ * considered connected since dcid has already
+ * been assigned; don't call l2cap_chan_close
+ * directly since that could lead to
+ * l2cap_chan_del and then removing the channel
+ * from the list while we're iterating over it.
+ */
+ __set_chan_timer(orig, 0);
+ l2cap_chan_unlock(orig);
+ }
continue;
}
@@ -5428,7 +5439,7 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn,
* configured, the MPS field may be less than the current MPS
* of that channel.
*/
- if (chan[i]->remote_mps >= mps && i) {
+ if (chan[i]->remote_mps > mps && num_scid > 1) {
BT_ERR("chan %p decreased MPS %u -> %u", chan[i],
chan[i]->remote_mps, mps);
result = L2CAP_RECONF_INVALID_MPS;
@@ -5466,14 +5477,20 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn,
BT_DBG("result 0x%4.4x", result);
- if (!result)
+ if (!result) {
+ list_for_each_entry(chan, &conn->chan_l, list) {
+ if (chan->ident == cmd->ident)
+ chan->ident = 0;
+ }
return 0;
+ }
list_for_each_entry_safe(chan, tmp, &conn->chan_l, list) {
if (chan->ident != cmd->ident)
continue;
- l2cap_chan_hold(chan);
+ if (!l2cap_chan_hold_unless_zero(chan))
+ continue;
l2cap_chan_lock(chan);
l2cap_chan_del(chan, ECONNRESET);
@@ -5626,6 +5643,15 @@ static inline void l2cap_sig_send_rej(struct l2cap_conn *conn, u16 ident)
l2cap_send_cmd(conn, ident, L2CAP_COMMAND_REJ, sizeof(rej), &rej);
}
+static inline void l2cap_sig_send_mtu_rej(struct l2cap_conn *conn, u8 ident)
+{
+ struct l2cap_cmd_rej_mtu rej;
+
+ rej.reason = cpu_to_le16(L2CAP_REJ_MTU_EXCEEDED);
+ rej.max_mtu = cpu_to_le16(L2CAP_SIG_MTU);
+ l2cap_send_cmd(conn, ident, L2CAP_COMMAND_REJ, sizeof(rej), &rej);
+}
+
static inline void l2cap_sig_channel(struct l2cap_conn *conn,
struct sk_buff *skb)
{
@@ -5638,6 +5664,43 @@ static inline void l2cap_sig_channel(struct l2cap_conn *conn,
if (hcon->type != ACL_LINK)
goto drop;
+ /*
+ * Bluetooth Core v5.4, Vol 3, Part A, Section 4: the BR/EDR
+ * signaling channel has a fixed signaling MTU (MTUsig) whose
+ * minimum and default is 48 octets. Section 4.1 says that on
+ * an MTUExceeded command reject the identifier "shall match
+ * the first request command in the L2CAP packet" and that
+ * packets containing only response commands "shall be
+ * silently discarded".
+ *
+ * Linux intentionally deviates from that prescription:
+ *
+ * 1. Silently discarding desynchronizes the peer. The
+ * remote stack never learns its responses were dropped,
+ * so any state machine waiting on a paired response
+ * stalls until its own timer fires.
+ *
+ * 2. Locating "the first request command" requires walking
+ * command headers past MTUsig, i.e. processing bytes
+ * from a packet we have already decided is too large to
+ * process.
+ *
+ * Reject every over-MTUsig signaling packet with one
+ * L2CAP_REJ_MTU_EXCEEDED command reject. The reject's
+ * reason field is what tells the peer that the whole packet
+ * was discarded; the identifier value is informational, so
+ * we use the identifier from the first command header, a
+ * single fixed-offset byte read.
+ */
+ if (skb->len > L2CAP_SIG_MTU) {
+ u8 ident = skb->data[1];
+
+ BT_DBG("signaling packet exceeds MTU: %u > %u",
+ skb->len, L2CAP_SIG_MTU);
+ l2cap_sig_send_mtu_rej(conn, ident);
+ goto drop;
+ }
+
while (skb->len >= L2CAP_CMD_HDR_SIZE) {
u16 len;
@@ -7282,7 +7345,7 @@ static void l2cap_ecred_reconfigure(struct l2cap_chan *chan)
chan->ident = l2cap_get_ident(conn);
l2cap_send_cmd(conn, chan->ident, L2CAP_ECRED_RECONF_REQ,
- sizeof(pdu), &pdu);
+ struct_size(pdu, scid, 1), pdu);
}
int l2cap_chan_reconfigure(struct l2cap_chan *chan, __u16 mtu)
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 71e8c1b..c138aa4 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -349,8 +349,13 @@ static int l2cap_sock_accept(struct socket *sock, struct socket *newsock,
}
nsk = bt_accept_dequeue(sk, newsock);
- if (nsk)
+ if (nsk) {
+ /* Drop the bridging ref from bt_accept_dequeue();
+ * the grafted socket keeps nsk alive from here.
+ */
+ sock_put(nsk);
break;
+ }
if (!timeo) {
err = -EAGAIN;
@@ -1475,22 +1480,56 @@ static void l2cap_sock_cleanup_listen(struct sock *parent)
BT_DBG("parent %p state %s", parent,
state_to_string(parent->sk_state));
- /* Close not yet accepted channels */
+ /* Close not yet accepted channels.
+ *
+ * bt_accept_dequeue() now returns sk with an extra reference held
+ * (taken while sk was still locked) so a concurrent l2cap_conn_del()
+ * -> l2cap_sock_kill() cannot free sk under us.
+ *
+ * cleanup_listen() runs under the parent sk lock, so unlike
+ * l2cap_sock_shutdown() we must NOT take conn->lock here: that would
+ * establish sk_lock -> conn->lock and invert the established
+ * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock).
+ *
+ * Instead, briefly take the child sk lock to fetch and pin its chan.
+ * l2cap_conn_del() reaches the chan free only via
+ * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes
+ * the child sk lock; holding it across l2cap_chan_hold_unless_zero()
+ * therefore guarantees the chan cannot be freed while we read and
+ * pin it (hold_unless_zero() additionally skips a chan already past
+ * its last reference). We then drop the sk lock before taking
+ * chan->lock, so sk and chan locks are never held together.
+ *
+ * Since we cannot call l2cap_chan_close() without conn->lock,
+ * schedule l2cap_chan_timeout to close the channel; it already
+ * acquires conn->lock -> chan->lock in the correct order.
+ */
while ((sk = bt_accept_dequeue(parent, NULL))) {
- struct l2cap_chan *chan = l2cap_pi(sk)->chan;
+ struct l2cap_chan *chan;
+
+ lock_sock_nested(sk, L2CAP_NESTING_NORMAL);
+ chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan);
+ release_sock(sk);
+ if (!chan) {
+ /* l2cap_conn_del() already tearing this child down */
+ sock_put(sk);
+ continue;
+ }
BT_DBG("child chan %p state %s", chan,
state_to_string(chan->state));
- l2cap_chan_hold(chan);
l2cap_chan_lock(chan);
-
- __clear_chan_timer(chan);
- l2cap_chan_close(chan, ECONNRESET);
- l2cap_sock_kill(sk);
-
+ /* Since we cannot call l2cap_chan_close() without
+ * conn->lock, schedule its timer to trigger the close
+ * and cleanup of this channel.
+ */
+ if (chan->conn)
+ __set_chan_timer(chan, 0);
l2cap_chan_unlock(chan);
+
l2cap_chan_put(chan);
+ sock_put(sk);
}
}
@@ -1498,6 +1537,9 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan)
{
struct sock *sk, *parent = chan->data;
+ if (!parent)
+ return NULL;
+
lock_sock(parent);
/* Check for backlog size */
@@ -1657,6 +1699,9 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state,
{
struct sock *sk = chan->data;
+ if (!sk)
+ return;
+
sk->sk_state = state;
if (err)
@@ -1758,6 +1803,9 @@ static long l2cap_sock_get_sndtimeo_cb(struct l2cap_chan *chan)
{
struct sock *sk = chan->data;
+ if (!sk)
+ return 0;
+
return READ_ONCE(sk->sk_sndtimeo);
}
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index b05bb38..f4aa814 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -8638,6 +8638,12 @@ static bool tlv_data_is_valid(struct hci_dev *hdev, u32 adv_flags, u8 *data,
if (!cur_len)
continue;
+ /* If the current field length would exceed the total data
+ * length, then it's invalid.
+ */
+ if (i + cur_len >= len)
+ return false;
+
if (data[i + 1] == EIR_FLAGS &&
(!is_adv_data || flags_managed(adv_flags)))
return false;
@@ -8654,12 +8660,6 @@ static bool tlv_data_is_valid(struct hci_dev *hdev, u32 adv_flags, u8 *data,
if (data[i + 1] == EIR_APPEARANCE &&
appearance_managed(adv_flags))
return false;
-
- /* If the current field length would exceed the total data
- * length, then it's invalid.
- */
- if (i + cur_len >= len)
- return false;
}
return true;
@@ -9110,9 +9110,16 @@ static int add_ext_adv_data(struct sock *sk, struct hci_dev *hdev, void *data,
struct adv_info *adv_instance;
int err = 0;
struct mgmt_pending_cmd *cmd;
+ u16 expected_len;
BT_DBG("%s", hdev->name);
+ expected_len = struct_size(cp, data, cp->adv_data_len +
+ cp->scan_rsp_len);
+ if (expected_len > data_len)
+ return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_DATA,
+ MGMT_STATUS_INVALID_PARAMS);
+
hci_dev_lock(hdev);
adv_instance = hci_find_adv_instance(hdev, cp->instance);
diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c
index 611a9a9..364b938 100644
--- a/net/bluetooth/rfcomm/core.c
+++ b/net/bluetooth/rfcomm/core.c
@@ -1431,10 +1431,15 @@ static int rfcomm_apply_pn(struct rfcomm_dlc *d, int cr, struct rfcomm_pn *pn)
static int rfcomm_recv_pn(struct rfcomm_session *s, int cr, struct sk_buff *skb)
{
- struct rfcomm_pn *pn = (void *) skb->data;
+ struct rfcomm_pn *pn;
struct rfcomm_dlc *d;
- u8 dlci = pn->dlci;
+ u8 dlci;
+ pn = skb_pull_data(skb, sizeof(*pn));
+ if (!pn)
+ return -EILSEQ;
+
+ dlci = pn->dlci;
BT_DBG("session %p state %ld dlci %d", s, s->state, dlci);
if (!dlci)
@@ -1483,8 +1488,8 @@ static int rfcomm_recv_pn(struct rfcomm_session *s, int cr, struct sk_buff *skb)
static int rfcomm_recv_rpn(struct rfcomm_session *s, int cr, int len, struct sk_buff *skb)
{
- struct rfcomm_rpn *rpn = (void *) skb->data;
- u8 dlci = __get_dlci(rpn->dlci);
+ struct rfcomm_rpn *rpn;
+ u8 dlci;
u8 bit_rate = 0;
u8 data_bits = 0;
@@ -1495,15 +1500,16 @@ static int rfcomm_recv_rpn(struct rfcomm_session *s, int cr, int len, struct sk_
u8 xoff_char = 0;
u16 rpn_mask = RFCOMM_RPN_PM_ALL;
- BT_DBG("dlci %d cr %d len 0x%x bitr 0x%x line 0x%x flow 0x%x xonc 0x%x xoffc 0x%x pm 0x%x",
- dlci, cr, len, rpn->bit_rate, rpn->line_settings, rpn->flow_ctrl,
- rpn->xon_char, rpn->xoff_char, rpn->param_mask);
-
- if (!cr)
- return 0;
-
if (len == 1) {
- /* This is a request, return default (according to ETSI TS 07.10) settings */
+ rpn = skb_pull_data(skb, 1);
+ if (!rpn)
+ return -EILSEQ;
+
+ dlci = __get_dlci(rpn->dlci);
+
+ if (!cr)
+ return 0;
+
bit_rate = RFCOMM_RPN_BR_9600;
data_bits = RFCOMM_RPN_DATA_8;
stop_bits = RFCOMM_RPN_STOP_1;
@@ -1514,6 +1520,19 @@ static int rfcomm_recv_rpn(struct rfcomm_session *s, int cr, int len, struct sk_
goto rpn_out;
}
+ rpn = skb_pull_data(skb, sizeof(*rpn));
+ if (!rpn)
+ return -EILSEQ;
+
+ dlci = __get_dlci(rpn->dlci);
+
+ BT_DBG("dlci %d cr %d len 0x%x bitr 0x%x line 0x%x flow 0x%x xonc 0x%x xoffc 0x%x pm 0x%x",
+ dlci, cr, len, rpn->bit_rate, rpn->line_settings, rpn->flow_ctrl,
+ rpn->xon_char, rpn->xoff_char, rpn->param_mask);
+
+ if (!cr)
+ return 0;
+
/* Check for sane values, ignore/accept bit_rate, 8 bits, 1 stop bit,
* no parity, no flow control lines, normal XON/XOFF chars */
@@ -1589,9 +1608,14 @@ static int rfcomm_recv_rpn(struct rfcomm_session *s, int cr, int len, struct sk_
static int rfcomm_recv_rls(struct rfcomm_session *s, int cr, struct sk_buff *skb)
{
- struct rfcomm_rls *rls = (void *) skb->data;
- u8 dlci = __get_dlci(rls->dlci);
+ struct rfcomm_rls *rls;
+ u8 dlci;
+ rls = skb_pull_data(skb, sizeof(*rls));
+ if (!rls)
+ return -EILSEQ;
+
+ dlci = __get_dlci(rls->dlci);
BT_DBG("dlci %d cr %d status 0x%x", dlci, cr, rls->status);
if (!cr)
@@ -1608,10 +1632,15 @@ static int rfcomm_recv_rls(struct rfcomm_session *s, int cr, struct sk_buff *skb
static int rfcomm_recv_msc(struct rfcomm_session *s, int cr, struct sk_buff *skb)
{
- struct rfcomm_msc *msc = (void *) skb->data;
+ struct rfcomm_msc *msc;
struct rfcomm_dlc *d;
- u8 dlci = __get_dlci(msc->dlci);
+ u8 dlci;
+ msc = skb_pull_data(skb, sizeof(*msc));
+ if (!msc)
+ return -EILSEQ;
+
+ dlci = __get_dlci(msc->dlci);
BT_DBG("dlci %d cr %d v24 0x%x", dlci, cr, msc->v24_sig);
d = rfcomm_dlc_get(s, dlci);
@@ -1644,17 +1673,19 @@ static int rfcomm_recv_msc(struct rfcomm_session *s, int cr, struct sk_buff *skb
static int rfcomm_recv_mcc(struct rfcomm_session *s, struct sk_buff *skb)
{
- struct rfcomm_mcc *mcc = (void *) skb->data;
+ struct rfcomm_mcc *mcc;
u8 type, cr, len;
+ mcc = skb_pull_data(skb, sizeof(*mcc));
+ if (!mcc)
+ return -EILSEQ;
+
cr = __test_cr(mcc->type);
type = __get_mcc_type(mcc->type);
len = __get_mcc_len(mcc->len);
BT_DBG("%p type 0x%x cr %d", s, type, cr);
- skb_pull(skb, 2);
-
switch (type) {
case RFCOMM_PN:
rfcomm_recv_pn(s, cr, skb);
@@ -1715,9 +1746,12 @@ static int rfcomm_recv_data(struct rfcomm_session *s, u8 dlci, int pf, struct sk
}
if (pf && d->cfc) {
- u8 credits = *(u8 *) skb->data; skb_pull(skb, 1);
+ u8 *credits = skb_pull_data(skb, 1);
- d->tx_credits += credits;
+ if (!credits)
+ goto drop;
+
+ d->tx_credits += *credits;
if (d->tx_credits)
clear_bit(RFCOMM_TX_THROTTLED, &d->flags);
}
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index be6639c..805ed5d 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -122,7 +122,7 @@ static struct sock *__rfcomm_get_listen_sock_by_addr(u8 channel, bdaddr_t *src)
}
/* Find socket with channel and source bdaddr.
- * Returns closest match.
+ * Returns closest match with an extra reference held.
*/
static struct sock *rfcomm_get_sock_by_channel(int state, u8 channel, bdaddr_t *src)
{
@@ -136,15 +136,25 @@ static struct sock *rfcomm_get_sock_by_channel(int state, u8 channel, bdaddr_t *
if (rfcomm_pi(sk)->channel == channel) {
/* Exact match. */
- if (!bacmp(&rfcomm_pi(sk)->src, src))
+ if (!bacmp(&rfcomm_pi(sk)->src, src)) {
+ sock_hold(sk);
break;
+ }
/* Closest match */
- if (!bacmp(&rfcomm_pi(sk)->src, BDADDR_ANY))
+ if (!bacmp(&rfcomm_pi(sk)->src, BDADDR_ANY)) {
+ if (sk1)
+ sock_put(sk1);
+
sk1 = sk;
+ sock_hold(sk1);
+ }
}
}
+ if (sk && sk1)
+ sock_put(sk1);
+
read_unlock(&rfcomm_sk_list.lock);
return sk ? sk : sk1;
@@ -180,6 +190,8 @@ static void rfcomm_sock_cleanup_listen(struct sock *parent)
while ((sk = bt_accept_dequeue(parent, NULL))) {
rfcomm_sock_close(sk);
rfcomm_sock_kill(sk);
+ /* Drop the reference handed back by bt_accept_dequeue(). */
+ sock_put(sk);
}
parent->sk_state = BT_CLOSED;
@@ -497,8 +509,13 @@ static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock,
}
nsk = bt_accept_dequeue(sk, newsock);
- if (nsk)
+ if (nsk) {
+ /* Drop the bridging ref from bt_accept_dequeue();
+ * the grafted socket keeps nsk alive from here.
+ */
+ sock_put(nsk);
break;
+ }
if (!timeo) {
err = -EAGAIN;
@@ -934,6 +951,7 @@ int rfcomm_connect_ind(struct rfcomm_session *s, u8 channel, struct rfcomm_dlc *
{
struct sock *sk, *parent;
bdaddr_t src, dst;
+ bool defer_setup = false;
int result = 0;
BT_DBG("session %p channel %d", s, channel);
@@ -947,6 +965,11 @@ int rfcomm_connect_ind(struct rfcomm_session *s, u8 channel, struct rfcomm_dlc *
lock_sock(parent);
+ if (parent->sk_state != BT_LISTEN)
+ goto done;
+
+ defer_setup = test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags);
+
/* Check for backlog size */
if (sk_acceptq_is_full(parent)) {
BT_DBG("backlog full %d", parent->sk_ack_backlog);
@@ -974,9 +997,11 @@ int rfcomm_connect_ind(struct rfcomm_session *s, u8 channel, struct rfcomm_dlc *
done:
release_sock(parent);
- if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags))
+ if (defer_setup)
parent->sk_state_change(parent);
+ sock_put(parent);
+
return result;
}
diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c
index 18826d4..140869e 100644
--- a/net/bluetooth/sco.c
+++ b/net/bluetooth/sco.c
@@ -312,11 +312,21 @@ static int sco_connect(struct sock *sk)
struct sco_conn *conn;
struct hci_conn *hcon;
struct hci_dev *hdev;
+ bdaddr_t src, dst;
+ struct bt_codec codec;
+ __u16 setting;
int err, type;
- BT_DBG("%pMR -> %pMR", &sco_pi(sk)->src, &sco_pi(sk)->dst);
+ lock_sock(sk);
+ bacpy(&src, &sco_pi(sk)->src);
+ bacpy(&dst, &sco_pi(sk)->dst);
+ setting = sco_pi(sk)->setting;
+ codec = sco_pi(sk)->codec;
+ release_sock(sk);
- hdev = hci_get_route(&sco_pi(sk)->dst, &sco_pi(sk)->src, BDADDR_BREDR);
+ BT_DBG("%pMR -> %pMR", &src, &dst);
+
+ hdev = hci_get_route(&dst, &src, BDADDR_BREDR);
if (!hdev)
return -EHOSTUNREACH;
@@ -327,7 +337,7 @@ static int sco_connect(struct sock *sk)
else
type = SCO_LINK;
- switch (sco_pi(sk)->setting & SCO_AIRMODE_MASK) {
+ switch (setting & SCO_AIRMODE_MASK) {
case SCO_AIRMODE_TRANSP:
if (!lmp_transp_capable(hdev) || !lmp_esco_capable(hdev)) {
err = -EOPNOTSUPP;
@@ -336,8 +346,8 @@ static int sco_connect(struct sock *sk)
break;
}
- hcon = hci_connect_sco(hdev, type, &sco_pi(sk)->dst,
- sco_pi(sk)->setting, &sco_pi(sk)->codec,
+ hcon = hci_connect_sco(hdev, type, &dst,
+ setting, &codec,
READ_ONCE(sk->sk_sndtimeo));
if (IS_ERR(hcon)) {
err = PTR_ERR(hcon);
@@ -472,9 +482,13 @@ static struct sock *sco_get_sock_listen(bdaddr_t *src)
sk1 = sk;
}
+ sk = sk ? sk : sk1;
+ if (sk)
+ sock_hold(sk);
+
read_unlock(&sco_sk_list.lock);
- return sk ? sk : sk1;
+ return sk;
}
static void sco_sock_destruct(struct sock *sk)
@@ -498,6 +512,8 @@ static void sco_sock_cleanup_listen(struct sock *parent)
while ((sk = bt_accept_dequeue(parent, NULL))) {
sco_sock_close(sk);
sco_sock_kill(sk);
+ /* Drop the reference handed back by bt_accept_dequeue(). */
+ sock_put(sk);
}
parent->sk_state = BT_CLOSED;
@@ -515,11 +531,13 @@ static void sco_sock_kill(struct sock *sk)
BT_DBG("sk %p state %d", sk, sk->sk_state);
/* Sock is dead, so set conn->sk to NULL to avoid possible UAF */
+ lock_sock(sk);
if (sco_pi(sk)->conn) {
sco_conn_lock(sco_pi(sk)->conn);
sco_pi(sk)->conn->sk = NULL;
sco_conn_unlock(sco_pi(sk)->conn);
}
+ release_sock(sk);
/* Kill poor orphan */
bt_sock_unlink(&sco_sk_list, sk);
@@ -759,8 +777,13 @@ static int sco_sock_accept(struct socket *sock, struct socket *newsock,
}
ch = bt_accept_dequeue(sk, newsock);
- if (ch)
+ if (ch) {
+ /* Drop the bridging ref from bt_accept_dequeue();
+ * the grafted socket keeps ch alive from here.
+ */
+ sock_put(ch);
break;
+ }
if (!timeo) {
err = -EAGAIN;
@@ -1365,40 +1388,51 @@ static int sco_sock_release(struct socket *sock)
static void sco_conn_ready(struct sco_conn *conn)
{
- struct sock *parent;
- struct sock *sk = conn->sk;
+ struct sock *parent, *sk;
+
+ sco_conn_lock(conn);
+ sk = sco_sock_hold(conn);
+ sco_conn_unlock(conn);
BT_DBG("conn %p", conn);
if (sk) {
lock_sock(sk);
- sco_sock_clear_timer(sk);
- sk->sk_state = BT_CONNECTED;
- sk->sk_state_change(sk);
- release_sock(sk);
- } else {
- sco_conn_lock(conn);
- if (!conn->hcon) {
- sco_conn_unlock(conn);
- return;
+ /* conn->sk may have become NULL if racing with sk close, but
+ * due to held hdev->lock, it can't become different sk.
+ */
+ if (conn->sk) {
+ sco_sock_clear_timer(sk);
+ sk->sk_state = BT_CONNECTED;
+ sk->sk_state_change(sk);
}
+ release_sock(sk);
+ sock_put(sk);
+ } else {
+ if (!conn->hcon)
+ return;
+
+ lockdep_assert_held(&conn->hcon->hdev->lock);
+
parent = sco_get_sock_listen(&conn->hcon->src);
- if (!parent) {
- sco_conn_unlock(conn);
+ if (!parent)
return;
- }
lock_sock(parent);
+ sco_conn_lock(conn);
+
+ /* hdev->lock guarantees conn->sk == NULL still here */
+
+ if (parent->sk_state != BT_LISTEN)
+ goto release;
+
sk = sco_sock_alloc(sock_net(parent), NULL,
BTPROTO_SCO, GFP_ATOMIC, 0);
- if (!sk) {
- release_sock(parent);
- sco_conn_unlock(conn);
- return;
- }
+ if (!sk)
+ goto release;
sco_sock_init(sk, parent);
@@ -1417,9 +1451,10 @@ static void sco_conn_ready(struct sco_conn *conn)
/* Wake up parent */
parent->sk_data_ready(parent);
- release_sock(parent);
-
+release:
sco_conn_unlock(conn);
+ release_sock(parent);
+ sock_put(parent);
}
}
diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c
index 881d866..2eef4f3 100644
--- a/net/bridge/br_multicast.c
+++ b/net/bridge/br_multicast.c
@@ -4640,10 +4640,24 @@ static void br_multicast_start_querier(struct net_bridge_mcast *brmctx,
rcu_read_unlock();
}
-static void br_multicast_del_grps(struct net_bridge *br)
+static void br_multicast_enable_all_ports(struct net_bridge *br)
{
struct net_bridge_port *port;
+ if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
+ return;
+
+ list_for_each_entry(port, &br->port_list, list)
+ __br_multicast_enable_port_ctx(&port->multicast_ctx);
+}
+
+static void br_multicast_disable_all_ports(struct net_bridge *br)
+{
+ struct net_bridge_port *port;
+
+ if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
+ return;
+
list_for_each_entry(port, &br->port_list, list)
__br_multicast_disable_port_ctx(&port->multicast_ctx);
}
@@ -4651,7 +4665,6 @@ static void br_multicast_del_grps(struct net_bridge *br)
int br_multicast_toggle(struct net_bridge *br, unsigned long val,
struct netlink_ext_ack *extack)
{
- struct net_bridge_port *port;
bool change_snoopers = false;
int err = 0;
@@ -4668,7 +4681,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val,
br_opt_toggle(br, BROPT_MULTICAST_ENABLED, !!val);
if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) {
change_snoopers = true;
- br_multicast_del_grps(br);
+ br_multicast_disable_all_ports(br);
goto unlock;
}
@@ -4676,8 +4689,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val,
goto unlock;
br_multicast_open(br);
- list_for_each_entry(port, &br->port_list, list)
- __br_multicast_enable_port_ctx(&port->multicast_ctx);
+ br_multicast_enable_all_ports(br);
change_snoopers = true;
diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c
index 0ab1c94..0a394e5 100644
--- a/net/bridge/br_netfilter_hooks.c
+++ b/net/bridge/br_netfilter_hooks.c
@@ -297,7 +297,11 @@ int br_nf_pre_routing_finish_bridge(struct net *net, struct sock *sk, struct sk_
goto free_skb;
}
- neigh_hh_bridge(&neigh->hh, skb);
+ if (neigh_hh_bridge(&neigh->hh, skb)) {
+ neigh_release(neigh);
+ goto free_skb;
+ }
+
skb->dev = br_indev;
ret = br_handle_frame_finish(net, sk, skb);
diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c
index 6fd5386..b9591dd 100644
--- a/net/bridge/br_netlink.c
+++ b/net/bridge/br_netlink.c
@@ -1000,19 +1000,25 @@ static int br_setport(struct net_bridge_port *p, struct nlattr *tb[],
br_port_flags_change(p, changed_mask);
if (tb[IFLA_BRPORT_COST]) {
+ spin_lock_bh(&p->br->lock);
err = br_stp_set_path_cost(p, nla_get_u32(tb[IFLA_BRPORT_COST]));
+ spin_unlock_bh(&p->br->lock);
if (err)
return err;
}
if (tb[IFLA_BRPORT_PRIORITY]) {
+ spin_lock_bh(&p->br->lock);
err = br_stp_set_port_priority(p, nla_get_u16(tb[IFLA_BRPORT_PRIORITY]));
+ spin_unlock_bh(&p->br->lock);
if (err)
return err;
}
if (tb[IFLA_BRPORT_STATE]) {
+ spin_lock_bh(&p->br->lock);
err = br_set_port_state(p, nla_get_u8(tb[IFLA_BRPORT_STATE]));
+ spin_unlock_bh(&p->br->lock);
if (err)
return err;
}
@@ -1114,9 +1120,7 @@ int br_setlink(struct net_device *dev, struct nlmsghdr *nlh, u16 flags,
if (err)
return err;
- spin_lock_bh(&p->br->lock);
err = br_setport(p, tb, extack);
- spin_unlock_bh(&p->br->lock);
} else {
/* Binary compatibility with old RSTP */
if (nla_len(protinfo) < sizeof(u8))
@@ -1203,17 +1207,10 @@ static int br_port_slave_changelink(struct net_device *brdev,
struct nlattr *data[],
struct netlink_ext_ack *extack)
{
- struct net_bridge *br = netdev_priv(brdev);
- int ret;
-
if (!data)
return 0;
- spin_lock_bh(&br->lock);
- ret = br_setport(br_port_get_rtnl(dev), data, extack);
- spin_unlock_bh(&br->lock);
-
- return ret;
+ return br_setport(br_port_get_rtnl(dev), data, extack);
}
static int br_port_fill_slave_info(struct sk_buff *skb,
@@ -1824,6 +1821,7 @@ static int br_fill_linkxstats(struct sk_buff *skb,
const struct net_device *dev,
int *prividx, int attr)
{
+ unsigned int limit = U16_MAX - nla_total_size(0);
struct nlattr *nla __maybe_unused;
struct net_bridge_port *p = NULL;
struct net_bridge_vlan_group *vg;
@@ -1841,6 +1839,7 @@ static int br_fill_linkxstats(struct sk_buff *skb,
p = br_port_get_rtnl(dev);
if (!p)
return 0;
+ limit -= nla_total_size_64bit(sizeof(p->stp_xstats));
br = p->br;
vg = nbp_vlan_group(p);
break;
@@ -1855,6 +1854,9 @@ static int br_fill_linkxstats(struct sk_buff *skb,
if (vg) {
u16 pvid;
+#ifdef CONFIG_BRIDGE_IGMP_SNOOPING
+ limit -= nla_total_size_64bit(sizeof(struct br_mcast_stats));
+#endif
pvid = br_get_pvid(vg);
list_for_each_entry(v, &vg->vlan_list, vlist) {
struct bridge_vlan_xstats vxi;
@@ -1862,6 +1864,11 @@ static int br_fill_linkxstats(struct sk_buff *skb,
if (++vl_idx < *prividx)
continue;
+
+ if (skb_tail_pointer(skb) - (unsigned char *)nest +
+ nla_total_size(sizeof(vxi)) >= limit)
+ goto nla_put_failure;
+
memset(&vxi, 0, sizeof(vxi));
vxi.vid = v->vid;
vxi.flags = v->flags;
diff --git a/net/bridge/br_switchdev.c b/net/bridge/br_switchdev.c
index 18b558a..ee3ad9d 100644
--- a/net/bridge/br_switchdev.c
+++ b/net/bridge/br_switchdev.c
@@ -99,7 +99,6 @@ int br_switchdev_set_port_flag(struct net_bridge_port *p,
attr.u.brport_flags.val = flags;
attr.u.brport_flags.mask = mask;
- /* We run from atomic context here */
err = call_switchdev_notifiers(SWITCHDEV_PORT_ATTR_SET, p->dev,
&info.info, extack);
err = notifier_to_errno(err);
diff --git a/net/bridge/br_sysfs_if.c b/net/bridge/br_sysfs_if.c
index 1f57c36..d6df81f 100644
--- a/net/bridge/br_sysfs_if.c
+++ b/net/bridge/br_sysfs_if.c
@@ -86,16 +86,34 @@ static ssize_t show_path_cost(struct net_bridge_port *p, char *buf)
return sysfs_emit(buf, "%d\n", p->path_cost);
}
-static BRPORT_ATTR(path_cost, 0644,
- show_path_cost, br_stp_set_path_cost);
+static int store_path_cost(struct net_bridge_port *p, unsigned long v)
+{
+ int ret;
+
+ spin_lock_bh(&p->br->lock);
+ ret = br_stp_set_path_cost(p, v);
+ spin_unlock_bh(&p->br->lock);
+ return ret;
+}
+
+static BRPORT_ATTR(path_cost, 0644, show_path_cost, store_path_cost);
static ssize_t show_priority(struct net_bridge_port *p, char *buf)
{
return sysfs_emit(buf, "%d\n", p->priority);
}
-static BRPORT_ATTR(priority, 0644,
- show_priority, br_stp_set_port_priority);
+static int store_priority(struct net_bridge_port *p, unsigned long v)
+{
+ int ret;
+
+ spin_lock_bh(&p->br->lock);
+ ret = br_stp_set_port_priority(p, v);
+ spin_unlock_bh(&p->br->lock);
+ return ret;
+}
+
+static BRPORT_ATTR(priority, 0644, show_priority, store_priority);
static ssize_t show_designated_root(struct net_bridge_port *p, char *buf)
{
@@ -334,17 +352,13 @@ static ssize_t brport_store(struct kobject *kobj,
ret = -ENOMEM;
goto out_unlock;
}
- spin_lock_bh(&p->br->lock);
ret = brport_attr->store_raw(p, buf_copy);
- spin_unlock_bh(&p->br->lock);
kfree(buf_copy);
} else if (brport_attr->store) {
val = simple_strtoul(buf, &endp, 0);
if (endp == buf)
goto out_unlock;
- spin_lock_bh(&p->br->lock);
ret = brport_attr->store(p, val);
- spin_unlock_bh(&p->br->lock);
}
if (!ret) {
diff --git a/net/bridge/netfilter/ebt_snat.c b/net/bridge/netfilter/ebt_snat.c
index 7dfbcdf..c9e229a 100644
--- a/net/bridge/netfilter/ebt_snat.c
+++ b/net/bridge/netfilter/ebt_snat.c
@@ -31,6 +31,9 @@ ebt_snat_tg(struct sk_buff *skb, const struct xt_action_param *par)
const struct arphdr *ap;
struct arphdr _ah;
+ if (skb_ensure_writable(skb, sizeof(_ah) + ETH_ALEN))
+ return EBT_DROP;
+
ap = skb_header_pointer(skb, 0, sizeof(_ah), &_ah);
if (ap == NULL)
return EBT_DROP;
diff --git a/net/bridge/netfilter/ebtable_broute.c b/net/bridge/netfilter/ebtable_broute.c
index 7413602..f05c79f 100644
--- a/net/bridge/netfilter/ebtable_broute.c
+++ b/net/bridge/netfilter/ebtable_broute.c
@@ -112,24 +112,22 @@ static struct pernet_operations broute_net_ops = {
static int __init ebtable_broute_init(void)
{
- int ret = ebt_register_template(&broute_table, broute_table_init);
+ int ret = register_pernet_subsys(&broute_net_ops);
if (ret)
return ret;
- ret = register_pernet_subsys(&broute_net_ops);
- if (ret) {
- ebt_unregister_template(&broute_table);
- return ret;
- }
+ ret = ebt_register_template(&broute_table, broute_table_init);
+ if (ret)
+ unregister_pernet_subsys(&broute_net_ops);
- return 0;
+ return ret;
}
static void __exit ebtable_broute_fini(void)
{
- unregister_pernet_subsys(&broute_net_ops);
ebt_unregister_template(&broute_table);
+ unregister_pernet_subsys(&broute_net_ops);
}
module_init(ebtable_broute_init);
diff --git a/net/bridge/netfilter/ebtable_filter.c b/net/bridge/netfilter/ebtable_filter.c
index dacd81b..0fc03b0 100644
--- a/net/bridge/netfilter/ebtable_filter.c
+++ b/net/bridge/netfilter/ebtable_filter.c
@@ -93,24 +93,22 @@ static struct pernet_operations frame_filter_net_ops = {
static int __init ebtable_filter_init(void)
{
- int ret = ebt_register_template(&frame_filter, frame_filter_table_init);
+ int ret = register_pernet_subsys(&frame_filter_net_ops);
if (ret)
return ret;
- ret = register_pernet_subsys(&frame_filter_net_ops);
- if (ret) {
- ebt_unregister_template(&frame_filter);
- return ret;
- }
+ ret = ebt_register_template(&frame_filter, frame_filter_table_init);
+ if (ret)
+ unregister_pernet_subsys(&frame_filter_net_ops);
- return 0;
+ return ret;
}
static void __exit ebtable_filter_fini(void)
{
- unregister_pernet_subsys(&frame_filter_net_ops);
ebt_unregister_template(&frame_filter);
+ unregister_pernet_subsys(&frame_filter_net_ops);
}
module_init(ebtable_filter_init);
diff --git a/net/bridge/netfilter/ebtable_nat.c b/net/bridge/netfilter/ebtable_nat.c
index 0f2a8c6..8a10375 100644
--- a/net/bridge/netfilter/ebtable_nat.c
+++ b/net/bridge/netfilter/ebtable_nat.c
@@ -93,24 +93,22 @@ static struct pernet_operations frame_nat_net_ops = {
static int __init ebtable_nat_init(void)
{
- int ret = ebt_register_template(&frame_nat, frame_nat_table_init);
+ int ret = register_pernet_subsys(&frame_nat_net_ops);
if (ret)
return ret;
- ret = register_pernet_subsys(&frame_nat_net_ops);
- if (ret) {
- ebt_unregister_template(&frame_nat);
- return ret;
- }
+ ret = ebt_register_template(&frame_nat, frame_nat_table_init);
+ if (ret)
+ unregister_pernet_subsys(&frame_nat_net_ops);
return ret;
}
static void __exit ebtable_nat_fini(void)
{
- unregister_pernet_subsys(&frame_nat_net_ops);
ebt_unregister_template(&frame_nat);
+ unregister_pernet_subsys(&frame_nat_net_ops);
}
module_init(ebtable_nat_init);
diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c
index aea3e19..8a6a069 100644
--- a/net/bridge/netfilter/ebtables.c
+++ b/net/bridge/netfilter/ebtables.c
@@ -42,6 +42,7 @@
struct ebt_pernet {
struct list_head tables;
+ struct list_head dead_tables;
};
struct ebt_template {
@@ -1162,11 +1163,6 @@ static int do_replace(struct net *net, sockptr_t arg, unsigned int len)
static void __ebt_unregister_table(struct net *net, struct ebt_table *table)
{
- mutex_lock(&ebt_mutex);
- list_del(&table->list);
- mutex_unlock(&ebt_mutex);
- audit_log_nfcfg(table->name, AF_BRIDGE, table->private->nentries,
- AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
EBT_ENTRY_ITERATE(table->private->entries, table->private->entries_size,
ebt_cleanup_entry, net, NULL);
if (table->private->nentries)
@@ -1267,13 +1263,15 @@ int ebt_register_table(struct net *net, const struct ebt_table *input_table,
for (i = 0; i < num_ops; i++)
ops[i].priv = table;
- list_add(&table->list, &ebt_net->tables);
- mutex_unlock(&ebt_mutex);
-
table->ops = ops;
ret = nf_register_net_hooks(net, ops, num_ops);
- if (ret)
+ if (ret) {
+ synchronize_rcu();
__ebt_unregister_table(net, table);
+ } else {
+ list_add(&table->list, &ebt_net->tables);
+ }
+ mutex_unlock(&ebt_mutex);
audit_log_nfcfg(repl->name, AF_BRIDGE, repl->nentries,
AUDIT_XT_OP_REGISTER, GFP_KERNEL);
@@ -1339,7 +1337,7 @@ void ebt_unregister_template(const struct ebt_table *t)
}
EXPORT_SYMBOL(ebt_unregister_template);
-static struct ebt_table *__ebt_find_table(struct net *net, const char *name)
+void ebt_unregister_table_pre_exit(struct net *net, const char *name)
{
struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
struct ebt_table *t;
@@ -1348,30 +1346,36 @@ static struct ebt_table *__ebt_find_table(struct net *net, const char *name)
list_for_each_entry(t, &ebt_net->tables, list) {
if (strcmp(t->name, name) == 0) {
+ list_move(&t->list, &ebt_net->dead_tables);
mutex_unlock(&ebt_mutex);
- return t;
+ nf_unregister_net_hooks(net, t->ops, hweight32(t->valid_hooks));
+ return;
}
}
mutex_unlock(&ebt_mutex);
- return NULL;
-}
-
-void ebt_unregister_table_pre_exit(struct net *net, const char *name)
-{
- struct ebt_table *table = __ebt_find_table(net, name);
-
- if (table)
- nf_unregister_net_hooks(net, table->ops, hweight32(table->valid_hooks));
}
EXPORT_SYMBOL(ebt_unregister_table_pre_exit);
void ebt_unregister_table(struct net *net, const char *name)
{
- struct ebt_table *table = __ebt_find_table(net, name);
+ struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
+ struct ebt_table *t;
- if (table)
- __ebt_unregister_table(net, table);
+ mutex_lock(&ebt_mutex);
+
+ list_for_each_entry(t, &ebt_net->dead_tables, list) {
+ if (strcmp(t->name, name) == 0) {
+ list_del(&t->list);
+ audit_log_nfcfg(t->name, AF_BRIDGE, t->private->nentries,
+ AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
+ __ebt_unregister_table(net, t);
+ mutex_unlock(&ebt_mutex);
+ return;
+ }
+ }
+
+ mutex_unlock(&ebt_mutex);
}
/* userspace just supplied us with counters */
@@ -1952,6 +1956,25 @@ enum compat_mwt {
EBT_COMPAT_TARGET,
};
+static bool match_size_ok(const struct xt_match *match, unsigned int match_size)
+{
+ u16 csize;
+
+ if (match->matchsize == -1) /* cannot validate ebt_among */
+ return true;
+
+ csize = match->compatsize ? : match->matchsize;
+
+ return match_size >= csize;
+}
+
+static bool tgt_size_ok(const struct xt_target *tgt, unsigned int tgt_size)
+{
+ u16 csize = tgt->compatsize ? : tgt->targetsize;
+
+ return tgt_size >= csize;
+}
+
static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt,
enum compat_mwt compat_mwt,
struct ebt_entries_buf_state *state,
@@ -1977,6 +2000,11 @@ static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt,
if (IS_ERR(match))
return PTR_ERR(match);
+ if (!match_size_ok(match, match_size)) {
+ module_put(match->me);
+ return -EINVAL;
+ }
+
off = ebt_compat_match_offset(match, match_size);
if (dst) {
if (match->compat_from_user)
@@ -1996,6 +2024,12 @@ static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt,
mwt->u.revision);
if (IS_ERR(wt))
return PTR_ERR(wt);
+
+ if (!tgt_size_ok(wt, match_size)) {
+ module_put(wt->me);
+ return -EINVAL;
+ }
+
off = xt_compat_target_offset(wt);
if (dst) {
@@ -2556,11 +2590,21 @@ static int __net_init ebt_pernet_init(struct net *net)
struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
INIT_LIST_HEAD(&ebt_net->tables);
+ INIT_LIST_HEAD(&ebt_net->dead_tables);
return 0;
}
+static void __net_exit ebt_pernet_exit(struct net *net)
+{
+ struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
+
+ WARN_ON_ONCE(!list_empty(&ebt_net->tables));
+ WARN_ON_ONCE(!list_empty(&ebt_net->dead_tables));
+}
+
static struct pernet_operations ebt_net_ops = {
.init = ebt_pernet_init,
+ .exit = ebt_pernet_exit,
.id = &ebt_pernet_id,
.size = sizeof(struct ebt_pernet),
};
@@ -2569,19 +2613,20 @@ static int __init ebtables_init(void)
{
int ret;
- ret = xt_register_target(&ebt_standard_target);
+ ret = register_pernet_subsys(&ebt_net_ops);
if (ret < 0)
return ret;
- ret = nf_register_sockopt(&ebt_sockopts);
+
+ ret = xt_register_target(&ebt_standard_target);
if (ret < 0) {
- xt_unregister_target(&ebt_standard_target);
+ unregister_pernet_subsys(&ebt_net_ops);
return ret;
}
- ret = register_pernet_subsys(&ebt_net_ops);
+ ret = nf_register_sockopt(&ebt_sockopts);
if (ret < 0) {
- nf_unregister_sockopt(&ebt_sockopts);
xt_unregister_target(&ebt_standard_target);
+ unregister_pernet_subsys(&ebt_net_ops);
return ret;
}
diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c
index 692e0b8..9e64e82 100644
--- a/net/ceph/auth_x.c
+++ b/net/ceph/auth_x.c
@@ -115,6 +115,11 @@ static int __ceph_x_decrypt(const struct ceph_crypto_key *key, int usage_slot,
if (ret)
return ret;
+ if (plaintext_len < sizeof(*hdr)) {
+ pr_err("%s plaintext too small %d\n", __func__, plaintext_len);
+ return -EINVAL;
+ }
+
hdr = p + ceph_crypt_data_offset(key);
if (le64_to_cpu(hdr->magic) != CEPHX_ENC_MAGIC) {
pr_err("%s bad magic\n", __func__);
diff --git a/net/ceph/crush/crush.c b/net/ceph/crush/crush.c
index 254ded0..521aec1 100644
--- a/net/ceph/crush/crush.c
+++ b/net/ceph/crush/crush.c
@@ -47,7 +47,6 @@ int crush_get_bucket_item_weight(const struct crush_bucket *b, int p)
void crush_destroy_bucket_uniform(struct crush_bucket_uniform *b)
{
kfree(b->h.items);
- kfree(b);
}
void crush_destroy_bucket_list(struct crush_bucket_list *b)
@@ -55,14 +54,12 @@ void crush_destroy_bucket_list(struct crush_bucket_list *b)
kfree(b->item_weights);
kfree(b->sum_weights);
kfree(b->h.items);
- kfree(b);
}
void crush_destroy_bucket_tree(struct crush_bucket_tree *b)
{
kfree(b->h.items);
kfree(b->node_weights);
- kfree(b);
}
void crush_destroy_bucket_straw(struct crush_bucket_straw *b)
@@ -70,14 +67,12 @@ void crush_destroy_bucket_straw(struct crush_bucket_straw *b)
kfree(b->straws);
kfree(b->item_weights);
kfree(b->h.items);
- kfree(b);
}
void crush_destroy_bucket_straw2(struct crush_bucket_straw2 *b)
{
kfree(b->item_weights);
kfree(b->h.items);
- kfree(b);
}
void crush_destroy_bucket(struct crush_bucket *b)
@@ -99,6 +94,7 @@ void crush_destroy_bucket(struct crush_bucket *b)
crush_destroy_bucket_straw2((struct crush_bucket_straw2 *)b);
break;
}
+ kfree(b);
}
/**
diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c
index c89e66d..8b5b058 100644
--- a/net/ceph/osdmap.c
+++ b/net/ceph/osdmap.c
@@ -72,8 +72,7 @@ static int crush_decode_uniform_bucket(void **p, void *end,
struct crush_bucket_uniform *b)
{
dout("crush_decode_uniform_bucket %p to %p\n", *p, end);
- ceph_decode_need(p, end, (1+b->h.size) * sizeof(u32), bad);
- b->item_weight = ceph_decode_32(p);
+ ceph_decode_32_safe(p, end, b->item_weight, bad);
return 0;
bad:
return -EINVAL;
@@ -389,11 +388,15 @@ static int decode_choose_args(void **p, void *end, struct crush_map *c)
goto fail;
if (arg->ids_size &&
- arg->ids_size != c->buckets[bucket_index]->size)
+ (!c->buckets[bucket_index] ||
+ arg->ids_size != c->buckets[bucket_index]->size))
goto e_inval;
}
- insert_choose_arg_map(&c->choose_args, arg_map);
+ if (!__insert_choose_arg_map(&c->choose_args, arg_map)) {
+ ret = -EEXIST;
+ goto fail;
+ }
}
return 0;
@@ -516,6 +519,10 @@ static struct crush_map *crush_decode(void *pbyval, void *end)
b->id = ceph_decode_32(p);
b->type = ceph_decode_16(p);
b->alg = ceph_decode_8(p);
+ if (b->alg != alg) {
+ b->alg = 0;
+ goto bad;
+ }
b->hash = ceph_decode_8(p);
b->weight = ceph_decode_32(p);
b->size = ceph_decode_32(p);
@@ -1702,7 +1709,7 @@ static int osdmap_decode(void **p, void *end, bool msgr2,
ceph_decode_need(p, end, 3*sizeof(u32) +
map->max_osd*(struct_v >= 5 ? sizeof(u32) :
sizeof(u8)) +
- sizeof(*map->osd_weight), e_inval);
+ map->max_osd*sizeof(*map->osd_weight), e_inval);
if (ceph_decode_32(p) != map->max_osd)
goto e_inval;
diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 14eb781..ecd659f 100644
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -172,7 +172,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
struct bpf_map *map;
smap = rcu_dereference(SDATA(selem)->smap);
- if (!(smap->map.map_flags & BPF_F_CLONE))
+ if (!smap || !(smap->map.map_flags & BPF_F_CLONE))
continue;
/* Note that for lockless listeners adding new element
@@ -531,10 +531,10 @@ bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
}
EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
-static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
+static int diag_get(struct bpf_local_storage_map *smap,
+ struct bpf_local_storage_data *sdata, struct sk_buff *skb)
{
struct nlattr *nla_stg, *nla_value;
- struct bpf_local_storage_map *smap;
/* It cannot exceed max nlattr's payload */
BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
@@ -543,7 +543,6 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
if (!nla_stg)
return -EMSGSIZE;
- smap = rcu_dereference(sdata->smap);
if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
goto errout;
@@ -558,6 +557,7 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
sdata->data, true);
else
copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
+ check_and_init_map_value(&smap->map, nla_data(nla_value));
nla_nest_end(skb, nla_stg);
return 0;
@@ -596,9 +596,11 @@ static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
saved_len = skb->len;
hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
smap = rcu_dereference(SDATA(selem)->smap);
+ if (!smap)
+ continue;
diag_size += nla_value_size(smap->map.value_size);
- if (nla_stgs && diag_get(SDATA(selem), skb))
+ if (nla_stgs && diag_get(smap, SDATA(selem), skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
@@ -665,7 +667,7 @@ int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
diag_size += nla_value_size(diag->maps[i]->value_size);
- if (nla_stgs && diag_get(sdata, skb))
+ if (nla_stgs && diag_get((struct bpf_local_storage_map *)diag->maps[i], sdata, skb))
/* Continue to learn diag_size */
err = -EMSGSIZE;
}
diff --git a/net/core/dev.c b/net/core/dev.c
index 06c1959..0c6c270 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -371,7 +371,7 @@ static void netdev_name_node_alt_free(struct rcu_head *head)
static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
{
netdev_name_node_del(name_node);
- list_del(&name_node->list);
+ list_del_rcu(&name_node->list);
call_rcu(&name_node->rcu, netdev_name_node_alt_free);
}
@@ -6862,9 +6862,9 @@ static void skb_defer_free_flush(void)
#if defined(CONFIG_NET_RX_BUSY_POLL)
-static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
+static void __busy_poll_stop(struct napi_struct *napi, unsigned long timeout)
{
- if (!skip_schedule) {
+ if (!timeout) {
gro_normal_list(&napi->gro);
__napi_schedule(napi);
return;
@@ -6874,6 +6874,8 @@ static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
gro_flush_normal(&napi->gro, HZ >= 1000);
clear_bit(NAPI_STATE_SCHED, &napi->state);
+ hrtimer_start(&napi->timer, ns_to_ktime(timeout),
+ HRTIMER_MODE_REL_PINNED);
}
enum {
@@ -6885,8 +6887,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
unsigned flags, u16 budget)
{
struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- bool skip_schedule = false;
- unsigned long timeout;
+ unsigned long timeout = 0;
int rc;
/* Busy polling means there is a high chance device driver hard irq
@@ -6906,10 +6907,12 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
if (flags & NAPI_F_PREFER_BUSY_POLL) {
napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
- timeout = napi_get_gro_flush_timeout(napi);
- if (napi->defer_hard_irqs_count && timeout) {
- hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
- skip_schedule = true;
+ if (napi->defer_hard_irqs_count) {
+ /* A short enough gro flush timeout and long enough
+ * poll can result in timer firing too early.
+ * Timer will be armed later if necessary.
+ */
+ timeout = napi_get_gro_flush_timeout(napi);
}
}
@@ -6924,7 +6927,7 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
trace_napi_poll(napi, rc, budget);
netpoll_poll_unlock(have_poll_lock);
if (rc == budget)
- __busy_poll_stop(napi, skip_schedule);
+ __busy_poll_stop(napi, timeout);
bpf_net_ctx_clear(bpf_net_ctx);
local_bh_enable();
}
diff --git a/net/core/devmem.c b/net/core/devmem.c
index cde4c89..4f71de4 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -241,6 +241,11 @@ net_devmem_bind_dmabuf(struct net_device *dev,
}
if (direction == DMA_TO_DEVICE) {
+ if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
+ err = -EINVAL;
+ NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
+ goto err_unmap;
+ }
binding->tx_vec = kvmalloc_objs(struct net_iov *,
dmabuf->size / PAGE_SIZE);
if (!binding->tx_vec) {
@@ -267,6 +272,12 @@ net_devmem_bind_dmabuf(struct net_device *dev,
size_t len = sg_dma_len(sg);
struct net_iov *niov;
+ if (!IS_ALIGNED(len, PAGE_SIZE)) {
+ err = -EINVAL;
+ NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned");
+ goto err_free_chunks;
+ }
+
owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
dev_to_node(&dev->dev));
if (!owner) {
@@ -297,8 +308,7 @@ net_devmem_bind_dmabuf(struct net_device *dev,
for (i = 0; i < owner->area.num_niovs; i++) {
niov = &owner->area.niovs[i];
- niov->type = NET_IOV_DMABUF;
- niov->owner = &owner->area;
+ net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
net_devmem_get_dma_addr(niov));
if (direction == DMA_TO_DEVICE)
diff --git a/net/core/failover.c b/net/core/failover.c
index 11bb183..e43c59c 100644
--- a/net/core/failover.c
+++ b/net/core/failover.c
@@ -12,6 +12,7 @@
#include <uapi/linux/if_arp.h>
#include <linux/rtnetlink.h>
#include <linux/if_vlan.h>
+#include <net/netdev_lock.h>
#include <net/failover.h>
static LIST_HEAD(failover_list);
@@ -221,8 +222,11 @@ failover_existing_slave_register(struct net_device *failover_dev)
for_each_netdev(net, dev) {
if (netif_is_failover(dev))
continue;
- if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr))
+ if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr)) {
+ netdev_lock_ops(dev);
failover_slave_register(dev);
+ netdev_unlock_ops(dev);
+ }
}
rtnl_unlock();
}
diff --git a/net/core/filter.c b/net/core/filter.c
index 80a3b70..8043976 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -1654,15 +1654,24 @@ int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
return err;
}
+static void sk_reuseport_prog_free_rcu(struct rcu_head *rcu)
+{
+ struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
+ struct bpf_prog *prog = aux->prog;
+
+ bpf_release_orig_filter(prog);
+ bpf_prog_free(prog);
+}
+
void sk_reuseport_prog_free(struct bpf_prog *prog)
{
if (!prog)
return;
- if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
- bpf_prog_put(prog);
+ if (bpf_prog_was_classic(prog))
+ call_rcu(&prog->aux->rcu, sk_reuseport_prog_free_rcu);
else
- bpf_prog_destroy(prog);
+ bpf_prog_put(prog);
}
static inline int __bpf_try_make_writable(struct sk_buff *skb,
@@ -2860,7 +2869,7 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
psge->length = start - offset;
rsge.length -= psge->length;
- rsge.offset += start;
+ rsge.offset += start - offset;
sk_msg_iter_var_next(i);
sg_unmark_end(psge);
@@ -5481,7 +5490,7 @@ static int sol_tcp_sockopt(struct sock *sk, int optname,
char *optval, int *optlen,
bool getopt)
{
- if (sk->sk_protocol != IPPROTO_TCP)
+ if (!sk_is_tcp(sk))
return -EINVAL;
switch (optname) {
@@ -5688,6 +5697,30 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = {
.arg5_type = ARG_CONST_SIZE,
};
+BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level,
+ int, optname, char *, optval, int, optlen)
+{
+ /*
+ * TCP_NODELAY triggers tcp_push_pending_frames() and re-enters
+ * CA_EVENT_TX_START in bpf_tcp_cc.
+ */
+ if (level == SOL_TCP && optname == TCP_NODELAY)
+ return -EOPNOTSUPP;
+
+ return _bpf_setsockopt(sk, level, optname, optval, optlen);
+}
+
+const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = {
+ .func = bpf_sk_setsockopt_nodelay,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
+ .arg5_type = ARG_CONST_SIZE,
+};
+
BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level,
int, optname, char *, optval, int, optlen)
{
@@ -5833,6 +5866,12 @@ BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
if (!is_locked_tcp_sock_ops(bpf_sock))
return -EOPNOTSUPP;
+ /* TCP_NODELAY triggers tcp_push_pending_frames() and re-enters these callbacks. */
+ if ((bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB ||
+ bpf_sock->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB) &&
+ level == SOL_TCP && optname == TCP_NODELAY)
+ return -EOPNOTSUPP;
+
return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen);
}
@@ -6443,6 +6482,8 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
* against MTU of FIB lookup resulting net_device
*/
dev = dev_get_by_index_rcu(net, params->ifindex);
+ if (unlikely(!dev))
+ return -ENODEV;
if (!is_skb_forwardable(dev, skb))
rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
@@ -7443,7 +7484,7 @@ u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
{
- if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
+ if (sk_fullsock(sk) && sk_is_tcp(sk))
return (unsigned long)sk;
return (unsigned long)NULL;
@@ -11915,7 +11956,7 @@ BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
*/
BTF_TYPE_EMIT(struct tcp6_sock);
if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
- sk->sk_family == AF_INET6)
+ sk->sk_type == SOCK_STREAM && sk->sk_family == AF_INET6)
return (unsigned long)sk;
return (unsigned long)NULL;
@@ -11931,7 +11972,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
{
- if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
+ if (sk && sk_fullsock(sk) && sk_is_tcp(sk))
return (unsigned long)sk;
return (unsigned long)NULL;
diff --git a/net/core/gro.c b/net/core/gro.c
index 31d21de..a847539 100644
--- a/net/core/gro.c
+++ b/net/core/gro.c
@@ -109,6 +109,9 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
if (p->pp_recycle != skb->pp_recycle)
return -ETOOMANYREFS;
+ if (skb_zcopy(p) || skb_zcopy(skb))
+ return -ETOOMANYREFS;
+
if (unlikely(p->len + len >= netif_get_gro_max_size(p->dev, p) ||
NAPI_GRO_CB(skb)->flush))
return -E2BIG;
@@ -213,10 +216,12 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
p->data_len += len;
p->truesize += delta_truesize;
p->len += len;
+ skb_shinfo(p)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
if (lp != p) {
lp->data_len += len;
lp->truesize += delta_truesize;
lp->len += len;
+ skb_shinfo(lp)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
}
NAPI_GRO_CB(skb)->same_flow = 1;
return 0;
@@ -244,6 +249,8 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
p->truesize += skb->truesize;
p->len += skb->len;
+ skb_shinfo(p)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
+
NAPI_GRO_CB(skb)->same_flow = 1;
return 0;
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 9e12524..5d92160 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -3210,8 +3210,10 @@ int neigh_xmit(int index, struct net_device *dev,
rcu_read_lock();
tbl = rcu_dereference(neigh_tables[index]);
- if (!tbl)
- goto out_unlock;
+ if (!tbl) {
+ rcu_read_unlock();
+ goto out_kfree_skb;
+ }
if (index == NEIGH_ARP_TABLE) {
u32 key = *((u32 *)addr);
@@ -3227,7 +3229,6 @@ int neigh_xmit(int index, struct net_device *dev,
goto out_kfree_skb;
}
err = READ_ONCE(neigh->output)(neigh, skb);
-out_unlock:
rcu_read_unlock();
}
else if (index == NEIGH_LINK_TABLE) {
@@ -3237,11 +3238,10 @@ int neigh_xmit(int index, struct net_device *dev,
goto out_kfree_skb;
err = dev_queue_xmit(skb);
}
-out:
return err;
out_kfree_skb:
kfree_skb(skb);
- goto out;
+ return err;
}
EXPORT_SYMBOL(neigh_xmit);
diff --git a/net/core/netmem_priv.h b/net/core/netmem_priv.h
index 3e6fde8..23175cb 100644
--- a/net/core/netmem_priv.h
+++ b/net/core/netmem_priv.h
@@ -8,18 +8,21 @@ static inline unsigned long netmem_get_pp_magic(netmem_ref netmem)
return netmem_to_nmdesc(netmem)->pp_magic & ~PP_DMA_INDEX_MASK;
}
+static inline void netmem_or_pp_magic(netmem_ref netmem, unsigned long pp_magic)
+{
+ netmem_to_nmdesc(netmem)->pp_magic |= pp_magic;
+}
+
+static inline void netmem_clear_pp_magic(netmem_ref netmem)
+{
+ WARN_ON_ONCE(netmem_to_nmdesc(netmem)->pp_magic & PP_DMA_INDEX_MASK);
+
+ netmem_to_nmdesc(netmem)->pp_magic = 0;
+}
+
static inline bool netmem_is_pp(netmem_ref netmem)
{
- struct page *page;
-
- /* XXX: Now that the offset of page_type is shared between
- * struct page and net_iov, just cast the netmem to struct page
- * unconditionally by clearing NET_IOV if any, no matter whether
- * it comes from struct net_iov or struct page. This should be
- * adjusted once the offset is no longer shared.
- */
- page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
- return PageNetpp(page);
+ return (netmem_get_pp_magic(netmem) & PP_MAGIC_MASK) == PP_SIGNATURE;
}
static inline void netmem_set_pp(netmem_ref netmem, struct page_pool *pool)
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index cd74bef..3f4a17f 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -319,6 +319,8 @@ static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
lockdep_assert_irqs_disabled();
dev = np->dev;
+ /* npinfo->txq belongs to np->dev, so retries must stay bound to it. */
+ skb->dev = dev;
rcu_read_lock();
npinfo = rcu_dereference_bh(dev->npinfo);
@@ -608,14 +610,16 @@ EXPORT_SYMBOL_GPL(__netpoll_setup);
/*
* Returns a pointer to a string representation of the identifier used
* to select the egress interface for the given netpoll instance. buf
- * must be a buffer of length at least MAC_ADDR_STR_LEN + 1.
+ * is used to format np->dev_mac when np->dev_name is empty; bufsz must
+ * be at least MAC_ADDR_STR_LEN + 1 to fit the formatted MAC address
+ * and its NUL terminator.
*/
-static char *egress_dev(struct netpoll *np, char *buf)
+static char *egress_dev(struct netpoll *np, char *buf, size_t bufsz)
{
if (np->dev_name[0])
return np->dev_name;
- snprintf(buf, MAC_ADDR_STR_LEN, "%pM", np->dev_mac);
+ snprintf(buf, bufsz, "%pM", np->dev_mac);
return buf;
}
@@ -645,7 +649,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev)
if (!IS_ENABLED(CONFIG_IPV6)) {
np_err(np, "IPv6 is not supported %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EINVAL;
}
@@ -667,7 +671,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev)
}
if (err) {
np_err(np, "no IPv6 address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return err;
}
@@ -687,14 +691,14 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev)
in_dev = __in_dev_get_rtnl(ndev);
if (!in_dev) {
np_err(np, "no IP address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EDESTADDRREQ;
}
ifa = rtnl_dereference(in_dev->ifa_list);
if (!ifa) {
np_err(np, "no IP address for %s, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
return -EDESTADDRREQ;
}
@@ -704,6 +708,23 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev)
return 0;
}
+/*
+ * Test whether the caller left np->local_ip unset, so that
+ * netpoll_setup() should auto-populate it from the egress device.
+ *
+ * np->local_ip is a union of __be32 (IPv4) and struct in6_addr (IPv6),
+ * so an IPv6 address whose first 4 bytes are zero (e.g. ::1, ::2,
+ * IPv4-mapped ::ffff:a.b.c.d) must not be tested via the IPv4 arm —
+ * doing so would misclassify a caller-supplied address as unset and
+ * silently overwrite it with whatever address the device exposes.
+ */
+static bool netpoll_local_ip_unset(const struct netpoll *np)
+{
+ if (np->ipv6)
+ return ipv6_addr_any(&np->local_ip.in6);
+ return !np->local_ip.ip;
+}
+
int netpoll_setup(struct netpoll *np)
{
struct net *net = current->nsproxy->net_ns;
@@ -719,7 +740,8 @@ int netpoll_setup(struct netpoll *np)
ndev = dev_getbyhwaddr(net, ARPHRD_ETHER, np->dev_mac);
if (!ndev) {
- np_err(np, "%s doesn't exist, aborting\n", egress_dev(np, buf));
+ np_err(np, "%s doesn't exist, aborting\n",
+ egress_dev(np, buf, sizeof(buf)));
err = -ENODEV;
goto unlock;
}
@@ -727,14 +749,14 @@ int netpoll_setup(struct netpoll *np)
if (netdev_master_upper_dev_get(ndev)) {
np_err(np, "%s is a slave device, aborting\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
err = -EBUSY;
goto put;
}
if (!netif_running(ndev)) {
np_info(np, "device %s not up yet, forcing it\n",
- egress_dev(np, buf));
+ egress_dev(np, buf, sizeof(buf)));
err = dev_open(ndev, NULL);
if (err) {
@@ -747,7 +769,7 @@ int netpoll_setup(struct netpoll *np)
rtnl_lock();
}
- if (!np->local_ip.ip) {
+ if (netpoll_local_ip_unset(np)) {
if (!np->ipv6) {
err = netpoll_take_ipv4(np, ndev);
if (err)
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 877bbf7..8171d11 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -327,6 +327,11 @@ static void page_pool_uninit(struct page_pool *pool)
if (!pool->system)
free_percpu(pool->recycle_stats);
#endif
+
+ if (pool->mp_ops) {
+ pool->mp_ops->destroy(pool);
+ static_branch_dec(&page_pool_mem_providers);
+ }
}
/**
@@ -702,18 +707,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict)
void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
{
- struct page *page;
-
netmem_set_pp(netmem, pool);
-
- /* XXX: Now that the offset of page_type is shared between
- * struct page and net_iov, just cast the netmem to struct page
- * unconditionally by clearing NET_IOV if any, no matter whether
- * it comes from struct net_iov or struct page. This should be
- * adjusted once the offset is no longer shared.
- */
- page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
- __SetPageNetpp(page);
+ netmem_or_pp_magic(netmem, PP_SIGNATURE);
/* Ensuring all pages have been split into one fragment initially:
* page_pool_set_pp_info() is only called once for every page when it
@@ -728,17 +723,7 @@ void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
void page_pool_clear_pp_info(netmem_ref netmem)
{
- struct page *page;
-
- /* XXX: Now that the offset of page_type is shared between
- * struct page and net_iov, just cast the netmem to struct page
- * unconditionally by clearing NET_IOV if any, no matter whether
- * it comes from struct net_iov or struct page. This should be
- * adjusted once the offset is no longer shared.
- */
- page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
- __ClearPageNetpp(page);
-
+ netmem_clear_pp_magic(netmem);
netmem_set_pp(netmem, NULL);
}
@@ -1146,11 +1131,6 @@ static void __page_pool_destroy(struct page_pool *pool)
page_pool_unlist(pool);
page_pool_uninit(pool);
- if (pool->mp_ops) {
- pool->mp_ops->destroy(pool);
- static_branch_dec(&page_pool_mem_providers);
- }
-
kfree(pool);
}
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index b613bb6..511c25b 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1572,6 +1572,7 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb,
port_guid.vf = ivi.vf;
memcpy(vf_mac.mac, ivi.mac, sizeof(ivi.mac));
+ memset(&vf_broadcast, 0, sizeof(vf_broadcast));
memcpy(vf_broadcast.broadcast, dev->broadcast, dev->addr_len);
vf_vlan.vlan = ivi.vlan;
vf_vlan.qos = ivi.qos;
@@ -6327,8 +6328,9 @@ static int rtnl_stats_get(struct sk_buff *skb, struct nlmsghdr *nlh,
NETLINK_CB(skb).portid, nlh->nlmsg_seq, 0,
0, &filters, &idxattr, &prividx, extack);
if (err < 0) {
- /* -EMSGSIZE implies BUG in if_nlmsg_stats_size */
- WARN_ON(err == -EMSGSIZE);
+ /* -EMSGSIZE implies BUG in if_nlmsg_stats_size
+ * or a too big nested attribute.
+ */
kfree_skb(nskb);
} else {
err = rtnl_unicast(nskb, net, NETLINK_CB(skb).portid);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 7dad68e..c02f0a5 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -2248,6 +2248,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
skb_frag_ref(skb, i);
}
skb_shinfo(n)->nr_frags = i;
+ skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
}
if (skb_has_frag_list(skb)) {
@@ -2786,6 +2787,8 @@ int ___pskb_trim(struct sk_buff *skb, unsigned int len)
skb->data_len = 0;
skb_set_tail_pointer(skb, len);
}
+ if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb))
+ skb->unreadable = 0;
if (!skb->sk || skb->destructor == sock_edemux)
skb_condense(skb);
@@ -2793,16 +2796,37 @@ int ___pskb_trim(struct sk_buff *skb, unsigned int len)
}
EXPORT_SYMBOL(___pskb_trim);
+static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len)
+{
+ int delta = skb->len - len;
+
+ if (skb_frags_readable(skb)) {
+ skb->csum = csum_block_sub(skb->csum,
+ skb_checksum(skb, len, delta, 0),
+ len);
+ return 0;
+ }
+
+ if (len > skb_headlen(skb))
+ return -EFAULT;
+
+ /* The trimmed bytes are unreadable, but the remaining packet can be
+ * checksummed by software after trimming.
+ */
+ skb->ip_summed = CHECKSUM_NONE;
+ return 0;
+}
+
/* Note : use pskb_trim_rcsum() instead of calling this directly
*/
int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
{
if (skb->ip_summed == CHECKSUM_COMPLETE) {
- int delta = skb->len - len;
+ int err;
- skb->csum = csum_block_sub(skb->csum,
- skb_checksum(skb, len, delta, 0),
- len);
+ err = pskb_trim_rcsum_complete(skb, len);
+ if (err)
+ return err;
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
@@ -4349,6 +4373,8 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
tgt->ip_summed = CHECKSUM_PARTIAL;
skb->ip_summed = CHECKSUM_PARTIAL;
+ skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
+
skb_len_add(skb, -shiftlen);
skb_len_add(tgt, shiftlen);
@@ -4959,7 +4985,8 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb,
skb_copy_from_linear_data_offset(head_skb, offset,
skb_put(nskb, hsize), hsize);
- skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags &
+ skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags |
+ skb_shinfo(frag_skb)->flags) &
SKBFL_SHARED_FRAG;
if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
@@ -4976,6 +5003,9 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb,
nfrags = skb_shinfo(list_skb)->nr_frags;
frag = skb_shinfo(list_skb)->frags;
frag_skb = list_skb;
+
+ skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG;
+
if (!skb_headlen(list_skb)) {
BUG_ON(!nfrags);
} else {
@@ -6200,6 +6230,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
from_shinfo->frags,
from_shinfo->nr_frags * sizeof(skb_frag_t));
to_shinfo->nr_frags += from_shinfo->nr_frags;
+ if (from_shinfo->nr_frags)
+ to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG;
if (!skb_cloned(from))
from_shinfo->nr_frags = 0;
@@ -6791,6 +6823,11 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
skb->len -= off;
+ /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
+ * while refcounting frags below.
+ */
+ skb_zcopy_downgrade_managed(skb);
+
memcpy((struct skb_shared_info *)(data + size),
skb_shinfo(skb),
offsetof(struct skb_shared_info,
@@ -6801,6 +6838,8 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
skb_kfree_head(data);
return -ENOMEM;
}
+ if (skb_zcopy(skb))
+ net_zcopy_get(skb_zcopy(skb));
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
skb_frag_ref(skb, i);
if (skb_has_frag_list(skb))
@@ -6902,6 +6941,11 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
return -ENOMEM;
size = SKB_WITH_OVERHEAD(size);
+ /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it
+ * while refcounting frags below.
+ */
+ skb_zcopy_downgrade_managed(skb);
+
memcpy((struct skb_shared_info *)(data + size),
skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
if (skb_orphan_frags(skb, gfp_mask)) {
@@ -6944,6 +6988,8 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
skb_kfree_head(data);
return -ENOMEM;
}
+ if (skb_zcopy(skb))
+ net_zcopy_get(skb_zcopy(skb));
skb_release_data(skb, SKB_CONSUMED);
skb->head = data;
diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 6187a83..e1850ca 100644
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -1268,12 +1268,19 @@ static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
static void sk_psock_verdict_data_ready(struct sock *sk)
{
const struct proto_ops *ops = NULL;
+ struct sk_psock *psock;
struct socket *sock;
int copied;
trace_sk_data_ready(sk);
rcu_read_lock();
+ psock = sk_psock(sk);
+ if (psock && tls_sw_has_ctx_rx(sk)) {
+ psock->saved_data_ready(sk);
+ rcu_read_unlock();
+ return;
+ }
sock = READ_ONCE(sk->sk_socket);
if (likely(sock))
ops = READ_ONCE(sock->ops);
@@ -1283,8 +1290,6 @@ static void sk_psock_verdict_data_ready(struct sock *sk)
copied = ops->read_skb(sk, sk_psock_verdict_recv);
if (copied >= 0) {
- struct sk_psock *psock;
-
rcu_read_lock();
psock = sk_psock(sk);
if (psock)
diff --git a/net/core/sock.c b/net/core/sock.c
index b37b664..d097025 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2676,8 +2676,12 @@ void sock_wfree(struct sk_buff *skb)
int old;
if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
+ void (*sk_write_space)(struct sock *sk);
+
+ sk_write_space = READ_ONCE(sk->sk_write_space);
+
if (sock_flag(sk, SOCK_RCU_FREE) &&
- sk->sk_write_space == sock_def_write_space) {
+ sk_write_space == sock_def_write_space) {
rcu_read_lock();
free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc,
&old);
@@ -2693,7 +2697,7 @@ void sock_wfree(struct sk_buff *skb)
* after sk_write_space() call
*/
WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
- sk->sk_write_space(sk);
+ sk_write_space(sk);
len = 1;
}
/*
diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index 02a68be..99e3789 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -1630,18 +1630,23 @@ void sock_map_unhash(struct sock *sk)
void (*saved_unhash)(struct sock *sk);
struct sk_psock *psock;
+retry:
rcu_read_lock();
psock = sk_psock(sk);
if (unlikely(!psock)) {
rcu_read_unlock();
saved_unhash = READ_ONCE(sk->sk_prot)->unhash;
+ if (unlikely(saved_unhash == sock_map_unhash))
+ goto retry;
} else {
saved_unhash = psock->saved_unhash;
sock_map_remove_links(sk, psock);
rcu_read_unlock();
+
+ if (WARN_ON_ONCE(saved_unhash == sock_map_unhash))
+ return;
}
- if (WARN_ON_ONCE(saved_unhash == sock_map_unhash))
- return;
+
if (saved_unhash)
saved_unhash(sk);
}
@@ -1652,20 +1657,25 @@ void sock_map_destroy(struct sock *sk)
void (*saved_destroy)(struct sock *sk);
struct sk_psock *psock;
+retry:
rcu_read_lock();
psock = sk_psock_get(sk);
if (unlikely(!psock)) {
rcu_read_unlock();
saved_destroy = READ_ONCE(sk->sk_prot)->destroy;
+ if (unlikely(saved_destroy == sock_map_destroy))
+ goto retry;
} else {
saved_destroy = psock->saved_destroy;
sock_map_remove_links(sk, psock);
rcu_read_unlock();
sk_psock_stop(psock);
sk_psock_put(sk, psock);
+
+ if (WARN_ON_ONCE(saved_destroy == sock_map_destroy))
+ return;
}
- if (WARN_ON_ONCE(saved_destroy == sock_map_destroy))
- return;
+
if (saved_destroy)
saved_destroy(sk);
}
@@ -1676,32 +1686,33 @@ void sock_map_close(struct sock *sk, long timeout)
void (*saved_close)(struct sock *sk, long timeout);
struct sk_psock *psock;
+retry:
lock_sock(sk);
rcu_read_lock();
- psock = sk_psock(sk);
+ psock = sk_psock_get(sk);
if (likely(psock)) {
saved_close = psock->saved_close;
sock_map_remove_links(sk, psock);
- psock = sk_psock_get(sk);
- if (unlikely(!psock))
- goto no_psock;
rcu_read_unlock();
sk_psock_stop(psock);
release_sock(sk);
cancel_delayed_work_sync(&psock->work);
sk_psock_put(sk, psock);
+
+ /* Make sure we do not recurse. This is a bug.
+ * Leak the socket instead of crashing on a stack overflow.
+ */
+ if (WARN_ON_ONCE(saved_close == sock_map_close))
+ return;
} else {
saved_close = READ_ONCE(sk->sk_prot)->close;
-no_psock:
rcu_read_unlock();
release_sock(sk);
+
+ if (unlikely(saved_close == sock_map_close))
+ goto retry;
}
- /* Make sure we do not recurse. This is a bug.
- * Leak the socket instead of crashing on a stack overflow.
- */
- if (WARN_ON_ONCE(saved_close == sock_map_close))
- return;
saved_close(sk, timeout);
}
EXPORT_SYMBOL_GPL(sock_map_close);
diff --git a/net/devlink/core.c b/net/devlink/core.c
index eeb6a71..fe9f6a0 100644
--- a/net/devlink/core.c
+++ b/net/devlink/core.c
@@ -518,6 +518,8 @@ void devlink_free(struct devlink *devlink)
{
ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+ devlink_rel_put(devlink);
+
WARN_ON(!list_empty(&devlink->trap_policer_list));
WARN_ON(!list_empty(&devlink->trap_group_list));
WARN_ON(!list_empty(&devlink->trap_list));
diff --git a/net/ethtool/bitset.c b/net/ethtool/bitset.c
index 8bb98d3..a3a2cc64 100644
--- a/net/ethtool/bitset.c
+++ b/net/ethtool/bitset.c
@@ -92,7 +92,7 @@ static bool ethnl_bitmap32_not_zero(const u32 *map, unsigned int start,
u32 mask;
if (end <= start)
- return true;
+ return false;
if (start % 32) {
mask = ethnl_upper_bits(start);
@@ -105,11 +105,11 @@ static bool ethnl_bitmap32_not_zero(const u32 *map, unsigned int start,
start_word++;
}
- if (!memchr_inv(map + start_word, '\0',
- (end_word - start_word) * sizeof(u32)))
+ if (memchr_inv(map + start_word, '\0',
+ (end_word - start_word) * sizeof(u32)))
return true;
if (end % 32 == 0)
- return true;
+ return false;
return map[end_word] & ethnl_lower_bits(end);
}
diff --git a/net/ethtool/cmis.h b/net/ethtool/cmis.h
index 4a9a946..778783a 100644
--- a/net/ethtool/cmis.h
+++ b/net/ethtool/cmis.h
@@ -63,9 +63,9 @@ struct ethtool_cmis_cdb_request {
* struct ethtool_cmis_cdb_cmd_args - CDB commands execution arguments
* @req: CDB command fields as described in the CMIS standard.
* @max_duration: Maximum duration time for command completion in msec.
+ * @msleep_pre_rpl: Waiting time before checking reply in msec.
* @read_write_len_ext: Allowable additional number of byte octets to the LPL
* in a READ or a WRITE commands.
- * @msleep_pre_rpl: Waiting time before checking reply in msec.
* @rpl_exp_len: Expected reply length in bytes.
* @flags: Validation flags for CDB commands.
* @err_msg: Error message to be sent to user space.
@@ -73,8 +73,8 @@ struct ethtool_cmis_cdb_request {
struct ethtool_cmis_cdb_cmd_args {
struct ethtool_cmis_cdb_request req;
u16 max_duration;
+ u16 msleep_pre_rpl;
u8 read_write_len_ext;
- u8 msleep_pre_rpl;
u8 rpl_exp_len;
u8 flags;
char *err_msg;
diff --git a/net/ethtool/cmis_cdb.c b/net/ethtool/cmis_cdb.c
index 3670ca42..f3a53a9 100644
--- a/net/ethtool/cmis_cdb.c
+++ b/net/ethtool/cmis_cdb.c
@@ -513,8 +513,13 @@ static int cmis_cdb_process_reply(struct net_device *dev,
}
rpl = (struct ethtool_cmis_cdb_rpl *)page_data->data;
- if ((args->rpl_exp_len > rpl->hdr.rpl_len + rpl_hdr_len) ||
- !rpl->hdr.rpl_chk_code) {
+ if (rpl->hdr.rpl_len != args->rpl_exp_len) {
+ netdev_warn(dev, "CDB reply length mismatch, expected %u got %u\n",
+ args->rpl_exp_len, rpl->hdr.rpl_len);
+ err = -EIO;
+ goto out;
+ }
+ if (!rpl->hdr.rpl_chk_code) {
err = -EIO;
goto out;
}
diff --git a/net/ethtool/cmis_fw_update.c b/net/ethtool/cmis_fw_update.c
index df5f344..291d04d 100644
--- a/net/ethtool/cmis_fw_update.c
+++ b/net/ethtool/cmis_fw_update.c
@@ -44,6 +44,20 @@ enum cmis_cdb_fw_write_mechanism {
CMIS_CDB_FW_WRITE_MECHANISM_BOTH = 0x11,
};
+/* See section 9.7.2 "CMD 0101h: Start Firmware Download" in CMIS standard
+ * revision 5.2.
+ * struct cmis_cdb_start_fw_download_pl is a structured layout of the
+ * flat array, ethtool_cmis_cdb_request::payload.
+ */
+struct cmis_cdb_start_fw_download_pl {
+ __struct_group(cmis_cdb_start_fw_download_pl_h, head, /* no attrs */,
+ __be32 image_size;
+ __be32 resv1;
+ );
+ u8 vendor_data[ETHTOOL_CMIS_CDB_LPL_MAX_PL_LENGTH -
+ sizeof(struct cmis_cdb_start_fw_download_pl_h)];
+};
+
static int
cmis_fw_update_fw_mng_features_get(struct ethtool_cmis_cdb *cdb,
struct net_device *dev,
@@ -86,6 +100,14 @@ cmis_fw_update_fw_mng_features_get(struct ethtool_cmis_cdb *cdb,
*/
cdb->read_write_len_ext = rpl->read_write_len_ext;
fw_mng->start_cmd_payload_size = rpl->start_cmd_payload_size;
+ if (fw_mng->start_cmd_payload_size >
+ sizeof_field(struct cmis_cdb_start_fw_download_pl, vendor_data)) {
+ ethnl_module_fw_flash_ntf_err(dev, ntf_params,
+ "Start cmd payload size exceeds max LPL payload",
+ NULL);
+ return -EINVAL;
+ }
+
fw_mng->write_mechanism =
rpl->write_mechanism == CMIS_CDB_FW_WRITE_MECHANISM_LPL ?
CMIS_CDB_FW_WRITE_MECHANISM_LPL :
@@ -97,20 +119,6 @@ cmis_fw_update_fw_mng_features_get(struct ethtool_cmis_cdb *cdb,
return 0;
}
-/* See section 9.7.2 "CMD 0101h: Start Firmware Download" in CMIS standard
- * revision 5.2.
- * struct cmis_cdb_start_fw_download_pl is a structured layout of the
- * flat array, ethtool_cmis_cdb_request::payload.
- */
-struct cmis_cdb_start_fw_download_pl {
- __struct_group(cmis_cdb_start_fw_download_pl_h, head, /* no attrs */,
- __be32 image_size;
- __be32 resv1;
- );
- u8 vendor_data[ETHTOOL_CMIS_CDB_LPL_MAX_PL_LENGTH -
- sizeof(struct cmis_cdb_start_fw_download_pl_h)];
-};
-
static int
cmis_fw_update_start_download(struct ethtool_cmis_cdb *cdb,
struct ethtool_cmis_fw_update_params *fw_update,
@@ -122,6 +130,14 @@ cmis_fw_update_start_download(struct ethtool_cmis_cdb *cdb,
u8 lpl_len;
int err;
+ if (fw_update->fw->size < vendor_data_size) {
+ ethnl_module_fw_flash_ntf_err(fw_update->dev,
+ &fw_update->ntf_params,
+ "Firmware image too small for module's start payload",
+ NULL);
+ return -EINVAL;
+ }
+
pl.image_size = cpu_to_be32(fw_update->fw->size);
memcpy(pl.vendor_data, fw_update->fw->data, vendor_data_size);
diff --git a/net/ethtool/coalesce.c b/net/ethtool/coalesce.c
index 1e2c5c7..e73fc3e 100644
--- a/net/ethtool/coalesce.c
+++ b/net/ethtool/coalesce.c
@@ -472,6 +472,12 @@ static int ethnl_update_profile(struct net_device *dev,
nla_for_each_nested_type(nest, ETHTOOL_A_PROFILE_IRQ_MODERATION,
nests, rem) {
+ if (i >= NET_DIM_PARAMS_NUM_PROFILES) {
+ NL_SET_BAD_ATTR(extack, nest);
+ ret = -E2BIG;
+ goto err_out;
+ }
+
ret = nla_parse_nested(tb, len_irq_moder - 1, nest,
coalesce_irq_moderation_policy,
extack);
diff --git a/net/ethtool/eeprom.c b/net/ethtool/eeprom.c
index a557e39..0b8cfed 100644
--- a/net/ethtool/eeprom.c
+++ b/net/ethtool/eeprom.c
@@ -44,6 +44,9 @@ static int fallback_set_params(struct eeprom_req_info *request,
if (offset >= modinfo->eeprom_len)
return -EINVAL;
+ if (length > modinfo->eeprom_len - offset)
+ return -EINVAL;
+
eeprom->cmd = ETHTOOL_GMODULEEEPROM;
eeprom->len = length;
eeprom->offset = offset;
@@ -69,7 +72,7 @@ static int eeprom_fallback(struct eeprom_req_info *request,
if (err < 0)
return err;
- data = kmalloc(eeprom.len, GFP_KERNEL);
+ data = kzalloc(eeprom.len, GFP_KERNEL);
if (!data)
return -ENOMEM;
err = ethtool_get_module_eeprom_call(dev, &eeprom, data);
@@ -141,12 +144,11 @@ static int eeprom_prepare_data(const struct ethnl_req_info *req_base,
return 0;
err_ops:
+ if (ret == -EOPNOTSUPP)
+ ret = eeprom_fallback(request, reply);
ethnl_ops_complete(dev);
err_free:
kfree(page_data.data);
-
- if (ret == -EOPNOTSUPP)
- return eeprom_fallback(request, reply);
return ret;
}
diff --git a/net/ethtool/linkstate.c b/net/ethtool/linkstate.c
index 8a5985f..24569e9 100644
--- a/net/ethtool/linkstate.c
+++ b/net/ethtool/linkstate.c
@@ -106,10 +106,8 @@ static int linkstate_prepare_data(const struct ethnl_req_info *req_base,
phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_LINKSTATE_HEADER,
info->extack);
- if (IS_ERR(phydev)) {
- ret = PTR_ERR(phydev);
- goto out;
- }
+ if (IS_ERR(phydev))
+ return PTR_ERR(phydev);
ret = ethnl_ops_begin(dev);
if (ret < 0)
diff --git a/net/ethtool/module.c b/net/ethtool/module.c
index cad2eb2..ea4fb2a 100644
--- a/net/ethtool/module.c
+++ b/net/ethtool/module.c
@@ -120,12 +120,6 @@ ethnl_set_module_validate(struct ethnl_req_info *req_info,
if (!tb[ETHTOOL_A_MODULE_POWER_MODE_POLICY])
return 0;
- if (req_info->dev->ethtool->module_fw_flash_in_progress) {
- NL_SET_ERR_MSG(info->extack,
- "Module firmware flashing is in progress");
- return -EBUSY;
- }
-
if (!ops->get_module_power_mode || !ops->set_module_power_mode) {
NL_SET_ERR_MSG_ATTR(info->extack,
tb[ETHTOOL_A_MODULE_POWER_MODE_POLICY],
@@ -148,6 +142,12 @@ ethnl_set_module(struct ethnl_req_info *req_info, struct genl_info *info)
ops = dev->ethtool_ops;
+ if (dev->ethtool->module_fw_flash_in_progress) {
+ NL_SET_ERR_MSG(info->extack,
+ "Module firmware flashing is in progress");
+ return -EBUSY;
+ }
+
power_new.policy = nla_get_u8(tb[ETHTOOL_A_MODULE_POWER_MODE_POLICY]);
ret = ops->get_module_power_mode(dev, &power, info->extack);
if (ret < 0)
@@ -221,14 +221,22 @@ static void module_flash_fw_work_list_del(struct list_head *list)
static void module_flash_fw_work(struct work_struct *work)
{
struct ethtool_module_fw_flash *module_fw;
+ struct net_device *dev;
module_fw = container_of(work, struct ethtool_module_fw_flash, work);
+ dev = module_fw->fw_update.dev;
ethtool_cmis_fw_update(&module_fw->fw_update);
module_flash_fw_work_list_del(&module_fw->list);
- module_fw->fw_update.dev->ethtool->module_fw_flash_in_progress = false;
- netdev_put(module_fw->fw_update.dev, &module_fw->dev_tracker);
+
+ rtnl_lock();
+ netdev_lock_ops(dev);
+ dev->ethtool->module_fw_flash_in_progress = false;
+ netdev_unlock_ops(dev);
+ rtnl_unlock();
+
+ netdev_put(dev, &module_fw->dev_tracker);
release_firmware(module_fw->fw_update.fw);
kfree(module_fw);
}
@@ -283,11 +291,9 @@ void ethnl_module_fw_flash_sock_destroy(struct ethnl_sock_priv *sk_priv)
spin_lock(&module_fw_flash_work_list_lock);
list_for_each_entry(work, &module_fw_flash_work_list, list) {
- if (work->fw_update.dev == sk_priv->dev &&
- work->fw_update.ntf_params.portid == sk_priv->portid) {
+ if (work->fw_update.ntf_params.portid == sk_priv->portid &&
+ dev_net(work->fw_update.dev) == sk_priv->net)
work->fw_update.ntf_params.closed_sock = true;
- break;
- }
}
spin_unlock(&module_fw_flash_work_list_lock);
}
@@ -319,14 +325,13 @@ module_flash_fw_schedule(struct net_device *dev, const char *file_name,
if (err < 0)
goto err_release_firmware;
- dev->ethtool->module_fw_flash_in_progress = true;
- netdev_hold(dev, &module_fw->dev_tracker, GFP_KERNEL);
fw_update->dev = dev;
fw_update->ntf_params.portid = info->snd_portid;
fw_update->ntf_params.seq = info->snd_seq;
fw_update->ntf_params.closed_sock = false;
- err = ethnl_sock_priv_set(skb, dev, fw_update->ntf_params.portid,
+ err = ethnl_sock_priv_set(skb, dev_net(dev),
+ fw_update->ntf_params.portid,
ETHTOOL_SOCK_TYPE_MODULE_FW_FLASH);
if (err < 0)
goto err_release_firmware;
@@ -335,6 +340,9 @@ module_flash_fw_schedule(struct net_device *dev, const char *file_name,
if (err < 0)
goto err_release_firmware;
+ dev->ethtool->module_fw_flash_in_progress = true;
+ netdev_hold(dev, &module_fw->dev_tracker, GFP_KERNEL);
+
schedule_work(&module_fw->work);
return 0;
@@ -427,10 +435,11 @@ int ethnl_act_module_fw_flash(struct sk_buff *skb, struct genl_info *info)
ret = ethnl_module_fw_flash_validate(dev, info->extack);
if (ret < 0)
- goto out_unlock;
+ goto out_complete;
ret = module_flash_fw(dev, tb, skb, info);
+out_complete:
ethnl_ops_complete(dev);
out_unlock:
diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c
index 5046023..7d45f9a 100644
--- a/net/ethtool/netlink.c
+++ b/net/ethtool/netlink.c
@@ -53,7 +53,7 @@ const struct nla_policy ethnl_header_policy_phy_stats[] = {
[ETHTOOL_A_HEADER_PHY_INDEX] = NLA_POLICY_MIN(NLA_U32, 1),
};
-int ethnl_sock_priv_set(struct sk_buff *skb, struct net_device *dev, u32 portid,
+int ethnl_sock_priv_set(struct sk_buff *skb, struct net *net, u32 portid,
enum ethnl_sock_type type)
{
struct ethnl_sock_priv *sk_priv;
@@ -62,7 +62,7 @@ int ethnl_sock_priv_set(struct sk_buff *skb, struct net_device *dev, u32 portid,
if (IS_ERR(sk_priv))
return PTR_ERR(sk_priv);
- sk_priv->dev = dev;
+ sk_priv->net = net;
sk_priv->portid = portid;
sk_priv->type = type;
diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h
index aaf6f24..fd2198e 100644
--- a/net/ethtool/netlink.h
+++ b/net/ethtool/netlink.h
@@ -318,12 +318,12 @@ enum ethnl_sock_type {
};
struct ethnl_sock_priv {
- struct net_device *dev;
+ struct net *net;
u32 portid;
enum ethnl_sock_type type;
};
-int ethnl_sock_priv_set(struct sk_buff *skb, struct net_device *dev, u32 portid,
+int ethnl_sock_priv_set(struct sk_buff *skb, struct net *net, u32 portid,
enum ethnl_sock_type type);
/**
diff --git a/net/ethtool/phy.c b/net/ethtool/phy.c
index d4e6887..ddc6eab 100644
--- a/net/ethtool/phy.c
+++ b/net/ethtool/phy.c
@@ -76,6 +76,7 @@ static int phy_prepare_data(const struct ethnl_req_info *req_info,
struct nlattr **tb = info->attrs;
struct phy_device_node *pdn;
struct phy_device *phydev;
+ int ret;
/* RTNL is held by the caller */
phydev = ethnl_req_get_phydev(req_info, tb, ETHTOOL_A_PHY_HEADER,
@@ -88,8 +89,19 @@ static int phy_prepare_data(const struct ethnl_req_info *req_info,
return -EOPNOTSUPP;
rep_data->phyindex = phydev->phyindex;
+
rep_data->name = kstrdup(dev_name(&phydev->mdio.dev), GFP_KERNEL);
- rep_data->drvname = kstrdup(phydev->drv->name, GFP_KERNEL);
+ if (!rep_data->name)
+ return -ENOMEM;
+
+ if (phydev->drv) {
+ rep_data->drvname = kstrdup(phydev->drv->name, GFP_KERNEL);
+ if (!rep_data->drvname) {
+ ret = -ENOMEM;
+ goto err_free_name;
+ }
+ }
+
rep_data->upstream_type = pdn->upstream_type;
if (pdn->upstream_type == PHY_UPSTREAM_PHY) {
@@ -97,15 +109,33 @@ static int phy_prepare_data(const struct ethnl_req_info *req_info,
rep_data->upstream_index = upstream->phyindex;
}
- if (pdn->parent_sfp_bus)
+ if (pdn->parent_sfp_bus) {
rep_data->upstream_sfp_name = kstrdup(sfp_get_name(pdn->parent_sfp_bus),
GFP_KERNEL);
+ if (!rep_data->upstream_sfp_name) {
+ ret = -ENOMEM;
+ goto err_free_drvname;
+ }
+ }
- if (phydev->sfp_bus)
+ if (phydev->sfp_bus) {
rep_data->downstream_sfp_name = kstrdup(sfp_get_name(phydev->sfp_bus),
GFP_KERNEL);
+ if (!rep_data->downstream_sfp_name) {
+ ret = -ENOMEM;
+ goto err_free_upstream_sfp;
+ }
+ }
return 0;
+
+err_free_upstream_sfp:
+ kfree(rep_data->upstream_sfp_name);
+err_free_drvname:
+ kfree(rep_data->drvname);
+err_free_name:
+ kfree(rep_data->name);
+ return ret;
}
static int phy_fill_reply(struct sk_buff *skb,
diff --git a/net/ethtool/pse-pd.c b/net/ethtool/pse-pd.c
index 2eb9bdc..757c9e0 100644
--- a/net/ethtool/pse-pd.c
+++ b/net/ethtool/pse-pd.c
@@ -62,14 +62,14 @@ static int pse_prepare_data(const struct ethnl_req_info *req_base,
struct phy_device *phydev;
int ret;
- ret = ethnl_ops_begin(dev);
- if (ret < 0)
- return ret;
-
phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_PSE_HEADER,
info->extack);
if (IS_ERR(phydev))
- return -ENODEV;
+ return PTR_ERR(phydev);
+
+ ret = ethnl_ops_begin(dev);
+ if (ret < 0)
+ return ret;
ret = pse_get_pse_attributes(phydev, info->extack, data);
diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c
index 353110b..53792f5 100644
--- a/net/ethtool/rss.c
+++ b/net/ethtool/rss.c
@@ -134,8 +134,7 @@ rss_get_data_alloc(struct net_device *dev, struct rss_reply_data *data)
if (!rss_config)
return -ENOMEM;
- if (data->indir_size)
- data->indir_table = (u32 *)rss_config;
+ data->indir_table = (u32 *)rss_config;
if (data->hkey_size)
data->hkey = rss_config + indir_bytes;
@@ -170,8 +169,10 @@ rss_prepare_get(const struct rss_req_info *request, struct net_device *dev,
rxfh.key = data->hkey;
ret = ops->get_rxfh(dev, &rxfh);
- if (ret)
+ if (ret) {
+ rss_get_data_free(data);
goto out_unlock;
+ }
data->hfunc = rxfh.hfunc;
data->input_xfrm = rxfh.input_xfrm;
@@ -686,7 +687,7 @@ rss_set_prep_indir(struct net_device *dev, struct genl_info *info,
ethtool_rxfh_indir_default(i, num_rx_rings);
}
- *mod |= memcmp(rxfh->indir, data->indir_table, data->indir_size);
+ *mod |= memcmp(rxfh->indir, data->indir_table, alloc_size);
return user_size;
@@ -981,11 +982,17 @@ ethnl_rss_create_validate(struct net_device *dev, struct genl_info *info)
}
static void
-ethnl_rss_create_send_ntf(struct sk_buff *rsp, struct net_device *dev)
+ethnl_rss_create_send_ntf(const struct sk_buff *rsp, struct net_device *dev)
{
- struct nlmsghdr *nlh = (void *)rsp->data;
struct genlmsghdr *genl_hdr;
+ struct nlmsghdr *nlh;
+ struct sk_buff *ntf;
+ ntf = skb_copy_expand(rsp, 0, 0, GFP_KERNEL);
+ if (!ntf)
+ return;
+
+ nlh = nlmsg_hdr(ntf);
/* Convert the reply into a notification */
nlh->nlmsg_pid = 0;
nlh->nlmsg_seq = ethnl_bcast_seq_next();
@@ -993,7 +1000,7 @@ ethnl_rss_create_send_ntf(struct sk_buff *rsp, struct net_device *dev)
genl_hdr = nlmsg_data(nlh);
genl_hdr->cmd = ETHTOOL_MSG_RSS_CREATE_NTF;
- ethnl_multicast(rsp, dev);
+ ethnl_multicast(ntf, dev);
}
int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info)
@@ -1099,17 +1106,13 @@ int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info)
ntf_fail |= rss_fill_reply(rsp, &req.base, &data.base);
if (WARN_ON(!hdr || ntf_fail)) {
ret = -EMSGSIZE;
- goto exit_unlock;
+ goto err_remove_ctx;
}
genlmsg_end(rsp, hdr);
- /* Use the same skb for the response and the notification,
- * genlmsg_reply() will copy the skb if it has elevated user count.
- */
- skb_get(rsp);
- ret = genlmsg_reply(rsp, info);
ethnl_rss_create_send_ntf(rsp, dev);
+ ret = genlmsg_reply(rsp, info);
rsp = NULL;
exit_unlock:
@@ -1131,6 +1134,10 @@ int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info)
nlmsg_free(rsp);
return ret;
+err_remove_ctx:
+ if (ops->remove_rxfh_context(dev, ctx, req.rss_context, NULL))
+ /* leave the context on failure, like ethnl_rss_delete_doit() */
+ goto exit_unlock;
err_ctx_id_free:
xa_erase(&dev->ethtool->rss_ctx, req.rss_context);
err_unlock_free_ctx:
@@ -1168,8 +1175,10 @@ int ethnl_rss_delete_doit(struct sk_buff *skb, struct genl_info *info)
dev = req.dev;
ops = dev->ethtool_ops;
- if (!ops->create_rxfh_context)
+ if (!ops->create_rxfh_context) {
+ ret = -EOPNOTSUPP;
goto exit_free_dev;
+ }
rtnl_lock();
netdev_lock_ops(dev);
diff --git a/net/ethtool/strset.c b/net/ethtool/strset.c
index bb1e829..94c4718 100644
--- a/net/ethtool/strset.c
+++ b/net/ethtool/strset.c
@@ -311,7 +311,7 @@ static int strset_prepare_data(const struct ethnl_req_info *req_base,
return 0;
}
- phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_HEADER_FLAGS,
+ phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_STRSET_HEADER,
info->extack);
/* phydev can be NULL, check for errors only */
diff --git a/net/ethtool/tsconfig.c b/net/ethtool/tsconfig.c
index e4f518e49..fc4f93c 100644
--- a/net/ethtool/tsconfig.c
+++ b/net/ethtool/tsconfig.c
@@ -69,8 +69,10 @@ static int tsconfig_prepare_data(const struct ethnl_req_info *req_base,
if (ret)
goto out;
- if (ts_info.phc_index == -1)
- return -ENODEV;
+ if (ts_info.phc_index == -1) {
+ ret = -ENODEV;
+ goto out;
+ }
data->hwprov_desc.index = ts_info.phc_index;
data->hwprov_desc.qualifier = ts_info.phc_qualifier;
@@ -224,16 +226,21 @@ static int tsconfig_send_reply(struct net_device *dev, struct genl_info *info)
reply_len = ret + ethnl_reply_header_size();
rskb = ethnl_reply_init(reply_len, dev, ETHTOOL_MSG_TSCONFIG_SET_REPLY,
ETHTOOL_A_TSCONFIG_HEADER, info, &reply_payload);
- if (!rskb)
+ if (!rskb) {
+ ret = -ENOMEM;
goto err_cleanup;
+ }
ret = tsconfig_fill_reply(rskb, &req_info->base, &reply_data->base);
if (ret < 0)
- goto err_cleanup;
+ goto err_free_msg;
genlmsg_end(rskb, reply_payload);
ret = genlmsg_reply(rskb, info);
+ rskb = NULL;
+err_free_msg:
+ nlmsg_free(rskb);
err_cleanup:
kfree(reply_data);
kfree(req_info);
diff --git a/net/ethtool/tsinfo.c b/net/ethtool/tsinfo.c
index a865f0f..14bf01e 100644
--- a/net/ethtool/tsinfo.c
+++ b/net/ethtool/tsinfo.c
@@ -83,6 +83,11 @@ tsinfo_parse_request(struct ethnl_req_info *req_base,
if (!tb[ETHTOOL_A_TSINFO_HWTSTAMP_PROVIDER])
return 0;
+ if (req_base->flags & ETHTOOL_FLAG_STATS) {
+ NL_SET_ERR_MSG(extack, "can't query statistics for a provider");
+ return -EOPNOTSUPP;
+ }
+
return ts_parse_hwtst_provider(tb[ETHTOOL_A_TSINFO_HWTSTAMP_PROVIDER],
&req->hwprov_desc, extack, &mod);
}
@@ -402,10 +407,8 @@ static int ethnl_tsinfo_dump_one_netdev(struct sk_buff *skb,
continue;
ehdr = ethnl_tsinfo_prepare_dump(skb, dev, reply_data, cb);
- if (IS_ERR(ehdr)) {
- ret = PTR_ERR(ehdr);
- goto err;
- }
+ if (IS_ERR(ehdr))
+ return PTR_ERR(ehdr);
reply_data->ts_info.phc_qualifier = ctx->pos_phcqualifier;
ret = ops->get_ts_info(dev, &reply_data->ts_info);
@@ -523,6 +526,12 @@ int ethnl_tsinfo_start(struct netlink_callback *cb)
if (ret < 0)
goto free_reply_data;
+ if (req_info->base.flags & ETHTOOL_FLAG_STATS) {
+ NL_SET_ERR_MSG(cb->extack, "stats not supported in dump");
+ ret = -EOPNOTSUPP;
+ goto err_dev_put;
+ }
+
ctx->req_info = req_info;
ctx->reply_data = reply_data;
ctx->pos_ifindex = 0;
@@ -532,6 +541,8 @@ int ethnl_tsinfo_start(struct netlink_callback *cb)
return 0;
+err_dev_put:
+ ethnl_parse_header_dev_put(&req_info->base);
free_reply_data:
kfree(reply_data);
free_req_info:
diff --git a/net/handshake/genl.c b/net/handshake/genl.c
index 8706126..4b20cd9 100644
--- a/net/handshake/genl.c
+++ b/net/handshake/genl.c
@@ -10,6 +10,7 @@
#include "genl.h"
#include <uapi/linux/handshake.h>
+#include <linux/err.h>
/* HANDSHAKE_CMD_ACCEPT - do */
static const struct nla_policy handshake_accept_nl_policy[HANDSHAKE_A_ACCEPT_HANDLER_CLASS + 1] = {
@@ -18,7 +19,7 @@ static const struct nla_policy handshake_accept_nl_policy[HANDSHAKE_A_ACCEPT_HAN
/* HANDSHAKE_CMD_DONE - do */
static const struct nla_policy handshake_done_nl_policy[HANDSHAKE_A_DONE_REMOTE_AUTH + 1] = {
- [HANDSHAKE_A_DONE_STATUS] = { .type = NLA_U32, },
+ [HANDSHAKE_A_DONE_STATUS] = NLA_POLICY_MAX(NLA_U32, MAX_ERRNO),
[HANDSHAKE_A_DONE_SOCKFD] = { .type = NLA_S32, },
[HANDSHAKE_A_DONE_REMOTE_AUTH] = { .type = NLA_U32, },
};
diff --git a/net/handshake/genl.h b/net/handshake/genl.h
index 8d3e186..46b65f1 100644
--- a/net/handshake/genl.h
+++ b/net/handshake/genl.h
@@ -11,6 +11,7 @@
#include <net/genetlink.h>
#include <uapi/linux/handshake.h>
+#include <linux/err.h>
int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info);
int handshake_nl_done_doit(struct sk_buff *skb, struct genl_info *info);
diff --git a/net/handshake/handshake-test.c b/net/handshake/handshake-test.c
index 55442b2..3dd5074 100644
--- a/net/handshake/handshake-test.c
+++ b/net/handshake/handshake-test.c
@@ -25,7 +25,7 @@ static int test_accept_func(struct handshake_req *req, struct genl_info *info,
return 0;
}
-static void test_done_func(struct handshake_req *req, unsigned int status,
+static void test_done_func(struct handshake_req *req, int status,
struct genl_info *info)
{
}
@@ -208,6 +208,7 @@ static void handshake_req_submit_test3(struct kunit *test)
static void handshake_req_submit_test4(struct kunit *test)
{
struct handshake_req *req, *result;
+ unsigned long fcount_before;
struct socket *sock;
struct file *filp;
int err;
@@ -224,8 +225,10 @@ static void handshake_req_submit_test4(struct kunit *test)
KUNIT_ASSERT_NOT_NULL(test, sock->sk);
sock->file = filp;
+ fcount_before = file_count(filp);
err = handshake_req_submit(sock, req, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
/* Act */
result = handshake_req_hash_lookup(sock->sk);
@@ -235,11 +238,13 @@ static void handshake_req_submit_test4(struct kunit *test)
KUNIT_EXPECT_PTR_EQ(test, req, result);
handshake_req_cancel(sock->sk);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
}
static void handshake_req_submit_test5(struct kunit *test)
{
+ unsigned long fcount_before;
struct handshake_req *req;
struct handshake_net *hn;
struct socket *sock;
@@ -265,12 +270,14 @@ static void handshake_req_submit_test5(struct kunit *test)
saved = hn->hn_pending;
hn->hn_pending = hn->hn_pending_max + 1;
+ fcount_before = file_count(filp);
/* Act */
err = handshake_req_submit(sock, req, GFP_KERNEL);
/* Assert */
KUNIT_EXPECT_EQ(test, err, -EAGAIN);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
hn->hn_pending = saved;
@@ -279,6 +286,7 @@ static void handshake_req_submit_test5(struct kunit *test)
static void handshake_req_submit_test6(struct kunit *test)
{
struct handshake_req *req1, *req2;
+ unsigned long fcount_before;
struct socket *sock;
struct file *filp;
int err;
@@ -296,21 +304,26 @@ static void handshake_req_submit_test6(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filp);
KUNIT_ASSERT_NOT_NULL(test, sock->sk);
sock->file = filp;
+ fcount_before = file_count(filp);
/* Act */
err = handshake_req_submit(sock, req1, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
err = handshake_req_submit(sock, req2, GFP_KERNEL);
/* Assert */
KUNIT_EXPECT_EQ(test, err, -EBUSY);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
handshake_req_cancel(sock->sk);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
}
static void handshake_req_cancel_test1(struct kunit *test)
{
+ unsigned long fcount_before;
struct handshake_req *req;
struct socket *sock;
struct file *filp;
@@ -329,8 +342,10 @@ static void handshake_req_cancel_test1(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filp);
sock->file = filp;
+ fcount_before = file_count(filp);
err = handshake_req_submit(sock, req, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
/* NB: handshake_req hasn't been accepted */
@@ -339,12 +354,14 @@ static void handshake_req_cancel_test1(struct kunit *test)
/* Assert */
KUNIT_EXPECT_TRUE(test, result);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
}
static void handshake_req_cancel_test2(struct kunit *test)
{
+ unsigned long fcount_before;
struct handshake_req *req, *next;
struct handshake_net *hn;
struct socket *sock;
@@ -365,8 +382,10 @@ static void handshake_req_cancel_test2(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filp);
sock->file = filp;
+ fcount_before = file_count(filp);
err = handshake_req_submit(sock, req, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
net = sock_net(sock->sk);
hn = handshake_pernet(net);
@@ -375,18 +394,24 @@ static void handshake_req_cancel_test2(struct kunit *test)
/* Pretend to accept this request */
next = handshake_req_next(hn, HANDSHAKE_HANDLER_CLASS_TLSHD);
KUNIT_ASSERT_PTR_EQ(test, req, next);
+ /* Simulate FD_PREPARE() consuming the file reference handed
+ * off by handshake_req_next(); see handshake_nl_accept_doit().
+ */
+ fput(filp);
/* Act */
result = handshake_req_cancel(sock->sk);
/* Assert */
KUNIT_EXPECT_TRUE(test, result);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
}
static void handshake_req_cancel_test3(struct kunit *test)
{
+ unsigned long fcount_before;
struct handshake_req *req, *next;
struct handshake_net *hn;
struct socket *sock;
@@ -407,8 +432,10 @@ static void handshake_req_cancel_test3(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filp);
sock->file = filp;
+ fcount_before = file_count(filp);
err = handshake_req_submit(sock, req, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before + 1);
net = sock_net(sock->sk);
hn = handshake_pernet(net);
@@ -417,15 +444,21 @@ static void handshake_req_cancel_test3(struct kunit *test)
/* Pretend to accept this request */
next = handshake_req_next(hn, HANDSHAKE_HANDLER_CLASS_TLSHD);
KUNIT_ASSERT_PTR_EQ(test, req, next);
+ /* Simulate FD_PREPARE() consuming the file reference handed
+ * off by handshake_req_next(); see handshake_nl_accept_doit().
+ */
+ fput(filp);
/* Pretend to complete this request */
handshake_complete(next, -ETIMEDOUT, NULL);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
/* Act */
result = handshake_req_cancel(sock->sk);
/* Assert */
KUNIT_EXPECT_FALSE(test, result);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
fput(filp);
}
@@ -446,6 +479,7 @@ static struct handshake_proto handshake_req_alloc_proto_destroy = {
static void handshake_req_destroy_test1(struct kunit *test)
{
+ unsigned long fcount_before;
struct handshake_req *req;
struct socket *sock;
struct file *filp;
@@ -465,10 +499,12 @@ static void handshake_req_destroy_test1(struct kunit *test)
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filp);
sock->file = filp;
+ fcount_before = file_count(filp);
err = handshake_req_submit(sock, req, GFP_KERNEL);
KUNIT_ASSERT_EQ(test, err, 0);
handshake_req_cancel(sock->sk);
+ KUNIT_EXPECT_EQ(test, file_count(filp), fcount_before);
/* Act */
/* Ensure the close/release/put process has run to
diff --git a/net/handshake/handshake.h b/net/handshake/handshake.h
index a481637..da61cad 100644
--- a/net/handshake/handshake.h
+++ b/net/handshake/handshake.h
@@ -24,6 +24,7 @@ enum hn_flags_bits {
HANDSHAKE_F_NET_DRAINING,
};
+struct file;
struct handshake_proto;
/* One handshake request */
@@ -32,6 +33,7 @@ struct handshake_req {
struct rhash_head hr_rhash;
unsigned long hr_flags;
const struct handshake_proto *hr_proto;
+ struct file *hr_file;
struct sock *hr_sk;
void (*hr_odestruct)(struct sock *sk);
@@ -57,7 +59,7 @@ struct handshake_proto {
int (*hp_accept)(struct handshake_req *req,
struct genl_info *info, int fd);
void (*hp_done)(struct handshake_req *req,
- unsigned int status,
+ int status,
struct genl_info *info);
void (*hp_destroy)(struct handshake_req *req);
};
@@ -86,7 +88,7 @@ struct handshake_req *handshake_req_hash_lookup(struct sock *sk);
struct handshake_req *handshake_req_next(struct handshake_net *hn, int class);
int handshake_req_submit(struct socket *sock, struct handshake_req *req,
gfp_t flags);
-void handshake_complete(struct handshake_req *req, unsigned int status,
+void handshake_complete(struct handshake_req *req, int status,
struct genl_info *info);
bool handshake_req_cancel(struct sock *sk);
diff --git a/net/handshake/netlink.c b/net/handshake/netlink.c
index b989456..3fd4fef 100644
--- a/net/handshake/netlink.c
+++ b/net/handshake/netlink.c
@@ -92,7 +92,6 @@ int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info)
struct net *net = sock_net(skb->sk);
struct handshake_net *hn = handshake_pernet(net);
struct handshake_req *req = NULL;
- struct socket *sock;
int class, err;
err = -EOPNOTSUPP;
@@ -107,15 +106,13 @@ int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info)
err = -EAGAIN;
req = handshake_req_next(hn, class);
if (req) {
- sock = req->hr_sk->sk_socket;
-
- FD_PREPARE(fdf, O_CLOEXEC, sock->file);
+ FD_PREPARE(fdf, O_CLOEXEC, req->hr_file);
if (fdf.err) {
+ fput(req->hr_file); /* drop ref from handshake_req_next() */
err = fdf.err;
goto out_complete;
}
- get_file(sock->file); /* FD_PREPARE() consumes a reference. */
err = req->hr_proto->hp_accept(req, info, fd_prepare_fd(fdf));
if (err)
goto out_complete; /* Automatic cleanup handles fput */
@@ -160,7 +157,7 @@ int handshake_nl_done_doit(struct sk_buff *skb, struct genl_info *info)
status = -EIO;
if (info->attrs[HANDSHAKE_A_DONE_STATUS])
- status = nla_get_u32(info->attrs[HANDSHAKE_A_DONE_STATUS]);
+ status = -(int)nla_get_u32(info->attrs[HANDSHAKE_A_DONE_STATUS]);
handshake_complete(req, status, info);
sockfd_put(sock);
@@ -202,21 +199,21 @@ static void __net_exit handshake_net_exit(struct net *net)
* accepted and are in progress will be destroyed when
* the socket is closed.
*/
- spin_lock(&hn->hn_lock);
+ spin_lock_bh(&hn->hn_lock);
set_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags);
- list_splice_init(&requests, &hn->hn_requests);
- spin_unlock(&hn->hn_lock);
+ list_splice_init(&hn->hn_requests, &requests);
+ list_for_each_entry(req, &requests, hr_list)
+ get_file(req->hr_file);
+ spin_unlock_bh(&hn->hn_lock);
while (!list_empty(&requests)) {
+ struct file *file;
+
req = list_first_entry(&requests, struct handshake_req, hr_list);
- list_del(&req->hr_list);
-
- /*
- * Requests on this list have not yet been
- * accepted, so they do not have an fd to put.
- */
-
+ file = req->hr_file;
+ list_del_init(&req->hr_list);
handshake_complete(req, -ETIMEDOUT, NULL);
+ fput(file);
}
}
diff --git a/net/handshake/request.c b/net/handshake/request.c
index 2829adb..cd30d54 100644
--- a/net/handshake/request.c
+++ b/net/handshake/request.c
@@ -13,6 +13,7 @@
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/inet.h>
+#include <linux/file.h>
#include <linux/rhashtable.h>
#include <net/sock.h>
@@ -162,35 +163,56 @@ static void __remove_pending_locked(struct handshake_net *hn,
* otherwise %false.
*
* If @req was on a pending list, it has not yet been accepted.
+ * Returns %false when the net namespace is draining; the drain
+ * loop has taken ownership of the pending list.
*/
static bool remove_pending(struct handshake_net *hn, struct handshake_req *req)
{
bool ret = false;
- spin_lock(&hn->hn_lock);
- if (!list_empty(&req->hr_list)) {
+ spin_lock_bh(&hn->hn_lock);
+ if (!test_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags) &&
+ !list_empty(&req->hr_list)) {
__remove_pending_locked(hn, req);
ret = true;
}
- spin_unlock(&hn->hn_lock);
+ spin_unlock_bh(&hn->hn_lock);
return ret;
}
+/**
+ * handshake_req_next - Return the next queued handshake request
+ * @hn: per-net handshake state
+ * @class: handler class to match
+ *
+ * On a non-NULL return, the caller owns an extra reference
+ * on @req->hr_file. FD_PREPARE() consumes it on success; on
+ * the FD_PREPARE() failure path the caller must fput() it.
+ *
+ * Return: pointer to a removed handshake_req, or NULL.
+ */
struct handshake_req *handshake_req_next(struct handshake_net *hn, int class)
{
struct handshake_req *req, *pos;
req = NULL;
- spin_lock(&hn->hn_lock);
+ spin_lock_bh(&hn->hn_lock);
list_for_each_entry(pos, &hn->hn_requests, hr_list) {
if (pos->hr_proto->hp_handler_class != class)
continue;
__remove_pending_locked(hn, pos);
+ /* Hand off a file reference to the accept side under
+ * hn_lock. A concurrent handshake_req_cancel() can drop
+ * hr_file before accept reaches FD_PREPARE(); this extra
+ * reference keeps the file alive until FD_PREPARE() takes
+ * ownership.
+ */
+ get_file(pos->hr_file);
req = pos;
break;
}
- spin_unlock(&hn->hn_lock);
+ spin_unlock_bh(&hn->hn_lock);
return req;
}
@@ -215,9 +237,16 @@ EXPORT_SYMBOL_IF_KUNIT(handshake_req_next);
* A zero return value from handshake_req_submit() means that
* exactly one subsequent completion callback is guaranteed.
*
- * A negative return value from handshake_req_submit() means that
- * no completion callback will be done and that @req has been
- * destroyed.
+ * A negative return value from handshake_req_submit() guarantees that
+ * no completion callback will occur and that @req is no longer owned by
+ * the caller. If cancellation wins the completion race after the request
+ * has been published, final destruction is deferred until socket teardown.
+ *
+ * The caller must hold a reference on @sock->file for the duration
+ * of this call. Once the request is published to the accept side, a
+ * concurrent completion or cancellation may release the request's pin on
+ * @sock->file; the caller's reference is what keeps @sock->sk valid until
+ * handshake_req_submit() returns.
*/
int handshake_req_submit(struct socket *sock, struct handshake_req *req,
gfp_t flags)
@@ -236,6 +265,14 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req,
kfree(req);
return -EINVAL;
}
+
+ /*
+ * Pin sock->file for the lifetime of the request so the
+ * accept side does not race a consumer that releases the
+ * socket while a handshake is pending.
+ */
+ req->hr_file = get_file(sock->file);
+
req->hr_odestruct = req->hr_sk->sk_destruct;
req->hr_sk->sk_destruct = handshake_sk_destruct;
@@ -249,7 +286,7 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req,
if (READ_ONCE(hn->hn_pending) >= hn->hn_pending_max)
goto out_err;
- spin_lock(&hn->hn_lock);
+ spin_lock_bh(&hn->hn_lock);
ret = -EOPNOTSUPP;
if (test_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags))
goto out_unlock;
@@ -258,7 +295,7 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req,
goto out_unlock;
if (!__add_pending_locked(hn, req))
goto out_unlock;
- spin_unlock(&hn->hn_lock);
+ spin_unlock_bh(&hn->hn_lock);
ret = handshake_genl_notify(net, req->hr_proto, flags);
if (ret) {
@@ -267,35 +304,36 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req,
goto out_err;
}
- /* Prevent socket release while a handshake request is pending */
- sock_hold(req->hr_sk);
-
trace_handshake_submit(net, req, req->hr_sk);
return 0;
out_unlock:
- spin_unlock(&hn->hn_lock);
+ spin_unlock_bh(&hn->hn_lock);
out_err:
- /* Restore original destructor so socket teardown still runs on failure */
- req->hr_sk->sk_destruct = req->hr_odestruct;
trace_handshake_submit_err(net, req, req->hr_sk, ret);
- handshake_req_destroy(req);
+ if (!test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
+ /* Restore original destructor so socket teardown still runs. */
+ req->hr_sk->sk_destruct = req->hr_odestruct;
+ fput(req->hr_file);
+ handshake_req_destroy(req);
+ }
return ret;
}
EXPORT_SYMBOL(handshake_req_submit);
-void handshake_complete(struct handshake_req *req, unsigned int status,
+void handshake_complete(struct handshake_req *req, int status,
struct genl_info *info)
{
struct sock *sk = req->hr_sk;
struct net *net = sock_net(sk);
if (!test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
+ struct file *file = req->hr_file;
+
trace_handshake_complete(net, req, sk, status);
req->hr_proto->hp_done(req, status, info);
- /* Handshake request is no longer pending */
- sock_put(sk);
+ fput(file);
}
}
EXPORT_SYMBOL_IF_KUNIT(handshake_complete);
@@ -342,8 +380,7 @@ bool handshake_req_cancel(struct sock *sk)
out_true:
trace_handshake_cancel(net, req, sk);
- /* Handshake request is no longer pending */
- sock_put(sk);
+ fput(req->hr_file);
return true;
}
EXPORT_SYMBOL(handshake_req_cancel);
diff --git a/net/handshake/tlshd.c b/net/handshake/tlshd.c
index 8f9532a..7567150 100644
--- a/net/handshake/tlshd.c
+++ b/net/handshake/tlshd.c
@@ -93,7 +93,7 @@ static void tls_handshake_remote_peerids(struct tls_handshake_req *treq,
*
*/
static void tls_handshake_done(struct handshake_req *req,
- unsigned int status, struct genl_info *info)
+ int status, struct genl_info *info)
{
struct tls_handshake_req *treq = handshake_req_private(req);
@@ -104,7 +104,7 @@ static void tls_handshake_done(struct handshake_req *req,
if (!status)
set_bit(HANDSHAKE_F_REQ_SESSION, &req->hr_flags);
- treq->th_consumer_done(treq->th_consumer_data, -status,
+ treq->th_consumer_done(treq->th_consumer_data, status,
treq->th_peerid[0]);
}
@@ -425,6 +425,8 @@ EXPORT_SYMBOL(tls_server_hello_psk);
* Request cancellation races with request completion. To determine
* who won, callers examine the return value from this function.
*
+ * Context: May be called from process or softirq context.
+ *
* Return values:
* %true - Uncompleted handshake request was canceled
* %false - Handshake request already completed or not found
diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c
index 0aca859..f669a22 100644
--- a/net/hsr/hsr_forward.c
+++ b/net/hsr/hsr_forward.c
@@ -84,7 +84,7 @@ static bool is_supervision_frame(struct hsr_priv *hsr, struct sk_buff *skb)
/* Get next tlv */
total_length += hsr_sup_tag->tlv.HSR_TLV_length;
- if (!pskb_may_pull(skb, total_length))
+ if (!pskb_may_pull(skb, total_length + sizeof(struct hsr_sup_tlv)))
return false;
skb_pull(skb, total_length);
hsr_sup_tlv = (struct hsr_sup_tlv *)skb->data;
@@ -100,7 +100,7 @@ static bool is_supervision_frame(struct hsr_priv *hsr, struct sk_buff *skb)
/* make sure another tlv follows */
total_length += sizeof(struct hsr_sup_tlv) + hsr_sup_tlv->HSR_TLV_length;
- if (!pskb_may_pull(skb, total_length))
+ if (!pskb_may_pull(skb, total_length + sizeof(struct hsr_sup_tlv)))
return false;
/* get next tlv */
diff --git a/net/hsr/hsr_framereg.c b/net/hsr/hsr_framereg.c
index d09875b..a28dfd8 100644
--- a/net/hsr/hsr_framereg.c
+++ b/net/hsr/hsr_framereg.c
@@ -35,10 +35,8 @@ bool hsr_addr_is_self(struct hsr_priv *hsr, unsigned char *addr)
rcu_read_lock();
sn = rcu_dereference(hsr->self_node);
- if (!sn) {
- WARN_ONCE(1, "HSR: No self node\n");
+ if (!sn)
goto out;
- }
if (ether_addr_equal(addr, sn->macaddress_A) ||
ether_addr_equal(addr, sn->macaddress_B))
@@ -163,8 +161,8 @@ void hsr_del_nodes(struct list_head *node_db)
struct hsr_node *tmp;
list_for_each_entry_safe(node, tmp, node_db, mac_list) {
- list_del(&node->mac_list);
- hsr_free_node(node);
+ list_del_rcu(&node->mac_list);
+ call_rcu(&node->rcu_head, hsr_free_node_rcu);
}
}
@@ -889,7 +887,10 @@ int hsr_get_node_data(struct hsr_priv *hsr,
if (node->addr_B_port != HSR_PT_NONE) {
port = hsr_port_get_hsr(hsr, node->addr_B_port);
- *addr_b_ifindex = port->dev->ifindex;
+ if (port)
+ *addr_b_ifindex = port->dev->ifindex;
+ else
+ *addr_b_ifindex = -1;
} else {
*addr_b_ifindex = -1;
}
diff --git a/net/ieee802154/6lowpan/tx.c b/net/ieee802154/6lowpan/tx.c
index 0c07662..4df76ff 100644
--- a/net/ieee802154/6lowpan/tx.c
+++ b/net/ieee802154/6lowpan/tx.c
@@ -255,6 +255,11 @@ netdev_tx_t lowpan_xmit(struct sk_buff *skb, struct net_device *ldev)
pr_debug("package xmit\n");
+ if (skb->protocol != htons(ETH_P_IPV6)) {
+ kfree_skb(skb);
+ return NET_XMIT_DROP;
+ }
+
WARN_ON_ONCE(skb->len > IPV6_MIN_MTU);
/* We must take a copy of the skb before we modify/replace the ipv6
diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c
index 5fb81244..6fd642d 100644
--- a/net/ipv4/ah4.c
+++ b/net/ipv4/ah4.c
@@ -124,9 +124,14 @@ static void ah_output_done(void *data, int err)
struct iphdr *top_iph = ip_hdr(skb);
struct ip_auth_hdr *ah = ip_auth_hdr(skb);
int ihl = ip_hdrlen(skb);
+ int seqhi_len = 0;
+ __be32 *seqhi;
+ if (x->props.flags & XFRM_STATE_ESN)
+ seqhi_len = sizeof(*seqhi);
iph = AH_SKB_CB(skb)->tmp;
- icv = ah_tmp_icv(iph, ihl);
+ seqhi = (__be32 *)((char *)iph + ihl);
+ icv = ah_tmp_icv(seqhi, seqhi_len);
memcpy(ah->auth_data, icv, ahp->icv_trunc_len);
top_iph->tos = iph->tos;
@@ -138,7 +143,7 @@ static void ah_output_done(void *data, int err)
}
kfree(AH_SKB_CB(skb)->tmp);
- xfrm_output_resume(skb->sk, skb, err);
+ xfrm_output_resume(skb_to_full_sk(skb), skb, err);
}
static int ah_output(struct xfrm_state *x, struct sk_buff *skb)
@@ -270,12 +275,17 @@ static void ah_input_done(void *data, int err)
struct ip_auth_hdr *ah = ip_auth_hdr(skb);
int ihl = ip_hdrlen(skb);
int ah_hlen = (ah->hdrlen + 2) << 2;
+ int seqhi_len = 0;
+ __be32 *seqhi;
if (err)
goto out;
+ if (x->props.flags & XFRM_STATE_ESN)
+ seqhi_len = sizeof(*seqhi);
work_iph = AH_SKB_CB(skb)->tmp;
- auth_data = ah_tmp_auth(work_iph, ihl);
+ seqhi = (__be32 *)((char *)work_iph + ihl);
+ auth_data = ah_tmp_auth(seqhi, seqhi_len);
icv = ah_tmp_icv(auth_data, ahp->icv_trunc_len);
err = crypto_memneq(icv, auth_data, ahp->icv_trunc_len) ? -EBADMSG : 0;
diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c
index 008edc7..791e150 100644
--- a/net/ipv4/bpf_tcp_ca.c
+++ b/net/ipv4/bpf_tcp_ca.c
@@ -168,7 +168,7 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
*/
if (prog_ops_moff(prog) !=
offsetof(struct tcp_congestion_ops, release))
- return &bpf_sk_setsockopt_proto;
+ return &bpf_sk_setsockopt_nodelay_proto;
return NULL;
case BPF_FUNC_getsockopt:
/* Since get/setsockopt is usually expected to
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 6dfc0bc..513c821 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -419,8 +419,8 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
return err;
}
- if (ALIGN(tailen, L1_CACHE_BYTES) > PAGE_SIZE ||
- ALIGN(skb->data_len, L1_CACHE_BYTES) > PAGE_SIZE)
+ if (ALIGN(skb->data_len + tailen, L1_CACHE_BYTES) >
+ PAGE_SIZE)
goto cow;
if (!skb_cloned(skb)) {
@@ -873,7 +873,8 @@ static int esp_input(struct xfrm_state *x, struct sk_buff *skb)
nfrags = 1;
goto skip_cow;
- } else if (!skb_has_frag_list(skb)) {
+ } else if (!skb_has_frag_list(skb) &&
+ !skb_has_shared_frag(skb)) {
nfrags = skb_shinfo(skb)->nr_frags;
nfrags++;
diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
index 7eeff65..23e921d 100644
--- a/net/ipv4/icmp.c
+++ b/net/ipv4/icmp.c
@@ -961,6 +961,9 @@ void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info,
if (IS_ERR(rt))
goto out_unlock;
+ if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
+ goto ende;
+
/* peer icmp_ratelimit */
if (!icmpv4_xrlim_allow(net, rt, &fl4, type, code, apply_ratelimit))
goto ende;
diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index a674fb4..a9ad390 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -122,16 +122,29 @@
* contradict to specs provided this delay is small enough.
*/
-#define IGMP_V1_SEEN(in_dev) \
- (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 1 || \
- IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 1 || \
- ((in_dev)->mr_v1_seen && \
- time_before(jiffies, (in_dev)->mr_v1_seen)))
-#define IGMP_V2_SEEN(in_dev) \
- (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 2 || \
- IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 2 || \
- ((in_dev)->mr_v2_seen && \
- time_before(jiffies, (in_dev)->mr_v2_seen)))
+static bool IGMP_V1_SEEN(const struct in_device *in_dev)
+{
+ unsigned long seen;
+
+ if (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 1)
+ return true;
+ if (IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 1)
+ return true;
+ seen = READ_ONCE(in_dev->mr_v1_seen);
+ return seen && time_before(jiffies, seen);
+}
+
+static bool IGMP_V2_SEEN(const struct in_device *in_dev)
+{
+ unsigned long seen;
+
+ if (IPV4_DEVCONF_ALL_RO(dev_net(in_dev->dev), FORCE_IGMP_VERSION) == 2)
+ return true;
+ if (IN_DEV_CONF_GET((in_dev), FORCE_IGMP_VERSION) == 2)
+ return true;
+ seen = READ_ONCE(in_dev->mr_v2_seen);
+ return seen && time_before(jiffies, seen);
+}
static int unsolicited_report_interval(struct in_device *in_dev)
{
@@ -954,23 +967,21 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
int max_delay;
int mark = 0;
struct net *net = dev_net(in_dev->dev);
-
+ unsigned long seen;
if (len == 8) {
+ seen = jiffies + READ_ONCE(in_dev->mr_qrv) * READ_ONCE(in_dev->mr_qi) +
+ READ_ONCE(in_dev->mr_qri);
if (ih->code == 0) {
/* Alas, old v1 router presents here. */
max_delay = IGMP_QUERY_RESPONSE_INTERVAL;
- in_dev->mr_v1_seen = jiffies +
- (in_dev->mr_qrv * in_dev->mr_qi) +
- in_dev->mr_qri;
+ WRITE_ONCE(in_dev->mr_v1_seen, seen);
group = 0;
} else {
/* v2 router present */
max_delay = ih->code*(HZ/IGMP_TIMER_SCALE);
- in_dev->mr_v2_seen = jiffies +
- (in_dev->mr_qrv * in_dev->mr_qi) +
- in_dev->mr_qri;
+ WRITE_ONCE(in_dev->mr_v2_seen, seen);
}
/* cancel the interface change timer */
WRITE_ONCE(in_dev->mr_ifc_count, 0);
@@ -995,6 +1006,8 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
if (!max_delay)
max_delay = 1; /* can't mod w/ 0 */
} else { /* v3 */
+ unsigned long mr_qi;
+
if (!pskb_may_pull(skb, sizeof(struct igmpv3_query)))
return true;
@@ -1015,15 +1028,16 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
* received value was zero, use the default or statically
* configured value.
*/
- in_dev->mr_qrv = ih3->qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv);
- in_dev->mr_qi = IGMPV3_QQIC(ih3->qqic)*HZ ?: IGMP_QUERY_INTERVAL;
-
+ WRITE_ONCE(in_dev->mr_qrv,
+ ih3->qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv));
+ mr_qi = IGMPV3_QQIC(ih3->qqic)*HZ ?: IGMP_QUERY_INTERVAL;
+ WRITE_ONCE(in_dev->mr_qi, mr_qi);
/* RFC3376, 8.3. Query Response Interval:
* The number of seconds represented by the [Query Response
* Interval] must be less than the [Query Interval].
*/
- if (in_dev->mr_qri >= in_dev->mr_qi)
- in_dev->mr_qri = (in_dev->mr_qi/HZ - 1)*HZ;
+ if (READ_ONCE(in_dev->mr_qri) >= mr_qi)
+ WRITE_ONCE(in_dev->mr_qri, (mr_qi/HZ - 1) * HZ);
if (!group) { /* general query */
if (ih3->nsrcs)
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 928654c..5b934ce 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -1108,7 +1108,7 @@ static void reqsk_timer_handler(struct timer_list *t)
if (!inet_ehash_insert(req_to_sk(nreq), req_to_sk(oreq), NULL)) {
/* delete timer */
- __inet_csk_reqsk_queue_drop(sk_listener, nreq, true);
+ __inet_csk_reqsk_queue_drop(sk_listener, nreq, false);
goto no_ownership;
}
@@ -1134,7 +1134,7 @@ static void reqsk_timer_handler(struct timer_list *t)
}
drop:
- __inet_csk_reqsk_queue_drop(sk_listener, oreq, true);
+ __inet_csk_reqsk_queue_drop(oreq->rsk_listener, oreq, true);
reqsk_put(oreq);
}
@@ -1148,6 +1148,9 @@ static bool reqsk_queue_hash_req(struct request_sock *req)
/* The timer needs to be setup after a successful insertion. */
req->timeout = tcp_timeout_init((struct sock *)req);
timer_setup(&req->rsk_timer, reqsk_timer_handler, TIMER_PINNED);
+
+ preempt_disable_nested();
+
mod_timer(&req->rsk_timer, jiffies + req->timeout);
/* before letting lookups find us, make sure all req fields
@@ -1155,6 +1158,9 @@ static bool reqsk_queue_hash_req(struct request_sock *req)
*/
smp_wmb();
refcount_set(&req->rsk_refcnt, 2 + 1);
+
+ preempt_enable_nested();
+
return true;
}
diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c
index d8083b9..5b957a8 100644
--- a/net/ipv4/inetpeer.c
+++ b/net/ipv4/inetpeer.c
@@ -179,7 +179,8 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base,
seq = read_seqbegin(&base->lock);
p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp);
- if (p)
+ /* Make sure tree was not modified during our lookup. */
+ if (p && !read_seqretry(&base->lock, seq))
return p;
/* retry an exact lookup, taking the lock before.
diff --git a/net/ipv4/ip_options.c b/net/ipv4/ip_options.c
index be8815ce..09d7451 100644
--- a/net/ipv4/ip_options.c
+++ b/net/ipv4/ip_options.c
@@ -530,6 +530,10 @@ int ip_options_get(struct net *net, struct ip_options_rcu **optp,
kfree(opt);
return -EINVAL;
}
+ if (opt->opt.srr && !ns_capable(net->user_ns, CAP_NET_RAW)) {
+ kfree(opt);
+ return -EPERM;
+ }
kfree(*optp);
*optp = opt;
return 0;
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index e4790cc..5bcd73c 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -1233,6 +1233,8 @@ static int __ip_append_data(struct sock *sk,
if (err < 0)
goto error;
copy = err;
+ if (!(flags & MSG_NO_SHARED_FRAGS))
+ skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
wmem_alloc_delta += copy;
} else if (!zc) {
int i = skb_shinfo(skb)->nr_frags;
diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
index 2667f53..d3c677e 100644
--- a/net/ipv4/ip_tunnel_core.c
+++ b/net/ipv4/ip_tunnel_core.c
@@ -212,7 +212,7 @@ EXPORT_SYMBOL_GPL(iptunnel_handle_offloads);
*/
static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu)
{
- const struct iphdr *iph = ip_hdr(skb);
+ const struct iphdr *iph;
struct icmphdr *icmph;
struct iphdr *niph;
struct ethhdr eh;
@@ -226,7 +226,6 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu)
skb_copy_bits(skb, skb_mac_offset(skb), &eh, ETH_HLEN);
pskb_pull(skb, ETH_HLEN);
- skb_reset_network_header(skb);
err = pskb_trim(skb, 576 - sizeof(*niph) - sizeof(*icmph));
if (err)
@@ -236,7 +235,7 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu)
err = skb_cow(skb, sizeof(*niph) + sizeof(*icmph) + ETH_HLEN);
if (err)
return err;
-
+ iph = ip_hdr(skb);
icmph = skb_push(skb, sizeof(*icmph));
*icmph = (struct icmphdr) {
.type = ICMP_DEST_UNREACH,
@@ -281,7 +280,6 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu)
*/
static int iptunnel_pmtud_check_icmp(struct sk_buff *skb, int mtu)
{
- const struct icmphdr *icmph = icmp_hdr(skb);
const struct iphdr *iph = ip_hdr(skb);
if (mtu < 576 || iph->frag_off != htons(IP_DF))
@@ -292,9 +290,17 @@ static int iptunnel_pmtud_check_icmp(struct sk_buff *skb, int mtu)
ipv4_is_lbcast(iph->saddr) || ipv4_is_multicast(iph->saddr))
return 0;
- if (iph->protocol == IPPROTO_ICMP && icmp_is_err(icmph->type))
- return 0;
+ if (iph->protocol == IPPROTO_ICMP) {
+ const struct icmphdr *icmph;
+ if (!pskb_network_may_pull(skb, iph->ihl * 4 +
+ offsetofend(struct icmphdr, type)))
+ return 0;
+ iph = ip_hdr(skb);
+ icmph = (void *)iph + iph->ihl * 4;
+ if (icmp_is_err(icmph->type))
+ return 0;
+ }
return iptunnel_pmtud_build_icmp(skb, mtu);
}
@@ -308,7 +314,7 @@ static int iptunnel_pmtud_check_icmp(struct sk_buff *skb, int mtu)
*/
static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu)
{
- const struct ipv6hdr *ip6h = ipv6_hdr(skb);
+ const struct ipv6hdr *ip6h;
struct icmp6hdr *icmp6h;
struct ipv6hdr *nip6h;
struct ethhdr eh;
@@ -323,7 +329,6 @@ static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu)
skb_copy_bits(skb, skb_mac_offset(skb), &eh, ETH_HLEN);
pskb_pull(skb, ETH_HLEN);
- skb_reset_network_header(skb);
err = pskb_trim(skb, IPV6_MIN_MTU - sizeof(*nip6h) - sizeof(*icmp6h));
if (err)
@@ -334,6 +339,7 @@ static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu)
if (err)
return err;
+ ip6h = ipv6_hdr(skb);
icmp6h = skb_push(skb, sizeof(*icmp6h));
*icmp6h = (struct icmp6hdr) {
.icmp6_type = ICMPV6_PKT_TOOBIG,
diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
index 8a08d09..2628cd3a 100644
--- a/net/ipv4/ipmr.c
+++ b/net/ipv4/ipmr.c
@@ -151,16 +151,6 @@ static struct mr_table *__ipmr_get_table(struct net *net, u32 id)
return NULL;
}
-static struct mr_table *ipmr_get_table(struct net *net, u32 id)
-{
- struct mr_table *mrt;
-
- rcu_read_lock();
- mrt = __ipmr_get_table(net, id);
- rcu_read_unlock();
- return mrt;
-}
-
static int ipmr_fib_lookup(struct net *net, struct flowi4 *flp4,
struct mr_table **mrt)
{
@@ -293,7 +283,7 @@ static void __net_exit ipmr_rules_exit_rtnl(struct net *net,
struct mr_table *mrt, *next;
list_for_each_entry_safe(mrt, next, &net->ipv4.mr_tables, list) {
- list_del(&mrt->list);
+ list_del_rcu(&mrt->list);
ipmr_free_table(mrt, dev_kill_list);
}
}
@@ -315,28 +305,30 @@ bool ipmr_rule_default(const struct fib_rule *rule)
}
EXPORT_SYMBOL(ipmr_rule_default);
#else
-#define ipmr_for_each_table(mrt, net) \
- for (mrt = net->ipv4.mrt; mrt; mrt = NULL)
-
static struct mr_table *ipmr_mr_table_iter(struct net *net,
struct mr_table *mrt)
{
if (!mrt)
- return net->ipv4.mrt;
+ return rcu_dereference(net->ipv4.mrt);
return NULL;
}
-static struct mr_table *ipmr_get_table(struct net *net, u32 id)
+static struct mr_table *__ipmr_get_table(struct net *net, u32 id)
{
- return net->ipv4.mrt;
+ return rcu_dereference_check(net->ipv4.mrt,
+ lockdep_rtnl_is_held() ||
+ !rcu_access_pointer(net->ipv4.mrt));
}
-#define __ipmr_get_table ipmr_get_table
+#define ipmr_for_each_table(mrt, net) \
+ for (mrt = __ipmr_get_table(net, 0); mrt; mrt = NULL)
static int ipmr_fib_lookup(struct net *net, struct flowi4 *flp4,
struct mr_table **mrt)
{
- *mrt = net->ipv4.mrt;
+ *mrt = rcu_dereference(net->ipv4.mrt);
+ if (!*mrt)
+ return -EAGAIN;
return 0;
}
@@ -347,7 +339,8 @@ static int __net_init ipmr_rules_init(struct net *net)
mrt = ipmr_new_table(net, RT_TABLE_DEFAULT);
if (IS_ERR(mrt))
return PTR_ERR(mrt);
- net->ipv4.mrt = mrt;
+
+ rcu_assign_pointer(net->ipv4.mrt, mrt);
return 0;
}
@@ -358,9 +351,10 @@ static void __net_exit ipmr_rules_exit(struct net *net)
static void __net_exit ipmr_rules_exit_rtnl(struct net *net,
struct list_head *dev_kill_list)
{
- ipmr_free_table(net->ipv4.mrt, dev_kill_list);
+ struct mr_table *mrt = rcu_dereference_protected(net->ipv4.mrt, 1);
- net->ipv4.mrt = NULL;
+ RCU_INIT_POINTER(net->ipv4.mrt, NULL);
+ ipmr_free_table(mrt, dev_kill_list);
}
static int ipmr_rules_dump(struct net *net, struct notifier_block *nb,
@@ -381,6 +375,17 @@ bool ipmr_rule_default(const struct fib_rule *rule)
EXPORT_SYMBOL(ipmr_rule_default);
#endif
+static struct mr_table *ipmr_get_table(struct net *net, u32 id)
+{
+ struct mr_table *mrt;
+
+ rcu_read_lock();
+ mrt = __ipmr_get_table(net, id);
+ rcu_read_unlock();
+
+ return mrt;
+}
+
static inline int ipmr_hash_cmp(struct rhashtable_compare_arg *arg,
const void *ptr)
{
@@ -441,12 +446,11 @@ static void ipmr_free_table(struct mr_table *mrt, struct list_head *dev_kill_lis
WARN_ON_ONCE(!mr_can_free_table(net));
- timer_shutdown_sync(&mrt->ipmr_expire_timer);
mroute_clean_tables(mrt, MRT_FLUSH_VIFS | MRT_FLUSH_VIFS_STATIC |
MRT_FLUSH_MFC | MRT_FLUSH_MFC_STATIC,
&ipmr_dev_kill_list);
- rhltable_destroy(&mrt->mfc_hash);
- kfree(mrt);
+ timer_shutdown_sync(&mrt->ipmr_expire_timer);
+ mr_table_free(mrt);
WARN_ON_ONCE(!net_initialized(net) && !list_empty(&ipmr_dev_kill_list));
list_splice(&ipmr_dev_kill_list, dev_kill_list);
@@ -533,15 +537,16 @@ static netdev_tx_t reg_vif_xmit(struct sk_buff *skb, struct net_device *dev)
};
int err;
+ rcu_read_lock();
err = ipmr_fib_lookup(net, &fl4, &mrt);
if (err < 0) {
+ rcu_read_unlock();
kfree_skb(skb);
return err;
}
DEV_STATS_ADD(dev, tx_bytes, skb->len);
DEV_STATS_INC(dev, tx_packets);
- rcu_read_lock();
/* Pairs with WRITE_ONCE() in vif_add() and vif_delete() */
ipmr_cache_report(mrt, skb, READ_ONCE(mrt->mroute_reg_vif_num),
@@ -1108,11 +1113,12 @@ static int ipmr_cache_report(const struct mr_table *mrt,
msg->im_vif_hi = vifi >> 8;
ipv4_pktinfo_prepare(mroute_sk, pkt, false);
memcpy(skb->cb, pkt->cb, sizeof(skb->cb));
- /* Add our header */
- igmp = skb_put(skb, sizeof(struct igmphdr));
+ /* Add our header.
+ * Note that code, csum and group fields are cleared.
+ */
+ igmp = skb_put_zero(skb, sizeof(struct igmphdr));
igmp->type = assert;
msg->im_msgtype = assert;
- igmp->code = 0;
ip_hdr(skb)->tot_len = htons(skb->len); /* Fix the length */
skb->transport_header = skb->network_header;
}
@@ -1135,12 +1141,19 @@ static int ipmr_cache_report(const struct mr_table *mrt,
static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi,
struct sk_buff *skb, struct net_device *dev)
{
+ struct net *net = read_pnet(&mrt->net);
const struct iphdr *iph = ip_hdr(skb);
- struct mfc_cache *c;
+ struct mfc_cache *c = NULL;
bool found = false;
int err;
spin_lock_bh(&mfc_unres_lock);
+
+ if (!check_net(net)) {
+ err = -EINVAL;
+ goto err;
+ }
+
list_for_each_entry(c, &mrt->mfc_unres_queue, _c.list) {
if (c->mfc_mcastgrp == iph->daddr &&
c->mfc_origin == iph->saddr) {
@@ -1153,10 +1166,8 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi,
/* Create a new entry if allowable */
c = ipmr_cache_alloc_unres();
if (!c) {
- spin_unlock_bh(&mfc_unres_lock);
-
- kfree_skb(skb);
- return -ENOBUFS;
+ err = -ENOBUFS;
+ goto err;
}
/* Fill in the new cache entry */
@@ -1166,17 +1177,8 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi,
/* Reflect first query at mrouted. */
err = ipmr_cache_report(mrt, skb, vifi, IGMPMSG_NOCACHE);
-
- if (err < 0) {
- /* If the report failed throw the cache entry
- out - Brad Parker
- */
- spin_unlock_bh(&mfc_unres_lock);
-
- ipmr_cache_free(c);
- kfree_skb(skb);
- return err;
- }
+ if (err < 0)
+ goto err;
atomic_inc(&mrt->cache_resolve_queue_len);
list_add(&c->_c.list, &mrt->mfc_unres_queue);
@@ -1189,18 +1191,26 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi,
/* See if we can append the packet */
if (c->_c.mfc_un.unres.unresolved.qlen > 3) {
- kfree_skb(skb);
+ c = NULL;
err = -ENOBUFS;
- } else {
- if (dev) {
- skb->dev = dev;
- skb->skb_iif = dev->ifindex;
- }
- skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb);
- err = 0;
+ goto err;
}
+ if (dev) {
+ skb->dev = dev;
+ skb->skb_iif = dev->ifindex;
+ }
+
+ skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb);
+
spin_unlock_bh(&mfc_unres_lock);
+ return 0;
+
+err:
+ spin_unlock_bh(&mfc_unres_lock);
+ if (c)
+ ipmr_cache_free(c);
+ kfree_skb(skb);
return err;
}
@@ -1346,7 +1356,7 @@ static void mroute_clean_tables(struct mr_table *mrt, int flags,
}
if (flags & MRT_FLUSH_MFC) {
- if (atomic_read(&mrt->cache_resolve_queue_len) != 0) {
+ if (atomic_read(&mrt->cache_resolve_queue_len) != 0 || !check_net(net)) {
spin_lock_bh(&mfc_unres_lock);
list_for_each_entry_safe(c, tmp, &mrt->mfc_unres_queue, list) {
list_del(&c->list);
diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c
index 37a3c14..3930d61 100644
--- a/net/ipv4/ipmr_base.c
+++ b/net/ipv4/ipmr_base.c
@@ -28,6 +28,20 @@ void vif_device_init(struct vif_device *v,
v->link = dev->ifindex;
}
+static void __mr_free_table(struct work_struct *work)
+{
+ struct mr_table *mrt = container_of(to_rcu_work(work),
+ struct mr_table, work);
+
+ rhltable_destroy(&mrt->mfc_hash);
+ kfree(mrt);
+}
+
+void mr_table_free(struct mr_table *mrt)
+{
+ queue_rcu_work(system_unbound_wq, &mrt->work);
+}
+
struct mr_table *
mr_table_alloc(struct net *net, u32 id,
struct mr_table_ops *ops,
@@ -50,6 +64,8 @@ mr_table_alloc(struct net *net, u32 id,
kfree(mrt);
return ERR_PTR(err);
}
+
+ INIT_RCU_WORK(&mrt->work, __mr_free_table);
INIT_LIST_HEAD(&mrt->mfc_cache_list);
INIT_LIST_HEAD(&mrt->mfc_unres_queue);
diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index 1cdd9c2..ad22596 100644
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -110,13 +110,25 @@ static inline int arp_packet_match(const struct arphdr *arphdr,
arpptr += dev->addr_len;
memcpy(&src_ipaddr, arpptr, sizeof(u32));
arpptr += sizeof(u32);
- tgt_devaddr = arpptr;
- arpptr += dev->addr_len;
+
+ if (IS_ENABLED(CONFIG_FIREWIRE_NET) && dev->type == ARPHRD_IEEE1394) {
+ if (unlikely(memchr_inv(arpinfo->tgt_devaddr.mask, 0,
+ sizeof(arpinfo->tgt_devaddr.mask))))
+ return 0;
+
+ tgt_devaddr = NULL;
+ } else {
+ tgt_devaddr = arpptr;
+ arpptr += dev->addr_len;
+ }
memcpy(&tgt_ipaddr, arpptr, sizeof(u32));
if (NF_INVF(arpinfo, ARPT_INV_SRCDEVADDR,
arp_devaddr_compare(&arpinfo->src_devaddr, src_devaddr,
- dev->addr_len)) ||
+ dev->addr_len)))
+ return 0;
+
+ if (tgt_devaddr &&
NF_INVF(arpinfo, ARPT_INV_TGTDEVADDR,
arp_devaddr_compare(&arpinfo->tgt_devaddr, tgt_devaddr,
dev->addr_len)))
@@ -1489,13 +1501,11 @@ static int do_arpt_get_ctl(struct sock *sk, int cmd, void __user *user, int *len
static void __arpt_unregister_table(struct net *net, struct xt_table *table)
{
- struct xt_table_info *private;
- void *loc_cpu_entry;
+ struct xt_table_info *private = table->private;
struct module *table_owner = table->me;
+ void *loc_cpu_entry;
struct arpt_entry *iter;
- private = xt_unregister_table(table);
-
/* Decrease module usage counts and free resources */
loc_cpu_entry = private->entries;
xt_entry_foreach(iter, loc_cpu_entry, private->size)
@@ -1503,6 +1513,7 @@ static void __arpt_unregister_table(struct net *net, struct xt_table *table)
if (private->number > private->initial_entries)
module_put(table_owner);
xt_free_table_info(private);
+ kfree(table);
}
int arpt_register_table(struct net *net,
@@ -1510,13 +1521,11 @@ int arpt_register_table(struct net *net,
const struct arpt_replace *repl,
const struct nf_hook_ops *template_ops)
{
- struct nf_hook_ops *ops;
- unsigned int num_ops;
- int ret, i;
- struct xt_table_info *newinfo;
struct xt_table_info bootstrap = {0};
- void *loc_cpu_entry;
+ struct xt_table_info *newinfo;
struct xt_table *new_table;
+ void *loc_cpu_entry;
+ int ret;
newinfo = xt_alloc_table_info(repl->size);
if (!newinfo)
@@ -1531,7 +1540,7 @@ int arpt_register_table(struct net *net,
return ret;
}
- new_table = xt_register_table(net, table, &bootstrap, newinfo);
+ new_table = xt_register_table(net, table, template_ops, &bootstrap, newinfo);
if (IS_ERR(new_table)) {
struct arpt_entry *iter;
@@ -1541,46 +1550,12 @@ int arpt_register_table(struct net *net,
return PTR_ERR(new_table);
}
- num_ops = hweight32(table->valid_hooks);
- if (num_ops == 0) {
- ret = -EINVAL;
- goto out_free;
- }
-
- ops = kmemdup_array(template_ops, num_ops, sizeof(*ops), GFP_KERNEL);
- if (!ops) {
- ret = -ENOMEM;
- goto out_free;
- }
-
- for (i = 0; i < num_ops; i++)
- ops[i].priv = new_table;
-
- new_table->ops = ops;
-
- ret = nf_register_net_hooks(net, ops, num_ops);
- if (ret != 0)
- goto out_free;
-
- return ret;
-
-out_free:
- __arpt_unregister_table(net, new_table);
return ret;
}
-void arpt_unregister_table_pre_exit(struct net *net, const char *name)
-{
- struct xt_table *table = xt_find_table(net, NFPROTO_ARP, name);
-
- if (table)
- nf_unregister_net_hooks(net, table->ops, hweight32(table->valid_hooks));
-}
-EXPORT_SYMBOL(arpt_unregister_table_pre_exit);
-
void arpt_unregister_table(struct net *net, const char *name)
{
- struct xt_table *table = xt_find_table(net, NFPROTO_ARP, name);
+ struct xt_table *table = xt_unregister_table_exit(net, NFPROTO_ARP, name);
if (table)
__arpt_unregister_table(net, table);
diff --git a/net/ipv4/netfilter/arpt_mangle.c b/net/ipv4/netfilter/arpt_mangle.c
index a4e07e5..f65dd33 100644
--- a/net/ipv4/netfilter/arpt_mangle.c
+++ b/net/ipv4/netfilter/arpt_mangle.c
@@ -40,6 +40,10 @@ target(struct sk_buff *skb, const struct xt_action_param *par)
}
arpptr += pln;
if (mangle->flags & ARPT_MANGLE_TDEV) {
+ if (unlikely(IS_ENABLED(CONFIG_FIREWIRE_NET) &&
+ skb->dev->type == ARPHRD_IEEE1394))
+ return NF_DROP;
+
if (ARPT_DEV_ADDR_LEN_MAX < hln ||
(arpptr + hln > skb_tail_pointer(skb)))
return NF_DROP;
@@ -47,6 +51,10 @@ target(struct sk_buff *skb, const struct xt_action_param *par)
}
arpptr += hln;
if (mangle->flags & ARPT_MANGLE_TIP) {
+ if (unlikely(IS_ENABLED(CONFIG_FIREWIRE_NET) &&
+ skb->dev->type == ARPHRD_IEEE1394))
+ return NF_DROP;
+
if (ARPT_MANGLE_ADDR_LEN_MAX < pln ||
(arpptr + pln > skb_tail_pointer(skb)))
return NF_DROP;
diff --git a/net/ipv4/netfilter/arptable_filter.c b/net/ipv4/netfilter/arptable_filter.c
index 78cd5ee..370b635 100644
--- a/net/ipv4/netfilter/arptable_filter.c
+++ b/net/ipv4/netfilter/arptable_filter.c
@@ -43,7 +43,7 @@ static int arptable_filter_table_init(struct net *net)
static void __net_exit arptable_filter_net_pre_exit(struct net *net)
{
- arpt_unregister_table_pre_exit(net, "filter");
+ xt_unregister_table_pre_exit(net, NFPROTO_ARP, "filter");
}
static void __net_exit arptable_filter_net_exit(struct net *net)
@@ -58,32 +58,33 @@ static struct pernet_operations arptable_filter_net_ops = {
static int __init arptable_filter_init(void)
{
- int ret = xt_register_template(&packet_filter,
- arptable_filter_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
arpfilter_ops = xt_hook_ops_alloc(&packet_filter, arpt_do_table);
- if (IS_ERR(arpfilter_ops)) {
- xt_unregister_template(&packet_filter);
+ if (IS_ERR(arpfilter_ops))
return PTR_ERR(arpfilter_ops);
- }
ret = register_pernet_subsys(&arptable_filter_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&packet_filter,
+ arptable_filter_table_init);
if (ret < 0) {
- xt_unregister_template(&packet_filter);
- kfree(arpfilter_ops);
- return ret;
+ unregister_pernet_subsys(&arptable_filter_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(arpfilter_ops);
return ret;
}
static void __exit arptable_filter_fini(void)
{
- unregister_pernet_subsys(&arptable_filter_net_ops);
xt_unregister_template(&packet_filter);
+ unregister_pernet_subsys(&arptable_filter_net_ops);
kfree(arpfilter_ops);
}
diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index 23c8def..5cbdb08 100644
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -1704,12 +1704,10 @@ do_ipt_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
static void __ipt_unregister_table(struct net *net, struct xt_table *table)
{
- struct xt_table_info *private;
- void *loc_cpu_entry;
+ struct xt_table_info *private = table->private;
struct module *table_owner = table->me;
struct ipt_entry *iter;
-
- private = xt_unregister_table(table);
+ void *loc_cpu_entry;
/* Decrease module usage counts and free resources */
loc_cpu_entry = private->entries;
@@ -1718,19 +1716,18 @@ static void __ipt_unregister_table(struct net *net, struct xt_table *table)
if (private->number > private->initial_entries)
module_put(table_owner);
xt_free_table_info(private);
+ kfree(table);
}
int ipt_register_table(struct net *net, const struct xt_table *table,
const struct ipt_replace *repl,
const struct nf_hook_ops *template_ops)
{
- struct nf_hook_ops *ops;
- unsigned int num_ops;
- int ret, i;
- struct xt_table_info *newinfo;
struct xt_table_info bootstrap = {0};
- void *loc_cpu_entry;
+ struct xt_table_info *newinfo;
struct xt_table *new_table;
+ void *loc_cpu_entry;
+ int ret;
newinfo = xt_alloc_table_info(repl->size);
if (!newinfo)
@@ -1745,7 +1742,7 @@ int ipt_register_table(struct net *net, const struct xt_table *table,
return ret;
}
- new_table = xt_register_table(net, table, &bootstrap, newinfo);
+ new_table = xt_register_table(net, table, template_ops, &bootstrap, newinfo);
if (IS_ERR(new_table)) {
struct ipt_entry *iter;
@@ -1755,51 +1752,12 @@ int ipt_register_table(struct net *net, const struct xt_table *table,
return PTR_ERR(new_table);
}
- /* No template? No need to do anything. This is used by 'nat' table, it registers
- * with the nat core instead of the netfilter core.
- */
- if (!template_ops)
- return 0;
-
- num_ops = hweight32(table->valid_hooks);
- if (num_ops == 0) {
- ret = -EINVAL;
- goto out_free;
- }
-
- ops = kmemdup_array(template_ops, num_ops, sizeof(*ops), GFP_KERNEL);
- if (!ops) {
- ret = -ENOMEM;
- goto out_free;
- }
-
- for (i = 0; i < num_ops; i++)
- ops[i].priv = new_table;
-
- new_table->ops = ops;
-
- ret = nf_register_net_hooks(net, ops, num_ops);
- if (ret != 0)
- goto out_free;
-
return ret;
-
-out_free:
- __ipt_unregister_table(net, new_table);
- return ret;
-}
-
-void ipt_unregister_table_pre_exit(struct net *net, const char *name)
-{
- struct xt_table *table = xt_find_table(net, NFPROTO_IPV4, name);
-
- if (table)
- nf_unregister_net_hooks(net, table->ops, hweight32(table->valid_hooks));
}
void ipt_unregister_table_exit(struct net *net, const char *name)
{
- struct xt_table *table = xt_find_table(net, NFPROTO_IPV4, name);
+ struct xt_table *table = xt_unregister_table_exit(net, NFPROTO_IPV4, name);
if (table)
__ipt_unregister_table(net, table);
@@ -1887,7 +1845,6 @@ static void __exit ip_tables_fini(void)
}
EXPORT_SYMBOL(ipt_register_table);
-EXPORT_SYMBOL(ipt_unregister_table_pre_exit);
EXPORT_SYMBOL(ipt_unregister_table_exit);
EXPORT_SYMBOL(ipt_do_table);
module_init(ip_tables_init);
diff --git a/net/ipv4/netfilter/iptable_filter.c b/net/ipv4/netfilter/iptable_filter.c
index 3ab908b..672d7da 100644
--- a/net/ipv4/netfilter/iptable_filter.c
+++ b/net/ipv4/netfilter/iptable_filter.c
@@ -61,7 +61,7 @@ static int __net_init iptable_filter_net_init(struct net *net)
static void __net_exit iptable_filter_net_pre_exit(struct net *net)
{
- ipt_unregister_table_pre_exit(net, "filter");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "filter");
}
static void __net_exit iptable_filter_net_exit(struct net *net)
@@ -77,32 +77,33 @@ static struct pernet_operations iptable_filter_net_ops = {
static int __init iptable_filter_init(void)
{
- int ret = xt_register_template(&packet_filter,
- iptable_filter_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
filter_ops = xt_hook_ops_alloc(&packet_filter, ipt_do_table);
- if (IS_ERR(filter_ops)) {
- xt_unregister_template(&packet_filter);
+ if (IS_ERR(filter_ops))
return PTR_ERR(filter_ops);
- }
ret = register_pernet_subsys(&iptable_filter_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&packet_filter,
+ iptable_filter_table_init);
if (ret < 0) {
- xt_unregister_template(&packet_filter);
- kfree(filter_ops);
- return ret;
+ unregister_pernet_subsys(&iptable_filter_net_ops);
+ goto err_free;
}
return 0;
+err_free:
+ kfree(filter_ops);
+ return ret;
}
static void __exit iptable_filter_fini(void)
{
- unregister_pernet_subsys(&iptable_filter_net_ops);
xt_unregister_template(&packet_filter);
+ unregister_pernet_subsys(&iptable_filter_net_ops);
kfree(filter_ops);
}
diff --git a/net/ipv4/netfilter/iptable_mangle.c b/net/ipv4/netfilter/iptable_mangle.c
index 385d945..13d25d9 100644
--- a/net/ipv4/netfilter/iptable_mangle.c
+++ b/net/ipv4/netfilter/iptable_mangle.c
@@ -96,7 +96,7 @@ static int iptable_mangle_table_init(struct net *net)
static void __net_exit iptable_mangle_net_pre_exit(struct net *net)
{
- ipt_unregister_table_pre_exit(net, "mangle");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "mangle");
}
static void __net_exit iptable_mangle_net_exit(struct net *net)
@@ -111,32 +111,33 @@ static struct pernet_operations iptable_mangle_net_ops = {
static int __init iptable_mangle_init(void)
{
- int ret = xt_register_template(&packet_mangler,
- iptable_mangle_table_init);
- if (ret < 0)
- return ret;
+ int ret;
mangle_ops = xt_hook_ops_alloc(&packet_mangler, iptable_mangle_hook);
- if (IS_ERR(mangle_ops)) {
- xt_unregister_template(&packet_mangler);
- ret = PTR_ERR(mangle_ops);
- return ret;
- }
+ if (IS_ERR(mangle_ops))
+ return PTR_ERR(mangle_ops);
ret = register_pernet_subsys(&iptable_mangle_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&packet_mangler,
+ iptable_mangle_table_init);
if (ret < 0) {
- xt_unregister_template(&packet_mangler);
- kfree(mangle_ops);
- return ret;
+ unregister_pernet_subsys(&iptable_mangle_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(mangle_ops);
return ret;
}
static void __exit iptable_mangle_fini(void)
{
- unregister_pernet_subsys(&iptable_mangle_net_ops);
xt_unregister_template(&packet_mangler);
+ unregister_pernet_subsys(&iptable_mangle_net_ops);
kfree(mangle_ops);
}
diff --git a/net/ipv4/netfilter/iptable_nat.c b/net/ipv4/netfilter/iptable_nat.c
index 625a1ca1..a0df725 100644
--- a/net/ipv4/netfilter/iptable_nat.c
+++ b/net/ipv4/netfilter/iptable_nat.c
@@ -119,8 +119,11 @@ static int iptable_nat_table_init(struct net *net)
}
ret = ipt_nat_register_lookups(net);
- if (ret < 0)
+ if (ret < 0) {
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "nat");
+ synchronize_rcu();
ipt_unregister_table_exit(net, "nat");
+ }
kfree(repl);
return ret;
@@ -129,6 +132,7 @@ static int iptable_nat_table_init(struct net *net)
static void __net_exit iptable_nat_net_pre_exit(struct net *net)
{
ipt_nat_unregister_lookups(net);
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "nat");
}
static void __net_exit iptable_nat_net_exit(struct net *net)
diff --git a/net/ipv4/netfilter/iptable_raw.c b/net/ipv4/netfilter/iptable_raw.c
index 0e7f539..2745c22 100644
--- a/net/ipv4/netfilter/iptable_raw.c
+++ b/net/ipv4/netfilter/iptable_raw.c
@@ -53,7 +53,7 @@ static int iptable_raw_table_init(struct net *net)
static void __net_exit iptable_raw_net_pre_exit(struct net *net)
{
- ipt_unregister_table_pre_exit(net, "raw");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "raw");
}
static void __net_exit iptable_raw_net_exit(struct net *net)
@@ -77,32 +77,32 @@ static int __init iptable_raw_init(void)
pr_info("Enabling raw table before defrag\n");
}
- ret = xt_register_template(table,
- iptable_raw_table_init);
- if (ret < 0)
- return ret;
-
rawtable_ops = xt_hook_ops_alloc(table, ipt_do_table);
- if (IS_ERR(rawtable_ops)) {
- xt_unregister_template(table);
+ if (IS_ERR(rawtable_ops))
return PTR_ERR(rawtable_ops);
- }
ret = register_pernet_subsys(&iptable_raw_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(table,
+ iptable_raw_table_init);
if (ret < 0) {
- xt_unregister_template(table);
- kfree(rawtable_ops);
- return ret;
+ unregister_pernet_subsys(&iptable_raw_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(rawtable_ops);
return ret;
}
static void __exit iptable_raw_fini(void)
{
+ xt_unregister_template(&packet_raw);
unregister_pernet_subsys(&iptable_raw_net_ops);
kfree(rawtable_ops);
- xt_unregister_template(&packet_raw);
}
module_init(iptable_raw_init);
diff --git a/net/ipv4/netfilter/iptable_security.c b/net/ipv4/netfilter/iptable_security.c
index d885443..4918945 100644
--- a/net/ipv4/netfilter/iptable_security.c
+++ b/net/ipv4/netfilter/iptable_security.c
@@ -50,7 +50,7 @@ static int iptable_security_table_init(struct net *net)
static void __net_exit iptable_security_net_pre_exit(struct net *net)
{
- ipt_unregister_table_pre_exit(net, "security");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV4, "security");
}
static void __net_exit iptable_security_net_exit(struct net *net)
@@ -65,33 +65,34 @@ static struct pernet_operations iptable_security_net_ops = {
static int __init iptable_security_init(void)
{
- int ret = xt_register_template(&security_table,
- iptable_security_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
sectbl_ops = xt_hook_ops_alloc(&security_table, ipt_do_table);
- if (IS_ERR(sectbl_ops)) {
- xt_unregister_template(&security_table);
+ if (IS_ERR(sectbl_ops))
return PTR_ERR(sectbl_ops);
- }
ret = register_pernet_subsys(&iptable_security_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&security_table,
+ iptable_security_table_init);
if (ret < 0) {
- xt_unregister_template(&security_table);
- kfree(sectbl_ops);
- return ret;
+ unregister_pernet_subsys(&iptable_security_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(sectbl_ops);
return ret;
}
static void __exit iptable_security_fini(void)
{
+ xt_unregister_template(&security_table);
unregister_pernet_subsys(&iptable_security_net_ops);
kfree(sectbl_ops);
- xt_unregister_template(&security_table);
}
module_init(iptable_security_init);
diff --git a/net/ipv4/netfilter/nf_socket_ipv4.c b/net/ipv4/netfilter/nf_socket_ipv4.c
index 5080fa5..f9c6755 100644
--- a/net/ipv4/netfilter/nf_socket_ipv4.c
+++ b/net/ipv4/netfilter/nf_socket_ipv4.c
@@ -94,6 +94,9 @@ struct sock *nf_sk_lookup_slow_v4(struct net *net, const struct sk_buff *skb,
#endif
int doff = 0;
+ if (ntohs(iph->frag_off) & IP_OFFSET)
+ return NULL;
+
if (iph->protocol == IPPROTO_UDP || iph->protocol == IPPROTO_TCP) {
struct tcphdr _hdr;
struct udphdr *hp;
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index 5aaf9c6..68e88cb 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -391,7 +391,7 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4,
* in, reject the frame as invalid
*/
err = -EINVAL;
- if (iphlen > length)
+ if (iphlen > length || iphlen < sizeof(*iph))
goto error_free;
if (iphlen >= sizeof(*iph)) {
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index bc1296f..3d62d45 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -1272,7 +1272,7 @@ static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
__func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
skb->dev ? skb->dev->name : "?");
kfree_skb(skb);
- WARN_ON(1);
+ WARN_ON_ONCE(1);
return 0;
}
diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
index d8bdb1b..c0e85cc 100644
--- a/net/ipv4/sysctl_net_ipv4.c
+++ b/net/ipv4/sysctl_net_ipv4.c
@@ -1705,10 +1705,10 @@ static __net_exit void ipv4_sysctl_exit_net(struct net *net)
{
const struct ctl_table *table;
- kfree(net->ipv4.sysctl_local_reserved_ports);
table = net->ipv4.ipv4_hdr->ctl_table_arg;
unregister_net_sysctl_table(net->ipv4.ipv4_hdr);
kfree(table);
+ kfree(net->ipv4.sysctl_local_reserved_ports);
}
static __net_initdata struct pernet_operations ipv4_sysctl_ops = {
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 432fa28..389a7cc 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -299,9 +299,6 @@ enum {
DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
-DEFINE_PER_CPU(u32, tcp_tw_isn);
-EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
-
long sysctl_tcp_mem[3] __read_mostly;
DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c
index a97cdf3..0a4b38b 100644
--- a/net/ipv4/tcp_ao.c
+++ b/net/ipv4/tcp_ao.c
@@ -116,7 +116,8 @@ struct tcp_ao_key *tcp_ao_established_key(const struct sock *sk,
{
struct tcp_ao_key *key;
- hlist_for_each_entry_rcu(key, &ao->head, node, lockdep_sock_is_held(sk)) {
+ hlist_for_each_entry_rcu(key, &ao->head, node,
+ sk_fullsock(sk) && lockdep_sock_is_held(sk)) {
if ((sndid >= 0 && key->sndid != sndid) ||
(rcvid >= 0 && key->rcvid != rcvid))
continue;
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index d5c9e65..de9f68a 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -7589,6 +7589,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
struct sock *sk, struct sk_buff *skb)
{
struct tcp_fastopen_cookie foc = { .len = -1 };
+ u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
struct tcp_options_received tmp_opt;
const struct tcp_sock *tp = tcp_sk(sk);
struct net *net = sock_net(sk);
@@ -7599,20 +7600,16 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
struct dst_entry *dst;
struct flowi fl;
u8 syncookies;
- u32 isn;
#ifdef CONFIG_TCP_AO
const struct tcp_ao_hdr *aoh;
#endif
- isn = __this_cpu_read(tcp_tw_isn);
- if (isn) {
- /* TW buckets are converted to open requests without
- * limitations, they conserve resources and peer is
- * evidently real one.
- */
- __this_cpu_write(tcp_tw_isn, 0);
- } else {
+ /* If isn is non-zero, this SYN originally matched a TIME_WAIT socket.
+ * TW sockets are converted to open requests without limitations,
+ * we skip the queue limits and syncookie checks in the block below.
+ */
+ if (!isn) {
syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
if (syncookies == 2 || inet_csk_reqsk_queue_is_full(sk)) {
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 8fc24c3..fdc8115 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1827,7 +1827,6 @@ INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
{
enum skb_drop_reason reason;
- struct sock *rsk;
reason = psp_sk_rx_policy_check(sk, skb);
if (reason)
@@ -1863,24 +1862,21 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
return 0;
if (nsk != sk) {
reason = tcp_child_process(sk, nsk, skb);
- if (reason) {
- rsk = nsk;
+ sock_put(nsk);
+ if (reason)
goto reset;
- }
return 0;
}
} else
sock_rps_save_rxhash(sk, skb);
reason = tcp_rcv_state_process(sk, skb);
- if (reason) {
- rsk = sk;
+ if (reason)
goto reset;
- }
return 0;
reset:
- tcp_v4_send_reset(rsk, skb, sk_rst_convert_drop_reason(reason));
+ tcp_v4_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
discard:
sk_skb_reason_drop(sk, skb, reason);
/* Be careful here. If this function gets more complicated and
@@ -2193,13 +2189,16 @@ int tcp_v4_rcv(struct sk_buff *skb)
rst_reason = sk_rst_convert_drop_reason(drop_reason);
tcp_v4_send_reset(nsk, skb, rst_reason);
+ sock_put(nsk);
goto discard_and_relse;
}
+ sock_put(nsk);
sock_put(sk);
return 0;
}
}
+ isn = 0;
process:
if (static_branch_unlikely(&ip4_min_ttl)) {
/* min_ttl can be changed concurrently from do_ip_setsockopt() */
@@ -2229,6 +2228,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
th = (const struct tcphdr *)skb->data;
iph = ip_hdr(skb);
tcp_v4_fill_cb(skb, iph, th);
+ TCP_SKB_CB(skb)->tcp_tw_isn = isn;
skb->dev = NULL;
@@ -2315,7 +2315,6 @@ int tcp_v4_rcv(struct sk_buff *skb)
sk = sk2;
tcp_v4_restore_cb(skb);
refcounted = false;
- __this_cpu_write(tcp_tw_isn, isn);
goto process;
}
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index 199f0b5..e6092c3 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -1012,6 +1012,6 @@ enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
}
bh_unlock_sock(child);
- sock_put(child);
+
return reason;
}
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index f9d8755..6e4bb41 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -2626,6 +2626,7 @@ static int tcp_clone_payload(struct sock *sk, struct sk_buff *to,
todo = min_t(int, skb_frag_size(fragfrom),
probe_size - len);
len += todo;
+ skb_shinfo(to)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
if (lastfrag &&
skb_frag_page(fragfrom) == skb_frag_page(lastfrag) &&
skb_frag_off(fragfrom) == skb_frag_off(lastfrag) +
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index 8d791a9..322db13 100644
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -50,7 +50,8 @@ static u32 tcp_clamp_rto_to_user_timeout(const struct sock *sk)
u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when)
{
const struct inet_connection_sock *icsk = inet_csk(sk);
- u32 remaining, user_timeout;
+ u32 user_timeout;
+ s32 remaining;
s32 elapsed;
user_timeout = READ_ONCE(icsk->icsk_user_timeout);
@@ -61,7 +62,7 @@ u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when)
if (unlikely(elapsed < 0))
elapsed = 0;
remaining = msecs_to_jiffies(user_timeout) - elapsed;
- remaining = max_t(u32, remaining, TCP_TIMEOUT_MIN);
+ remaining = max_t(int, remaining, TCP_TIMEOUT_MIN);
return min_t(u32, remaining, when);
}
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 0ac2bf4..70f6cbd 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -2011,6 +2011,14 @@ int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
}
WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
+
+ /*
+ * skb->dev still aliases the UDP rx dev_scratch (its charge was freed
+ * on dequeue above); a sockmap verdict program may deref it via
+ * bpf_sk_lookup_*(), so clear it -> bpf_skc_lookup() uses skb->sk
+ */
+ skb->dev = NULL;
+
return recv_actor(sk, skb);
}
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index a0813d4..29651b1 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -482,11 +482,11 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
struct sock *sk = gso_skb->sk;
unsigned int sum_truesize = 0;
struct sk_buff *segs, *seg;
- __be16 newlen, msslen;
struct udphdr *uh;
unsigned int mss;
bool copy_dtor;
__sum16 check;
+ __be16 newlen;
int ret = 0;
mss = skb_shinfo(gso_skb)->gso_size;
@@ -555,15 +555,6 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
return segs;
}
- msslen = htons(sizeof(*uh) + mss);
-
- /* GSO partial and frag_list segmentation only requires splitting
- * the frame into an MSS multiple and possibly a remainder, both
- * cases return a GSO skb. So update the mss now.
- */
- if (skb_is_gso(segs))
- mss *= skb_shinfo(segs)->gso_segs;
-
seg = segs;
uh = udp_hdr(seg);
@@ -586,7 +577,7 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
if (!seg->next)
break;
- uh->len = msslen;
+ uh->len = newlen;
uh->check = check;
if (seg->ip_summed == CHECKSUM_PARTIAL)
@@ -599,9 +590,12 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
uh = udp_hdr(seg);
}
- /* last packet can be partial gso_size, account for that in checksum */
- newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
- seg->data_len);
+ /* Unless skb fits perfectly as GSO_PARTIAL, the trailing
+ * segment may not be full MSS, account for that in the checksum
+ */
+ if (!skb_is_gso(seg))
+ newlen = htons(skb_tail_pointer(seg) -
+ skb_transport_header(seg) + seg->data_len);
check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
uh->len = newlen;
diff --git a/net/ipv6/Kconfig b/net/ipv6/Kconfig
index c024aa7..c3806c6a 100644
--- a/net/ipv6/Kconfig
+++ b/net/ipv6/Kconfig
@@ -164,7 +164,7 @@
select INET_TUNNEL
select NET_IP_TUNNEL
select IPV6_NDISC_NODETYPE
- default y
+ default m
help
Tunneling means encapsulating data of one protocol type within
another protocol and sending it over a channel that understands the
@@ -172,7 +172,7 @@
into IPv4 packets. This is useful if you want to connect two IPv6
networks over an IPv4-only path.
- Saying M here will produce a module called sit. If unsure, say Y.
+ Saying M here will produce a module called sit. If unsure, say M.
config IPV6_SIT_6RD
bool "IPv6: IPv6 Rapid Deployment (6RD)"
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 5476b65..bb84a78 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -1013,7 +1013,7 @@ ipv6_link_dev_addr(struct inet6_dev *idev, struct inet6_ifaddr *ifp)
list_for_each(p, &idev->addr_list) {
struct inet6_ifaddr *ifa
= list_entry(p, struct inet6_ifaddr, if_list);
- if (ifp_scope > ipv6_addr_src_scope(&ifa->addr))
+ if (ifp_scope >= ipv6_addr_src_scope(&ifa->addr))
break;
}
diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c
index cb26bee..76f7a2d 100644
--- a/net/ipv6/ah6.c
+++ b/net/ipv6/ah6.c
@@ -317,14 +317,19 @@ static void ah6_output_done(void *data, int err)
struct ipv6hdr *top_iph = ipv6_hdr(skb);
struct ip_auth_hdr *ah = ip_auth_hdr(skb);
struct tmp_ext *iph_ext;
+ int seqhi_len = 0;
+ __be32 *seqhi;
extlen = skb_network_header_len(skb) - sizeof(struct ipv6hdr);
if (extlen)
extlen += sizeof(*iph_ext);
+ if (x->props.flags & XFRM_STATE_ESN)
+ seqhi_len = sizeof(*seqhi);
iph_base = AH_SKB_CB(skb)->tmp;
iph_ext = ah_tmp_ext(iph_base);
- icv = ah_tmp_icv(iph_ext, extlen);
+ seqhi = (__be32 *)((char *)iph_ext + extlen);
+ icv = ah_tmp_icv(seqhi, seqhi_len);
memcpy(ah->auth_data, icv, ahp->icv_trunc_len);
memcpy(top_iph, iph_base, IPV6HDR_BASELEN);
@@ -332,7 +337,7 @@ static void ah6_output_done(void *data, int err)
ah6_restore_hdrs(top_iph, iph_ext, extlen);
kfree(AH_SKB_CB(skb)->tmp);
- xfrm_output_resume(skb->sk, skb, err);
+ xfrm_output_resume(skb_to_full_sk(skb), skb, err);
}
static int ah6_output(struct xfrm_state *x, struct sk_buff *skb)
@@ -471,13 +476,18 @@ static void ah6_input_done(void *data, int err)
struct ip_auth_hdr *ah = ip_auth_hdr(skb);
int hdr_len = skb_network_header_len(skb);
int ah_hlen = ipv6_authlen(ah);
+ int seqhi_len = 0;
+ __be32 *seqhi;
if (err)
goto out;
+ if (x->props.flags & XFRM_STATE_ESN)
+ seqhi_len = sizeof(*seqhi);
work_iph = AH_SKB_CB(skb)->tmp;
auth_data = ah_tmp_auth(work_iph, hdr_len);
- icv = ah_tmp_icv(auth_data, ahp->icv_trunc_len);
+ seqhi = (__be32 *)(auth_data + ahp->icv_trunc_len);
+ icv = ah_tmp_icv(seqhi, seqhi_len);
err = crypto_memneq(icv, auth_data, ahp->icv_trunc_len) ? -EBADMSG : 0;
if (err)
diff --git a/net/ipv6/anycast.c b/net/ipv6/anycast.c
index 67a42e0..be6dac8 100644
--- a/net/ipv6/anycast.c
+++ b/net/ipv6/anycast.c
@@ -243,16 +243,16 @@ static void ipv6_add_acaddr_hash(struct net *net, struct ifacaddr6 *aca)
{
unsigned int hash = inet6_acaddr_hash(net, &aca->aca_addr);
- spin_lock(&acaddr_hash_lock);
+ spin_lock_bh(&acaddr_hash_lock);
hlist_add_head_rcu(&aca->aca_addr_lst, &inet6_acaddr_lst[hash]);
- spin_unlock(&acaddr_hash_lock);
+ spin_unlock_bh(&acaddr_hash_lock);
}
static void ipv6_del_acaddr_hash(struct ifacaddr6 *aca)
{
- spin_lock(&acaddr_hash_lock);
+ spin_lock_bh(&acaddr_hash_lock);
hlist_del_init_rcu(&aca->aca_addr_lst);
- spin_unlock(&acaddr_hash_lock);
+ spin_unlock_bh(&acaddr_hash_lock);
}
static void aca_get(struct ifacaddr6 *aca)
@@ -371,10 +371,10 @@ int __ipv6_dev_ac_inc(struct inet6_dev *idev, const struct in6_addr *addr)
aca->aca_next = idev->ac_list;
rcu_assign_pointer(idev->ac_list, aca);
- write_unlock_bh(&idev->lock);
-
ipv6_add_acaddr_hash(net, aca);
+ write_unlock_bh(&idev->lock);
+
ip6_ins_rt(net, f6i);
addrconf_join_solict(idev->dev, &aca->aca_addr);
@@ -649,8 +649,8 @@ void ipv6_anycast_cleanup(void)
{
int i;
- spin_lock(&acaddr_hash_lock);
+ spin_lock_bh(&acaddr_hash_lock);
for (i = 0; i < IN6_ADDR_HSIZE; i++)
WARN_ON(!hlist_empty(&inet6_acaddr_lst[i]));
- spin_unlock(&acaddr_hash_lock);
+ spin_unlock_bh(&acaddr_hash_lock);
}
diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c
index ca3605a..38d7b48 100644
--- a/net/ipv6/datagram.c
+++ b/net/ipv6/datagram.c
@@ -617,6 +617,18 @@ void ip6_datagram_recv_common_ctl(struct sock *sk, struct msghdr *msg,
}
}
+static u16 ipv6_get_exthdr_len(const struct sk_buff *skb, const u8 *ptr)
+{
+ u16 len;
+
+ if (ptr + 2 > skb_tail_pointer(skb))
+ return 0;
+
+ len = (ptr[1] + 1) << 3;
+
+ return (len <= skb_tail_pointer(skb) - ptr) ? len : 0;
+}
+
void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg,
struct sk_buff *skb)
{
@@ -643,7 +655,10 @@ void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg,
/* HbH is allowed only once */
if (np->rxopt.bits.hopopts && (opt->flags & IP6SKB_HOPBYHOP)) {
u8 *ptr = nh + sizeof(struct ipv6hdr);
- put_cmsg(msg, SOL_IPV6, IPV6_HOPOPTS, (ptr[1]+1)<<3, ptr);
+ u16 len = ipv6_get_exthdr_len(skb, ptr);
+
+ if (len)
+ put_cmsg(msg, SOL_IPV6, IPV6_HOPOPTS, len, ptr);
}
if (opt->lastopt &&
@@ -664,26 +679,37 @@ void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg,
unsigned int len;
u8 *ptr = nh + off;
+ if (ptr + 2 > skb_tail_pointer(skb))
+ return;
+
switch (nexthdr) {
case IPPROTO_DSTOPTS:
nexthdr = ptr[0];
- len = (ptr[1] + 1) << 3;
+ len = ipv6_get_exthdr_len(skb, ptr);
+ if (!len)
+ return;
if (np->rxopt.bits.dstopts)
put_cmsg(msg, SOL_IPV6, IPV6_DSTOPTS, len, ptr);
break;
case IPPROTO_ROUTING:
nexthdr = ptr[0];
- len = (ptr[1] + 1) << 3;
+ len = ipv6_get_exthdr_len(skb, ptr);
+ if (!len)
+ return;
if (np->rxopt.bits.srcrt)
put_cmsg(msg, SOL_IPV6, IPV6_RTHDR, len, ptr);
break;
case IPPROTO_AH:
nexthdr = ptr[0];
len = (ptr[1] + 2) << 2;
+ if (ptr + len > skb_tail_pointer(skb))
+ return;
break;
default:
nexthdr = ptr[0];
- len = (ptr[1] + 1) << 3;
+ len = ipv6_get_exthdr_len(skb, ptr);
+ if (!len)
+ return;
break;
}
@@ -705,19 +731,31 @@ void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg,
}
if (np->rxopt.bits.ohopopts && (opt->flags & IP6SKB_HOPBYHOP)) {
u8 *ptr = nh + sizeof(struct ipv6hdr);
- put_cmsg(msg, SOL_IPV6, IPV6_2292HOPOPTS, (ptr[1]+1)<<3, ptr);
+ u16 len = ipv6_get_exthdr_len(skb, ptr);
+
+ if (len)
+ put_cmsg(msg, SOL_IPV6, IPV6_2292HOPOPTS, len, ptr);
}
if (np->rxopt.bits.odstopts && opt->dst0) {
u8 *ptr = nh + opt->dst0;
- put_cmsg(msg, SOL_IPV6, IPV6_2292DSTOPTS, (ptr[1]+1)<<3, ptr);
+ u16 len = ipv6_get_exthdr_len(skb, ptr);
+
+ if (len)
+ put_cmsg(msg, SOL_IPV6, IPV6_2292DSTOPTS, len, ptr);
}
if (np->rxopt.bits.osrcrt && opt->srcrt) {
struct ipv6_rt_hdr *rthdr = (struct ipv6_rt_hdr *)(nh + opt->srcrt);
- put_cmsg(msg, SOL_IPV6, IPV6_2292RTHDR, (rthdr->hdrlen+1) << 3, rthdr);
+ u16 len = ipv6_get_exthdr_len(skb, (u8 *)rthdr);
+
+ if (len)
+ put_cmsg(msg, SOL_IPV6, IPV6_2292RTHDR, len, rthdr);
}
if (np->rxopt.bits.odstopts && opt->dst1) {
u8 *ptr = nh + opt->dst1;
- put_cmsg(msg, SOL_IPV6, IPV6_2292DSTOPTS, (ptr[1]+1)<<3, ptr);
+ u16 len = ipv6_get_exthdr_len(skb, ptr);
+
+ if (len)
+ put_cmsg(msg, SOL_IPV6, IPV6_2292DSTOPTS, len, ptr);
}
if (np->rxopt.bits.rxorigdstaddr) {
struct sockaddr_in6 sin6;
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index 9f75313..57481e42 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -448,8 +448,8 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
return err;
}
- if (ALIGN(tailen, L1_CACHE_BYTES) > PAGE_SIZE ||
- ALIGN(skb->data_len, L1_CACHE_BYTES) > PAGE_SIZE)
+ if (ALIGN(skb->data_len + tailen, L1_CACHE_BYTES) >
+ PAGE_SIZE)
goto cow;
if (!skb_cloned(skb)) {
@@ -915,7 +915,8 @@ static int esp6_input(struct xfrm_state *x, struct sk_buff *skb)
nfrags = 1;
goto skip_cow;
- } else if (!skb_has_frag_list(skb)) {
+ } else if (!skb_has_frag_list(skb) &&
+ !skb_has_shared_frag(skb)) {
nfrags = skb_shinfo(skb)->nr_frags;
nfrags++;
diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c
index 95558fd6..43f46ef 100644
--- a/net/ipv6/exthdrs.c
+++ b/net/ipv6/exthdrs.c
@@ -184,6 +184,8 @@ static bool ip6_parse_tlv(bool hopbyhop,
case IPV6_TLV_JUMBO:
if (!ipv6_hop_jumbo(skb, off))
return false;
+
+ nh = skb_network_header(skb);
break;
case IPV6_TLV_CALIPSO:
if (!ipv6_hop_calipso(skb, off))
@@ -201,6 +203,8 @@ static bool ip6_parse_tlv(bool hopbyhop,
case IPV6_TLV_HAO:
if (!ipv6_dest_hao(skb, off))
return false;
+
+ nh = skb_network_header(skb);
break;
#endif
default:
@@ -491,6 +495,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb)
struct net *net = dev_net(skb->dev);
struct inet6_dev *idev;
struct ipv6hdr *oldhdr;
+ unsigned int chdr_len;
unsigned char *buf;
int accept_rpl_seg;
int i, err;
@@ -543,7 +548,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb)
* unsigned char which is segments_left field. Should not be
* higher than that.
*/
- if (r || (n + 1) > 255) {
+ if (r || (n + 1) > 127) {
kfree_skb(skb);
return -1;
}
@@ -592,8 +597,10 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb)
skb_pull(skb, ((hdr->hdrlen + 1) << 3));
skb_postpull_rcsum(skb, oldhdr,
sizeof(struct ipv6hdr) + ((hdr->hdrlen + 1) << 3));
- if (unlikely(!hdr->segments_left)) {
- if (pskb_expand_head(skb, sizeof(struct ipv6hdr) + ((chdr->hdrlen + 1) << 3), 0,
+ chdr_len = sizeof(struct ipv6hdr) + ((chdr->hdrlen + 1) << 3);
+ if (unlikely(!hdr->segments_left ||
+ skb_headroom(skb) < chdr_len + skb->mac_len)) {
+ if (pskb_expand_head(skb, chdr_len + skb->mac_len, 0,
GFP_ATOMIC)) {
__IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_OUTDISCARDS);
kfree_skb(skb);
@@ -603,7 +610,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb)
oldhdr = ipv6_hdr(skb);
}
- skb_push(skb, ((chdr->hdrlen + 1) << 3) + sizeof(struct ipv6hdr));
+ skb_push(skb, chdr_len);
skb_reset_network_header(skb);
skb_mac_header_rebuild(skb);
skb_set_transport_header(skb, sizeof(struct ipv6hdr));
@@ -907,16 +914,27 @@ static bool ipv6_hop_ra(struct sk_buff *skb, int optoff)
static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff)
{
+ enum skb_drop_reason drop_reason;
struct ioam6_trace_hdr *trace;
struct ioam6_namespace *ns;
+ struct inet6_dev *idev;
struct ioam6_hdr *hdr;
+ drop_reason = SKB_DROP_REASON_IP_INHDR;
+
/* Bad alignment (must be 4n-aligned) */
if (optoff & 3)
goto drop;
+ /* Does the device still have IPv6 configuration? */
+ idev = __in6_dev_get(skb->dev);
+ if (!idev) {
+ drop_reason = SKB_DROP_REASON_IPV6DISABLED;
+ goto drop;
+ }
+
/* Ignore if IOAM is not enabled on ingress */
- if (!READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_enabled))
+ if (!READ_ONCE(idev->cnf.ioam6_enabled))
goto ignore;
/* Truncated Option header */
@@ -952,9 +970,9 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff)
if (skb_ensure_writable(skb, optoff + 2 + hdr->opt_len))
goto drop;
- /* Trace pointer may have changed */
- trace = (struct ioam6_trace_hdr *)(skb_network_header(skb)
- + optoff + sizeof(*hdr));
+ /* Trace and hdr pointers may have changed */
+ hdr = (struct ioam6_hdr *)(skb_network_header(skb) + optoff);
+ trace = (struct ioam6_trace_hdr *)((u8 *)hdr + sizeof(*hdr));
ioam6_fill_trace_data(skb, ns, trace, true);
@@ -969,7 +987,7 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff)
return true;
drop:
- kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
+ kfree_skb_reason(skb, drop_reason);
return false;
}
diff --git a/net/ipv6/exthdrs_core.c b/net/ipv6/exthdrs_core.c
index 49e31e4..9d06d48 100644
--- a/net/ipv6/exthdrs_core.c
+++ b/net/ipv6/exthdrs_core.c
@@ -73,6 +73,7 @@ int ipv6_skip_exthdr(const struct sk_buff *skb, int start, u8 *nexthdrp,
__be16 *frag_offp)
{
u8 nexthdr = *nexthdrp;
+ int exthdr_cnt = 0;
*frag_offp = 0;
@@ -82,6 +83,8 @@ int ipv6_skip_exthdr(const struct sk_buff *skb, int start, u8 *nexthdrp,
if (nexthdr == NEXTHDR_NONE)
return -1;
+ if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT))
+ return -1;
hp = skb_header_pointer(skb, start, sizeof(_hdr), &_hdr);
if (!hp)
return -1;
@@ -190,6 +193,7 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset,
{
unsigned int start = skb_network_offset(skb) + sizeof(struct ipv6hdr);
u8 nexthdr = ipv6_hdr(skb)->nexthdr;
+ int exthdr_cnt = 0;
bool found;
if (fragoff)
@@ -216,6 +220,9 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset,
return -ENOENT;
}
+ if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT))
+ return -EBADMSG;
+
hp = skb_header_pointer(skb, start, sizeof(_hdr), &_hdr);
if (!hp)
return -EBADMSG;
diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c
index c92f98c..b1ccdf0 100644
--- a/net/ipv6/ip6_flowlabel.c
+++ b/net/ipv6/ip6_flowlabel.c
@@ -36,11 +36,11 @@
/* FL hash table */
#define FL_MAX_PER_SOCK 32
-#define FL_MAX_SIZE 4096
+#define FL_MAX_SIZE 8192
#define FL_HASH_MASK 255
#define FL_HASH(l) (ntohl(l)&FL_HASH_MASK)
-static atomic_t fl_size = ATOMIC_INIT(0);
+static int fl_size;
static struct ip6_flowlabel __rcu *fl_ht[FL_HASH_MASK+1];
static void ip6_fl_gc(struct timer_list *unused);
@@ -162,8 +162,9 @@ static void ip6_fl_gc(struct timer_list *unused)
ttd = fl->expires;
if (time_after_eq(now, ttd)) {
*flp = fl->next;
+ fl_size--;
+ fl->fl_net->ipv6.flowlabel_count--;
fl_free(fl);
- atomic_dec(&fl_size);
continue;
}
if (!sched || time_before(ttd, sched))
@@ -172,7 +173,7 @@ static void ip6_fl_gc(struct timer_list *unused)
flp = &fl->next;
}
}
- if (!sched && atomic_read(&fl_size))
+ if (!sched && fl_size)
sched = now + FL_MAX_LINGER;
if (sched) {
mod_timer(&ip6_fl_gc_timer, sched);
@@ -196,7 +197,8 @@ static void __net_exit ip6_fl_purge(struct net *net)
atomic_read(&fl->users) == 0) {
*flp = fl->next;
fl_free(fl);
- atomic_dec(&fl_size);
+ fl_size--;
+ net->ipv6.flowlabel_count--;
continue;
}
flp = &fl->next;
@@ -210,10 +212,10 @@ static struct ip6_flowlabel *fl_intern(struct net *net,
{
struct ip6_flowlabel *lfl;
+ lockdep_assert_held(&ip6_fl_lock);
+
fl->label = label & IPV6_FLOWLABEL_MASK;
- rcu_read_lock();
- spin_lock_bh(&ip6_fl_lock);
if (label == 0) {
for (;;) {
fl->label = htonl(get_random_u32())&IPV6_FLOWLABEL_MASK;
@@ -235,8 +237,6 @@ static struct ip6_flowlabel *fl_intern(struct net *net,
lfl = __fl_lookup(net, fl->label);
if (lfl) {
atomic_inc(&lfl->users);
- spin_unlock_bh(&ip6_fl_lock);
- rcu_read_unlock();
return lfl;
}
}
@@ -244,9 +244,8 @@ static struct ip6_flowlabel *fl_intern(struct net *net,
fl->lastuse = jiffies;
fl->next = fl_ht[FL_HASH(fl->label)];
rcu_assign_pointer(fl_ht[FL_HASH(fl->label)], fl);
- atomic_inc(&fl_size);
- spin_unlock_bh(&ip6_fl_lock);
- rcu_read_unlock();
+ fl_size++;
+ net->ipv6.flowlabel_count++;
return NULL;
}
@@ -464,10 +463,17 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq,
static int mem_check(struct sock *sk)
{
- int room = FL_MAX_SIZE - atomic_read(&fl_size);
+ const int unpriv_total_limit = FL_MAX_SIZE - (FL_MAX_SIZE / 4);
+ const int unpriv_user_limit = unpriv_total_limit / 2;
+ struct net *net = sock_net(sk);
+ int room;
struct ipv6_fl_socklist *sfl;
int count = 0;
+ lockdep_assert_held(&ip6_fl_lock);
+
+ room = FL_MAX_SIZE - fl_size;
+
if (room > FL_MAX_SIZE - FL_MAX_PER_SOCK)
return 0;
@@ -478,7 +484,9 @@ static int mem_check(struct sock *sk)
if (room <= 0 ||
((count >= FL_MAX_PER_SOCK ||
- (count > 0 && room < FL_MAX_SIZE/2) || room < FL_MAX_SIZE/4) &&
+ (count > 0 && room < FL_MAX_SIZE / 2) ||
+ room < FL_MAX_SIZE / 4 ||
+ net->ipv6.flowlabel_count >= unpriv_user_limit) &&
!capable(CAP_NET_ADMIN)))
return -ENOBUFS;
@@ -692,11 +700,19 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq,
if (!sfl1)
goto done;
+ rcu_read_lock();
+ spin_lock_bh(&ip6_fl_lock);
err = mem_check(sk);
+ if (err == 0)
+ fl1 = fl_intern(net, fl, freq->flr_label);
+ else
+ fl1 = NULL;
+ spin_unlock_bh(&ip6_fl_lock);
+ rcu_read_unlock();
+
if (err != 0)
goto done;
- fl1 = fl_intern(net, fl, freq->flr_label);
if (fl1)
goto recheck;
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
index 63fc855..365b405 100644
--- a/net/ipv6/ip6_gre.c
+++ b/net/ipv6/ip6_gre.c
@@ -2262,10 +2262,11 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[],
struct nlattr *data[],
struct netlink_ext_ack *extack)
{
- struct ip6gre_net *ign = net_generic(dev_net(dev), ip6gre_net_id);
+ struct ip6_tnl *t = netdev_priv(dev);
struct __ip6_tnl_parm p;
- struct ip6_tnl *t;
+ struct ip6gre_net *ign;
+ ign = net_generic(t->net, ip6gre_net_id);
t = ip6gre_changelink_common(dev, tb, data, &p, extack);
if (IS_ERR(t))
return PTR_ERR(t);
diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c
index 967b07a..8972863 100644
--- a/net/ipv6/ip6_input.c
+++ b/net/ipv6/ip6_input.c
@@ -403,6 +403,7 @@ INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *));
void ip6_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int nexthdr,
bool have_final)
{
+ int exthdr_cnt = IP6CB(skb)->flags & IP6SKB_HOPBYHOP ? 1 : 0;
const struct inet6_protocol *ipprot;
struct inet6_dev *idev;
unsigned int nhoff;
@@ -487,6 +488,10 @@ void ip6_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int nexthdr,
nexthdr = ret;
goto resubmit_final;
} else {
+ if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT)) {
+ SKB_DR_SET(reason, IPV6_TOO_MANY_EXTHDRS);
+ goto discard;
+ }
goto resubmit;
}
} else if (ret == 0) {
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 7e92909..c14adcd 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -468,6 +468,7 @@ static int ip6_forward_proxy_check(struct sk_buff *skb)
default:
break;
}
+ hdr = ipv6_hdr(skb);
}
/*
@@ -582,6 +583,8 @@ int ip6_forward(struct sk_buff *skb)
if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) {
int proxied = ip6_forward_proxy_check(skb);
+
+ hdr = ipv6_hdr(skb);
if (proxied > 0) {
/* It's tempting to decrease the hop limit
* here by 1, as we do at the end of the
@@ -1794,6 +1797,8 @@ static int __ip6_append_data(struct sock *sk,
if (err < 0)
goto error;
copy = err;
+ if (!(flags & MSG_NO_SHARED_FRAGS))
+ skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
wmem_alloc_delta += copy;
} else if (!zc) {
int i = skb_shinfo(skb)->nr_frags;
diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
index c468c83..9d1037a 100644
--- a/net/ipv6/ip6_tunnel.c
+++ b/net/ipv6/ip6_tunnel.c
@@ -399,11 +399,15 @@ __u16 ip6_tnl_parse_tlv_enc_lim(struct sk_buff *skb, __u8 *raw)
unsigned int nhoff = raw - skb->data;
unsigned int off = nhoff + sizeof(*ipv6h);
u8 nexthdr = ipv6h->nexthdr;
+ int exthdr_cnt = 0;
while (ipv6_ext_hdr(nexthdr) && nexthdr != NEXTHDR_NONE) {
struct ipv6_opt_hdr *hdr;
u16 optlen;
+ if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT))
+ break;
+
if (!pskb_may_pull(skb, off + sizeof(*hdr)))
break;
diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c
index ad5290b..df793c8 100644
--- a/net/ipv6/ip6_vti.c
+++ b/net/ipv6/ip6_vti.c
@@ -722,10 +722,11 @@ vti6_tnl_change(struct ip6_tnl *t, const struct __ip6_tnl_parm *p,
static int vti6_update(struct ip6_tnl *t, struct __ip6_tnl_parm *p,
bool keep_mtu)
{
- struct net *net = dev_net(t->dev);
- struct vti6_net *ip6n = net_generic(net, vti6_net_id);
+ struct net *net = t->net;
+ struct vti6_net *ip6n;
int err;
+ ip6n = net_generic(net, vti6_net_id);
vti6_tnl_unlink(ip6n, t);
synchronize_net();
err = vti6_tnl_change(t, p, keep_mtu);
@@ -834,17 +835,24 @@ vti6_siocdevprivate(struct net_device *dev, struct ifreq *ifr, void __user *data
if (p.proto != IPPROTO_IPV6 && p.proto != 0)
break;
vti6_parm_from_user(&p1, &p);
- t = vti6_locate(net, &p1, cmd == SIOCADDTUNNEL);
if (dev != ip6n->fb_tnl_dev && cmd == SIOCCHGTUNNEL) {
+ struct ip6_tnl *self = netdev_priv(dev);
+
+ err = -EPERM;
+ if (!ns_capable(self->net->user_ns, CAP_NET_ADMIN))
+ break;
+ t = vti6_locate(self->net, &p1, false);
if (t) {
if (t->dev != dev) {
err = -EEXIST;
break;
}
} else
- t = netdev_priv(dev);
+ t = self;
err = vti6_update(t, &p1, false);
+ } else {
+ t = vti6_locate(net, &p1, cmd == SIOCADDTUNNEL);
}
if (t) {
err = 0;
@@ -1031,11 +1039,12 @@ static int vti6_changelink(struct net_device *dev, struct nlattr *tb[],
struct nlattr *data[],
struct netlink_ext_ack *extack)
{
- struct ip6_tnl *t;
+ struct ip6_tnl *t = netdev_priv(dev);
+ struct net *net = t->net;
struct __ip6_tnl_parm p;
- struct net *net = dev_net(dev);
- struct vti6_net *ip6n = net_generic(net, vti6_net_id);
+ struct vti6_net *ip6n;
+ ip6n = net_generic(net, vti6_net_id);
if (dev == ip6n->fb_tnl_dev)
return -EINVAL;
diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c
index 3330adc..d9b855d 100644
--- a/net/ipv6/mcast.c
+++ b/net/ipv6/mcast.c
@@ -1424,9 +1424,9 @@ void igmp6_event_query(struct sk_buff *skb)
static void __mld_query_work(struct sk_buff *skb)
{
struct mld2_query *mlh2 = NULL;
- const struct in6_addr *group;
unsigned long max_delay;
struct inet6_dev *idev;
+ struct in6_addr group;
struct ifmcaddr6 *ma;
struct mld_msg *mld;
int group_type;
@@ -1458,8 +1458,8 @@ static void __mld_query_work(struct sk_buff *skb)
goto kfree_skb;
mld = (struct mld_msg *)icmp6_hdr(skb);
- group = &mld->mld_mca;
- group_type = ipv6_addr_type(group);
+ group = mld->mld_mca;
+ group_type = ipv6_addr_type(&group);
if (group_type != IPV6_ADDR_ANY &&
!(group_type&IPV6_ADDR_MULTICAST))
@@ -1509,7 +1509,7 @@ static void __mld_query_work(struct sk_buff *skb)
}
} else {
for_each_mc_mclock(idev, ma) {
- if (!ipv6_addr_equal(group, &ma->mca_addr))
+ if (!ipv6_addr_equal(&group, &ma->mca_addr))
continue;
if (ma->mca_flags & MAF_TIMER_RUNNING) {
/* gsquery <- gsquery && mark */
diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c
index d585ac3..9d9c376 100644
--- a/net/ipv6/netfilter/ip6_tables.c
+++ b/net/ipv6/netfilter/ip6_tables.c
@@ -1713,12 +1713,10 @@ do_ip6t_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
static void __ip6t_unregister_table(struct net *net, struct xt_table *table)
{
- struct xt_table_info *private;
- void *loc_cpu_entry;
+ struct xt_table_info *private = table->private;
struct module *table_owner = table->me;
struct ip6t_entry *iter;
-
- private = xt_unregister_table(table);
+ void *loc_cpu_entry;
/* Decrease module usage counts and free resources */
loc_cpu_entry = private->entries;
@@ -1727,19 +1725,18 @@ static void __ip6t_unregister_table(struct net *net, struct xt_table *table)
if (private->number > private->initial_entries)
module_put(table_owner);
xt_free_table_info(private);
+ kfree(table);
}
int ip6t_register_table(struct net *net, const struct xt_table *table,
const struct ip6t_replace *repl,
const struct nf_hook_ops *template_ops)
{
- struct nf_hook_ops *ops;
- unsigned int num_ops;
- int ret, i;
- struct xt_table_info *newinfo;
struct xt_table_info bootstrap = {0};
- void *loc_cpu_entry;
+ struct xt_table_info *newinfo;
struct xt_table *new_table;
+ void *loc_cpu_entry;
+ int ret;
newinfo = xt_alloc_table_info(repl->size);
if (!newinfo)
@@ -1754,7 +1751,7 @@ int ip6t_register_table(struct net *net, const struct xt_table *table,
return ret;
}
- new_table = xt_register_table(net, table, &bootstrap, newinfo);
+ new_table = xt_register_table(net, table, template_ops, &bootstrap, newinfo);
if (IS_ERR(new_table)) {
struct ip6t_entry *iter;
@@ -1764,48 +1761,12 @@ int ip6t_register_table(struct net *net, const struct xt_table *table,
return PTR_ERR(new_table);
}
- if (!template_ops)
- return 0;
-
- num_ops = hweight32(table->valid_hooks);
- if (num_ops == 0) {
- ret = -EINVAL;
- goto out_free;
- }
-
- ops = kmemdup_array(template_ops, num_ops, sizeof(*ops), GFP_KERNEL);
- if (!ops) {
- ret = -ENOMEM;
- goto out_free;
- }
-
- for (i = 0; i < num_ops; i++)
- ops[i].priv = new_table;
-
- new_table->ops = ops;
-
- ret = nf_register_net_hooks(net, ops, num_ops);
- if (ret != 0)
- goto out_free;
-
return ret;
-
-out_free:
- __ip6t_unregister_table(net, new_table);
- return ret;
-}
-
-void ip6t_unregister_table_pre_exit(struct net *net, const char *name)
-{
- struct xt_table *table = xt_find_table(net, NFPROTO_IPV6, name);
-
- if (table)
- nf_unregister_net_hooks(net, table->ops, hweight32(table->valid_hooks));
}
void ip6t_unregister_table_exit(struct net *net, const char *name)
{
- struct xt_table *table = xt_find_table(net, NFPROTO_IPV6, name);
+ struct xt_table *table = xt_unregister_table_exit(net, NFPROTO_IPV6, name);
if (table)
__ip6t_unregister_table(net, table);
@@ -1894,7 +1855,6 @@ static void __exit ip6_tables_fini(void)
}
EXPORT_SYMBOL(ip6t_register_table);
-EXPORT_SYMBOL(ip6t_unregister_table_pre_exit);
EXPORT_SYMBOL(ip6t_unregister_table_exit);
EXPORT_SYMBOL(ip6t_do_table);
diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c
index e7a3fb9..450dd53 100644
--- a/net/ipv6/netfilter/ip6t_hbh.c
+++ b/net/ipv6/netfilter/ip6t_hbh.c
@@ -168,6 +168,10 @@ static int hbh_mt6_check(const struct xt_mtchk_param *par)
pr_debug("unknown flags %X\n", optsinfo->invflags);
return -EINVAL;
}
+ if (optsinfo->optsnr > IP6T_OPTS_OPTSNR) {
+ pr_debug("too many supported opts specified\n");
+ return -EINVAL;
+ }
if (optsinfo->flags & IP6T_OPTS_NSTRICT) {
pr_debug("Not strict - not implemented");
diff --git a/net/ipv6/netfilter/ip6table_filter.c b/net/ipv6/netfilter/ip6table_filter.c
index e899269..b074fc4 100644
--- a/net/ipv6/netfilter/ip6table_filter.c
+++ b/net/ipv6/netfilter/ip6table_filter.c
@@ -60,7 +60,7 @@ static int __net_init ip6table_filter_net_init(struct net *net)
static void __net_exit ip6table_filter_net_pre_exit(struct net *net)
{
- ip6t_unregister_table_pre_exit(net, "filter");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "filter");
}
static void __net_exit ip6table_filter_net_exit(struct net *net)
@@ -76,32 +76,32 @@ static struct pernet_operations ip6table_filter_net_ops = {
static int __init ip6table_filter_init(void)
{
- int ret = xt_register_template(&packet_filter,
- ip6table_filter_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
filter_ops = xt_hook_ops_alloc(&packet_filter, ip6t_do_table);
- if (IS_ERR(filter_ops)) {
- xt_unregister_template(&packet_filter);
+ if (IS_ERR(filter_ops))
return PTR_ERR(filter_ops);
- }
ret = register_pernet_subsys(&ip6table_filter_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&packet_filter, ip6table_filter_table_init);
if (ret < 0) {
- xt_unregister_template(&packet_filter);
- kfree(filter_ops);
- return ret;
+ unregister_pernet_subsys(&ip6table_filter_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(filter_ops);
return ret;
}
static void __exit ip6table_filter_fini(void)
{
- unregister_pernet_subsys(&ip6table_filter_net_ops);
xt_unregister_template(&packet_filter);
+ unregister_pernet_subsys(&ip6table_filter_net_ops);
kfree(filter_ops);
}
diff --git a/net/ipv6/netfilter/ip6table_mangle.c b/net/ipv6/netfilter/ip6table_mangle.c
index 8dd4cd0..e6ee036 100644
--- a/net/ipv6/netfilter/ip6table_mangle.c
+++ b/net/ipv6/netfilter/ip6table_mangle.c
@@ -89,7 +89,7 @@ static int ip6table_mangle_table_init(struct net *net)
static void __net_exit ip6table_mangle_net_pre_exit(struct net *net)
{
- ip6t_unregister_table_pre_exit(net, "mangle");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "mangle");
}
static void __net_exit ip6table_mangle_net_exit(struct net *net)
@@ -104,32 +104,33 @@ static struct pernet_operations ip6table_mangle_net_ops = {
static int __init ip6table_mangle_init(void)
{
- int ret = xt_register_template(&packet_mangler,
- ip6table_mangle_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
mangle_ops = xt_hook_ops_alloc(&packet_mangler, ip6table_mangle_hook);
- if (IS_ERR(mangle_ops)) {
- xt_unregister_template(&packet_mangler);
+ if (IS_ERR(mangle_ops))
return PTR_ERR(mangle_ops);
- }
ret = register_pernet_subsys(&ip6table_mangle_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&packet_mangler,
+ ip6table_mangle_table_init);
if (ret < 0) {
- xt_unregister_template(&packet_mangler);
- kfree(mangle_ops);
- return ret;
+ unregister_pernet_subsys(&ip6table_mangle_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(mangle_ops);
return ret;
}
static void __exit ip6table_mangle_fini(void)
{
- unregister_pernet_subsys(&ip6table_mangle_net_ops);
xt_unregister_template(&packet_mangler);
+ unregister_pernet_subsys(&ip6table_mangle_net_ops);
kfree(mangle_ops);
}
diff --git a/net/ipv6/netfilter/ip6table_nat.c b/net/ipv6/netfilter/ip6table_nat.c
index 5be7232..c2394e2 100644
--- a/net/ipv6/netfilter/ip6table_nat.c
+++ b/net/ipv6/netfilter/ip6table_nat.c
@@ -121,8 +121,11 @@ static int ip6table_nat_table_init(struct net *net)
}
ret = ip6t_nat_register_lookups(net);
- if (ret < 0)
+ if (ret < 0) {
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "nat");
+ synchronize_rcu();
ip6t_unregister_table_exit(net, "nat");
+ }
kfree(repl);
return ret;
@@ -131,6 +134,7 @@ static int ip6table_nat_table_init(struct net *net)
static void __net_exit ip6table_nat_net_pre_exit(struct net *net)
{
ip6t_nat_unregister_lookups(net);
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "nat");
}
static void __net_exit ip6table_nat_net_exit(struct net *net)
diff --git a/net/ipv6/netfilter/ip6table_raw.c b/net/ipv6/netfilter/ip6table_raw.c
index fc9f675..3b161ee 100644
--- a/net/ipv6/netfilter/ip6table_raw.c
+++ b/net/ipv6/netfilter/ip6table_raw.c
@@ -52,7 +52,7 @@ static int ip6table_raw_table_init(struct net *net)
static void __net_exit ip6table_raw_net_pre_exit(struct net *net)
{
- ip6t_unregister_table_pre_exit(net, "raw");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "raw");
}
static void __net_exit ip6table_raw_net_exit(struct net *net)
@@ -75,31 +75,31 @@ static int __init ip6table_raw_init(void)
pr_info("Enabling raw table before defrag\n");
}
- ret = xt_register_template(table, ip6table_raw_table_init);
- if (ret < 0)
- return ret;
-
/* Register hooks */
rawtable_ops = xt_hook_ops_alloc(table, ip6t_do_table);
- if (IS_ERR(rawtable_ops)) {
- xt_unregister_template(table);
+ if (IS_ERR(rawtable_ops))
return PTR_ERR(rawtable_ops);
- }
ret = register_pernet_subsys(&ip6table_raw_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(table, ip6table_raw_table_init);
if (ret < 0) {
- kfree(rawtable_ops);
- xt_unregister_template(table);
- return ret;
+ unregister_pernet_subsys(&ip6table_raw_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(rawtable_ops);
return ret;
}
static void __exit ip6table_raw_fini(void)
{
- unregister_pernet_subsys(&ip6table_raw_net_ops);
xt_unregister_template(&packet_raw);
+ unregister_pernet_subsys(&ip6table_raw_net_ops);
kfree(rawtable_ops);
}
diff --git a/net/ipv6/netfilter/ip6table_security.c b/net/ipv6/netfilter/ip6table_security.c
index 4df14a9..4bd5d97 100644
--- a/net/ipv6/netfilter/ip6table_security.c
+++ b/net/ipv6/netfilter/ip6table_security.c
@@ -49,7 +49,7 @@ static int ip6table_security_table_init(struct net *net)
static void __net_exit ip6table_security_net_pre_exit(struct net *net)
{
- ip6t_unregister_table_pre_exit(net, "security");
+ xt_unregister_table_pre_exit(net, NFPROTO_IPV6, "security");
}
static void __net_exit ip6table_security_net_exit(struct net *net)
@@ -64,32 +64,33 @@ static struct pernet_operations ip6table_security_net_ops = {
static int __init ip6table_security_init(void)
{
- int ret = xt_register_template(&security_table,
- ip6table_security_table_init);
-
- if (ret < 0)
- return ret;
+ int ret;
sectbl_ops = xt_hook_ops_alloc(&security_table, ip6t_do_table);
- if (IS_ERR(sectbl_ops)) {
- xt_unregister_template(&security_table);
+ if (IS_ERR(sectbl_ops))
return PTR_ERR(sectbl_ops);
- }
ret = register_pernet_subsys(&ip6table_security_net_ops);
+ if (ret < 0)
+ goto err_free;
+
+ ret = xt_register_template(&security_table,
+ ip6table_security_table_init);
if (ret < 0) {
- kfree(sectbl_ops);
- xt_unregister_template(&security_table);
- return ret;
+ unregister_pernet_subsys(&ip6table_security_net_ops);
+ goto err_free;
}
+ return 0;
+err_free:
+ kfree(sectbl_ops);
return ret;
}
static void __exit ip6table_security_fini(void)
{
- unregister_pernet_subsys(&ip6table_security_net_ops);
xt_unregister_template(&security_table);
+ unregister_pernet_subsys(&ip6table_security_net_ops);
kfree(sectbl_ops);
}
diff --git a/net/ipv6/netfilter/nf_socket_ipv6.c b/net/ipv6/netfilter/nf_socket_ipv6.c
index ced8bd4..893f2ae 100644
--- a/net/ipv6/netfilter/nf_socket_ipv6.c
+++ b/net/ipv6/netfilter/nf_socket_ipv6.c
@@ -100,6 +100,7 @@ struct sock *nf_sk_lookup_slow_v6(struct net *net, const struct sk_buff *skb,
const struct in6_addr *daddr = NULL, *saddr = NULL;
struct ipv6hdr *iph = ipv6_hdr(skb), ipv6_var;
struct sk_buff *data_skb = NULL;
+ unsigned short fragoff = 0;
int doff = 0;
int thoff = 0, tproto;
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
@@ -107,8 +108,8 @@ struct sock *nf_sk_lookup_slow_v6(struct net *net, const struct sk_buff *skb,
struct nf_conn const *ct;
#endif
- tproto = ipv6_find_hdr(skb, &thoff, -1, NULL, NULL);
- if (tproto < 0) {
+ tproto = ipv6_find_hdr(skb, &thoff, -1, &fragoff, NULL);
+ if (tproto < 0 || fragoff) {
pr_debug("unable to find transport header in IPv6 packet, dropping\n");
return NULL;
}
diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c
index 8b2dba8..2dbe447 100644
--- a/net/ipv6/netfilter/nft_fib_ipv6.c
+++ b/net/ipv6/netfilter/nft_fib_ipv6.c
@@ -160,21 +160,40 @@ static bool nft_fib6_info_nh_dev_match(const struct net_device *nh_dev,
l3mdev_master_ifindex_rcu(nh_dev) == dev->ifindex;
}
+static int nft_fib6_nh_match_dev_cb(struct fib6_nh *nh, void *arg)
+{
+ const struct net_device *dev = arg;
+
+ return nft_fib6_info_nh_dev_match(nh->fib_nh_dev, dev);
+}
+
static bool nft_fib6_info_nh_uses_dev(struct fib6_info *rt,
const struct net_device *dev)
{
const struct net_device *nh_dev;
struct fib6_info *iter;
+ /* External nexthop: fib6_siblings slot aliases nh_list, walk via nh. */
+ if (rt->nh)
+ return nexthop_for_each_fib6_nh(rt->nh,
+ nft_fib6_nh_match_dev_cb,
+ (void *)dev);
+
nh_dev = fib6_info_nh_dev(rt);
if (nft_fib6_info_nh_dev_match(nh_dev, dev))
return true;
- list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
+ if (!READ_ONCE(rt->fib6_nsiblings))
+ return false;
+
+ list_for_each_entry_rcu(iter, &rt->fib6_siblings, fib6_siblings) {
nh_dev = fib6_info_nh_dev(iter);
if (nft_fib6_info_nh_dev_match(nh_dev, dev))
return true;
+
+ if (!READ_ONCE(rt->fib6_nsiblings))
+ return false;
}
return false;
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 19eb6b7..636f012 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -481,6 +481,9 @@ void fib6_select_path(const struct net *net, struct fib6_result *res,
const struct fib6_nh *nh = sibling->fib6_nh;
int nh_upper_bound;
+ if (!READ_ONCE(first->fib6_nsiblings))
+ break;
+
nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);
if (hash > nh_upper_bound)
continue;
@@ -1645,6 +1648,10 @@ static unsigned int fib6_mtu(const struct fib6_result *res)
rcu_read_lock();
idev = __in6_dev_get(dev);
+ if (!idev) {
+ rcu_read_unlock();
+ return 0;
+ }
mtu = READ_ONCE(idev->cnf.mtu6);
rcu_read_unlock();
}
@@ -4995,6 +5002,7 @@ static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
break;
rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
+ fib6_update_sernum(net, rt);
rt6_multipath_rebalance(rt);
break;
}
@@ -5897,6 +5905,8 @@ static int rt6_fill_node(struct net *net, struct sk_buff *skb,
goto nla_put_failure;
}
+ if (!READ_ONCE(rt->fib6_nsiblings))
+ break;
}
rcu_read_unlock();
@@ -6928,7 +6938,7 @@ int __init ip6_route_init(void)
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
ret = bpf_iter_register();
if (ret)
- goto out_register_late_subsys;
+ goto out_register_notifier;
#endif
for_each_possible_cpu(cpu) {
@@ -6941,6 +6951,10 @@ int __init ip6_route_init(void)
out:
return ret;
+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
+out_register_notifier:
+ unregister_netdevice_notifier(&ip6_route_dev_notifier);
+#endif
out_register_late_subsys:
rtnl_unregister_all(PF_INET6);
unregister_pernet_subsys(&ip6_route_net_late_ops);
diff --git a/net/ipv6/rpl_iptunnel.c b/net/ipv6/rpl_iptunnel.c
index c7942cf..4e10adc 100644
--- a/net/ipv6/rpl_iptunnel.c
+++ b/net/ipv6/rpl_iptunnel.c
@@ -287,7 +287,16 @@ static int rpl_input(struct sk_buff *skb)
if (!dst) {
ip6_route_input(skb);
+
+ /* ip6_route_input() sets a NOREF dst; force a refcount on it
+ * before caching or further use.
+ */
+ skb_dst_force(skb);
dst = skb_dst(skb);
+ if (unlikely(!dst)) {
+ err = -ENETUNREACH;
+ goto drop;
+ }
/* cache only if we don't create a dst reference loop */
if (!dst->error && lwtst != dst->lwtstate) {
diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c
index 9b64343..4c45c0a 100644
--- a/net/ipv6/seg6_iptunnel.c
+++ b/net/ipv6/seg6_iptunnel.c
@@ -515,7 +515,16 @@ static int seg6_input_core(struct net *net, struct sock *sk,
if (!dst) {
ip6_route_input(skb);
+
+ /* ip6_route_input() sets a NOREF dst; force a refcount on it
+ * before caching or further use.
+ */
+ skb_dst_force(skb);
dst = skb_dst(skb);
+ if (unlikely(!dst)) {
+ err = -ENETUNREACH;
+ goto drop;
+ }
/* cache only if we don't create a dst reference loop */
if (!dst->error && lwtst != dst->lwtstate) {
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 2c3f7a7..36d75fb 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -288,8 +288,10 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
saddr = &fl6->saddr;
err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
- if (err)
+ if (err) {
+ dst_release(dst);
goto failure;
+ }
}
/* set the source address */
@@ -1617,12 +1619,13 @@ int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
if (sk->sk_state == TCP_LISTEN) {
struct sock *nsk = tcp_v6_cookie_check(sk, skb);
+ if (!nsk)
+ return 0;
if (nsk != sk) {
- if (nsk) {
- reason = tcp_child_process(sk, nsk, skb);
- if (reason)
- goto reset;
- }
+ reason = tcp_child_process(sk, nsk, skb);
+ sock_put(nsk);
+ if (reason)
+ goto reset;
return 0;
}
} else
@@ -1827,13 +1830,16 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
rst_reason = sk_rst_convert_drop_reason(drop_reason);
tcp_v6_send_reset(nsk, skb, rst_reason);
+ sock_put(nsk);
goto discard_and_relse;
}
+ sock_put(nsk);
sock_put(sk);
return 0;
}
}
+ isn = 0;
process:
if (static_branch_unlikely(&ip6_min_hopcount)) {
/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
@@ -1863,6 +1869,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
th = (const struct tcphdr *)skb->data;
hdr = ipv6_hdr(skb);
tcp_v6_fill_cb(skb, hdr, th);
+ TCP_SKB_CB(skb)->tcp_tw_isn = isn;
skb->dev = NULL;
@@ -1951,7 +1958,6 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
sk = sk2;
tcp_v6_restore_cb(skb);
refcounted = false;
- __this_cpu_write(tcp_tw_isn, isn);
goto process;
}
diff --git a/net/ipv6/xfrm6_protocol.c b/net/ipv6/xfrm6_protocol.c
index ea2f805..9b586fc 100644
--- a/net/ipv6/xfrm6_protocol.c
+++ b/net/ipv6/xfrm6_protocol.c
@@ -88,8 +88,10 @@ int xfrm6_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
dst = ip6_route_input_lookup(dev_net(skb->dev), skb->dev, &fl6,
skb, flags);
- if (dst->error)
+ if (dst->error) {
+ dst_release(dst);
goto drop;
+ }
skb_dst_set(skb, dst);
}
diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index 72dfccd..c2dc333 100644
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -1540,7 +1540,7 @@ static int iucv_sock_getsockopt(struct socket *sock, int level, int optname,
struct sock *sk = sock->sk;
struct iucv_sock *iucv = iucv_sk(sk);
unsigned int val;
- int len;
+ int len, rc;
if (level != SOL_IUCV)
return -ENOPROTOOPT;
@@ -1553,26 +1553,34 @@ static int iucv_sock_getsockopt(struct socket *sock, int level, int optname,
len = min_t(unsigned int, len, sizeof(int));
+ rc = 0;
+
+ lock_sock(sk);
switch (optname) {
case SO_IPRMDATA_MSG:
val = (iucv->flags & IUCV_IPRMDATA) ? 1 : 0;
break;
case SO_MSGLIMIT:
- lock_sock(sk);
val = (iucv->path != NULL) ? iucv->path->msglim /* connected */
: iucv->msglimit; /* default */
- release_sock(sk);
break;
case SO_MSGSIZE:
- if (sk->sk_state == IUCV_OPEN)
- return -EBADFD;
+ if (sk->sk_state == IUCV_OPEN) {
+ rc = -EBADFD;
+ break;
+ }
val = (iucv->hs_dev) ? iucv->hs_dev->mtu -
sizeof(struct af_iucv_trans_hdr) - ETH_HLEN :
0x7fffffff;
break;
default:
- return -ENOPROTOOPT;
+ rc = -ENOPROTOOPT;
+ break;
}
+ release_sock(sk);
+
+ if (rc)
+ return rc;
if (put_user(len, optlen))
return -EFAULT;
diff --git a/net/key/af_key.c b/net/key/af_key.c
index a166a88..9cffeef 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -3564,7 +3564,7 @@ static int set_ipsecrequest(struct sk_buff *skb,
#ifdef CONFIG_NET_KEY_MIGRATE
static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_bundles,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap)
{
int i;
@@ -3669,7 +3669,7 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
}
/* broadcast migrate message to sockets */
- pfkey_broadcast(skb, GFP_ATOMIC, BROADCAST_ALL, NULL, &init_net);
+ pfkey_broadcast(skb, GFP_ATOMIC, BROADCAST_ALL, NULL, net);
return 0;
@@ -3680,7 +3680,7 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
#else
static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_bundles,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap)
{
return -ENOPROTOOPT;
diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c
index 157fc23..9419c85 100644
--- a/net/l2tp/l2tp_core.c
+++ b/net/l2tp/l2tp_core.c
@@ -441,12 +441,13 @@ struct l2tp_session *l2tp_session_get_by_ifname(const struct net *net,
idr_for_each_entry_ul(&pn->l2tp_tunnel_idr, tunnel, tmp, tunnel_id) {
if (tunnel) {
list_for_each_entry_rcu(session, &tunnel->session_list, list) {
- if (!strcmp(session->ifname, ifname)) {
- refcount_inc(&session->ref_count);
- rcu_read_unlock_bh();
+ if (strcmp(session->ifname, ifname))
+ continue;
+ if (!refcount_inc_not_zero(&session->ref_count))
+ continue;
+ rcu_read_unlock_bh();
- return session;
- }
+ return session;
}
}
}
@@ -1360,7 +1361,7 @@ static void l2tp_session_unhash(struct l2tp_session *session)
spin_lock_bh(&pn->l2tp_session_idr_lock);
/* Remove from the per-tunnel list */
- list_del_init(&session->list);
+ list_del_rcu(&session->list);
/* Remove from per-net IDR */
if (tunnel->version == L2TP_HDR_VER_3) {
diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c
index 99d6582..e0b1915 100644
--- a/net/l2tp/l2tp_ppp.c
+++ b/net/l2tp/l2tp_ppp.c
@@ -1045,64 +1045,76 @@ static int pppol2tp_ioctl(struct socket *sock, unsigned int cmd,
{
struct pppol2tp_ioc_stats stats;
struct l2tp_session *session;
+ int err = 0;
+
+ session = pppol2tp_sock_to_session(sock->sk);
+
+ /* Validate session presence and magic integrity ONLY for commands
+ * that belong to L2TP and require a valid session.
+ */
+ switch (cmd) {
+ case PPPIOCGMRU:
+ case PPPIOCGFLAGS:
+ case PPPIOCSMRU:
+ case PPPIOCSFLAGS:
+ case PPPIOCGL2TPSTATS:
+ if (!session)
+ return -ENOTCONN;
+
+ if (session->magic != L2TP_SESSION_MAGIC) {
+ l2tp_session_put(session);
+ return -EBADF;
+ }
+ break;
+ default:
+ break;
+ }
switch (cmd) {
case PPPIOCGMRU:
case PPPIOCGFLAGS:
- session = sock->sk->sk_user_data;
- if (!session)
- return -ENOTCONN;
-
- if (WARN_ON(session->magic != L2TP_SESSION_MAGIC))
- return -EBADF;
-
/* Not defined for tunnels */
- if (!session->session_id && !session->peer_session_id)
- return -ENOSYS;
+ if (!session->session_id && !session->peer_session_id) {
+ err = -ENOSYS;
+ break;
+ }
- if (put_user(0, (int __user *)arg))
- return -EFAULT;
+ if (put_user(0, (int __user *)arg)) {
+ err = -EFAULT;
+ break;
+ }
break;
case PPPIOCSMRU:
case PPPIOCSFLAGS:
- session = sock->sk->sk_user_data;
- if (!session)
- return -ENOTCONN;
-
- if (WARN_ON(session->magic != L2TP_SESSION_MAGIC))
- return -EBADF;
-
/* Not defined for tunnels */
- if (!session->session_id && !session->peer_session_id)
- return -ENOSYS;
+ if (!session->session_id && !session->peer_session_id) {
+ err = -ENOSYS;
+ break;
+ }
- if (!access_ok((int __user *)arg, sizeof(int)))
- return -EFAULT;
+ if (!access_ok((int __user *)arg, sizeof(int))) {
+ err = -EFAULT;
+ break;
+ }
break;
case PPPIOCGL2TPSTATS:
- session = sock->sk->sk_user_data;
- if (!session)
- return -ENOTCONN;
-
- if (WARN_ON(session->magic != L2TP_SESSION_MAGIC))
- return -EBADF;
-
/* Session 0 represents the parent tunnel */
if (!session->session_id && !session->peer_session_id) {
u32 session_id;
- int err;
if (copy_from_user(&stats, (void __user *)arg,
- sizeof(stats)))
- return -EFAULT;
+ sizeof(stats))) {
+ err = -EFAULT;
+ break;
+ }
session_id = stats.session_id;
err = pppol2tp_tunnel_copy_stats(&stats,
session->tunnel);
if (err < 0)
- return err;
+ break;
stats.session_id = session_id;
} else {
@@ -1112,15 +1124,21 @@ static int pppol2tp_ioctl(struct socket *sock, unsigned int cmd,
stats.tunnel_id = session->tunnel->tunnel_id;
stats.using_ipsec = l2tp_tunnel_uses_xfrm(session->tunnel);
- if (copy_to_user((void __user *)arg, &stats, sizeof(stats)))
- return -EFAULT;
+ if (copy_to_user((void __user *)arg, &stats, sizeof(stats))) {
+ err = -EFAULT;
+ break;
+ }
break;
default:
- return -ENOIOCTLCMD;
+ err = -ENOIOCTLCMD;
+ break;
}
- return 0;
+ if (session)
+ l2tp_session_put(session);
+
+ return err;
}
/*****************************************************************************
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index 7b77d57..f9ee994 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -2344,8 +2344,9 @@ static int sta_apply_parameters(struct ieee80211_local *local,
sta->sta.max_sp = params->max_sp;
}
- ieee80211_sta_set_max_amsdu_subframes(sta, params->ext_capab,
- params->ext_capab_len);
+ if (params->ext_capab)
+ ieee80211_sta_set_max_amsdu_subframes(sta, params->ext_capab,
+ params->ext_capab_len);
/*
* cfg80211 validates this (1-2007) and allows setting the AID
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index 160ae65..b98ddfa 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -438,6 +438,15 @@ ieee80211_verify_sta_ht_mcs_support(struct ieee80211_sub_if_data *sdata,
ieee80211_apply_htcap_overrides(sdata, &sta_ht_cap);
/*
+ * Some Xfinity XB8 firmware advertises >1 spatial stream MCS indexes in
+ * their basic HT-MCS set. On cards with lower spatial streams, the check
+ * would fail, and we'd be stuck with no HT when it in fact work fine with
+ * its own supported rate. So check it only in strict mode.
+ */
+ if (!ieee80211_hw_check(&sdata->local->hw, STRICT))
+ return true;
+
+ /*
* P802.11REVme/D7.0 - 6.5.4.2.4
* ...
* If the MLME of an HT STA receives an MLME-JOIN.request primitive
@@ -8155,6 +8164,7 @@ ieee80211_parse_neg_ttlm(struct ieee80211_sub_if_data *sdata,
"No active links for TID %d", tid);
return -EINVAL;
}
+ pos += map_size;
} else {
map = 0;
}
@@ -8173,7 +8183,6 @@ ieee80211_parse_neg_ttlm(struct ieee80211_sub_if_data *sdata,
default:
return -EINVAL;
}
- pos += map_size;
}
return 0;
}
@@ -9140,7 +9149,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
struct ieee80211_bss *bss = (void *)cbss->priv;
struct sta_info *new_sta = NULL;
struct ieee80211_link_data *link;
- bool have_sta = false;
+ struct sta_info *have_sta = NULL;
bool mlo;
int err;
u16 new_links;
@@ -9159,11 +9168,8 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
mlo = false;
}
- if (assoc) {
- rcu_read_lock();
+ if (assoc)
have_sta = sta_info_get(sdata, ap_mld_addr);
- rcu_read_unlock();
- }
if (mlo && !have_sta &&
WARN_ON(sdata->vif.valid_links || sdata->vif.active_links))
@@ -9327,6 +9333,8 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
out_release_chan:
ieee80211_link_release_channel(link);
out_err:
+ if (mlo && have_sta)
+ WARN_ON(__sta_info_destroy(have_sta));
ieee80211_vif_set_links(sdata, 0, 0);
return err;
}
@@ -11224,6 +11232,9 @@ static void ieee80211_ml_epcs(struct ieee80211_sub_if_data *sdata,
control = get_unaligned_le16(pos);
link_id = control & IEEE80211_MLE_STA_EPCS_CONTROL_LINK_ID;
+ if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS)
+ continue;
+
link = sdata_dereference(sdata->link[link_id], sdata);
if (!link)
continue;
diff --git a/net/mac80211/parse.c b/net/mac80211/parse.c
index 2b3632c..77894d9 100644
--- a/net/mac80211/parse.c
+++ b/net/mac80211/parse.c
@@ -34,6 +34,22 @@
#include "led.h"
#include "wep.h"
+static const u8 empty_non_inheritance[] = {
+ WLAN_EID_EXTENSION, 1, WLAN_EID_EXT_NON_INHERITANCE,
+ /*
+ * cfg80211_is_element_inherited() hardcodes elements that
+ * cannot be inherited, so we just need an empty one to be
+ * calling it at all.
+ */
+};
+
+struct ieee80211_elem_defrag {
+ const struct element *elem;
+ /* container start/len */
+ const u8 *start;
+ size_t len;
+};
+
struct ieee80211_elems_parse {
/* must be first for kfree to work */
struct ieee802_11_elems elems;
@@ -41,11 +57,7 @@ struct ieee80211_elems_parse {
/* The basic Multi-Link element in the original elements */
const struct element *ml_basic_elem;
- /* The reconfiguration Multi-Link element in the original elements */
- const struct element *ml_reconf_elem;
-
- /* The EPCS Multi-Link element in the original elements */
- const struct element *ml_epcs_elem;
+ struct ieee80211_elem_defrag ml_reconf, ml_epcs;
bool multi_link_inner;
bool skip_vendor;
@@ -162,10 +174,14 @@ ieee80211_parse_extension_element(u32 *crc,
}
break;
case IEEE80211_ML_CONTROL_TYPE_RECONF:
- elems_parse->ml_reconf_elem = elem;
+ elems_parse->ml_reconf.elem = elem;
+ elems_parse->ml_reconf.start = params->start;
+ elems_parse->ml_reconf.len = params->len;
break;
case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
- elems_parse->ml_epcs_elem = elem;
+ elems_parse->ml_epcs.elem = elem;
+ elems_parse->ml_epcs.start = params->start;
+ elems_parse->ml_epcs.len = params->len;
break;
default:
break;
@@ -916,7 +932,7 @@ ieee80211_prep_mle_link_parse(struct ieee80211_elems_parse *elems_parse,
{
struct ieee802_11_elems *elems = &elems_parse->elems;
struct ieee80211_mle_per_sta_profile *prof;
- const struct element *tmp;
+ const struct element *tmp, *ret;
ssize_t ml_len;
const u8 *end;
@@ -986,50 +1002,40 @@ ieee80211_prep_mle_link_parse(struct ieee80211_elems_parse *elems_parse,
sub->from_ap = params->from_ap;
sub->link_id = -1;
- return cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
- sub->start, sub->len);
+ ret = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
+ sub->start, sub->len);
+ if (ret)
+ return ret;
+
+ /*
+ * Since we know we want and found a profile, apply an empty
+ * non-inheritance if the profile didn't have one, so that any
+ * element that shouldn't be inherited by spec isn't.
+ */
+ return (const void *)empty_non_inheritance;
}
-static void
-ieee80211_mle_defrag_reconf(struct ieee80211_elems_parse *elems_parse)
+static const void *
+ieee80211_mle_defrag(struct ieee80211_elems_parse *elems_parse,
+ struct ieee80211_elem_defrag *defrag,
+ size_t *out_len)
{
- struct ieee802_11_elems *elems = &elems_parse->elems;
+ const void *ret;
ssize_t ml_len;
- ml_len = cfg80211_defragment_element(elems_parse->ml_reconf_elem,
- elems->ie_start,
- elems->total_len,
+ ml_len = cfg80211_defragment_element(defrag->elem,
+ defrag->start, defrag->len,
elems_parse->scratch_pos,
elems_parse->scratch +
elems_parse->scratch_len -
elems_parse->scratch_pos,
WLAN_EID_FRAGMENT);
if (ml_len < 0)
- return;
- elems->ml_reconf = (void *)elems_parse->scratch_pos;
- elems->ml_reconf_len = ml_len;
+ return NULL;
+ ret = elems_parse->scratch_pos;
+ *out_len = ml_len;
elems_parse->scratch_pos += ml_len;
-}
-
-static void
-ieee80211_mle_defrag_epcs(struct ieee80211_elems_parse *elems_parse)
-{
- struct ieee802_11_elems *elems = &elems_parse->elems;
- ssize_t ml_len;
-
- ml_len = cfg80211_defragment_element(elems_parse->ml_epcs_elem,
- elems->ie_start,
- elems->total_len,
- elems_parse->scratch_pos,
- elems_parse->scratch +
- elems_parse->scratch_len -
- elems_parse->scratch_pos,
- WLAN_EID_FRAGMENT);
- if (ml_len < 0)
- return;
- elems->ml_epcs = (void *)elems_parse->scratch_pos;
- elems->ml_epcs_len = ml_len;
- elems_parse->scratch_pos += ml_len;
+ return ret;
}
struct ieee802_11_elems *
@@ -1042,6 +1048,7 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
size_t scratch_len = 3 * params->len;
bool multi_link_inner = false;
+ BUILD_BUG_ON(sizeof(empty_non_inheritance) != empty_non_inheritance[1] + 2);
BUILD_BUG_ON(offsetof(typeof(*elems_parse), elems) != 0);
/* cannot parse for both a specific link and non-transmitted BSS */
@@ -1089,6 +1096,17 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
non_inherit = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
sub.start, nontx_len);
+ /*
+ * If it's a non-transmitted BSS, we shouldn't pick
+ * any elements in the outer parsing that shouldn't
+ * be inherited. If the profile has a non-inheritance
+ * element this automatically happens, but if not then
+ * provide an empty one so that the hard-coded elements
+ * in cfg80211_is_element_inherited() are ignored, but
+ * it must be called.
+ */
+ if (params->bss->transmitted_bss && !non_inherit)
+ non_inherit = (const void *)empty_non_inheritance;
} else {
/* must always parse to get elems_parse->ml_basic_elem */
non_inherit = ieee80211_prep_mle_link_parse(elems_parse, params,
@@ -1109,9 +1127,12 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
_ieee802_11_parse_elems_full(&sub, elems_parse, NULL);
}
- ieee80211_mle_defrag_reconf(elems_parse);
-
- ieee80211_mle_defrag_epcs(elems_parse);
+ elems->ml_reconf = ieee80211_mle_defrag(elems_parse,
+ &elems_parse->ml_reconf,
+ &elems->ml_reconf_len);
+ elems->ml_epcs = ieee80211_mle_defrag(elems_parse,
+ &elems_parse->ml_epcs,
+ &elems->ml_epcs_len);
if (elems->tim && !elems->parse_error) {
const struct ieee80211_tim_ie *tim_ie = elems->tim;
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index 3e5d1c4..3fb4044 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -4971,7 +4971,7 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
struct sk_buff *skb = rx->skb;
struct ieee80211_hdr *hdr = (void *)skb->data;
struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
- static ieee80211_rx_result res;
+ ieee80211_rx_result res;
int orig_len = skb->len;
int hdrlen = ieee80211_hdrlen(hdr->frame_control);
int snap_offs = hdrlen;
@@ -4984,6 +4984,7 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
u8 sa[ETH_ALEN];
} addrs __aligned(2);
struct ieee80211_sta_rx_stats *stats;
+ u32 encoded_rate;
/* for parallel-rx, we need to have DUP_VALIDATED, otherwise we write
* to a common data structure; drivers can implement that per queue
@@ -5091,11 +5092,14 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
/* push the addresses in front */
memcpy(skb_push(skb, sizeof(addrs)), &addrs, sizeof(addrs));
+ /* capture before mesh forward may memset or free skb->cb */
+ encoded_rate = sta_stats_encode_rate(status);
+
res = ieee80211_rx_mesh_data(rx->sdata, rx->sta, rx->skb);
switch (res) {
case RX_QUEUED:
stats->last_rx = jiffies;
- stats->last_rate = sta_stats_encode_rate(status);
+ stats->last_rate = encoded_rate;
return true;
case RX_CONTINUE:
break;
@@ -5380,7 +5384,9 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
if (!link_sta)
goto out;
- ieee80211_rx_data_set_link(&rx, link_sta->link_id);
+ if (!ieee80211_rx_data_set_link(&rx,
+ link_sta->link_id))
+ goto out;
}
if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
diff --git a/net/mac80211/tests/chan-mode.c b/net/mac80211/tests/chan-mode.c
index adc0690..fa37083 100644
--- a/net/mac80211/tests/chan-mode.c
+++ b/net/mac80211/tests/chan-mode.c
@@ -65,6 +65,7 @@ static const struct determine_chan_mode_case {
.ht_capa_mask = {
.mcs.rx_mask[0] = 0xf7,
},
+ .strict = true,
}, {
.desc = "Masking out a RX rate in VHT capabilities",
.conn_mode = IEEE80211_CONN_MODE_EHT,
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index b487d23..ea7f63e 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -2181,7 +2181,9 @@ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb,
case IEEE80211_RADIOTAP_ANTENNA:
/* this can appear multiple times, keep a bitmap */
- info->control.antennas |= BIT(*iterator.this_arg);
+ /* control.antennas is only a 2-bit bitmap */
+ if (*iterator.this_arg < 2)
+ info->control.antennas |= BIT(*iterator.this_arg);
break;
case IEEE80211_RADIOTAP_DATA_RETRIES:
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index b093bc2..2529b01 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -3700,11 +3700,11 @@ void ieee80211_dfs_radar_detected_work(struct wiphy *wiphy,
struct ieee80211_local *local =
container_of(work, struct ieee80211_local, radar_detected_work);
struct cfg80211_chan_def chandef;
- struct ieee80211_chanctx *ctx;
+ struct ieee80211_chanctx *ctx, *tmp;
lockdep_assert_wiphy(local->hw.wiphy);
- list_for_each_entry(ctx, &local->chanctx_list, list) {
+ list_for_each_entry_safe(ctx, tmp, &local->chanctx_list, list) {
if (ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER)
continue;
diff --git a/net/mctp/test/route-test.c b/net/mctp/test/route-test.c
index e103364..e4b230e 100644
--- a/net/mctp/test/route-test.c
+++ b/net/mctp/test/route-test.c
@@ -920,9 +920,9 @@ static void mctp_test_route_input_cloned_frag(struct kunit *test)
static void mctp_test_route_input_null_eid(struct kunit *test)
{
struct mctp_hdr hdr = RX_HDR(1, 10, 0, FL_S | FL_E | FL_TO);
+ struct sockaddr_mctp addr = { 0 };
struct sk_buff *skb_pkt, *skb_sk;
struct mctp_test_dev *dev;
- struct sockaddr_mctp addr;
struct socket *sock;
u8 type = 0;
int rc;
diff --git a/net/mctp/test/utils.c b/net/mctp/test/utils.c
index c3987d5..6eef8d4 100644
--- a/net/mctp/test/utils.c
+++ b/net/mctp/test/utils.c
@@ -116,7 +116,7 @@ void mctp_test_destroy_dev(struct mctp_test_dev *dev)
static int mctp_test_dst_output(struct mctp_dst *dst, struct sk_buff *skb)
{
skb->dev = dst->dev->dev;
- dev_queue_xmit(skb);
+ dev_direct_xmit(skb, 0);
return 0;
}
diff --git a/net/mptcp/bpf.c b/net/mptcp/bpf.c
index 8a16672..4cc16cb 100644
--- a/net/mptcp/bpf.c
+++ b/net/mptcp/bpf.c
@@ -14,7 +14,7 @@
struct mptcp_sock *bpf_mptcp_sock_from_subflow(struct sock *sk)
{
- if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP && sk_is_mptcp(sk))
+ if (sk && sk_fullsock(sk) && sk_is_tcp(sk) && sk_is_mptcp(sk))
return mptcp_sk(mptcp_subflow_ctx(sk)->conn);
return NULL;
diff --git a/net/mptcp/fastopen.c b/net/mptcp/fastopen.c
index 82ec15b..082c46c 100644
--- a/net/mptcp/fastopen.c
+++ b/net/mptcp/fastopen.c
@@ -12,6 +12,7 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf
struct sock *sk, *ssk;
struct sk_buff *skb;
struct tcp_sock *tp;
+ bool has_rxtstamp;
/* on early fallback the subflow context is deleted by
* subflow_syn_recv_sock()
@@ -40,12 +41,13 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf
*/
tp->copied_seq += skb->len;
subflow->ssn_offset += skb->len;
+ has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp;
/* Only the sequence delta is relevant */
MPTCP_SKB_CB(skb)->map_seq = -skb->len;
MPTCP_SKB_CB(skb)->end_seq = 0;
MPTCP_SKB_CB(skb)->offset = 0;
- MPTCP_SKB_CB(skb)->has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp;
+ MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp;
MPTCP_SKB_CB(skb)->cant_coalesce = 1;
mptcp_data_lock(sk);
diff --git a/net/mptcp/options.c b/net/mptcp/options.c
index 8a1c569..b3ea785 100644
--- a/net/mptcp/options.c
+++ b/net/mptcp/options.c
@@ -566,12 +566,17 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
{
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
struct mptcp_sock *msk = mptcp_sk(subflow->conn);
+ struct tcp_sock *tp = tcp_sk(sk);
unsigned int dss_size = 0;
struct mptcp_ext *mpext;
unsigned int ack_size;
bool ret = false;
- u64 ack_seq;
+ /* Zero `use_ack` and `use_map` flags with one shot. */
+ BUILD_BUG_ON(sizeof_field(struct mptcp_ext, flags) != sizeof(u16));
+ BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct mptcp_ext, flags),
+ sizeof(u16)));
+ *(u16 *)&opts->ext_copy.flags = 0;
opts->csum_reqd = READ_ONCE(msk->csum_enabled);
mpext = skb ? mptcp_get_ext(skb) : NULL;
@@ -595,20 +600,16 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
/* passive sockets msk will set the 'can_ack' after accept(), even
* if the first subflow may have the already the remote key handy
*/
- opts->ext_copy.use_ack = 0;
if (!READ_ONCE(msk->can_ack)) {
*size = ALIGN(dss_size, 4);
return ret;
}
- ack_seq = READ_ONCE(msk->ack_seq);
if (READ_ONCE(msk->use_64bit_ack)) {
ack_size = TCPOLEN_MPTCP_DSS_ACK64;
- opts->ext_copy.data_ack = ack_seq;
opts->ext_copy.ack64 = 1;
} else {
ack_size = TCPOLEN_MPTCP_DSS_ACK32;
- opts->ext_copy.data_ack32 = (uint32_t)ack_seq;
opts->ext_copy.ack64 = 0;
}
opts->ext_copy.use_ack = 1;
@@ -618,6 +619,12 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
if (dss_size == 0)
ack_size += TCPOLEN_MPTCP_DSS_BASE;
+ /* The caller is __tcp_transmit_skb(), and will compute the new rcv
+ * wnd soon: ensure that the window can shrink.
+ */
+ if (skb)
+ tp->rcv_wnd = tp->rcv_nxt - tp->rcv_wup;
+
dss_size += ack_size;
*size = ALIGN(dss_size, 4);
@@ -658,7 +665,6 @@ static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *
{
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
struct mptcp_sock *msk = mptcp_sk(subflow->conn);
- bool drop_other_suboptions = false;
unsigned int opt_size = *size;
struct mptcp_addr_info addr;
bool echo;
@@ -669,36 +675,20 @@ static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *
*/
if (!mptcp_pm_should_add_signal(msk) ||
(opts->suboptions & (OPTION_MPTCP_MPJ_ACK | OPTION_MPTCP_MPC_ACK)) ||
- !mptcp_pm_add_addr_signal(msk, skb, opt_size, remaining, &addr,
- &echo, &drop_other_suboptions))
+ !skb || !skb_is_tcp_pure_ack(skb) ||
+ !mptcp_pm_add_addr_signal(msk, opt_size, remaining, &addr, &echo))
return false;
- /*
- * Later on, mptcp_write_options() will enforce mutually exclusion with
- * DSS, bail out if such option is set and we can't drop it.
- */
- if (drop_other_suboptions)
- remaining += opt_size;
- else if (opts->suboptions & OPTION_MPTCP_DSS)
- return false;
+ remaining += opt_size;
len = mptcp_add_addr_len(addr.family, echo, !!addr.port);
if (remaining < len)
return false;
*size = len;
- if (drop_other_suboptions) {
- pr_debug("drop other suboptions\n");
- opts->suboptions = 0;
-
- /* note that e.g. DSS could have written into the memory
- * aliased by ahmac, we must reset the field here
- * to avoid appending the hmac even for ADD_ADDR echo
- * options
- */
- opts->ahmac = 0;
- *size -= opt_size;
- }
+ pr_debug("drop other suboptions\n");
+ opts->suboptions = 0;
+ *size -= opt_size;
opts->addr = addr;
opts->suboptions |= OPTION_MPTCP_ADD_ADDR;
if (!echo) {
@@ -708,6 +698,7 @@ static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *
&opts->addr);
} else {
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADDTX);
+ opts->ahmac = 0;
}
pr_debug("addr_id=%d, ahmac=%llu, echo=%d, port=%d\n",
opts->addr.id, opts->ahmac, echo, ntohs(opts->addr.port));
@@ -1297,19 +1288,14 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
return true;
}
-static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th)
+static u64 mptcp_set_rwin(struct mptcp_sock *msk, struct tcp_sock *tp,
+ struct tcphdr *th, u64 ack_seq)
{
const struct sock *ssk = (const struct sock *)tp;
- struct mptcp_subflow_context *subflow;
- u64 ack_seq, rcv_wnd_old, rcv_wnd_new;
- struct mptcp_sock *msk;
+ u64 rcv_wnd_old, rcv_wnd_new;
u32 new_win;
u64 win;
- subflow = mptcp_subflow_ctx(ssk);
- msk = mptcp_sk(subflow->conn);
-
- ack_seq = READ_ONCE(msk->ack_seq);
rcv_wnd_new = ack_seq + tp->rcv_wnd;
rcv_wnd_old = atomic64_read(&msk->rcv_wnd_sent);
@@ -1362,7 +1348,7 @@ static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th)
update_wspace:
WRITE_ONCE(msk->old_wspace, tp->rcv_wnd);
- subflow->rcv_wnd_sent = rcv_wnd_new;
+ return rcv_wnd_new;
}
static void mptcp_track_rwin(struct tcp_sock *tp)
@@ -1474,13 +1460,25 @@ void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp,
*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
if (mpext->use_ack) {
+ struct mptcp_sock *msk;
+ u64 ack_seq;
+
+ /* DSS option is set only by mptcp_established_options,
+ * the caller is __tcp_transmit_skb() and ssk is always
+ * not NULL.
+ */
+ subflow = mptcp_subflow_ctx(ssk);
+ msk = mptcp_sk(subflow->conn);
+ ack_seq = READ_ONCE(msk->ack_seq);
if (mpext->ack64) {
- put_unaligned_be64(mpext->data_ack, ptr);
+ put_unaligned_be64(ack_seq, ptr);
ptr += 2;
} else {
- put_unaligned_be32(mpext->data_ack32, ptr);
+ put_unaligned_be32(ack_seq, ptr);
ptr += 1;
}
+ subflow->rcv_wnd_sent = mptcp_set_rwin(msk, tp, th,
+ ack_seq);
}
if (mpext->use_map) {
@@ -1708,9 +1706,6 @@ void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp,
i += 4;
}
}
-
- if (tp)
- mptcp_set_rwin(tp, th);
}
__be32 mptcp_get_reset_option(const struct sk_buff *skb)
diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index 57a4566..4705014 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -16,6 +16,7 @@ struct mptcp_pm_add_entry {
struct list_head list;
struct mptcp_addr_info addr;
u8 retrans_times;
+ bool timer_done;
struct timer_list add_timer;
struct mptcp_sock *sock;
struct rcu_head rcu;
@@ -283,6 +284,9 @@ int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk,
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
struct mptcp_addr_info local, remote;
+ if (!__mptcp_subflow_active(subflow))
+ continue;
+
mptcp_local_address((struct sock_common *)ssk, &local);
if (!mptcp_addresses_equal(&local, addr, addr->port))
continue;
@@ -305,18 +309,31 @@ static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk)
const struct net *net = sock_net((struct sock *)msk);
unsigned int rto = mptcp_get_add_addr_timeout(net);
struct mptcp_subflow_context *subflow;
- unsigned int max = 0;
+ unsigned int max = 0, max_stale = 0;
+
+ if (!rto)
+ return 0;
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
struct inet_connection_sock *icsk = inet_csk(ssk);
- if (icsk->icsk_rto > max)
+ if (!__mptcp_subflow_active(subflow))
+ continue;
+
+ if (unlikely(subflow->stale)) {
+ if (icsk->icsk_rto > max_stale)
+ max_stale = icsk->icsk_rto;
+ } else if (icsk->icsk_rto > max) {
max = icsk->icsk_rto;
+ }
}
- if (max && max < rto)
- rto = max;
+ if (max)
+ return min(max, rto);
+
+ if (max_stale)
+ return min(max_stale, rto);
return rto;
}
@@ -327,31 +344,33 @@ static void mptcp_pm_add_timer(struct timer_list *timer)
add_timer);
struct mptcp_sock *msk = entry->sock;
struct sock *sk = (struct sock *)msk;
- unsigned int timeout;
+ unsigned int timeout = 0;
pr_debug("msk=%p\n", msk);
- if (!msk)
- return;
+ bh_lock_sock(sk);
+ if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE))
+ goto out;
- if (inet_sk_state_load(sk) == TCP_CLOSE)
- return;
-
- if (!entry->addr.id)
- return;
-
- if (mptcp_pm_should_add_signal_addr(msk)) {
- sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8);
+ if (sock_owned_by_user(sk)) {
+ /* Try again later. */
+ timeout = HZ / 20;
goto out;
}
timeout = mptcp_adjust_add_addr_timeout(msk);
- if (!timeout)
+ if (!timeout || mptcp_pm_should_add_signal_addr(msk))
goto out;
spin_lock_bh(&msk->pm.lock);
- if (!mptcp_pm_should_add_signal_addr(msk)) {
+ /* The cancel path (mptcp_pm_del_add_timer()) can race with this
+ * callback. Once cancel updates retrans_times to MAX, suppress further
+ * retransmissions here. If this callback acquires pm.lock first, one
+ * final transmit attempt is still possible.
+ */
+ if (entry->retrans_times < ADD_ADDR_RETRANS_MAX &&
+ !mptcp_pm_should_add_signal_addr(msk)) {
pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id);
mptcp_pm_announce_addr(msk, &entry->addr, false);
mptcp_pm_add_addr_send_ack(msk);
@@ -359,8 +378,9 @@ static void mptcp_pm_add_timer(struct timer_list *timer)
}
if (entry->retrans_times < ADD_ADDR_RETRANS_MAX)
- sk_reset_timer(sk, timer,
- jiffies + (timeout << entry->retrans_times));
+ timeout <<= entry->retrans_times;
+ else
+ timeout = 0;
spin_unlock_bh(&msk->pm.lock);
@@ -368,7 +388,13 @@ static void mptcp_pm_add_timer(struct timer_list *timer)
mptcp_pm_subflow_established(msk);
out:
- __sock_put(sk);
+ if (timeout)
+ sk_reset_timer(sk, timer, jiffies + timeout);
+ else
+ /* if sock_put calls sk_free: avoid waiting for this timer */
+ entry->timer_done = true;
+ bh_unlock_sock(sk);
+ sock_put(sk);
}
struct mptcp_pm_add_entry *
@@ -394,8 +420,12 @@ mptcp_pm_del_add_timer(struct mptcp_sock *msk,
/* Note: entry might have been removed by another thread.
* We hold rcu_read_lock() to ensure it is not freed under us.
*/
- if (stop_timer)
- sk_stop_timer_sync(sk, &entry->add_timer);
+ if (stop_timer) {
+ if (check_id)
+ sk_stop_timer(sk, &entry->add_timer);
+ else
+ sk_stop_timer_sync(sk, &entry->add_timer);
+ }
rcu_read_unlock();
return entry;
@@ -431,6 +461,7 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk,
timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0);
reset_timer:
+ add_entry->timer_done = false;
timeout = mptcp_adjust_add_addr_timeout(msk);
if (timeout)
sk_reset_timer(sk, &add_entry->add_timer, jiffies + timeout);
@@ -451,7 +482,8 @@ static void mptcp_pm_free_anno_list(struct mptcp_sock *msk)
spin_unlock_bh(&msk->pm.lock);
list_for_each_entry_safe(entry, tmp, &free_list, list) {
- sk_stop_timer_sync(sk, &entry->add_timer);
+ if (!entry->timer_done)
+ sk_stop_timer_sync(sk, &entry->add_timer);
kfree_rcu(entry, rcu);
}
}
@@ -855,11 +887,11 @@ void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
}
}
-bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
- unsigned int opt_size, unsigned int remaining,
- struct mptcp_addr_info *addr, bool *echo,
- bool *drop_other_suboptions)
+bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, unsigned int opt_size,
+ unsigned int remaining,
+ struct mptcp_addr_info *addr, bool *echo)
{
+ bool skip_add_addr = false;
int ret = false;
u8 add_addr;
u8 family;
@@ -875,30 +907,49 @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
* plain dup-ack from TCP perspective. The other MPTCP-relevant info,
* if any, will be carried by the 'original' TCP ack
*/
- if (skb && skb_is_tcp_pure_ack(skb)) {
- remaining += opt_size;
- *drop_other_suboptions = true;
- }
+ remaining += opt_size;
*echo = mptcp_pm_should_add_signal_echo(msk);
- port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);
-
- family = *echo ? msk->pm.remote.family : msk->pm.local.family;
- if (remaining < mptcp_add_addr_len(family, *echo, port))
- goto out_unlock;
-
if (*echo) {
*addr = msk->pm.remote;
add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
+ port = !!msk->pm.remote.port;
+ family = msk->pm.remote.family;
} else {
*addr = msk->pm.local;
add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
+ port = !!msk->pm.local.port;
+ family = msk->pm.local.family;
}
- WRITE_ONCE(msk->pm.addr_signal, add_addr);
+
+ if (remaining < mptcp_add_addr_len(family, *echo, port)) {
+ struct net *net = sock_net((struct sock *)msk);
+
+ if (*echo) {
+ MPTCP_INC_STATS(net, MPTCP_MIB_ECHOADDTXDROP);
+ } else {
+ skip_add_addr = true;
+ MPTCP_INC_STATS(net, MPTCP_MIB_ADDADDRTXDROP);
+ }
+ goto drop_signal_mark;
+ }
+
ret = true;
+drop_signal_mark:
+ WRITE_ONCE(msk->pm.addr_signal, add_addr);
+
out_unlock:
spin_unlock_bh(&msk->pm.lock);
+
+ /* On pure-ACK option-space exhaustion, stop retrying this ADD_ADDR:
+ * clear the signal bit, cancel the matching retransmission timer, and
+ * let the PM state machine progress.
+ */
+ if (skip_add_addr) {
+ mptcp_pm_del_add_timer(msk, addr, true);
+ mptcp_pm_subflow_established(msk);
+ }
return ret;
}
diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c
index 0ebf43b..fc818b6 100644
--- a/net/mptcp/pm_kernel.c
+++ b/net/mptcp/pm_kernel.c
@@ -347,6 +347,8 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
/* check first for announce */
if (msk->pm.add_addr_signaled < endp_signal_max) {
+ u8 endp_id;
+
/* due to racing events on both ends we can reach here while
* previous add address is still running: if we invoke now
* mptcp_pm_announce_addr(), that will fail and the
@@ -360,19 +362,20 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
if (!select_signal_address(pernet, msk, &local))
goto subflow;
+ /* Special case for ID0: set the correct ID */
+ endp_id = local.addr.id;
+ if (endp_id == msk->mpc_endpoint_id)
+ local.addr.id = 0;
+
/* If the alloc fails, we are on memory pressure, not worth
* continuing, and trying to create subflows.
*/
if (!mptcp_pm_alloc_anno_list(msk, &local.addr))
return;
- __clear_bit(local.addr.id, msk->pm.id_avail_bitmap);
+ __clear_bit(endp_id, msk->pm.id_avail_bitmap);
msk->pm.add_addr_signaled++;
- /* Special case for ID0: set the correct ID */
- if (local.addr.id == msk->mpc_endpoint_id)
- local.addr.id = 0;
-
mptcp_pm_announce_addr(msk, &local.addr, false);
mptcp_pm_addr_send_ack(msk);
@@ -1278,6 +1281,7 @@ static void __reset_counters(struct pm_nl_pernet *pernet)
WRITE_ONCE(pernet->endp_signal_max, 0);
WRITE_ONCE(pernet->endp_subflow_max, 0);
WRITE_ONCE(pernet->endp_laminar_max, 0);
+ WRITE_ONCE(pernet->endp_fullmesh_max, 0);
pernet->endpoints = 0;
}
diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c
index 8cbc192..0d3a95e 100644
--- a/net/mptcp/pm_userspace.c
+++ b/net/mptcp/pm_userspace.c
@@ -408,19 +408,21 @@ int mptcp_pm_nl_subflow_create_doit(struct sk_buff *skb, struct genl_info *info)
local.flags = entry.flags;
local.ifindex = entry.ifindex;
+ spin_lock_bh(&msk->pm.lock);
+ msk->pm.extra_subflows++;
+ spin_unlock_bh(&msk->pm.lock);
+
lock_sock(sk);
err = __mptcp_subflow_connect(sk, &local, &addr_r);
release_sock(sk);
- if (err)
+ if (err) {
GENL_SET_ERR_MSG_FMT(info, "connect error: %d", err);
- spin_lock_bh(&msk->pm.lock);
- if (err)
+ spin_lock_bh(&msk->pm.lock);
mptcp_userspace_pm_delete_local_addr(msk, &entry);
- else
- msk->pm.extra_subflows++;
- spin_unlock_bh(&msk->pm.lock);
+ spin_unlock_bh(&msk->pm.lock);
+ }
create_err:
sock_put(sk);
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index 718e910..cb9515f 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -397,12 +397,26 @@ static bool __mptcp_move_skb(struct sock *sk, struct sk_buff *skb)
return false;
}
- /* old data, keep it simple and drop the whole pkt, sender
- * will retransmit as needed, if needed.
+ /* Completely old data? */
+ if (!after64(MPTCP_SKB_CB(skb)->end_seq, msk->ack_seq)) {
+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
+ mptcp_drop(sk, skb);
+ return false;
+ }
+
+ /* Partial packet: map_seq < ack_seq < end_seq.
+ * Skip the already-acked bytes and enqueue the new data.
*/
- MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
- mptcp_drop(sk, skb);
- return false;
+ copy_len = MPTCP_SKB_CB(skb)->end_seq - msk->ack_seq;
+ MPTCP_SKB_CB(skb)->offset += msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;
+ MPTCP_SKB_CB(skb)->map_seq += msk->ack_seq -
+ MPTCP_SKB_CB(skb)->map_seq;
+ msk->bytes_received += copy_len;
+ WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
+
+ skb_set_owner_r(skb, sk);
+ __skb_queue_tail(&sk->sk_receive_queue, skb);
+ return true;
}
static void mptcp_stop_rtx_timer(struct sock *sk)
@@ -2262,6 +2276,10 @@ static bool mptcp_move_skbs(struct sock *sk)
mptcp_backlog_spooled(sk, moved, &skbs);
}
mptcp_data_unlock(sk);
+
+ if (enqueued && mptcp_epollin_ready(sk))
+ sk->sk_data_ready(sk);
+
return enqueued;
}
@@ -2851,6 +2869,10 @@ static void __mptcp_retrans(struct sock *sk)
msk->bytes_retrans += len;
dfrag->already_sent = max(dfrag->already_sent, len);
+ /* With csum enabled retransmission can send new data. */
+ if (after64(dfrag->already_sent + dfrag->data_seq, msk->snd_nxt))
+ WRITE_ONCE(msk->snd_nxt, dfrag->already_sent + dfrag->data_seq);
+
reset_timer:
mptcp_check_and_set_pending(sk);
@@ -3302,7 +3324,8 @@ bool __mptcp_close(struct sock *sk, long timeout)
goto cleanup;
}
- if (mptcp_data_avail(msk) || timeout < 0) {
+ if (mptcp_data_avail(msk) || timeout < 0 ||
+ (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime)) {
/* If the msk has read data, or the caller explicitly ask it,
* do the MPTCP equivalent of TCP reset, aka MPTCP fastclose
*/
@@ -3472,6 +3495,7 @@ static int mptcp_disconnect(struct sock *sk, int flags)
/* for fallback's sake */
WRITE_ONCE(msk->ack_seq, 0);
+ atomic64_set(&msk->rcv_wnd_sent, 0);
WRITE_ONCE(sk->sk_shutdown, 0);
sk_error_report(sk);
@@ -4404,6 +4428,8 @@ static int __mptcp_read_sock(struct sock *sk, read_descriptor_t *desc,
}
mptcp_eat_recv_skb(sk, skb);
+ if (!desc->count)
+ break;
}
if (noack)
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index e4f5aba..b93b878 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -1229,10 +1229,9 @@ static inline int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list)
return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1;
}
-bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
- unsigned int opt_size, unsigned int remaining,
- struct mptcp_addr_info *addr, bool *echo,
- bool *drop_other_suboptions);
+bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, unsigned int opt_size,
+ unsigned int remaining,
+ struct mptcp_addr_info *addr, bool *echo);
bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
struct mptcp_rm_list *rm_list);
int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc);
diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c
index de90a28..fcf6feb 100644
--- a/net/mptcp/sockopt.c
+++ b/net/mptcp/sockopt.c
@@ -67,6 +67,12 @@ static int mptcp_get_int_option(struct mptcp_sock *msk, sockptr_t optval,
return 0;
}
+static void __mptcp_subflow_set_rcvbuf(struct sock *ssk, int val)
+{
+ WRITE_ONCE(ssk->sk_rcvbuf, val);
+ tcp_set_rcvbuf(ssk, val);
+}
+
static void mptcp_sol_socket_sync_intval(struct mptcp_sock *msk, int optname, int val)
{
struct mptcp_subflow_context *subflow;
@@ -100,7 +106,7 @@ static void mptcp_sol_socket_sync_intval(struct mptcp_sock *msk, int optname, in
case SO_RCVBUF:
case SO_RCVBUFFORCE:
ssk->sk_userlocks |= SOCK_RCVBUF_LOCK;
- WRITE_ONCE(ssk->sk_rcvbuf, sk->sk_rcvbuf);
+ __mptcp_subflow_set_rcvbuf(ssk, sk->sk_rcvbuf);
break;
case SO_MARK:
if (READ_ONCE(ssk->sk_mark) != sk->sk_mark) {
@@ -159,10 +165,10 @@ static int mptcp_setsockopt_sol_socket_tstamp(struct mptcp_sock *msk, int optnam
lock_sock(sk);
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
- bool slow = lock_sock_fast(ssk);
- sock_set_timestamp(sk, optname, !!val);
- unlock_sock_fast(ssk, slow);
+ lock_sock(ssk);
+ sock_set_timestamp(ssk, optname, !!val);
+ release_sock(ssk);
}
release_sock(sk);
@@ -235,15 +241,19 @@ static int mptcp_setsockopt_sol_socket_timestamping(struct mptcp_sock *msk,
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
- bool slow = lock_sock_fast(ssk);
+ int err;
- sock_set_timestamping(sk, optname, timestamping);
- unlock_sock_fast(ssk, slow);
+ lock_sock(ssk);
+ err = sock_set_timestamping(ssk, optname, timestamping);
+ release_sock(ssk);
+
+ if (err < 0 && ret == 0)
+ ret = err;
}
release_sock(sk);
- return 0;
+ return ret;
}
static int mptcp_setsockopt_sol_socket_linger(struct mptcp_sock *msk, sockptr_t optval,
@@ -807,11 +817,16 @@ static int mptcp_setsockopt_all_sf(struct mptcp_sock *msk, int level,
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
+ int err;
- ret = tcp_setsockopt(ssk, level, optname, optval, optlen);
- if (ret)
- break;
+ err = tcp_setsockopt(ssk, level, optname, optval, optlen);
+ if (err < 0 && ret == 0)
+ ret = err;
}
+
+ if (!ret)
+ sockopt_seq_inc(msk);
+
return ret;
}
@@ -1556,7 +1571,7 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk)
mptcp_subflow_ctx(ssk)->cached_sndbuf = sk->sk_sndbuf;
}
if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
- WRITE_ONCE(ssk->sk_rcvbuf, sk->sk_rcvbuf);
+ __mptcp_subflow_set_rcvbuf(ssk, sk->sk_rcvbuf);
}
if (sock_flag(sk, SOCK_LINGER)) {
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index e2cb9d2..d562e14 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -581,7 +581,7 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
subflow->backup);
if (!subflow_thmac_valid(subflow)) {
- MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKMAC);
subflow->reset_reason = MPTCP_RST_EMPTCP;
goto do_reset;
}
@@ -908,7 +908,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
if (!subflow_hmac_valid(subflow_req, &mp_opt)) {
SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
- subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
+ subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
goto dispose_child;
}
diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c
index c5a2623..3706b4a 100644
--- a/net/netfilter/ipset/ip_set_core.c
+++ b/net/netfilter/ipset/ip_set_core.c
@@ -1613,6 +1613,7 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb)
((dump_type == DUMP_ALL) ==
!!(set->type->features & IPSET_DUMP_LAST))) {
write_unlock_bh(&ip_set_ref_lock);
+ set = NULL;
continue;
}
pr_debug("List set: %s\n", set->name);
@@ -1648,13 +1649,13 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb)
if (cb->args[IPSET_CB_PROTO] > IPSET_PROTOCOL_MIN &&
nla_put_net16(skb, IPSET_ATTR_INDEX, htons(index)))
goto nla_put_failure;
+ if (set->variant->uref)
+ set->variant->uref(set, cb, true);
ret = set->variant->head(set, skb);
if (ret < 0)
goto release_refcount;
if (dump_flags & IPSET_FLAG_LIST_HEADER)
goto next_set;
- if (set->variant->uref)
- set->variant->uref(set, cb, true);
fallthrough;
default:
ret = set->variant->list(set, skb, cb);
diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h
index b79e5dd..04e4627 100644
--- a/net/netfilter/ipset/ip_set_hash_gen.h
+++ b/net/netfilter/ipset/ip_set_hash_gen.h
@@ -386,8 +386,9 @@ static void
mtype_ext_cleanup(struct ip_set *set, struct hbucket *n)
{
int i;
+ u8 pos = smp_load_acquire(&n->pos);
- for (i = 0; i < n->pos; i++)
+ for (i = 0; i < pos; i++)
if (test_bit(i, n->used))
ip_set_ext_destroy(set, ahash_data(n, i, set->dsize));
}
@@ -490,7 +491,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
#ifdef IP_SET_HASH_WITH_NETS
u8 k;
#endif
- u8 htable_bits = t->htable_bits;
+ u8 pos, htable_bits = t->htable_bits;
spin_lock_bh(&t->hregion[r].lock);
for (i = ahash_bucket_start(r, htable_bits);
@@ -498,7 +499,8 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
n = __ipset_dereference(hbucket(t, i));
if (!n)
continue;
- for (j = 0, d = 0; j < n->pos; j++) {
+ pos = smp_load_acquire(&n->pos);
+ for (j = 0, d = 0; j < pos; j++) {
if (!test_bit(j, n->used)) {
d++;
continue;
@@ -534,7 +536,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
/* Still try to delete expired elements. */
continue;
tmp->size = n->size - AHASH_INIT_SIZE;
- for (j = 0, d = 0; j < n->pos; j++) {
+ for (j = 0, d = 0; j < pos; j++) {
if (!test_bit(j, n->used))
continue;
data = ahash_data(n, j, dsize);
@@ -623,7 +625,7 @@ mtype_resize(struct ip_set *set, bool retried)
{
struct htype *h = set->data;
struct htable *t, *orig;
- u8 htable_bits;
+ u8 pos, htable_bits;
size_t hsize, dsize = set->dsize;
#ifdef IP_SET_HASH_WITH_NETS
u8 flags;
@@ -685,7 +687,8 @@ mtype_resize(struct ip_set *set, bool retried)
n = __ipset_dereference(hbucket(orig, i));
if (!n)
continue;
- for (j = 0; j < n->pos; j++) {
+ pos = smp_load_acquire(&n->pos);
+ for (j = 0; j < pos; j++) {
if (!test_bit(j, n->used))
continue;
data = ahash_data(n, j, dsize);
@@ -809,9 +812,10 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)
{
struct htype *h = set->data;
const struct htable *t;
- u32 i, j, r;
struct hbucket *n;
struct mtype_elem *data;
+ u32 i, j, r;
+ u8 pos;
t = rcu_dereference_bh(h->table);
for (r = 0; r < ahash_numof_locks(t->htable_bits); r++) {
@@ -820,7 +824,8 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)
n = rcu_dereference_bh(hbucket(t, i));
if (!n)
continue;
- for (j = 0; j < n->pos; j++) {
+ pos = smp_load_acquire(&n->pos);
+ for (j = 0; j < pos; j++) {
if (!test_bit(j, n->used))
continue;
data = ahash_data(n, j, set->dsize);
@@ -848,6 +853,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
bool flag_exist = flags & IPSET_FLAG_EXIST;
bool deleted = false, forceadd = false, reuse = false;
u32 r, key, multi = 0, elements, maxelem;
+ u8 npos = 0;
rcu_read_lock_bh();
t = rcu_dereference_bh(h->table);
@@ -889,7 +895,8 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
ext_size(AHASH_INIT_SIZE, set->dsize);
goto copy_elem;
}
- for (i = 0; i < n->pos; i++) {
+ npos = smp_load_acquire(&n->pos);
+ for (i = 0; i < npos; i++) {
if (!test_bit(i, n->used)) {
/* Reuse first deleted entry */
if (j == -1) {
@@ -933,7 +940,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
if (elements >= maxelem)
goto set_full;
/* Create a new slot */
- if (n->pos >= n->size) {
+ if (npos >= n->size) {
#ifdef IP_SET_HASH_WITH_MULTI
if (h->bucketsize >= AHASH_MAX_TUNED)
goto set_full;
@@ -962,7 +969,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
}
copy_elem:
- j = n->pos++;
+ j = npos++;
data = ahash_data(n, j, set->dsize);
copy_data:
t->hregion[r].elements++;
@@ -985,6 +992,8 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
if (SET_WITH_TIMEOUT(set))
ip_set_timeout_set(ext_timeout(data, set), ext->timeout);
smp_mb__before_atomic();
+ /* Ensure all data writes are visible before updating position */
+ smp_store_release(&n->pos, npos);
set_bit(j, n->used);
if (old != ERR_PTR(-ENOENT)) {
rcu_assign_pointer(hbucket(t, key), n);
@@ -1043,6 +1052,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
int i, j, k, r, ret = -IPSET_ERR_EXIST;
u32 key, multi = 0;
size_t dsize = set->dsize;
+ u8 pos;
/* Userspace add and resize is excluded by the mutex.
* Kernespace add does not trigger resize.
@@ -1058,7 +1068,8 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
n = rcu_dereference_bh(hbucket(t, key));
if (!n)
goto out;
- for (i = 0, k = 0; i < n->pos; i++) {
+ pos = smp_load_acquire(&n->pos);
+ for (i = 0, k = 0; i < pos; i++) {
if (!test_bit(i, n->used)) {
k++;
continue;
@@ -1072,8 +1083,8 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
ret = 0;
clear_bit(i, n->used);
smp_mb__after_atomic();
- if (i + 1 == n->pos)
- n->pos--;
+ if (i + 1 == pos)
+ smp_store_release(&n->pos, --pos);
t->hregion[r].elements--;
#ifdef IP_SET_HASH_WITH_NETS
for (j = 0; j < IPSET_NET_COUNT; j++)
@@ -1094,11 +1105,11 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
x->flags = flags;
}
}
- for (; i < n->pos; i++) {
+ for (; i < pos; i++) {
if (!test_bit(i, n->used))
k++;
}
- if (k == n->pos) {
+ if (k == pos) {
t->hregion[r].ext_size -= ext_size(n->size, dsize);
rcu_assign_pointer(hbucket(t, key), NULL);
kfree_rcu(n, rcu);
@@ -1109,7 +1120,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
if (!tmp)
goto out;
tmp->size = n->size - AHASH_INIT_SIZE;
- for (j = 0, k = 0; j < n->pos; j++) {
+ for (j = 0, k = 0; j < pos; j++) {
if (!test_bit(j, n->used))
continue;
data = ahash_data(n, j, dsize);
@@ -1170,6 +1181,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
int ret, i, j = 0;
#endif
u32 key, multi = 0;
+ u8 pos;
pr_debug("test by nets\n");
for (; j < NLEN && h->nets[j].cidr[0] && !multi; j++) {
@@ -1187,7 +1199,8 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
n = rcu_dereference_bh(hbucket(t, key));
if (!n)
continue;
- for (i = 0; i < n->pos; i++) {
+ pos = smp_load_acquire(&n->pos);
+ for (i = 0; i < pos; i++) {
if (!test_bit(i, n->used))
continue;
data = ahash_data(n, i, set->dsize);
@@ -1221,6 +1234,7 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
struct mtype_elem *data;
int i, ret = 0;
u32 key, multi = 0;
+ u8 pos;
rcu_read_lock_bh();
t = rcu_dereference_bh(h->table);
@@ -1243,7 +1257,8 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
ret = 0;
goto out;
}
- for (i = 0; i < n->pos; i++) {
+ pos = smp_load_acquire(&n->pos);
+ for (i = 0; i < pos; i++) {
if (!test_bit(i, n->used))
continue;
data = ahash_data(n, i, set->dsize);
@@ -1360,6 +1375,7 @@ mtype_list(const struct ip_set *set,
/* We assume that one hash bucket fills into one page */
void *incomplete;
int i, ret = 0;
+ u8 pos;
atd = nla_nest_start(skb, IPSET_ATTR_ADT);
if (!atd)
@@ -1378,7 +1394,8 @@ mtype_list(const struct ip_set *set,
cb->args[IPSET_CB_ARG0], t, n);
if (!n)
continue;
- for (i = 0; i < n->pos; i++) {
+ pos = smp_load_acquire(&n->pos);
+ for (i = 0; i < pos; i++) {
if (!test_bit(i, n->used))
continue;
e = ahash_data(n, i, set->dsize);
diff --git a/net/netfilter/ipset/ip_set_hash_ipmark.c b/net/netfilter/ipset/ip_set_hash_ipmark.c
index a22ec1a..e26ca2a 100644
--- a/net/netfilter/ipset/ip_set_hash_ipmark.c
+++ b/net/netfilter/ipset/ip_set_hash_ipmark.c
@@ -150,7 +150,7 @@ hash_ipmark4_uadt(struct ip_set *set, struct nlattr *tb[],
if (retried)
ip = ntohl(h->next.ip);
- for (; ip <= ip_to; ip++, i++) {
+ for (; ip <= ip_to; i++) {
e.ip = htonl(ip);
if (i > IPSET_MAX_RANGE) {
hash_ipmark4_data_next(&h->next, &e);
@@ -162,6 +162,10 @@ hash_ipmark4_uadt(struct ip_set *set, struct nlattr *tb[],
return ret;
ret = 0;
+
+ if (ip == ip_to)
+ break;
+ ip++;
}
return ret;
}
diff --git a/net/netfilter/ipset/ip_set_hash_ipport.c b/net/netfilter/ipset/ip_set_hash_ipport.c
index e977b5a..41ca24a 100644
--- a/net/netfilter/ipset/ip_set_hash_ipport.c
+++ b/net/netfilter/ipset/ip_set_hash_ipport.c
@@ -186,7 +186,7 @@ hash_ipport4_uadt(struct ip_set *set, struct nlattr *tb[],
if (retried)
ip = ntohl(h->next.ip);
- for (; ip <= ip_to; ip++) {
+ for (; ip <= ip_to;) {
p = retried && ip == ntohl(h->next.ip) ? ntohs(h->next.port)
: port;
for (; p <= port_to; p++, i++) {
@@ -203,6 +203,9 @@ hash_ipport4_uadt(struct ip_set *set, struct nlattr *tb[],
ret = 0;
}
+ if (ip == ip_to)
+ break;
+ ip++;
}
return ret;
}
diff --git a/net/netfilter/ipset/ip_set_hash_ipportip.c b/net/netfilter/ipset/ip_set_hash_ipportip.c
index 39a0193..b9ac2ef 100644
--- a/net/netfilter/ipset/ip_set_hash_ipportip.c
+++ b/net/netfilter/ipset/ip_set_hash_ipportip.c
@@ -182,7 +182,7 @@ hash_ipportip4_uadt(struct ip_set *set, struct nlattr *tb[],
if (retried)
ip = ntohl(h->next.ip);
- for (; ip <= ip_to; ip++) {
+ for (; ip <= ip_to;) {
p = retried && ip == ntohl(h->next.ip) ? ntohs(h->next.port)
: port;
for (; p <= port_to; p++, i++) {
@@ -199,6 +199,9 @@ hash_ipportip4_uadt(struct ip_set *set, struct nlattr *tb[],
ret = 0;
}
+ if (ip == ip_to)
+ break;
+ ip++;
}
return ret;
}
diff --git a/net/netfilter/ipset/ip_set_hash_ipportnet.c b/net/netfilter/ipset/ip_set_hash_ipportnet.c
index 5c6de60..2d6652d 100644
--- a/net/netfilter/ipset/ip_set_hash_ipportnet.c
+++ b/net/netfilter/ipset/ip_set_hash_ipportnet.c
@@ -274,7 +274,7 @@ hash_ipportnet4_uadt(struct ip_set *set, struct nlattr *tb[],
p = port;
ip2 = ip2_from;
}
- for (; ip <= ip_to; ip++) {
+ for (; ip <= ip_to;) {
e.ip = htonl(ip);
for (; p <= port_to; p++) {
e.port = htons(p);
@@ -298,6 +298,9 @@ hash_ipportnet4_uadt(struct ip_set *set, struct nlattr *tb[],
ip2 = ip2_from;
}
p = port;
+ if (ip == ip_to)
+ break;
+ ip++;
}
return ret;
}
diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c
index 2082bfb..9ea6b4f 100644
--- a/net/netfilter/ipvs/ip_vs_conn.c
+++ b/net/netfilter/ipvs/ip_vs_conn.c
@@ -267,27 +267,20 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
hash_key2 = hash_key;
use2 = false;
}
+
conn_tab_lock(t, cp, hash_key, hash_key2, use2, true /* new_hash */,
&head, &head2);
- spin_lock(&cp->lock);
- if (!(cp->flags & IP_VS_CONN_F_HASHED)) {
- cp->flags |= IP_VS_CONN_F_HASHED;
- WRITE_ONCE(cp->hn0.hash_key, hash_key);
- WRITE_ONCE(cp->hn1.hash_key, hash_key2);
- refcount_inc(&cp->refcnt);
- hlist_bl_add_head_rcu(&cp->hn0.node, head);
- if (use2)
- hlist_bl_add_head_rcu(&cp->hn1.node, head2);
- ret = 1;
- } else {
- pr_err("%s(): request for already hashed, called from %pS\n",
- __func__, __builtin_return_address(0));
- ret = 0;
- }
+ cp->flags |= IP_VS_CONN_F_HASHED;
+ WRITE_ONCE(cp->hn0.hash_key, hash_key);
+ WRITE_ONCE(cp->hn1.hash_key, hash_key2);
+ refcount_inc(&cp->refcnt);
+ hlist_bl_add_head_rcu(&cp->hn0.node, head);
+ if (use2)
+ hlist_bl_add_head_rcu(&cp->hn1.node, head2);
- spin_unlock(&cp->lock);
conn_tab_unlock(head, head2);
+ ret = 1;
/* Schedule resizing if load increases */
if (atomic_read(&ipvs->conn_count) > t->u_thresh &&
@@ -321,7 +314,6 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)
conn_tab_lock(t, cp, hash_key, hash_key2, use2, false /* new_hash */,
&head, &head2);
- spin_lock(&cp->lock);
if (cp->flags & IP_VS_CONN_F_HASHED) {
/* Decrease refcnt and unlink conn only if we are last user */
@@ -334,7 +326,6 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)
}
}
- spin_unlock(&cp->lock);
conn_tab_unlock(head, head2);
rcu_read_unlock();
@@ -637,6 +628,7 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
struct ip_vs_conn_hnode *hn;
u32 hash_key, hash_key_new;
struct ip_vs_conn_param p;
+ bool by_me = false;
int ntbl;
int dir;
@@ -664,8 +656,16 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
t = rcu_dereference(t->new_tbl);
ntbl++;
/* We are lost? */
- if (ntbl >= 2)
+ if (ntbl >= 2) {
+ spin_lock_bh(&cp->lock);
+ if (cp->flags & IP_VS_CONN_F_NO_CPORT && by_me)
+ cp->cport = 0;
+ /* hn1 will be rehashed on next packet */
+ spin_unlock_bh(&cp->lock);
+ IP_VS_ERR_RL("%s(): Too many ht changes for dir %d\n",
+ __func__, dir);
return;
+ }
}
/* Rehashing during resize? Use the recent table for adds */
@@ -683,10 +683,13 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
if (head > head2 && t == t2)
swap(head, head2);
+ /* Protect the cp->flags modification */
+ spin_lock_bh(&cp->lock);
+
/* Lock seqcount only for the old bucket, even if we are on new table
* because it affects the del operation, not the adding.
*/
- spin_lock_bh(&t->lock[hash_key & t->lock_mask].l);
+ spin_lock(&t->lock[hash_key & t->lock_mask].l);
preempt_disable_nested();
write_seqcount_begin(&t->seqc[hash_key & t->seqc_mask]);
@@ -704,14 +707,23 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
hlist_bl_unlock(head);
write_seqcount_end(&t->seqc[hash_key & t->seqc_mask]);
preempt_enable_nested();
- spin_unlock_bh(&t->lock[hash_key & t->lock_mask].l);
+ spin_unlock(&t->lock[hash_key & t->lock_mask].l);
+ spin_unlock_bh(&cp->lock);
hash_key = hash_key_new;
goto retry;
}
- spin_lock(&cp->lock);
- if ((cp->flags & IP_VS_CONN_F_NO_CPORT) &&
- (cp->flags & IP_VS_CONN_F_HASHED)) {
+ /* Fill cport once, even if multiple packets try to do it */
+ if (cp->flags & IP_VS_CONN_F_NO_CPORT && (!cp->cport || by_me)) {
+ /* If we race with resizing make sure cport is set for dir 1 */
+ if (!cp->cport) {
+ cp->cport = cport;
+ by_me = true;
+ }
+ if (!dir) {
+ atomic_dec(&ipvs->no_cport_conns[af_id]);
+ cp->flags &= ~IP_VS_CONN_F_NO_CPORT;
+ }
/* We do not recalc hash_key_r under lock, we assume the
* parameters in cp do not change, i.e. cport is
* the only possible change.
@@ -726,21 +738,17 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
hlist_bl_del_rcu(&hn->node);
hlist_bl_add_head_rcu(&hn->node, head_new);
}
- if (!dir) {
- atomic_dec(&ipvs->no_cport_conns[af_id]);
- cp->flags &= ~IP_VS_CONN_F_NO_CPORT;
- cp->cport = cport;
- }
}
- spin_unlock(&cp->lock);
if (head != head2)
hlist_bl_unlock(head2);
hlist_bl_unlock(head);
write_seqcount_end(&t->seqc[hash_key & t->seqc_mask]);
preempt_enable_nested();
- spin_unlock_bh(&t->lock[hash_key & t->lock_mask].l);
- if (dir--)
+ spin_unlock(&t->lock[hash_key & t->lock_mask].l);
+
+ spin_unlock_bh(&cp->lock);
+ if (dir-- && by_me)
goto next_dir;
}
@@ -1835,7 +1843,7 @@ static void ip_vs_conn_flush(struct netns_ipvs *ipvs)
if (!rcu_dereference_protected(ipvs->conn_tab, 1))
return;
- cancel_delayed_work_sync(&ipvs->conn_resize_work);
+ disable_delayed_work_sync(&ipvs->conn_resize_work);
if (!atomic_read(&ipvs->conn_count))
goto unreg;
diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c
index f5b7a20..d40b404 100644
--- a/net/netfilter/ipvs/ip_vs_core.c
+++ b/net/netfilter/ipvs/ip_vs_core.c
@@ -237,7 +237,7 @@ int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n,
{
if (!t)
return 1 << min_bits;
- n = roundup_pow_of_two(n);
+ n = n > 0 ? roundup_pow_of_two(n) : 1;
if (lfactor < 0) {
int factor = min(-lfactor, max_bits);
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c
index 6632daa..16daba8 100644
--- a/net/netfilter/ipvs/ip_vs_ctl.c
+++ b/net/netfilter/ipvs/ip_vs_ctl.c
@@ -261,12 +261,28 @@ static void est_reload_work_handler(struct work_struct *work)
if (!kd)
continue;
/* New config ? Stop kthread tasks */
- if (genid != genid_done)
- ip_vs_est_kthread_stop(kd);
+ if (genid != genid_done) {
+ if (!id) {
+ /* Only we can stop kt 0 but not under mutex */
+ mutex_unlock(&ipvs->est_mutex);
+ ip_vs_est_kthread_stop(kd);
+ mutex_lock(&ipvs->est_mutex);
+ if (!READ_ONCE(ipvs->enable))
+ goto unlock;
+ /* kd for kt 0 is never destroyed */
+ } else {
+ ip_vs_est_kthread_stop(kd);
+ }
+ }
if (!kd->task && !ip_vs_est_stopped(ipvs)) {
+ bool start;
+
/* Do not start kthreads above 0 in calc phase */
- if ((!id || !ipvs->est_calc_phase) &&
- ip_vs_est_kthread_start(ipvs, kd) < 0)
+ if (id)
+ start = !ipvs->est_calc_phase;
+ else
+ start = kd->needed;
+ if (start && ip_vs_est_kthread_start(ipvs, kd) < 0)
repeat = true;
}
}
@@ -311,18 +327,22 @@ ip_vs_use_count_dec(void)
/* Service hashing:
* Operation Locking order
* ---------------------------------------------------------------------------
- * add table service_mutex, svc_resize_sem(W)
- * del table service_mutex
- * move between tables svc_resize_sem(W), seqcount_t(W), bit lock
- * add/del service service_mutex, bit lock
+ * add first table service_mutex
+ * attach new table service_mutex
+ * add/del service service_mutex, RCU, bit lock
+ * move between tables (rehash) svc_resize_sem(W), seqcount_t(W), bit lock
+ * replace old with attached svc_resize_sem(W), svc_replace_sem(W)
* find service RCU, seqcount_t(R)
* walk services(blocking) service_mutex, svc_resize_sem(R)
* walk services(non-blocking) RCU, seqcount_t(R)
+ * walk services(non-blocking) svc_resize_sem(R), RCU, seqcount_t(R)
+ * walk services(non-blocking) svc_replace_sem(R), RCU, seqcount_t(R)
+ * del table service_mutex after stopped work
*
- * - new tables are linked/unlinked under service_mutex and svc_resize_sem
- * - new table is linked on resizing and all operations can run in parallel
- * in 2 tables until the new table is registered as current one
- * - two contexts can modify buckets: config and table resize, both in
+ * - new table is attached on resizing under service_mutex and all operations
+ * can run in parallel in 2 tables until the new table is registered as current
+ * one
+ * - two contexts can modify buckets: config and table resize (work), both in
* process context
* - only table resizer can move entries, so we do not protect t->seqc[]
* items with t->lock[]
@@ -330,9 +350,13 @@ ip_vs_use_count_dec(void)
* services are moved to new table
* - move operations may disturb readers: find operation will not miss entries
* but walkers may see same entry twice if they are forced to retry chains
- * - walkers using cond_resched_rcu() on !PREEMPT_RCU may need to hold
- * service_mutex to disallow new tables to be installed or to check
+ * or to walk the newly attached second table
+ * - walkers using cond_resched_rcu() on !PREEMPT_RCU may need to check
* svc_table_changes and repeat the RCU read section if new table is installed
+ * - walkers may serialize with the whole resizing process (svc_resize_sem)
+ * to prevent seeing same service twice or just with the svc_table
+ * replace (svc_replace_sem) when we can see entries twice but we
+ * prefer to run concurrently with the rehashing.
*/
/*
@@ -371,9 +395,16 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc)
/* increase its refcnt because it is referenced by the svc table */
atomic_inc(&svc->refcnt);
+ /* We know if new table is attached under service_mutex but rely on
+ * RCU to hold the old table to be freed in resizer
+ */
+ rcu_read_lock();
+
+ /* This can be the old or the new table */
+ t = rcu_dereference(ipvs->svc_table);
+
/* New entries go into recent table */
- t = rcu_dereference_protected(ipvs->svc_table, 1);
- t = rcu_dereference_protected(t->new_tbl, 1);
+ t = rcu_dereference(t->new_tbl);
if (svc->fwmark == 0) {
/*
@@ -394,6 +425,8 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc)
hlist_bl_add_head_rcu(&svc->s_list, head);
hlist_bl_unlock(head);
+ rcu_read_unlock();
+
return 1;
}
@@ -416,7 +449,13 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
return 0;
}
- t = rcu_dereference_protected(ipvs->svc_table, 1);
+ /* We know if new table is attached under service_mutex but rely on
+ * RCU to hold the old table to be freed in resizer
+ */
+ rcu_read_lock();
+
+ /* This can be the old or the new table */
+ t = rcu_dereference(ipvs->svc_table);
hash_key = READ_ONCE(svc->hash_key);
/* We need to lock the bucket in the right table */
if (ip_vs_rht_same_table(t, hash_key)) {
@@ -427,13 +466,13 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
/* Moved to new table ? */
if (hash_key != hash_key2) {
hlist_bl_unlock(head);
- t = rcu_dereference_protected(t->new_tbl, 1);
+ t = rcu_dereference(t->new_tbl);
head = t->buckets + (hash_key2 & t->mask);
hlist_bl_lock(head);
}
} else {
/* It is already moved to new table */
- t = rcu_dereference_protected(t->new_tbl, 1);
+ t = rcu_dereference(t->new_tbl);
head = t->buckets + (hash_key & t->mask);
hlist_bl_lock(head);
}
@@ -443,6 +482,8 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
svc->flags &= ~IP_VS_SVC_F_HASHED;
atomic_dec(&svc->refcnt);
hlist_bl_unlock(head);
+
+ rcu_read_unlock();
return 1;
}
@@ -650,15 +691,14 @@ static void svc_resize_work_handler(struct work_struct *work)
goto unlock_sem;
more_work = false;
clear_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags);
- if (!READ_ONCE(ipvs->enable) ||
- test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
+ if (!READ_ONCE(ipvs->enable))
goto unlock_m;
t = rcu_dereference_protected(ipvs->svc_table, 1);
/* Do nothing if table is removed */
if (!t)
goto unlock_m;
- /* New table needs to be registered? BUG! */
- if (t != rcu_dereference_protected(t->new_tbl, 1))
+ /* New table already attached? BUG! */
+ if (t != rcu_access_pointer(t->new_tbl))
goto unlock_m;
lfactor = sysctl_svc_lfactor(ipvs);
@@ -675,6 +715,7 @@ static void svc_resize_work_handler(struct work_struct *work)
/* Flip the table_id */
t_new->table_id = t->table_id ^ IP_VS_RHT_TABLE_ID_MASK;
+ /* Attach new table */
rcu_assign_pointer(t->new_tbl, t_new);
/* Allow add/del to new_tbl while moving from old table */
mutex_unlock(&ipvs->service_mutex);
@@ -682,8 +723,8 @@ static void svc_resize_work_handler(struct work_struct *work)
ip_vs_rht_for_each_bucket(t, bucket, head) {
same_bucket:
if (++limit >= 16) {
- if (!READ_ONCE(ipvs->enable) ||
- test_bit(IP_VS_WORK_SVC_NORESIZE,
+ /* Check if work is stopped */
+ if (test_bit(IP_VS_WORK_SVC_NORESIZE,
&ipvs->work_flags))
goto unlock_sem;
if (resched_score >= 100) {
@@ -748,16 +789,12 @@ static void svc_resize_work_handler(struct work_struct *work)
goto same_bucket;
}
- /* Tables can be switched only under service_mutex */
- while (!mutex_trylock(&ipvs->service_mutex)) {
- cond_resched();
- if (!READ_ONCE(ipvs->enable) ||
- test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
- goto unlock_sem;
- }
- if (!READ_ONCE(ipvs->enable) ||
- test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
- goto unlock_m;
+ /* Serialize with readers that don't like svc_table changes */
+ down_write(&ipvs->svc_replace_sem);
+
+ /* Check if work is stopped to avoid synchronize_rcu() */
+ if (test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
+ goto unlock_repl;
rcu_assign_pointer(ipvs->svc_table, t_new);
/* Inform readers that new table is installed */
@@ -765,8 +802,8 @@ static void svc_resize_work_handler(struct work_struct *work)
atomic_inc(&ipvs->svc_table_changes);
t_free = t;
-unlock_m:
- mutex_unlock(&ipvs->service_mutex);
+unlock_repl:
+ up_write(&ipvs->svc_replace_sem);
unlock_sem:
up_write(&ipvs->svc_resize_sem);
@@ -785,6 +822,11 @@ static void svc_resize_work_handler(struct work_struct *work)
test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
return;
queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work, 1);
+ return;
+
+unlock_m:
+ mutex_unlock(&ipvs->service_mutex);
+ goto unlock_sem;
}
static inline void
@@ -1102,6 +1144,24 @@ ip_vs_trash_get_dest(struct ip_vs_service *svc, int dest_af,
return dest;
}
+/* Put destination in trash */
+static void ip_vs_trash_put_dest(struct netns_ipvs *ipvs,
+ struct ip_vs_dest *dest, unsigned long istart,
+ bool cleanup)
+{
+ spin_lock_bh(&ipvs->dest_trash_lock);
+ IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n",
+ IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port),
+ refcount_read(&dest->refcnt));
+ if (list_empty(&ipvs->dest_trash) && !cleanup)
+ mod_timer(&ipvs->dest_trash_timer,
+ jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1));
+ /* dest lives in trash with reference */
+ list_add(&dest->t_list, &ipvs->dest_trash);
+ dest->idle_start = istart;
+ spin_unlock_bh(&ipvs->dest_trash_lock);
+}
+
static void ip_vs_dest_rcu_free(struct rcu_head *head)
{
struct ip_vs_dest *dest;
@@ -1461,9 +1521,12 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest)
ntohs(dest->vport));
ret = ip_vs_start_estimator(svc->ipvs, &dest->stats);
+ /* On error put back dest into the trash */
if (ret < 0)
- return ret;
- __ip_vs_update_dest(svc, dest, udest, 1);
+ ip_vs_trash_put_dest(svc->ipvs, dest, dest->idle_start,
+ false);
+ else
+ __ip_vs_update_dest(svc, dest, udest, 1);
} else {
/*
* Allocate and initialize the dest structure
@@ -1533,17 +1596,7 @@ static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest,
*/
ip_vs_rs_unhash(dest);
- spin_lock_bh(&ipvs->dest_trash_lock);
- IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n",
- IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port),
- refcount_read(&dest->refcnt));
- if (list_empty(&ipvs->dest_trash) && !cleanup)
- mod_timer(&ipvs->dest_trash_timer,
- jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1));
- /* dest lives in trash with reference */
- list_add(&dest->t_list, &ipvs->dest_trash);
- dest->idle_start = 0;
- spin_unlock_bh(&ipvs->dest_trash_lock);
+ ip_vs_trash_put_dest(ipvs, dest, 0, cleanup);
/* Queue up delayed work to expire all no destination connections.
* No-op when CONFIG_SYSCTL is disabled.
@@ -1664,6 +1717,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
struct ip_vs_pe *pe = NULL;
int ret_hooks = -1;
int ret = 0;
+ bool grow;
/* increase the module use count */
if (!ip_vs_use_count_inc())
@@ -1705,16 +1759,25 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
}
#endif
- t = rcu_dereference_protected(ipvs->svc_table, 1);
+ /* The old table can be freed, protect it with RCU */
+ rcu_read_lock();
+ t = rcu_dereference(ipvs->svc_table);
if (!t) {
int lfactor = sysctl_svc_lfactor(ipvs);
int new_size = ip_vs_svc_desired_size(ipvs, NULL, lfactor);
+ rcu_read_unlock();
t_new = ip_vs_svc_table_alloc(ipvs, new_size, lfactor);
if (!t_new) {
ret = -ENOMEM;
goto out_err;
}
+ grow = false;
+ } else {
+ /* Even the currently attached new table may need to grow */
+ t = rcu_dereference(t->new_tbl);
+ grow = ip_vs_get_num_services(ipvs) + 1 > t->u_thresh;
+ rcu_read_unlock();
}
if (!rcu_dereference_protected(ipvs->conn_tab, 1)) {
@@ -1773,6 +1836,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
goto out_err;
if (t_new) {
+ /* Add table for first time */
clear_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags);
rcu_assign_pointer(ipvs->svc_table, t_new);
t_new = NULL;
@@ -1804,19 +1868,23 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
ip_vs_svc_hash(svc);
/* Schedule resize work */
- if (t && ip_vs_get_num_services(ipvs) > t->u_thresh &&
- !test_and_set_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags))
+ if (grow && !test_and_set_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags))
queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work,
1);
*svc_p = svc;
if (!READ_ONCE(ipvs->enable)) {
+ mutex_lock(&ipvs->est_mutex);
+
/* Now there is a service - full throttle */
WRITE_ONCE(ipvs->enable, 1);
+ ipvs->est_max_threads = ip_vs_est_max_threads(ipvs);
+
/* Start estimation for first time */
- ip_vs_est_reload_start(ipvs);
+ ip_vs_est_reload_start(ipvs, true);
+ mutex_unlock(&ipvs->est_mutex);
}
return 0;
@@ -1830,7 +1898,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
if (ret_hooks >= 0)
ip_vs_unregister_hooks(ipvs, u->af);
if (svc != NULL) {
- ip_vs_unbind_scheduler(svc, sched);
+ ip_vs_unbind_scheduler(svc);
ip_vs_service_free(svc);
}
ip_vs_scheduler_put(sched);
@@ -1894,9 +1962,8 @@ ip_vs_edit_service(struct ip_vs_service *svc, struct ip_vs_service_user_kern *u)
old_sched = rcu_dereference_protected(svc->scheduler, 1);
if (sched != old_sched) {
if (old_sched) {
- ip_vs_unbind_scheduler(svc, old_sched);
- RCU_INIT_POINTER(svc->scheduler, NULL);
- /* Wait all svc->sched_data users */
+ ip_vs_unbind_scheduler(svc);
+ /* Wait all svc->scheduler/sched_data users */
synchronize_rcu();
}
/* Bind the new scheduler */
@@ -1904,6 +1971,10 @@ ip_vs_edit_service(struct ip_vs_service *svc, struct ip_vs_service_user_kern *u)
ret = ip_vs_bind_scheduler(svc, sched);
if (ret) {
ip_vs_scheduler_put(sched);
+ /* Try to restore the old_sched */
+ if (old_sched &&
+ !ip_vs_bind_scheduler(svc, old_sched))
+ old_sched = NULL;
goto out;
}
}
@@ -1959,7 +2030,7 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup)
/* Unbind scheduler */
old_sched = rcu_dereference_protected(svc->scheduler, 1);
- ip_vs_unbind_scheduler(svc, old_sched);
+ ip_vs_unbind_scheduler(svc);
ip_vs_scheduler_put(old_sched);
/* Unbind persistence engine, keep svc->pe */
@@ -2022,7 +2093,6 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
return -EEXIST;
ipvs = svc->ipvs;
ip_vs_unlink_service(svc, false);
- t = rcu_dereference_protected(ipvs->svc_table, 1);
/* Drop the table if no more services */
ns = ip_vs_get_num_services(ipvs);
@@ -2030,8 +2100,12 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
/* Stop the resizer and drop the tables */
set_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags);
cancel_delayed_work_sync(&ipvs->svc_resize_work);
+ t = rcu_dereference_protected(ipvs->svc_table, 1);
if (t) {
rcu_assign_pointer(ipvs->svc_table, NULL);
+ /* Inform readers that table is removed */
+ smp_mb__before_atomic();
+ atomic_inc(&ipvs->svc_table_changes);
while (1) {
p = rcu_dereference_protected(t->new_tbl, 1);
call_rcu(&t->rcu_head, ip_vs_rht_rcu_free);
@@ -2040,11 +2114,19 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
t = p;
}
}
- } else if (ns <= t->l_thresh &&
- !test_and_set_bit(IP_VS_WORK_SVC_RESIZE,
- &ipvs->work_flags)) {
- queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work,
- 1);
+ } else {
+ bool shrink;
+
+ rcu_read_lock();
+ t = rcu_dereference(ipvs->svc_table);
+ /* Even the currently attached new table may need to shrink */
+ t = rcu_dereference(t->new_tbl);
+ shrink = ns <= t->l_thresh;
+ rcu_read_unlock();
+ if (shrink && !test_and_set_bit(IP_VS_WORK_SVC_RESIZE,
+ &ipvs->work_flags))
+ queue_delayed_work(system_unbound_wq,
+ &ipvs->svc_resize_work, 1);
}
return 0;
}
@@ -2078,6 +2160,9 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup)
t = rcu_dereference_protected(ipvs->svc_table, 1);
if (t) {
rcu_assign_pointer(ipvs->svc_table, NULL);
+ /* Inform readers that table is removed */
+ smp_mb__before_atomic();
+ atomic_inc(&ipvs->svc_table_changes);
while (1) {
p = rcu_dereference_protected(t->new_tbl, 1);
call_rcu(&t->rcu_head, ip_vs_rht_rcu_free);
@@ -2086,6 +2171,11 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup)
t = p;
}
}
+ /* Stop the tot_stats estimator early under service_mutex
+ * to avoid locking it again later.
+ */
+ if (cleanup)
+ ip_vs_stop_estimator_tot_stats(ipvs);
return 0;
}
@@ -2141,17 +2231,21 @@ static int ip_vs_dst_event(struct notifier_block *this, unsigned long event,
struct ip_vs_service *svc;
struct hlist_bl_node *e;
struct ip_vs_dest *dest;
- int old_gen, new_gen;
+ int old_gen;
if (event != NETDEV_DOWN || !ipvs)
return NOTIFY_DONE;
IP_VS_DBG(3, "%s() dev=%s\n", __func__, dev->name);
+ /* Allow concurrent rehashing on resize but to avoid loop
+ * serialize with installing the new table.
+ */
+ down_read(&ipvs->svc_replace_sem);
+
old_gen = atomic_read(&ipvs->svc_table_changes);
rcu_read_lock();
-repeat:
smp_rmb(); /* ipvs->svc_table and svc_table_changes */
ip_vs_rht_walk_buckets_rcu(ipvs->svc_table, head) {
hlist_bl_for_each_entry_rcu(svc, e, head, s_list) {
@@ -2164,17 +2258,17 @@ static int ip_vs_dst_event(struct notifier_block *this, unsigned long event,
}
resched_score++;
if (resched_score >= 100) {
- resched_score = 0;
cond_resched_rcu();
- new_gen = atomic_read(&ipvs->svc_table_changes);
- /* New table installed ? */
- if (old_gen != new_gen) {
- old_gen = new_gen;
- goto repeat;
- }
+ /* Flushed? So no more dev refs */
+ if (atomic_read(&ipvs->svc_table_changes) != old_gen)
+ goto done;
+ resched_score = 0;
}
}
+
+done:
rcu_read_unlock();
+ up_read(&ipvs->svc_replace_sem);
return NOTIFY_DONE;
}
@@ -2201,6 +2295,10 @@ static int ip_vs_zero_all(struct netns_ipvs *ipvs)
struct ip_vs_service *svc;
struct hlist_bl_node *e;
+ /* svc_table can not be replaced (svc_replace_sem) or
+ * removed (service_mutex)
+ */
+ down_read(&ipvs->svc_replace_sem);
rcu_read_lock();
ip_vs_rht_walk_buckets_rcu(ipvs->svc_table, head) {
@@ -2216,6 +2314,7 @@ static int ip_vs_zero_all(struct netns_ipvs *ipvs)
}
rcu_read_unlock();
+ up_read(&ipvs->svc_replace_sem);
ip_vs_zero_stats(&ipvs->tot_stats->s);
return 0;
@@ -2331,7 +2430,7 @@ static int ipvs_proc_est_cpumask_set(const struct ctl_table *table,
/* est_max_threads may depend on cpulist size */
ipvs->est_max_threads = ip_vs_est_max_threads(ipvs);
ipvs->est_calc_phase = 1;
- ip_vs_est_reload_start(ipvs);
+ ip_vs_est_reload_start(ipvs, true);
unlock:
mutex_unlock(&ipvs->est_mutex);
@@ -2351,11 +2450,14 @@ static int ipvs_proc_est_cpumask_get(const struct ctl_table *table,
mutex_lock(&ipvs->est_mutex);
- if (ipvs->est_cpulist_valid)
- mask = *valp;
- else
- mask = (struct cpumask *)housekeeping_cpumask(HK_TYPE_KTHREAD);
- ret = scnprintf(buffer, size, "%*pbl\n", cpumask_pr_args(mask));
+ /* HK_TYPE_KTHREAD cpumask needs RCU protection */
+ scoped_guard(rcu) {
+ if (ipvs->est_cpulist_valid)
+ mask = *valp;
+ else
+ mask = (struct cpumask *)housekeeping_cpumask(HK_TYPE_KTHREAD);
+ ret = scnprintf(buffer, size, "%*pbl\n", cpumask_pr_args(mask));
+ }
mutex_unlock(&ipvs->est_mutex);
@@ -2411,7 +2513,7 @@ static int ipvs_proc_est_nice(const struct ctl_table *table, int write,
mutex_lock(&ipvs->est_mutex);
if (*valp != val) {
*valp = val;
- ip_vs_est_reload_start(ipvs);
+ ip_vs_est_reload_start(ipvs, true);
}
mutex_unlock(&ipvs->est_mutex);
}
@@ -2438,7 +2540,7 @@ static int ipvs_proc_run_estimation(const struct ctl_table *table, int write,
mutex_lock(&ipvs->est_mutex);
if (*valp != val) {
*valp = val;
- ip_vs_est_reload_start(ipvs);
+ ip_vs_est_reload_start(ipvs, true);
}
mutex_unlock(&ipvs->est_mutex);
}
@@ -2463,7 +2565,7 @@ static int ipvs_proc_conn_lfactor(const struct ctl_table *table, int write,
if (val < -8 || val > 8) {
ret = -EINVAL;
} else {
- *valp = val;
+ WRITE_ONCE(*valp, val);
if (rcu_access_pointer(ipvs->conn_tab))
mod_delayed_work(system_unbound_wq,
&ipvs->conn_resize_work, 0);
@@ -2490,10 +2592,16 @@ static int ipvs_proc_svc_lfactor(const struct ctl_table *table, int write,
if (val < -8 || val > 8) {
ret = -EINVAL;
} else {
- *valp = val;
- if (rcu_access_pointer(ipvs->svc_table))
+ mutex_lock(&ipvs->service_mutex);
+ WRITE_ONCE(*valp, val);
+ /* Make sure the services are present */
+ if (rcu_access_pointer(ipvs->svc_table) &&
+ READ_ONCE(ipvs->enable) &&
+ !test_bit(IP_VS_WORK_SVC_NORESIZE,
+ &ipvs->work_flags))
mod_delayed_work(system_unbound_wq,
&ipvs->svc_resize_work, 0);
+ mutex_unlock(&ipvs->service_mutex);
}
}
return ret;
@@ -3004,11 +3112,13 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
int old_gen, new_gen;
u32 counts[8];
u32 bucket;
- int count;
+ u32 count;
+ int loops;
u32 sum1;
u32 sum;
int i;
+ /* Info for conns */
rcu_read_lock();
t = rcu_dereference(ipvs->conn_tab);
@@ -3020,6 +3130,7 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
if (!atomic_read(&ipvs->conn_count))
goto after_conns;
old_gen = atomic_read(&ipvs->conn_tab_changes);
+ loops = 0;
repeat_conn:
smp_rmb(); /* ipvs->conn_tab and conn_tab_changes */
@@ -3032,8 +3143,11 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
resched_score++;
ip_vs_rht_walk_bucket_rcu(t, bucket, head) {
count = 0;
- hlist_bl_for_each_entry_rcu(hn, e, head, node)
+ hlist_bl_for_each_entry_rcu(hn, e, head, node) {
count++;
+ if (count >= ARRAY_SIZE(counts) - 1)
+ break;
+ }
}
resched_score += count;
if (resched_score >= 100) {
@@ -3042,31 +3156,40 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
new_gen = atomic_read(&ipvs->conn_tab_changes);
/* New table installed ? */
if (old_gen != new_gen) {
+ /* Too many changes? */
+ if (++loops >= 5)
+ goto after_conns;
old_gen = new_gen;
goto repeat_conn;
}
}
- counts[min(count, (int)ARRAY_SIZE(counts) - 1)]++;
+ counts[count]++;
}
}
for (sum = 0, i = 0; i < ARRAY_SIZE(counts); i++)
sum += counts[i];
sum1 = sum - counts[0];
- seq_printf(seq, "Conn buckets empty:\t%u (%lu%%)\n",
- counts[0], (unsigned long)counts[0] * 100 / max(sum, 1U));
+ seq_printf(seq, "Conn buckets empty:\t%u (%llu%%)\n",
+ counts[0], div_u64((u64)counts[0] * 100U, max(sum, 1U)));
for (i = 1; i < ARRAY_SIZE(counts); i++) {
if (!counts[i])
continue;
- seq_printf(seq, "Conn buckets len-%d:\t%u (%lu%%)\n",
+ seq_printf(seq, "Conn buckets len-%d:\t%u (%llu%%)\n",
i, counts[i],
- (unsigned long)counts[i] * 100 / max(sum1, 1U));
+ div_u64((u64)counts[i] * 100U, max(sum1, 1U)));
}
after_conns:
+ rcu_read_unlock();
+
+ /* Info for services */
+ down_read(&ipvs->svc_replace_sem);
+ rcu_read_lock();
+
t = rcu_dereference(ipvs->svc_table);
count = ip_vs_get_num_services(ipvs);
- seq_printf(seq, "Services:\t%d\n", count);
+ seq_printf(seq, "Services:\t%u\n", count);
seq_printf(seq, "Service buckets:\t%d (%d bits, lfactor %d)\n",
t ? t->size : 0, t ? t->bits : 0, t ? t->lfactor : 0);
@@ -3074,7 +3197,6 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
goto after_svc;
old_gen = atomic_read(&ipvs->svc_table_changes);
-repeat_svc:
smp_rmb(); /* ipvs->svc_table and svc_table_changes */
memset(counts, 0, sizeof(counts));
ip_vs_rht_for_each_table_rcu(ipvs->svc_table, t, pt) {
@@ -3086,37 +3208,41 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
ip_vs_rht_walk_bucket_rcu(t, bucket, head) {
count = 0;
hlist_bl_for_each_entry_rcu(svc, e, head,
- s_list)
+ s_list) {
count++;
+ if (count >= ARRAY_SIZE(counts) - 1)
+ break;
+ }
}
resched_score += count;
if (resched_score >= 100) {
resched_score = 0;
cond_resched_rcu();
- new_gen = atomic_read(&ipvs->svc_table_changes);
- /* New table installed ? */
- if (old_gen != new_gen) {
- old_gen = new_gen;
- goto repeat_svc;
- }
+ /* Flushed? */
+ if (atomic_read(&ipvs->svc_table_changes) !=
+ old_gen)
+ goto after_svc;
}
- counts[min(count, (int)ARRAY_SIZE(counts) - 1)]++;
+ counts[count]++;
}
}
for (sum = 0, i = 0; i < ARRAY_SIZE(counts); i++)
sum += counts[i];
sum1 = sum - counts[0];
- seq_printf(seq, "Service buckets empty:\t%u (%lu%%)\n",
- counts[0], (unsigned long)counts[0] * 100 / max(sum, 1U));
+ seq_printf(seq, "Service buckets empty:\t%u (%llu%%)\n",
+ counts[0], div_u64((u64)counts[0] * 100U, max(sum, 1U)));
for (i = 1; i < ARRAY_SIZE(counts); i++) {
if (!counts[i])
continue;
- seq_printf(seq, "Service buckets len-%d:\t%u (%lu%%)\n",
+ seq_printf(seq, "Service buckets len-%d:\t%u (%llu%%)\n",
i, counts[i],
- (unsigned long)counts[i] * 100 / max(sum1, 1U));
+ div_u64((u64)counts[i] * 100U, max(sum1, 1U)));
}
after_svc:
+ rcu_read_unlock();
+ up_read(&ipvs->svc_replace_sem);
+
seq_printf(seq, "Stats thread slots:\t%d (max %lu)\n",
ipvs->est_kt_count, ipvs->est_max_threads);
seq_printf(seq, "Stats chain max len:\t%d\n", ipvs->est_chain_max);
@@ -3124,7 +3250,6 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
ipvs->est_chain_max * IPVS_EST_CHAIN_FACTOR *
IPVS_EST_NTICKS);
- rcu_read_unlock();
return 0;
}
@@ -3436,7 +3561,7 @@ __ip_vs_get_service_entries(struct netns_ipvs *ipvs,
int ret = 0;
lockdep_assert_held(&ipvs->svc_resize_sem);
- /* All service modifications are disabled, go ahead */
+ /* All svc_table modifications are disabled, go ahead */
ip_vs_rht_walk_buckets(ipvs->svc_table, head) {
hlist_bl_for_each_entry(svc, e, head, s_list) {
/* Only expose IPv4 entries to old interface */
@@ -3620,7 +3745,7 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
pr_err("length: %u != %zu\n", *len, size);
return -EINVAL;
}
- /* Protect against table resizer moving the entries.
+ /* Prevent modifications to the list with services.
* Try reverse locking, so that we do not hold the mutex
* while waiting for semaphore.
*/
@@ -3962,6 +4087,7 @@ static int ip_vs_genl_dump_services(struct sk_buff *skb,
int start = cb->args[0];
int idx = 0;
+ /* Make sure we do not see same service twice during resize */
down_read(&ipvs->svc_resize_sem);
rcu_read_lock();
ip_vs_rht_walk_buckets_safe_rcu(ipvs->svc_table, head) {
@@ -4967,7 +5093,14 @@ static void __net_exit ip_vs_control_net_cleanup_sysctl(struct netns_ipvs *ipvs)
cancel_delayed_work_sync(&ipvs->defense_work);
cancel_work_sync(&ipvs->defense_work.work);
unregister_net_sysctl_table(ipvs->sysctl_hdr);
- ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s);
+ if (ipvs->tot_stats->s.est.ktid != -2) {
+ /* Not stopped yet? This happens only on netns init error and
+ * we even do not need to lock the service_mutex for this case.
+ */
+ mutex_lock(&ipvs->service_mutex);
+ ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s);
+ mutex_unlock(&ipvs->service_mutex);
+ }
if (ipvs->est_cpulist_valid)
free_cpumask_var(ipvs->sysctl_est_cpulist);
@@ -4998,6 +5131,7 @@ int __net_init ip_vs_control_net_init(struct netns_ipvs *ipvs)
/* Initialize service_mutex, svc_table per netns */
__mutex_init(&ipvs->service_mutex, "ipvs->service_mutex", &__ipvs_service_key);
init_rwsem(&ipvs->svc_resize_sem);
+ init_rwsem(&ipvs->svc_replace_sem);
INIT_DELAYED_WORK(&ipvs->svc_resize_work, svc_resize_work_handler);
atomic_set(&ipvs->svc_table_changes, 0);
RCU_INIT_POINTER(ipvs->svc_table, NULL);
@@ -5039,7 +5173,7 @@ int __net_init ip_vs_control_net_init(struct netns_ipvs *ipvs)
ipvs->net->proc_net,
ip_vs_stats_percpu_show, NULL))
goto err_percpu;
- if (!proc_create_net_single("ip_vs_status", 0, ipvs->net->proc_net,
+ if (!proc_create_net_single("ip_vs_status", 0440, ipvs->net->proc_net,
ip_vs_status_show, NULL))
goto err_status;
#endif
diff --git a/net/netfilter/ipvs/ip_vs_est.c b/net/netfilter/ipvs/ip_vs_est.c
index 433ba3c..ab09f51 100644
--- a/net/netfilter/ipvs/ip_vs_est.c
+++ b/net/netfilter/ipvs/ip_vs_est.c
@@ -68,6 +68,11 @@
and the limit of estimators per kthread
- est_add_ktid: ktid where to add new ests, can point to empty slot where
we should add kt data
+ - data protected by service_mutex: est_temp_list, est_add_ktid,
+ est_kt_count(R/W), est_kt_arr(R/W), est_genid_done, kd->needed(R/W)
+ - data protected by est_mutex: est_genid, est_max_threads, sysctl_est_cpulist,
+ est_cpulist_valid, sysctl_est_nice, est_stopped, sysctl_run_estimation,
+ est_kt_count(R), est_kt_arr(R), kd->needed(R), kd->task (id > 0)
*/
static struct lock_class_key __ipvs_est_key;
@@ -227,14 +232,17 @@ static int ip_vs_estimation_kthread(void *data)
}
/* Schedule stop/start for kthread tasks */
-void ip_vs_est_reload_start(struct netns_ipvs *ipvs)
+void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart)
{
+ lockdep_assert_held(&ipvs->est_mutex);
+
/* Ignore reloads before first service is added */
if (!READ_ONCE(ipvs->enable))
return;
ip_vs_est_stopped_recalc(ipvs);
- /* Bump the kthread configuration genid */
- atomic_inc(&ipvs->est_genid);
+ /* Bump the kthread configuration genid if stopping is requested */
+ if (restart)
+ atomic_inc(&ipvs->est_genid);
queue_delayed_work(system_long_wq, &ipvs->est_reload_work, 0);
}
@@ -304,12 +312,17 @@ static int ip_vs_est_add_kthread(struct netns_ipvs *ipvs)
void *arr = NULL;
int i;
- if ((unsigned long)ipvs->est_kt_count >= ipvs->est_max_threads &&
- READ_ONCE(ipvs->enable) && ipvs->est_max_threads)
- return -EINVAL;
-
mutex_lock(&ipvs->est_mutex);
+ /* Allow kt 0 data to be created before the services are added
+ * and limit the kthreads when services are present.
+ */
+ if ((unsigned long)ipvs->est_kt_count >= ipvs->est_max_threads &&
+ READ_ONCE(ipvs->enable) && ipvs->est_max_threads) {
+ ret = -EINVAL;
+ goto out;
+ }
+
for (i = 0; i < id; i++) {
if (!ipvs->est_kt_arr[i])
break;
@@ -333,6 +346,7 @@ static int ip_vs_est_add_kthread(struct netns_ipvs *ipvs)
kd->est_timer = jiffies;
kd->id = id;
ip_vs_est_set_params(ipvs, kd);
+ kd->needed = 1;
/* Pre-allocate stats used in calc phase */
if (!id && !kd->calc_stats) {
@@ -341,12 +355,8 @@ static int ip_vs_est_add_kthread(struct netns_ipvs *ipvs)
goto out;
}
- /* Start kthread tasks only when services are present */
- if (READ_ONCE(ipvs->enable) && !ip_vs_est_stopped(ipvs)) {
- ret = ip_vs_est_kthread_start(ipvs, kd);
- if (ret < 0)
- goto out;
- }
+ /* Request kthread to be started */
+ ip_vs_est_reload_start(ipvs, false);
if (arr)
ipvs->est_kt_count++;
@@ -482,12 +492,11 @@ static int ip_vs_enqueue_estimator(struct netns_ipvs *ipvs,
/* Start estimation for stats */
int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats)
{
+ struct ip_vs_est_kt_data *kd = ipvs->est_kt_count > 0 ?
+ ipvs->est_kt_arr[0] : NULL;
struct ip_vs_estimator *est = &stats->est;
int ret;
- if (!ipvs->est_max_threads && READ_ONCE(ipvs->enable))
- ipvs->est_max_threads = ip_vs_est_max_threads(ipvs);
-
est->ktid = -1;
est->ktrow = IPVS_EST_NTICKS - 1; /* Initial delay */
@@ -496,8 +505,15 @@ int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats)
* will not allocate much memory, just for kt 0.
*/
ret = 0;
- if (!ipvs->est_kt_count || !ipvs->est_kt_arr[0])
+ if (!kd) {
ret = ip_vs_est_add_kthread(ipvs);
+ } else if (!kd->needed) {
+ mutex_lock(&ipvs->est_mutex);
+ /* We have job for the kt 0 task */
+ kd->needed = 1;
+ ip_vs_est_reload_start(ipvs, true);
+ mutex_unlock(&ipvs->est_mutex);
+ }
if (ret >= 0)
hlist_add_head(&est->list, &ipvs->est_temp_list);
else
@@ -578,16 +594,14 @@ void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats)
}
end_kt0:
- /* kt 0 is freed after all other kthreads and chains are empty */
+ /* kt 0 task is stopped after all other kt slots and chains are empty */
if (ipvs->est_kt_count == 1 && hlist_empty(&ipvs->est_temp_list)) {
kd = ipvs->est_kt_arr[0];
- if (!kd || !kd->est_count) {
+ if (kd && !kd->est_count) {
mutex_lock(&ipvs->est_mutex);
- if (kd) {
- ip_vs_est_kthread_destroy(kd);
- ipvs->est_kt_arr[0] = NULL;
- }
- ipvs->est_kt_count--;
+ /* Keep the kt0 data but request kthread_stop */
+ kd->needed = 0;
+ ip_vs_est_reload_start(ipvs, true);
mutex_unlock(&ipvs->est_mutex);
ipvs->est_add_ktid = 0;
}
@@ -647,9 +661,9 @@ static int ip_vs_est_calc_limits(struct netns_ipvs *ipvs, int *chain_max)
u64 val;
INIT_HLIST_HEAD(&chain);
- mutex_lock(&ipvs->service_mutex);
+ mutex_lock(&ipvs->est_mutex);
kd = ipvs->est_kt_arr[0];
- mutex_unlock(&ipvs->service_mutex);
+ mutex_unlock(&ipvs->est_mutex);
s = kd ? kd->calc_stats : NULL;
if (!s)
goto out;
@@ -748,16 +762,16 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs)
if (!ip_vs_est_calc_limits(ipvs, &chain_max))
return;
- mutex_lock(&ipvs->service_mutex);
-
/* Stop all other tasks, so that we can immediately move the
* estimators to est_temp_list without RCU grace period
*/
mutex_lock(&ipvs->est_mutex);
for (id = 1; id < ipvs->est_kt_count; id++) {
/* netns clean up started, abort */
- if (!READ_ONCE(ipvs->enable))
- goto unlock2;
+ if (kthread_should_stop() || !READ_ONCE(ipvs->enable)) {
+ mutex_unlock(&ipvs->est_mutex);
+ return;
+ }
kd = ipvs->est_kt_arr[id];
if (!kd)
continue;
@@ -765,9 +779,11 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs)
}
mutex_unlock(&ipvs->est_mutex);
+ mutex_lock(&ipvs->service_mutex);
+
/* Move all estimators to est_temp_list but carefully,
* all estimators and kthread data can be released while
- * we reschedule. Even for kthread 0.
+ * we reschedule.
*/
step = 0;
@@ -849,9 +865,7 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs)
ip_vs_stop_estimator(ipvs, stats);
/* Tasks are stopped, move without RCU grace period */
est->ktid = -1;
- est->ktrow = row - kd->est_row;
- if (est->ktrow < 0)
- est->ktrow += IPVS_EST_NTICKS;
+ est->ktrow = delay;
hlist_add_head(&est->list, &ipvs->est_temp_list);
/* kd freed ? */
if (last)
@@ -889,7 +903,6 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs)
if (genid == atomic_read(&ipvs->est_genid))
ipvs->est_calc_phase = 0;
-unlock2:
mutex_unlock(&ipvs->est_mutex);
unlock:
diff --git a/net/netfilter/ipvs/ip_vs_sched.c b/net/netfilter/ipvs/ip_vs_sched.c
index c6e421c..24adc38 100644
--- a/net/netfilter/ipvs/ip_vs_sched.c
+++ b/net/netfilter/ipvs/ip_vs_sched.c
@@ -56,19 +56,19 @@ int ip_vs_bind_scheduler(struct ip_vs_service *svc,
/*
* Unbind a service with its scheduler
*/
-void ip_vs_unbind_scheduler(struct ip_vs_service *svc,
- struct ip_vs_scheduler *sched)
+void ip_vs_unbind_scheduler(struct ip_vs_service *svc)
{
- struct ip_vs_scheduler *cur_sched;
+ struct ip_vs_scheduler *sched;
- cur_sched = rcu_dereference_protected(svc->scheduler, 1);
- /* This check proves that old 'sched' was installed */
- if (!cur_sched)
+ sched = rcu_dereference_protected(svc->scheduler, 1);
+ if (!sched)
return;
+ /* Reset the scheduler before initiating any RCU callbacks */
+ rcu_assign_pointer(svc->scheduler, NULL);
+ smp_wmb(); /* paired with smp_rmb() in ip_vs_schedule() */
if (sched->done_service)
sched->done_service(svc);
- /* svc->scheduler can be set to NULL only by caller */
}
diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c
index 4f39bf7..75e53fd 100644
--- a/net/netfilter/nf_conntrack_broadcast.c
+++ b/net/netfilter/nf_conntrack_broadcast.c
@@ -72,6 +72,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb,
exp->flags = NF_CT_EXPECT_PERMANENT;
exp->class = NF_CT_EXPECT_CLASS_DEFAULT;
rcu_assign_pointer(exp->helper, helper);
+ rcu_assign_pointer(exp->assign_helper, NULL);
write_pnet(&exp->net, net);
#ifdef CONFIG_NF_CONNTRACK_ZONES
exp->zone = ct->zone;
diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
index b081892..b521b5e 100644
--- a/net/netfilter/nf_conntrack_core.c
+++ b/net/netfilter/nf_conntrack_core.c
@@ -568,6 +568,13 @@ static void destroy_gre_conntrack(struct nf_conn *ct)
#endif
}
+static void warn_on_keymap_list_leak(const struct net *net)
+{
+#ifdef CONFIG_NF_CT_PROTO_GRE
+ WARN_ON_ONCE(!list_empty(&net->ct.nf_ct_proto.gre.keymap_list));
+#endif
+}
+
void nf_ct_destroy(struct nf_conntrack *nfct)
{
struct nf_conn *ct = (struct nf_conn *)nfct;
@@ -1811,14 +1818,17 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,
spin_lock_bh(&nf_conntrack_expect_lock);
exp = nf_ct_find_expectation(net, zone, tuple, !tmpl || nf_ct_is_confirmed(tmpl));
if (exp) {
+ struct nf_conntrack_helper *assign_helper;
+
/* Welcome, Mr. Bond. We've been expecting you... */
__set_bit(IPS_EXPECTED_BIT, &ct->status);
/* exp->master safe, refcnt bumped in nf_ct_find_expectation */
ct->master = exp->master;
- if (exp->helper) {
+ assign_helper = rcu_dereference(exp->assign_helper);
+ if (assign_helper) {
help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
if (help)
- rcu_assign_pointer(help->helper, exp->helper);
+ rcu_assign_pointer(help->helper, assign_helper);
}
#ifdef CONFIG_NF_CONNTRACK_MARK
@@ -2507,6 +2517,7 @@ void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
}
list_for_each_entry(net, net_exit_list, exit_list) {
+ warn_on_keymap_list_leak(net);
nf_conntrack_ecache_pernet_fini(net);
nf_conntrack_expect_pernet_fini(net);
free_percpu(net->ct.stat);
diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c
index 24d0576..8e943ef 100644
--- a/net/netfilter/nf_conntrack_expect.c
+++ b/net/netfilter/nf_conntrack_expect.c
@@ -344,6 +344,7 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class,
helper = rcu_dereference(help->helper);
rcu_assign_pointer(exp->helper, helper);
+ rcu_assign_pointer(exp->assign_helper, NULL);
write_pnet(&exp->net, net);
#ifdef CONFIG_NF_CONNTRACK_ZONES
exp->zone = ct->zone;
diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c
index 3f5c504..b2fe655 100644
--- a/net/netfilter/nf_conntrack_h323_main.c
+++ b/net/netfilter/nf_conntrack_h323_main.c
@@ -643,7 +643,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct,
&ct->tuplehash[!dir].tuple.src.u3,
&ct->tuplehash[!dir].tuple.dst.u3,
IPPROTO_TCP, NULL, &port);
- rcu_assign_pointer(exp->helper, &nf_conntrack_helper_h245);
+ rcu_assign_pointer(exp->assign_helper, &nf_conntrack_helper_h245);
nathook = rcu_dereference(nfct_h323_nat_hook);
if (memcmp(&ct->tuplehash[dir].tuple.src.u3,
@@ -767,7 +767,7 @@ static int expect_callforwarding(struct sk_buff *skb,
nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct),
&ct->tuplehash[!dir].tuple.src.u3, &addr,
IPPROTO_TCP, NULL, &port);
- rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931);
+ rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931);
nathook = rcu_dereference(nfct_h323_nat_hook);
if (memcmp(&ct->tuplehash[dir].tuple.src.u3,
@@ -1234,7 +1234,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct,
&ct->tuplehash[!dir].tuple.src.u3 : NULL,
&ct->tuplehash[!dir].tuple.dst.u3,
IPPROTO_TCP, NULL, &port);
- rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931);
+ rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931);
exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */
nathook = rcu_dereference(nfct_h323_nat_hook);
@@ -1306,7 +1306,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct,
nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct),
&ct->tuplehash[!dir].tuple.src.u3, &addr,
IPPROTO_UDP, NULL, &port);
- rcu_assign_pointer(exp->helper, nf_conntrack_helper_ras);
+ rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_ras);
if (nf_ct_expect_related(exp, 0) == 0) {
pr_debug("nf_ct_ras: expect RAS ");
@@ -1523,7 +1523,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct,
&ct->tuplehash[!dir].tuple.src.u3, &addr,
IPPROTO_TCP, NULL, &port);
exp->flags = NF_CT_EXPECT_PERMANENT;
- rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931);
+ rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931);
if (nf_ct_expect_related(exp, 0) == 0) {
pr_debug("nf_ct_ras: expect Q.931 ");
@@ -1577,7 +1577,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct,
&ct->tuplehash[!dir].tuple.src.u3, &addr,
IPPROTO_TCP, NULL, &port);
exp->flags = NF_CT_EXPECT_PERMANENT;
- rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931);
+ rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931);
if (nf_ct_expect_related(exp, 0) == 0) {
pr_debug("nf_ct_ras: expect Q.931 ");
diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c
index a715304..17e971b 100644
--- a/net/netfilter/nf_conntrack_helper.c
+++ b/net/netfilter/nf_conntrack_helper.c
@@ -321,8 +321,8 @@ __printf(3, 4)
void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct,
const char *fmt, ...)
{
+ const char *helper_name = "(null)";
const struct nf_conn_help *help;
- const struct nf_conntrack_helper *helper;
struct va_format vaf;
va_list args;
@@ -331,14 +331,17 @@ void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct,
vaf.fmt = fmt;
vaf.va = &args;
- /* Called from the helper function, this call never fails */
help = nfct_help(ct);
+ if (help) {
+ const struct nf_conntrack_helper *helper;
- /* rcu_read_lock()ed by nf_hook_thresh */
- helper = rcu_dereference(help->helper);
+ helper = rcu_dereference(help->helper);
+ if (helper)
+ helper_name = helper->name;
+ }
nf_log_packet(nf_ct_net(ct), nf_ct_l3num(ct), 0, skb, NULL, NULL, NULL,
- "nf_ct_%s: dropping packet: %pV ", helper->name, &vaf);
+ "helper %s dropping packet: %pV ", helper_name, &vaf);
va_end(args);
}
@@ -400,6 +403,11 @@ static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data)
this = rcu_dereference_protected(exp->helper,
lockdep_is_held(&nf_conntrack_expect_lock));
+ if (this == me)
+ return true;
+
+ this = rcu_dereference_protected(exp->assign_helper,
+ lockdep_is_held(&nf_conntrack_expect_lock));
return this == me;
}
diff --git a/net/netfilter/nf_conntrack_irc.c b/net/netfilter/nf_conntrack_irc.c
index 522183b..2ebe4cb 100644
--- a/net/netfilter/nf_conntrack_irc.c
+++ b/net/netfilter/nf_conntrack_irc.c
@@ -203,7 +203,7 @@ static int help(struct sk_buff *skb, unsigned int protoff,
if (parse_dcc(data, data_limit, &dcc_ip,
&dcc_port, &addr_beg_p, &addr_end_p)) {
pr_debug("unable to parse dcc command\n");
- continue;
+ goto out;
}
pr_debug("DCC bound ip/port: %pI4:%u\n",
@@ -217,7 +217,7 @@ static int help(struct sk_buff *skb, unsigned int protoff,
net_warn_ratelimited("Forged DCC command from %pI4: %pI4:%u\n",
&tuple->src.u3.ip,
&dcc_ip, dcc_port);
- continue;
+ goto out;
}
exp = nf_ct_expect_alloc(ct);
diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c
index eda5fe4..befa7e8 100644
--- a/net/netfilter/nf_conntrack_netlink.c
+++ b/net/netfilter/nf_conntrack_netlink.c
@@ -2634,6 +2634,7 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = {
static struct nf_conntrack_expect *
ctnetlink_alloc_expect(const struct nlattr *const cda[], struct nf_conn *ct,
+ const struct nf_conntrack_helper *assign_helper,
struct nf_conntrack_tuple *tuple,
struct nf_conntrack_tuple *mask);
@@ -2860,6 +2861,7 @@ static int
ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct,
u32 portid, u32 report)
{
+ struct nf_conntrack_helper *assign_helper = NULL;
struct nlattr *cda[CTA_EXPECT_MAX+1];
struct nf_conntrack_tuple tuple, mask;
struct nf_conntrack_expect *exp;
@@ -2870,13 +2872,26 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct,
if (err < 0)
return err;
+ if (!cda[CTA_EXPECT_TUPLE] || !cda[CTA_EXPECT_MASK])
+ return -EINVAL;
+
err = ctnetlink_glue_exp_parse((const struct nlattr * const *)cda,
ct, &tuple, &mask);
if (err < 0)
return err;
+ if (cda[CTA_EXPECT_HELP_NAME]) {
+ const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]);
+
+ assign_helper = __nf_conntrack_helper_find(helpname,
+ nf_ct_l3num(ct),
+ tuple.dst.protonum);
+ if (!assign_helper)
+ return -EOPNOTSUPP;
+ }
+
exp = ctnetlink_alloc_expect((const struct nlattr * const *)cda, ct,
- &tuple, &mask);
+ assign_helper, &tuple, &mask);
if (IS_ERR(exp))
return PTR_ERR(exp);
@@ -3515,6 +3530,7 @@ ctnetlink_parse_expect_nat(const struct nlattr *attr,
static struct nf_conntrack_expect *
ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct,
+ const struct nf_conntrack_helper *assign_helper,
struct nf_conntrack_tuple *tuple,
struct nf_conntrack_tuple *mask)
{
@@ -3568,6 +3584,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct,
exp->zone = ct->zone;
#endif
rcu_assign_pointer(exp->helper, helper);
+ rcu_assign_pointer(exp->assign_helper, assign_helper);
exp->tuple = *tuple;
exp->mask.src.u3 = mask->src.u3;
exp->mask.src.u.all = mask->src.u.all;
@@ -3623,7 +3640,7 @@ ctnetlink_create_expect(struct net *net,
ct = nf_ct_tuplehash_to_ctrack(h);
rcu_read_lock();
- exp = ctnetlink_alloc_expect(cda, ct, &tuple, &mask);
+ exp = ctnetlink_alloc_expect(cda, ct, NULL, &tuple, &mask);
if (IS_ERR(exp)) {
err = PTR_ERR(exp);
goto err_rcu;
diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c
index 4c67963..dc23e41 100644
--- a/net/netfilter/nf_conntrack_pptp.c
+++ b/net/netfilter/nf_conntrack_pptp.c
@@ -225,13 +225,9 @@ static int exp_gre(struct nf_conn *ct, __be16 callid, __be16 peer_callid)
if (nf_ct_expect_related(exp_reply, 0) != 0)
goto out_unexpect_orig;
- /* Add GRE keymap entries */
- if (nf_ct_gre_keymap_add(ct, IP_CT_DIR_ORIGINAL, &exp_orig->tuple) != 0)
+ if (!nf_ct_gre_keymap_add(ct, &exp_orig->tuple,
+ &exp_reply->tuple))
goto out_unexpect_both;
- if (nf_ct_gre_keymap_add(ct, IP_CT_DIR_REPLY, &exp_reply->tuple) != 0) {
- nf_ct_gre_keymap_destroy(ct);
- goto out_unexpect_both;
- }
ret = 0;
out_put_both:
diff --git a/net/netfilter/nf_conntrack_proto_gre.c b/net/netfilter/nf_conntrack_proto_gre.c
index 94c19bc..35e2208 100644
--- a/net/netfilter/nf_conntrack_proto_gre.c
+++ b/net/netfilter/nf_conntrack_proto_gre.c
@@ -87,41 +87,97 @@ static __be16 gre_keymap_lookup(struct net *net, struct nf_conntrack_tuple *t)
return key;
}
-/* add a single keymap entry, associate with specified master ct */
-int nf_ct_gre_keymap_add(struct nf_conn *ct, enum ip_conntrack_dir dir,
- struct nf_conntrack_tuple *t)
+enum nf_ct_gre_km_act {
+ NF_CT_GRE_KM_NEW,
+ NF_CT_GRE_KM_BAD,
+ NF_CT_GRE_KM_DUP
+};
+
+static enum nf_ct_gre_km_act
+nf_ct_gre_km_acceptable(const struct nf_ct_pptp_master *ct_pptp_info,
+ const struct nf_conntrack_tuple *orig,
+ const struct nf_conntrack_tuple *repl)
+{
+ struct nf_ct_gre_keymap *km_orig, *km_repl;
+
+ lockdep_assert_held(&keymap_lock);
+
+ km_orig = ct_pptp_info->keymap[IP_CT_DIR_ORIGINAL];
+ km_repl = ct_pptp_info->keymap[IP_CT_DIR_REPLY];
+
+ if (km_orig && km_repl) {
+ if (!gre_key_cmpfn(km_orig, orig))
+ return NF_CT_GRE_KM_BAD;
+
+ if (!gre_key_cmpfn(km_repl, repl))
+ return NF_CT_GRE_KM_BAD;
+
+ return NF_CT_GRE_KM_DUP;
+ }
+
+ DEBUG_NET_WARN_ON_ONCE(km_orig);
+ DEBUG_NET_WARN_ON_ONCE(km_repl);
+ return NF_CT_GRE_KM_NEW;
+}
+
+/* add keymap entries, associate with specified master ct */
+bool nf_ct_gre_keymap_add(struct nf_conn *ct,
+ const struct nf_conntrack_tuple *orig,
+ const struct nf_conntrack_tuple *repl)
{
struct net *net = nf_ct_net(ct);
struct nf_gre_net *net_gre = gre_pernet(net);
struct nf_ct_pptp_master *ct_pptp_info = nfct_help_data(ct);
- struct nf_ct_gre_keymap **kmp, *km;
+ struct nf_ct_gre_keymap *km_orig, *km_repl;
+ bool ret = false;
- kmp = &ct_pptp_info->keymap[dir];
- if (*kmp) {
- /* check whether it's a retransmission */
- list_for_each_entry_rcu(km, &net_gre->keymap_list, list) {
- if (gre_key_cmpfn(km, t) && km == *kmp)
- return 0;
- }
- pr_debug("trying to override keymap_%s for ct %p\n",
- dir == IP_CT_DIR_REPLY ? "reply" : "orig", ct);
- return -EEXIST;
- }
+ km_orig = kmalloc_obj(*km_orig, GFP_ATOMIC);
+ if (!km_orig)
+ return false;
+ km_repl = kmalloc_obj(*km_repl, GFP_ATOMIC);
+ if (!km_repl)
+ goto km_free;
- km = kmalloc_obj(*km, GFP_ATOMIC);
- if (!km)
- return -ENOMEM;
- memcpy(&km->tuple, t, sizeof(*t));
- *kmp = km;
-
- pr_debug("adding new entry %p: ", km);
- nf_ct_dump_tuple(&km->tuple);
+ memcpy(&km_orig->tuple, orig, sizeof(*orig));
+ memcpy(&km_repl->tuple, repl, sizeof(*repl));
spin_lock_bh(&keymap_lock);
- list_add_tail(&km->list, &net_gre->keymap_list);
+ if (nf_ct_is_dying(ct))
+ goto unlock_free;
+
+ switch (nf_ct_gre_km_acceptable(ct_pptp_info, orig, repl)) {
+ case NF_CT_GRE_KM_NEW:
+ break;
+ case NF_CT_GRE_KM_DUP:
+ ret = true;
+ goto unlock_free;
+ case NF_CT_GRE_KM_BAD:
+ pr_debug("trying to override keymap for ct %p\n", ct);
+ goto unlock_free;
+ }
+
+ if (ct_pptp_info->keymap[IP_CT_DIR_ORIGINAL] ||
+ ct_pptp_info->keymap[IP_CT_DIR_REPLY])
+ goto unlock_free;
+
+ pr_debug("adding new entries %p,%p: ", km_orig, km_repl);
+ nf_ct_dump_tuple(&km_orig->tuple);
+ nf_ct_dump_tuple(&km_repl->tuple);
+
+ list_add_tail_rcu(&km_orig->list, &net_gre->keymap_list);
+ list_add_tail_rcu(&km_repl->list, &net_gre->keymap_list);
+ ct_pptp_info->keymap[IP_CT_DIR_ORIGINAL] = km_orig;
+ ct_pptp_info->keymap[IP_CT_DIR_REPLY] = km_repl;
spin_unlock_bh(&keymap_lock);
- return 0;
+ return true;
+
+unlock_free:
+ spin_unlock_bh(&keymap_lock);
+km_free:
+ kfree(km_orig);
+ kfree(km_repl);
+ return ret;
}
EXPORT_SYMBOL_GPL(nf_ct_gre_keymap_add);
diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c
index 645d2c4..7e10fa6 100644
--- a/net/netfilter/nf_conntrack_proto_sctp.c
+++ b/net/netfilter/nf_conntrack_proto_sctp.c
@@ -466,9 +466,13 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct,
if (!ih)
goto out_unlock;
- if (ct->proto.sctp.init[dir] && ct->proto.sctp.init[!dir])
- ct->proto.sctp.init[!dir] = 0;
- ct->proto.sctp.init[dir] = 1;
+ /* Do not record INIT matching peer vtag (stale or retransmitted INIT). */
+ if (old_state == SCTP_CONNTRACK_NONE ||
+ ct->proto.sctp.vtag[!dir] != ih->init_tag) {
+ if (ct->proto.sctp.init[dir] && ct->proto.sctp.init[!dir])
+ ct->proto.sctp.init[!dir] = 0;
+ ct->proto.sctp.init[dir] = 1;
+ }
pr_debug("Setting vtag %x for dir %d\n", ih->init_tag, !dir);
ct->proto.sctp.vtag[!dir] = ih->init_tag;
diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c
index b67426c..e99ab1e 100644
--- a/net/netfilter/nf_conntrack_proto_tcp.c
+++ b/net/netfilter/nf_conntrack_proto_tcp.c
@@ -1221,7 +1221,8 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct,
new_state = old_state;
}
if (((test_bit(IPS_SEEN_REPLY_BIT, &ct->status)
- && ct->proto.tcp.last_index == TCP_SYN_SET)
+ && ct->proto.tcp.last_index == TCP_SYN_SET
+ && ct->proto.tcp.last_dir != dir)
|| (!test_bit(IPS_ASSURED_BIT, &ct->status)
&& ct->proto.tcp.last_index == TCP_ACK_SET))
&& ntohl(th->ack_seq) == ct->proto.tcp.last_end) {
diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c
index 182cfb1..e69941f 100644
--- a/net/netfilter/nf_conntrack_sip.c
+++ b/net/netfilter/nf_conntrack_sip.c
@@ -181,6 +181,57 @@ static int sip_parse_addr(const struct nf_conn *ct, const char *cp,
return 1;
}
+/* Parse optional port number after IP address.
+ * Returns false on malformed input, true otherwise.
+ * If port is non-NULL, stores parsed port in network byte order.
+ * If no port is present, sets *port to default SIP port.
+ */
+static bool sip_parse_port(const char *dptr, const char **endp,
+ const char *limit, __be16 *port)
+{
+ unsigned int p = 0;
+ int len = 0;
+
+ if (dptr >= limit)
+ return false;
+
+ if (*dptr != ':') {
+ if (port)
+ *port = htons(SIP_PORT);
+ if (endp)
+ *endp = dptr;
+ return true;
+ }
+
+ dptr++; /* skip ':' */
+
+ while (dptr < limit && isdigit(*dptr)) {
+ p = p * 10 + (*dptr - '0');
+ dptr++;
+ len++;
+ if (len > 5) /* max "65535" */
+ return false;
+ }
+
+ if (len == 0)
+ return false;
+
+ /* reached limit while parsing port */
+ if (dptr >= limit)
+ return false;
+
+ if (p < 1024 || p > 65535)
+ return false;
+
+ if (port)
+ *port = htons(p);
+
+ if (endp)
+ *endp = dptr;
+
+ return true;
+}
+
/* skip ip address. returns its length. */
static int epaddr_len(const struct nf_conn *ct, const char *dptr,
const char *limit, int *shift)
@@ -193,11 +244,8 @@ static int epaddr_len(const struct nf_conn *ct, const char *dptr,
return 0;
}
- /* Port number */
- if (*dptr == ':') {
- dptr++;
- dptr += digits_len(ct, dptr, limit, shift);
- }
+ if (!sip_parse_port(dptr, &dptr, limit, NULL))
+ return 0;
return dptr - aux;
}
@@ -228,6 +276,51 @@ static int skp_epaddr_len(const struct nf_conn *ct, const char *dptr,
return epaddr_len(ct, dptr, limit, shift);
}
+/* simple_strtoul stops after first non-number character.
+ * But as we're not dealing with c-strings, we can't rely on
+ * hitting \r,\n,\0 etc. before moving past end of buffer.
+ *
+ * This is a variant of simple_strtoul, but doesn't require
+ * a c-string.
+ *
+ * If value exceeds UINT_MAX, 0 is returned.
+ */
+static unsigned int sip_strtouint(const char *cp, unsigned int len, char **endp)
+{
+ const unsigned int max = sizeof("4294967295");
+ unsigned int olen = len;
+ const char *s = cp;
+ u64 result = 0;
+
+ if (len > max)
+ len = max;
+
+ while (olen > 0 && isdigit(*s)) {
+ unsigned int value;
+
+ if (len == 0)
+ goto err;
+
+ value = *s - '0';
+ result = result * 10 + value;
+
+ if (result > UINT_MAX)
+ goto err;
+ s++;
+ len--;
+ olen--;
+ }
+
+ if (endp)
+ *endp = (char *)s;
+
+ return result;
+err:
+ if (endp)
+ *endp = (char *)cp;
+ return 0;
+}
+
/* Parse a SIP request line of the form:
*
* Request-Line = Method SP Request-URI SP SIP-Version CRLF
@@ -241,7 +334,6 @@ int ct_sip_parse_request(const struct nf_conn *ct,
{
const char *start = dptr, *limit = dptr + datalen, *end;
unsigned int mlen;
- unsigned int p;
int shift = 0;
/* Skip method and following whitespace */
@@ -267,14 +359,8 @@ int ct_sip_parse_request(const struct nf_conn *ct,
if (!sip_parse_addr(ct, dptr, &end, addr, limit, true))
return -1;
- if (end < limit && *end == ':') {
- end++;
- p = simple_strtoul(end, (char **)&end, 10);
- if (p < 1024 || p > 65535)
- return -1;
- *port = htons(p);
- } else
- *port = htons(SIP_PORT);
+ if (!sip_parse_port(end, &end, limit, port))
+ return -1;
if (end == dptr)
return 0;
@@ -509,7 +595,6 @@ int ct_sip_parse_header_uri(const struct nf_conn *ct, const char *dptr,
union nf_inet_addr *addr, __be16 *port)
{
const char *c, *limit = dptr + datalen;
- unsigned int p;
int ret;
ret = ct_sip_walk_headers(ct, dptr, dataoff ? *dataoff : 0, datalen,
@@ -520,14 +605,8 @@ int ct_sip_parse_header_uri(const struct nf_conn *ct, const char *dptr,
if (!sip_parse_addr(ct, dptr + *matchoff, &c, addr, limit, true))
return -1;
- if (*c == ':') {
- c++;
- p = simple_strtoul(c, (char **)&c, 10);
- if (p < 1024 || p > 65535)
- return -1;
- *port = htons(p);
- } else
- *port = htons(SIP_PORT);
+ if (!sip_parse_port(c, &c, limit, port))
+ return -1;
if (dataoff)
*dataoff = c - dptr;
@@ -609,7 +688,7 @@ int ct_sip_parse_numerical_param(const struct nf_conn *ct, const char *dptr,
return 0;
start += strlen(name);
- *val = simple_strtoul(start, &end, 0);
+ *val = sip_strtouint(start, limit - start, (char **)&end);
if (start == end)
return -1;
if (matchoff && matchlen) {
@@ -1064,6 +1143,8 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff,
mediaoff = sdpoff;
for (i = 0; i < ARRAY_SIZE(sdp_media_types); ) {
+ char *end;
+
if (ct_sip_get_sdp_header(ct, *dptr, mediaoff, *datalen,
SDP_HDR_MEDIA, SDP_HDR_UNSPEC,
&mediaoff, &medialen) <= 0)
@@ -1079,8 +1160,8 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff,
mediaoff += t->len;
medialen -= t->len;
- port = simple_strtoul(*dptr + mediaoff, NULL, 10);
- if (port == 0)
+ port = sip_strtouint(*dptr + mediaoff, *datalen - mediaoff, (char **)&end);
+ if (port == 0 || *dptr + mediaoff == end)
continue;
if (port < 1024 || port > 65535) {
nf_ct_helper_log(skb, ct, "wrong port %u", port);
@@ -1254,7 +1335,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff,
*/
if (ct_sip_get_header(ct, *dptr, 0, *datalen, SIP_HDR_EXPIRES,
&matchoff, &matchlen) > 0)
- expires = simple_strtoul(*dptr + matchoff, NULL, 10);
+ expires = sip_strtouint(*dptr + matchoff, *datalen - matchoff, NULL);
ret = ct_sip_parse_header_uri(ct, *dptr, NULL, *datalen,
SIP_HDR_CONTACT, NULL,
@@ -1285,6 +1366,10 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff,
goto store_cseq;
}
+ helper = rcu_dereference(nfct_help(ct)->helper);
+ if (!helper)
+ return NF_DROP;
+
exp = nf_ct_expect_alloc(ct);
if (!exp) {
nf_ct_helper_log(skb, ct, "cannot alloc expectation");
@@ -1295,14 +1380,10 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff,
if (sip_direct_signalling)
saddr = &ct->tuplehash[!dir].tuple.src.u3;
- helper = rcu_dereference(nfct_help(ct)->helper);
- if (!helper)
- return NF_DROP;
-
nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct),
saddr, &daddr, proto, NULL, &port);
exp->timeout.expires = sip_timeout * HZ;
- rcu_assign_pointer(exp->helper, helper);
+ rcu_assign_pointer(exp->assign_helper, helper);
exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE;
hooks = rcu_dereference(nf_nat_sip_hooks);
@@ -1358,7 +1439,7 @@ static int process_register_response(struct sk_buff *skb, unsigned int protoff,
if (ct_sip_get_header(ct, *dptr, 0, *datalen, SIP_HDR_EXPIRES,
&matchoff, &matchlen) > 0)
- expires = simple_strtoul(*dptr + matchoff, NULL, 10);
+ expires = sip_strtouint(*dptr + matchoff, *datalen - matchoff, NULL);
while (1) {
unsigned int c_expires = expires;
@@ -1418,10 +1499,12 @@ static int process_sip_response(struct sk_buff *skb, unsigned int protoff,
struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
unsigned int matchoff, matchlen, matchend;
unsigned int code, cseq, i;
+ char *end;
if (*datalen < strlen("SIP/2.0 200"))
return NF_ACCEPT;
- code = simple_strtoul(*dptr + strlen("SIP/2.0 "), NULL, 10);
+ code = sip_strtouint(*dptr + strlen("SIP/2.0 "),
+ *datalen - strlen("SIP/2.0 "), NULL);
if (!code) {
nf_ct_helper_log(skb, ct, "cannot get code");
return NF_DROP;
@@ -1432,8 +1515,8 @@ static int process_sip_response(struct sk_buff *skb, unsigned int protoff,
nf_ct_helper_log(skb, ct, "cannot parse cseq");
return NF_DROP;
}
- cseq = simple_strtoul(*dptr + matchoff, NULL, 10);
- if (!cseq && *(*dptr + matchoff) != '0') {
+ cseq = sip_strtouint(*dptr + matchoff, *datalen - matchoff, (char **)&end);
+ if (*dptr + matchoff == end) {
nf_ct_helper_log(skb, ct, "cannot get cseq");
return NF_DROP;
}
@@ -1482,6 +1565,7 @@ static int process_sip_request(struct sk_buff *skb, unsigned int protoff,
for (i = 0; i < ARRAY_SIZE(sip_handlers); i++) {
const struct sip_handler *handler;
+ char *end;
handler = &sip_handlers[i];
if (handler->request == NULL)
@@ -1498,8 +1582,8 @@ static int process_sip_request(struct sk_buff *skb, unsigned int protoff,
nf_ct_helper_log(skb, ct, "cannot parse cseq");
return NF_DROP;
}
- cseq = simple_strtoul(*dptr + matchoff, NULL, 10);
- if (!cseq && *(*dptr + matchoff) != '0') {
+ cseq = sip_strtouint(*dptr + matchoff, *datalen - matchoff, (char **)&end);
+ if (*dptr + matchoff == end) {
nf_ct_helper_log(skb, ct, "cannot get cseq");
return NF_DROP;
}
@@ -1575,7 +1659,7 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff,
&matchoff, &matchlen) <= 0)
break;
- clen = simple_strtoul(dptr + matchoff, (char **)&end, 10);
+ clen = sip_strtouint(dptr + matchoff, datalen - matchoff, (char **)&end);
if (dptr + matchoff == end)
break;
diff --git a/net/netfilter/nf_dup_netdev.c b/net/netfilter/nf_dup_netdev.c
index e348fb9..3b0a70e1 100644
--- a/net/netfilter/nf_dup_netdev.c
+++ b/net/netfilter/nf_dup_netdev.c
@@ -13,22 +13,6 @@
#include <net/netfilter/nf_tables_offload.h>
#include <net/netfilter/nf_dup_netdev.h>
-#define NF_RECURSION_LIMIT 2
-
-#ifndef CONFIG_PREEMPT_RT
-static u8 *nf_get_nf_dup_skb_recursion(void)
-{
- return this_cpu_ptr(&softnet_data.xmit.nf_dup_skb_recursion);
-}
-#else
-
-static u8 *nf_get_nf_dup_skb_recursion(void)
-{
- return ¤t->net_xmit.nf_dup_skb_recursion;
-}
-
-#endif
-
static void nf_do_netdev_egress(struct sk_buff *skb, struct net_device *dev,
enum nf_dev_hooks hook)
{
diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
index 2c4140e..785d8c2 100644
--- a/net/netfilter/nf_flow_table_core.c
+++ b/net/netfilter/nf_flow_table_core.c
@@ -122,6 +122,7 @@ static int flow_offload_fill_route(struct flow_offload *flow,
flow_tuple->tun = route->tuple[dir].in.tun;
flow_tuple->encap_num = route->tuple[dir].in.num_encaps;
+ flow_tuple->needs_gso_segment = route->tuple[dir].out.needs_gso_segment;
flow_tuple->tun_num = route->tuple[dir].in.num_tuns;
switch (route->tuple[dir].xmit_type) {
diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c
index fd56d66..9c05a50 100644
--- a/net/netfilter/nf_flow_table_ip.c
+++ b/net/netfilter/nf_flow_table_ip.c
@@ -445,13 +445,13 @@ static void nf_flow_encap_pop(struct nf_flowtable_ctx *ctx,
switch (skb->protocol) {
case htons(ETH_P_8021Q):
vlan_hdr = (struct vlan_hdr *)skb->data;
- __skb_pull(skb, VLAN_HLEN);
+ skb_pull_rcsum(skb, VLAN_HLEN);
vlan_set_encap_proto(skb, vlan_hdr);
skb_reset_network_header(skb);
break;
case htons(ETH_P_PPP_SES):
skb->protocol = __nf_flow_pppoe_proto(skb);
- skb_pull(skb, PPPOE_SES_HLEN);
+ skb_pull_rcsum(skb, PPPOE_SES_HLEN);
skb_reset_network_header(skb);
break;
}
@@ -462,23 +462,6 @@ static void nf_flow_encap_pop(struct nf_flowtable_ctx *ctx,
nf_flow_ip_tunnel_pop(ctx, skb);
}
-struct nf_flow_xmit {
- const void *dest;
- const void *source;
- struct net_device *outdev;
-};
-
-static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
- struct nf_flow_xmit *xmit)
-{
- skb->dev = xmit->outdev;
- dev_hard_header(skb, skb->dev, ntohs(skb->protocol),
- xmit->dest, xmit->source, skb->len);
- dev_queue_xmit(skb);
-
- return NF_STOLEN;
-}
-
static struct flow_offload_tuple_rhash *
nf_flow_offload_lookup(struct nf_flowtable_ctx *ctx,
struct nf_flowtable *flow_table, struct sk_buff *skb)
@@ -524,7 +507,7 @@ static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx,
return 0;
}
- if (skb_try_make_writable(skb, thoff + ctx->hdrsize))
+ if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
return -1;
flow_offload_refresh(flow_table, flow, false);
@@ -544,7 +527,34 @@ static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx,
return 1;
}
-static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id)
+/* Similar to skb_vlan_push. */
+static int nf_flow_vlan_push(struct sk_buff *skb, __be16 proto, u16 id,
+ u32 needed_headroom)
+{
+ if (skb_vlan_tag_present(skb)) {
+ struct vlan_hdr *vhdr;
+
+ if (skb_cow_head(skb, needed_headroom + VLAN_HLEN))
+ return -1;
+
+ __skb_push(skb, VLAN_HLEN);
+ if (skb_mac_header_was_set(skb))
+ skb->mac_header -= VLAN_HLEN;
+
+ vhdr = (struct vlan_hdr *)skb->data;
+ skb->network_header -= VLAN_HLEN;
+ vhdr->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
+ vhdr->h_vlan_encapsulated_proto = skb->protocol;
+ skb->protocol = skb->vlan_proto;
+ skb_postpush_rcsum(skb, skb->data, VLAN_HLEN);
+ }
+ __vlan_hwaccel_put_tag(skb, proto, id);
+
+ return 0;
+}
+
+static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id,
+ u32 needed_headroom)
{
int data_len = skb->len + sizeof(__be16);
struct ppp_hdr {
@@ -553,7 +563,7 @@ static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id)
} *ph;
__be16 proto;
- if (skb_cow_head(skb, PPPOE_SES_HLEN))
+ if (skb_cow_head(skb, needed_headroom + PPPOE_SES_HLEN))
return -1;
switch (skb->protocol) {
@@ -730,21 +740,24 @@ static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb,
}
static int nf_flow_encap_push(struct sk_buff *skb,
- struct flow_offload_tuple *tuple)
+ struct flow_offload_tuple *tuple,
+ struct net_device *outdev)
{
+ u32 needed_headroom = LL_RESERVED_SPACE(outdev);
int i;
- for (i = 0; i < tuple->encap_num; i++) {
+ for (i = tuple->encap_num - 1; i >= 0; i--) {
switch (tuple->encap[i].proto) {
case htons(ETH_P_8021Q):
case htons(ETH_P_8021AD):
- skb_reset_mac_header(skb);
- if (skb_vlan_push(skb, tuple->encap[i].proto,
- tuple->encap[i].id) < 0)
+ if (nf_flow_vlan_push(skb, tuple->encap[i].proto,
+ tuple->encap[i].id,
+ needed_headroom) < 0)
return -1;
break;
case htons(ETH_P_PPP_SES):
- if (nf_flow_pppoe_push(skb, tuple->encap[i].id) < 0)
+ if (nf_flow_pppoe_push(skb, tuple->encap[i].id,
+ needed_headroom) < 0)
return -1;
break;
}
@@ -753,6 +766,76 @@ static int nf_flow_encap_push(struct sk_buff *skb,
return 0;
}
+struct nf_flow_xmit {
+ const void *dest;
+ const void *source;
+ struct net_device *outdev;
+ struct flow_offload_tuple *tuple;
+ bool needs_gso_segment;
+};
+
+static void __nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
+ struct nf_flow_xmit *xmit)
+{
+ struct net_device *dev = xmit->outdev;
+ unsigned int hh_len = LL_RESERVED_SPACE(dev);
+
+ if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
+ skb = skb_expand_head(skb, hh_len);
+ if (!skb)
+ return;
+ }
+
+ skb->dev = dev;
+ dev_hard_header(skb, dev, ntohs(skb->protocol),
+ xmit->dest, xmit->source, skb->len);
+ dev_queue_xmit(skb);
+}
+
+static unsigned int nf_flow_encap_gso_xmit(struct net *net, struct sk_buff *skb,
+ struct nf_flow_xmit *xmit)
+{
+ struct sk_buff *segs, *nskb;
+
+ segs = skb_gso_segment(skb, 0);
+ if (IS_ERR(segs))
+ return NF_DROP;
+
+ if (segs)
+ consume_skb(skb);
+ else
+ segs = skb;
+
+ skb_list_walk_safe(segs, segs, nskb) {
+ skb_mark_not_on_list(segs);
+
+ if (nf_flow_encap_push(segs, xmit->tuple, xmit->outdev) < 0) {
+ kfree_skb(segs);
+ kfree_skb_list(nskb);
+ return NF_STOLEN;
+ }
+ __nf_flow_queue_xmit(net, segs, xmit);
+ }
+
+ return NF_STOLEN;
+}
+
+static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
+ struct nf_flow_xmit *xmit)
+{
+ if (xmit->tuple->encap_num) {
+ if (skb_is_gso(skb) && xmit->needs_gso_segment)
+ return nf_flow_encap_gso_xmit(net, skb, xmit);
+
+ if (nf_flow_encap_push(skb, xmit->tuple, xmit->outdev) < 0)
+ return NF_DROP;
+ }
+
+ __nf_flow_queue_xmit(net, skb, xmit);
+
+ return NF_STOLEN;
+}
+
unsigned int
nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
const struct nf_hook_state *state)
@@ -797,9 +880,6 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, &ip_daddr) < 0)
return NF_DROP;
- if (nf_flow_encap_push(skb, other_tuple) < 0)
- return NF_DROP;
-
switch (tuplehash->tuple.xmit_type) {
case FLOW_OFFLOAD_XMIT_NEIGH:
rt = dst_rtable(tuplehash->tuple.dst_cache);
@@ -829,6 +909,8 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
WARN_ON_ONCE(1);
return NF_DROP;
}
+ xmit.tuple = other_tuple;
+ xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
return nf_flow_queue_xmit(state->net, skb, &xmit);
}
@@ -1037,7 +1119,7 @@ static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx,
return 0;
}
- if (skb_try_make_writable(skb, thoff + ctx->hdrsize))
+ if (skb_ensure_writable(skb, thoff + ctx->hdrsize))
return -1;
flow_offload_refresh(flow_table, flow, false);
@@ -1119,9 +1201,6 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
&ip6_daddr, encap_limit) < 0)
return NF_DROP;
- if (nf_flow_encap_push(skb, other_tuple) < 0)
- return NF_DROP;
-
switch (tuplehash->tuple.xmit_type) {
case FLOW_OFFLOAD_XMIT_NEIGH:
rt = dst_rt6_info(tuplehash->tuple.dst_cache);
@@ -1151,6 +1230,8 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
WARN_ON_ONCE(1);
return NF_DROP;
}
+ xmit.tuple = other_tuple;
+ xmit.needs_gso_segment = tuplehash->tuple.needs_gso_segment;
return nf_flow_queue_xmit(state->net, skb, &xmit);
}
diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c
index 6bb9579..9e88ea6 100644
--- a/net/netfilter/nf_flow_table_path.c
+++ b/net/netfilter/nf_flow_table_path.c
@@ -86,6 +86,7 @@ struct nft_forward_info {
u8 ingress_vlans;
u8 h_source[ETH_ALEN];
u8 h_dest[ETH_ALEN];
+ bool needs_gso_segment;
enum flow_offload_xmit_type xmit_type;
};
@@ -138,8 +139,11 @@ static void nft_dev_path_info(const struct net_device_path_stack *stack,
path->encap.proto;
info->num_encaps++;
}
- if (path->type == DEV_PATH_PPPOE)
+ if (path->type == DEV_PATH_PPPOE) {
memcpy(info->h_dest, path->encap.h_dest, ETH_ALEN);
+ info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
+ info->needs_gso_segment = 1;
+ }
break;
case DEV_PATH_BRIDGE:
if (is_zero_ether_addr(info->h_source))
@@ -279,6 +283,7 @@ static void nft_dev_forward_path(const struct nft_pktinfo *pkt,
memcpy(route->tuple[dir].out.h_dest, info.h_dest, ETH_ALEN);
route->tuple[dir].xmit_type = info.xmit_type;
}
+ route->tuple[dir].out.needs_gso_segment = info.needs_gso_segment;
}
int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct,
diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c
index c845b6d..9fbfc6b 100644
--- a/net/netfilter/nf_nat_sip.c
+++ b/net/netfilter/nf_nat_sip.c
@@ -246,6 +246,7 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff,
if (ct_sip_parse_numerical_param(ct, *dptr, matchend, *datalen,
"rport=", &poff, &plen,
&n) > 0 &&
+ n >= 1024 && n <= 65535 &&
htons(n) == ct->tuplehash[dir].tuple.dst.u.udp.port &&
htons(n) != ct->tuplehash[!dir].tuple.src.u.udp.port) {
__be16 p = ct->tuplehash[!dir].tuple.src.u.udp.port;
diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c
index a6c81c0..57b4500 100644
--- a/net/netfilter/nf_queue.c
+++ b/net/netfilter/nf_queue.c
@@ -61,6 +61,7 @@ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry)
struct nf_hook_state *state = &entry->state;
/* Release those devices we held, or Alexey will kill me. */
+ dev_put(entry->skb_dev);
dev_put(state->in);
dev_put(state->out);
if (state->sk)
@@ -102,6 +103,7 @@ bool nf_queue_entry_get_refs(struct nf_queue_entry *entry)
if (state->sk && !refcount_inc_not_zero(&state->sk->sk_refcnt))
return false;
+ dev_hold(entry->skb_dev);
dev_hold(state->in);
dev_hold(state->out);
@@ -202,11 +204,11 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
*entry = (struct nf_queue_entry) {
.skb = skb,
+ .skb_dev = skb->dev,
.state = *state,
.hook_index = index,
.size = sizeof(*entry) + route_key_size,
};
-
__nf_queue_entry_init_physdevs(entry);
if (!nf_queue_entry_get_refs(entry)) {
diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c
index 57f57e2..ed00114 100644
--- a/net/netfilter/nf_synproxy_core.c
+++ b/net/netfilter/nf_synproxy_core.c
@@ -22,6 +22,8 @@
#include <net/netfilter/nf_conntrack_zones.h>
#include <net/netfilter/nf_synproxy.h>
+static DEFINE_MUTEX(synproxy_mutex);
+
unsigned int synproxy_net_id;
EXPORT_SYMBOL_GPL(synproxy_net_id);
@@ -200,6 +202,8 @@ synproxy_tstamp_adjust(struct sk_buff *skb, unsigned int protoff,
if (skb_ensure_writable(skb, optend))
return 0;
+ th = (struct tcphdr *)(skb->data + protoff);
+
while (optoff < optend) {
unsigned char *op = skb->data + optoff;
@@ -767,26 +771,31 @@ static const struct nf_hook_ops ipv4_synproxy_ops[] = {
int nf_synproxy_ipv4_init(struct synproxy_net *snet, struct net *net)
{
- int err;
+ int err = 0;
+ mutex_lock(&synproxy_mutex);
if (snet->hook_ref4 == 0) {
err = nf_register_net_hooks(net, ipv4_synproxy_ops,
ARRAY_SIZE(ipv4_synproxy_ops));
if (err)
- return err;
+ goto out;
}
snet->hook_ref4++;
- return 0;
+out:
+ mutex_unlock(&synproxy_mutex);
+ return err;
}
EXPORT_SYMBOL_GPL(nf_synproxy_ipv4_init);
void nf_synproxy_ipv4_fini(struct synproxy_net *snet, struct net *net)
{
+ mutex_lock(&synproxy_mutex);
snet->hook_ref4--;
if (snet->hook_ref4 == 0)
nf_unregister_net_hooks(net, ipv4_synproxy_ops,
ARRAY_SIZE(ipv4_synproxy_ops));
+ mutex_unlock(&synproxy_mutex);
}
EXPORT_SYMBOL_GPL(nf_synproxy_ipv4_fini);
@@ -1191,27 +1200,32 @@ static const struct nf_hook_ops ipv6_synproxy_ops[] = {
int
nf_synproxy_ipv6_init(struct synproxy_net *snet, struct net *net)
{
- int err;
+ int err = 0;
+ mutex_lock(&synproxy_mutex);
if (snet->hook_ref6 == 0) {
err = nf_register_net_hooks(net, ipv6_synproxy_ops,
ARRAY_SIZE(ipv6_synproxy_ops));
if (err)
- return err;
+ goto out;
}
snet->hook_ref6++;
- return 0;
+out:
+ mutex_unlock(&synproxy_mutex);
+ return err;
}
EXPORT_SYMBOL_GPL(nf_synproxy_ipv6_init);
void
nf_synproxy_ipv6_fini(struct synproxy_net *snet, struct net *net)
{
+ mutex_lock(&synproxy_mutex);
snet->hook_ref6--;
if (snet->hook_ref6 == 0)
nf_unregister_net_hooks(net, ipv6_synproxy_ops,
ARRAY_SIZE(ipv6_synproxy_ops));
+ mutex_unlock(&synproxy_mutex);
}
EXPORT_SYMBOL_GPL(nf_synproxy_ipv6_fini);
#endif /* CONFIG_IPV6 */
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index 8537b94..87387ad 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -374,7 +374,40 @@ static void nft_netdev_hook_free_rcu(struct nft_hook *hook)
call_rcu(&hook->rcu, __nft_netdev_hook_free_rcu);
}
+static void nft_netdev_hook_unlink_free_rcu(struct nft_hook *hook)
+{
+ list_del_rcu(&hook->list);
+ nft_netdev_hook_free_rcu(hook);
+}
+
+static void nft_trans_hook_destroy(struct nft_trans_hook *trans_hook)
+{
+ list_del(&trans_hook->list);
+ kfree(trans_hook);
+}
+
+static void nft_netdev_unregister_trans_hook(struct net *net,
+ const struct nft_table *table,
+ struct list_head *hook_list)
+{
+ struct nft_trans_hook *trans_hook, *next;
+ struct nf_hook_ops *ops;
+ struct nft_hook *hook;
+
+ list_for_each_entry_safe(trans_hook, next, hook_list, list) {
+ hook = trans_hook->hook;
+
+ if (!(table->flags & NFT_TABLE_F_DORMANT)) {
+ list_for_each_entry(ops, &hook->ops_list, list)
+ nf_unregister_net_hook(net, ops);
+ }
+ nft_netdev_hook_unlink_free_rcu(hook);
+ nft_trans_hook_destroy(trans_hook);
+ }
+}
+
static void nft_netdev_unregister_hooks(struct net *net,
+ const struct nft_table *table,
struct list_head *hook_list,
bool release_netdev)
{
@@ -382,12 +415,12 @@ static void nft_netdev_unregister_hooks(struct net *net,
struct nf_hook_ops *ops;
list_for_each_entry_safe(hook, next, hook_list, list) {
- list_for_each_entry(ops, &hook->ops_list, list)
- nf_unregister_net_hook(net, ops);
- if (release_netdev) {
- list_del(&hook->list);
- nft_netdev_hook_free_rcu(hook);
+ if (!(table->flags & NFT_TABLE_F_DORMANT)) {
+ list_for_each_entry(ops, &hook->ops_list, list)
+ nf_unregister_net_hook(net, ops);
}
+ if (release_netdev)
+ nft_netdev_hook_unlink_free_rcu(hook);
}
}
@@ -422,20 +455,25 @@ static void __nf_tables_unregister_hook(struct net *net,
struct nft_base_chain *basechain;
const struct nf_hook_ops *ops;
- if (table->flags & NFT_TABLE_F_DORMANT ||
- !nft_is_base_chain(chain))
+ if (!nft_is_base_chain(chain))
return;
basechain = nft_base_chain(chain);
ops = &basechain->ops;
+ /* must also be called for dormant tables */
+ if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
+ nft_netdev_unregister_hooks(net, table, &basechain->hook_list,
+ release_netdev);
+ return;
+ }
+
+ if (table->flags & NFT_TABLE_F_DORMANT)
+ return;
+
if (basechain->type->ops_unregister)
return basechain->type->ops_unregister(net, ops);
- if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
- nft_netdev_unregister_hooks(net, &basechain->hook_list,
- release_netdev);
- else
- nf_unregister_net_hook(net, &basechain->ops);
+ nf_unregister_net_hook(net, &basechain->ops);
}
static void nf_tables_unregister_hook(struct net *net,
@@ -1942,15 +1980,69 @@ static int nft_nla_put_hook_dev(struct sk_buff *skb, struct nft_hook *hook)
return nla_put_string(skb, attr, hook->ifname);
}
+struct nft_hook_dump_ctx {
+ struct nft_hook *first;
+ int n;
+};
+
+static int nft_dump_basechain_hook_one(struct sk_buff *skb,
+ struct nft_hook *hook,
+ struct nft_hook_dump_ctx *dump_ctx)
+{
+ if (!dump_ctx->first)
+ dump_ctx->first = hook;
+
+ if (nft_nla_put_hook_dev(skb, hook))
+ return -1;
+
+ dump_ctx->n++;
+
+ return 0;
+}
+
+static int nft_dump_basechain_hook_list(struct sk_buff *skb,
+ const struct net *net,
+ const struct list_head *hook_list,
+ struct nft_hook_dump_ctx *dump_ctx)
+{
+ struct nft_hook *hook;
+ int err;
+
+ list_for_each_entry_rcu(hook, hook_list, list,
+ lockdep_commit_lock_is_held(net)) {
+ err = nft_dump_basechain_hook_one(skb, hook, dump_ctx);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+
+static int nft_dump_basechain_trans_hook_list(struct sk_buff *skb,
+ const struct list_head *trans_hook_list,
+ struct nft_hook_dump_ctx *dump_ctx)
+{
+ struct nft_trans_hook *trans_hook;
+ int err;
+
+ list_for_each_entry(trans_hook, trans_hook_list, list) {
+ err = nft_dump_basechain_hook_one(skb, trans_hook->hook, dump_ctx);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+
static int nft_dump_basechain_hook(struct sk_buff *skb,
const struct net *net, int family,
const struct nft_base_chain *basechain,
- const struct list_head *hook_list)
+ const struct list_head *hook_list,
+ const struct list_head *trans_hook_list)
{
const struct nf_hook_ops *ops = &basechain->ops;
- struct nft_hook *hook, *first = NULL;
+ struct nft_hook_dump_ctx dump_hook_ctx = {};
struct nlattr *nest, *nest_devs;
- int n = 0;
nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
if (nest == NULL)
@@ -1965,23 +2057,23 @@ static int nft_dump_basechain_hook(struct sk_buff *skb,
if (!nest_devs)
goto nla_put_failure;
- if (!hook_list)
+ if (!hook_list && !trans_hook_list)
hook_list = &basechain->hook_list;
- list_for_each_entry_rcu(hook, hook_list, list,
- lockdep_commit_lock_is_held(net)) {
- if (!first)
- first = hook;
-
- if (nft_nla_put_hook_dev(skb, hook))
- goto nla_put_failure;
- n++;
+ if (hook_list &&
+ nft_dump_basechain_hook_list(skb, net, hook_list, &dump_hook_ctx)) {
+ goto nla_put_failure;
+ } else if (trans_hook_list &&
+ nft_dump_basechain_trans_hook_list(skb, trans_hook_list,
+ &dump_hook_ctx)) {
+ goto nla_put_failure;
}
+
nla_nest_end(skb, nest_devs);
- if (n == 1 &&
- !hook_is_prefix(first) &&
- nla_put_string(skb, NFTA_HOOK_DEV, first->ifname))
+ if (dump_hook_ctx.n == 1 &&
+ !hook_is_prefix(dump_hook_ctx.first) &&
+ nla_put_string(skb, NFTA_HOOK_DEV, dump_hook_ctx.first->ifname))
goto nla_put_failure;
}
nla_nest_end(skb, nest);
@@ -1995,7 +2087,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
u32 portid, u32 seq, int event, u32 flags,
int family, const struct nft_table *table,
const struct nft_chain *chain,
- const struct list_head *hook_list)
+ const struct list_head *hook_list,
+ const struct list_head *trans_hook_list)
{
struct nlmsghdr *nlh;
@@ -2011,7 +2104,7 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
NFTA_CHAIN_PAD))
goto nla_put_failure;
- if (!hook_list &&
+ if (!hook_list && !trans_hook_list &&
(event == NFT_MSG_DELCHAIN ||
event == NFT_MSG_DESTROYCHAIN)) {
nlmsg_end(skb, nlh);
@@ -2022,7 +2115,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
const struct nft_base_chain *basechain = nft_base_chain(chain);
struct nft_stats __percpu *stats;
- if (nft_dump_basechain_hook(skb, net, family, basechain, hook_list))
+ if (nft_dump_basechain_hook(skb, net, family, basechain,
+ hook_list, trans_hook_list))
goto nla_put_failure;
if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
@@ -2058,7 +2152,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
}
static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
- const struct list_head *hook_list)
+ const struct list_head *hook_list,
+ const struct list_head *trans_hook_list)
{
struct nftables_pernet *nft_net;
struct sk_buff *skb;
@@ -2078,7 +2173,7 @@ static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
event, flags, ctx->family, ctx->table,
- ctx->chain, hook_list);
+ ctx->chain, hook_list, trans_hook_list);
if (err < 0) {
kfree_skb(skb);
goto err;
@@ -2124,7 +2219,7 @@ static int nf_tables_dump_chains(struct sk_buff *skb,
NFT_MSG_NEWCHAIN,
NLM_F_MULTI,
table->family, table,
- chain, NULL) < 0)
+ chain, NULL, NULL) < 0)
goto done;
nl_dump_check_consistent(cb, nlmsg_hdr(skb));
@@ -2178,7 +2273,7 @@ static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
- 0, family, table, chain, NULL);
+ 0, family, table, chain, NULL, NULL);
if (err < 0)
goto err_fill_chain_info;
@@ -2271,10 +2366,8 @@ void nf_tables_chain_destroy(struct nft_chain *chain)
if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
list_for_each_entry_safe(hook, next,
- &basechain->hook_list, list) {
- list_del_rcu(&hook->list);
- nft_netdev_hook_free_rcu(hook);
- }
+ &basechain->hook_list, list)
+ nft_netdev_hook_unlink_free_rcu(hook);
}
module_put(basechain->type->owner);
if (rcu_access_pointer(basechain->stats)) {
@@ -2343,8 +2436,12 @@ static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
list_for_each_entry(hook, hook_list, list) {
if (!strncmp(hook->ifname, this->ifname,
- min(hook->ifnamelen, this->ifnamelen)))
+ min(hook->ifnamelen, this->ifnamelen))) {
+ if (hook->flags & NFT_HOOK_REMOVE)
+ continue;
+
return hook;
+ }
}
return NULL;
@@ -2974,6 +3071,7 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
list_for_each_entry(ops, &h->ops_list, list)
nf_unregister_net_hook(ctx->net, ops);
}
+ /* hook.list is on stack, no need for list_del_rcu() */
list_del(&h->list);
nft_netdev_hook_free_rcu(h);
}
@@ -3102,6 +3200,32 @@ static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
return nf_tables_addchain(&ctx, family, policy, flags, extack);
}
+static int nft_trans_delhook(struct nft_hook *hook,
+ struct list_head *del_list)
+{
+ struct nft_trans_hook *trans_hook;
+
+ trans_hook = kmalloc_obj(*trans_hook, GFP_KERNEL);
+ if (!trans_hook)
+ return -ENOMEM;
+
+ trans_hook->hook = hook;
+ list_add_tail(&trans_hook->list, del_list);
+ hook->flags |= NFT_HOOK_REMOVE;
+
+ return 0;
+}
+
+static void nft_trans_delhook_abort(struct list_head *del_list)
+{
+ struct nft_trans_hook *trans_hook, *next;
+
+ list_for_each_entry_safe(trans_hook, next, del_list, list) {
+ trans_hook->hook->flags &= ~NFT_HOOK_REMOVE;
+ nft_trans_hook_destroy(trans_hook);
+ }
+}
+
static int nft_delchain_hook(struct nft_ctx *ctx,
struct nft_base_chain *basechain,
struct netlink_ext_ack *extack)
@@ -3128,7 +3252,10 @@ static int nft_delchain_hook(struct nft_ctx *ctx,
err = -ENOENT;
goto err_chain_del_hook;
}
- list_move(&hook->list, &chain_del_list);
+ if (nft_trans_delhook(hook, &chain_del_list) < 0) {
+ err = -ENOMEM;
+ goto err_chain_del_hook;
+ }
}
trans = nft_trans_alloc_chain(ctx, NFT_MSG_DELCHAIN);
@@ -3148,7 +3275,7 @@ static int nft_delchain_hook(struct nft_ctx *ctx,
return 0;
err_chain_del_hook:
- list_splice(&chain_del_list, &basechain->hook_list);
+ nft_trans_delhook_abort(&chain_del_list);
nft_chain_release_hook(&chain_hook);
return err;
@@ -4086,6 +4213,7 @@ static int nft_table_validate(struct net *net, const struct nft_table *table)
struct nft_chain *chain;
struct nft_ctx ctx = {
.net = net,
+ .table = (struct nft_table *)table,
.family = table->family,
};
int err = 0;
@@ -8852,10 +8980,8 @@ static void __nft_unregister_flowtable_net_hooks(struct net *net,
list_for_each_entry_safe(hook, next, hook_list, list) {
list_for_each_entry(ops, &hook->ops_list, list)
nft_unregister_flowtable_ops(net, flowtable, ops);
- if (release_netdev) {
- list_del(&hook->list);
- nft_netdev_hook_free_rcu(hook);
- }
+ if (release_netdev)
+ nft_netdev_hook_unlink_free_rcu(hook);
}
}
@@ -8926,8 +9052,7 @@ static int nft_register_flowtable_net_hooks(struct net *net,
nft_unregister_flowtable_ops(net, flowtable, ops);
}
- list_del_rcu(&hook->list);
- nft_netdev_hook_free_rcu(hook);
+ nft_netdev_hook_unlink_free_rcu(hook);
}
return err;
@@ -8937,9 +9062,25 @@ static void nft_hooks_destroy(struct list_head *hook_list)
{
struct nft_hook *hook, *next;
- list_for_each_entry_safe(hook, next, hook_list, list) {
- list_del_rcu(&hook->list);
- nft_netdev_hook_free_rcu(hook);
+ list_for_each_entry_safe(hook, next, hook_list, list)
+ nft_netdev_hook_unlink_free_rcu(hook);
+}
+
+static void nft_flowtable_unregister_trans_hook(struct net *net,
+ struct nft_flowtable *flowtable,
+ struct list_head *hook_list)
+{
+ struct nft_trans_hook *trans_hook, *next;
+ struct nf_hook_ops *ops;
+ struct nft_hook *hook;
+
+ list_for_each_entry_safe(trans_hook, next, hook_list, list) {
+ hook = trans_hook->hook;
+ list_for_each_entry(ops, &hook->ops_list, list)
+ nft_unregister_flowtable_ops(net, flowtable, ops);
+
+ nft_netdev_hook_unlink_free_rcu(hook);
+ nft_trans_hook_destroy(trans_hook);
}
}
@@ -9028,8 +9169,7 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
nft_unregister_flowtable_ops(ctx->net,
flowtable, ops);
}
- list_del_rcu(&hook->list);
- nft_netdev_hook_free_rcu(hook);
+ nft_netdev_hook_unlink_free_rcu(hook);
}
return err;
@@ -9202,7 +9342,10 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx,
err = -ENOENT;
goto err_flowtable_del_hook;
}
- list_move(&hook->list, &flowtable_del_list);
+ if (nft_trans_delhook(hook, &flowtable_del_list) < 0) {
+ err = -ENOMEM;
+ goto err_flowtable_del_hook;
+ }
}
trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
@@ -9223,7 +9366,7 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx,
return 0;
err_flowtable_del_hook:
- list_splice(&flowtable_del_list, &flowtable->hook_list);
+ nft_trans_delhook_abort(&flowtable_del_list);
nft_flowtable_hook_release(&flowtable_hook);
return err;
@@ -9288,8 +9431,10 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
u32 portid, u32 seq, int event,
u32 flags, int family,
struct nft_flowtable *flowtable,
- struct list_head *hook_list)
+ struct list_head *hook_list,
+ struct list_head *trans_hook_list)
{
+ struct nft_trans_hook *trans_hook;
struct nlattr *nest, *nest_devs;
struct nft_hook *hook;
struct nlmsghdr *nlh;
@@ -9306,7 +9451,7 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
NFTA_FLOWTABLE_PAD))
goto nla_put_failure;
- if (!hook_list &&
+ if (!hook_list && !trans_hook_list &&
(event == NFT_MSG_DELFLOWTABLE ||
event == NFT_MSG_DESTROYFLOWTABLE)) {
nlmsg_end(skb, nlh);
@@ -9328,13 +9473,20 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
if (!nest_devs)
goto nla_put_failure;
- if (!hook_list)
+ if (!hook_list && !trans_hook_list)
hook_list = &flowtable->hook_list;
- list_for_each_entry_rcu(hook, hook_list, list,
- lockdep_commit_lock_is_held(net)) {
- if (nft_nla_put_hook_dev(skb, hook))
- goto nla_put_failure;
+ if (hook_list) {
+ list_for_each_entry_rcu(hook, hook_list, list,
+ lockdep_commit_lock_is_held(net)) {
+ if (nft_nla_put_hook_dev(skb, hook))
+ goto nla_put_failure;
+ }
+ } else if (trans_hook_list) {
+ list_for_each_entry(trans_hook, trans_hook_list, list) {
+ if (nft_nla_put_hook_dev(skb, trans_hook->hook))
+ goto nla_put_failure;
+ }
}
nla_nest_end(skb, nest_devs);
nla_nest_end(skb, nest);
@@ -9388,7 +9540,7 @@ static int nf_tables_dump_flowtable(struct sk_buff *skb,
NFT_MSG_NEWFLOWTABLE,
NLM_F_MULTI | NLM_F_APPEND,
table->family,
- flowtable, NULL) < 0)
+ flowtable, NULL, NULL) < 0)
goto done;
nl_dump_check_consistent(cb, nlmsg_hdr(skb));
@@ -9488,7 +9640,7 @@ static int nf_tables_getflowtable(struct sk_buff *skb,
err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
info->nlh->nlmsg_seq,
NFT_MSG_NEWFLOWTABLE, 0, family,
- flowtable, NULL);
+ flowtable, NULL, NULL);
if (err < 0)
goto err_fill_flowtable_info;
@@ -9501,7 +9653,9 @@ static int nf_tables_getflowtable(struct sk_buff *skb,
static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
struct nft_flowtable *flowtable,
- struct list_head *hook_list, int event)
+ struct list_head *hook_list,
+ struct list_head *trans_hook_list,
+ int event)
{
struct nftables_pernet *nft_net = nft_pernet(ctx->net);
struct sk_buff *skb;
@@ -9521,7 +9675,8 @@ static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
ctx->seq, event, flags,
- ctx->family, flowtable, hook_list);
+ ctx->family, flowtable,
+ hook_list, trans_hook_list);
if (err < 0) {
kfree_skb(skb);
goto err;
@@ -9535,13 +9690,8 @@ static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
{
- struct nft_hook *hook, *next;
-
flowtable->data.type->free(&flowtable->data);
- list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
- list_del_rcu(&hook->list);
- nft_netdev_hook_free_rcu(hook);
- }
+ nft_hooks_destroy(&flowtable->hook_list);
kfree(flowtable->name);
module_put(flowtable->data.type->owner);
kfree(flowtable);
@@ -10060,9 +10210,7 @@ static void nft_commit_release(struct nft_trans *trans)
break;
case NFT_MSG_DELCHAIN:
case NFT_MSG_DESTROYCHAIN:
- if (nft_trans_chain_update(trans))
- nft_hooks_destroy(&nft_trans_chain_hooks(trans));
- else
+ if (!nft_trans_chain_update(trans))
nf_tables_chain_destroy(nft_trans_chain(trans));
break;
case NFT_MSG_DELRULE:
@@ -10083,9 +10231,7 @@ static void nft_commit_release(struct nft_trans *trans)
break;
case NFT_MSG_DELFLOWTABLE:
case NFT_MSG_DESTROYFLOWTABLE:
- if (nft_trans_flowtable_update(trans))
- nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
- else
+ if (!nft_trans_flowtable_update(trans))
nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
break;
}
@@ -10845,31 +10991,28 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
if (nft_trans_chain_update(trans)) {
nft_chain_commit_update(nft_trans_container_chain(trans));
nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN,
- &nft_trans_chain_hooks(trans));
- list_splice(&nft_trans_chain_hooks(trans),
- &nft_trans_basechain(trans)->hook_list);
+ &nft_trans_chain_hooks(trans), NULL);
+ list_splice_rcu(&nft_trans_chain_hooks(trans),
+ &nft_trans_basechain(trans)->hook_list);
/* trans destroyed after rcu grace period */
} else {
nft_chain_commit_drop_policy(nft_trans_container_chain(trans));
nft_clear(net, nft_trans_chain(trans));
- nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL);
+ nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL, NULL);
nft_trans_destroy(trans);
}
break;
case NFT_MSG_DELCHAIN:
case NFT_MSG_DESTROYCHAIN:
if (nft_trans_chain_update(trans)) {
- nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
+ nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN, NULL,
&nft_trans_chain_hooks(trans));
- if (!(table->flags & NFT_TABLE_F_DORMANT)) {
- nft_netdev_unregister_hooks(net,
- &nft_trans_chain_hooks(trans),
- true);
- }
+ nft_netdev_unregister_trans_hook(net, table,
+ &nft_trans_chain_hooks(trans));
} else {
nft_chain_del(nft_trans_chain(trans));
nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
- NULL);
+ NULL, NULL);
nf_tables_unregister_hook(ctx.net, ctx.table,
nft_trans_chain(trans));
}
@@ -10975,14 +11118,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
nf_tables_flowtable_notify(&ctx,
nft_trans_flowtable(trans),
&nft_trans_flowtable_hooks(trans),
+ NULL,
NFT_MSG_NEWFLOWTABLE);
- list_splice(&nft_trans_flowtable_hooks(trans),
- &nft_trans_flowtable(trans)->hook_list);
+ list_splice_rcu(&nft_trans_flowtable_hooks(trans),
+ &nft_trans_flowtable(trans)->hook_list);
} else {
nft_clear(net, nft_trans_flowtable(trans));
nf_tables_flowtable_notify(&ctx,
nft_trans_flowtable(trans),
NULL,
+ NULL,
NFT_MSG_NEWFLOWTABLE);
}
nft_trans_destroy(trans);
@@ -10992,16 +11137,18 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
if (nft_trans_flowtable_update(trans)) {
nf_tables_flowtable_notify(&ctx,
nft_trans_flowtable(trans),
+ NULL,
&nft_trans_flowtable_hooks(trans),
trans->msg_type);
- nft_unregister_flowtable_net_hooks(net,
- nft_trans_flowtable(trans),
- &nft_trans_flowtable_hooks(trans));
+ nft_flowtable_unregister_trans_hook(net,
+ nft_trans_flowtable(trans),
+ &nft_trans_flowtable_hooks(trans));
} else {
list_del_rcu(&nft_trans_flowtable(trans)->list);
nf_tables_flowtable_notify(&ctx,
nft_trans_flowtable(trans),
NULL,
+ NULL,
trans->msg_type);
nft_unregister_flowtable_net_hooks(net,
nft_trans_flowtable(trans),
@@ -11143,11 +11290,9 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
break;
case NFT_MSG_NEWCHAIN:
if (nft_trans_chain_update(trans)) {
- if (!(table->flags & NFT_TABLE_F_DORMANT)) {
- nft_netdev_unregister_hooks(net,
- &nft_trans_chain_hooks(trans),
- true);
- }
+ nft_netdev_unregister_hooks(net, table,
+ &nft_trans_chain_hooks(trans),
+ true);
free_percpu(nft_trans_chain_stats(trans));
kfree(nft_trans_chain_name(trans));
nft_trans_destroy(trans);
@@ -11165,8 +11310,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
case NFT_MSG_DELCHAIN:
case NFT_MSG_DESTROYCHAIN:
if (nft_trans_chain_update(trans)) {
- list_splice(&nft_trans_chain_hooks(trans),
- &nft_trans_basechain(trans)->hook_list);
+ nft_trans_delhook_abort(&nft_trans_chain_hooks(trans));
} else {
nft_use_inc_restore(&table->use);
nft_clear(trans->net, nft_trans_chain(trans));
@@ -11280,8 +11424,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
case NFT_MSG_DELFLOWTABLE:
case NFT_MSG_DESTROYFLOWTABLE:
if (nft_trans_flowtable_update(trans)) {
- list_splice(&nft_trans_flowtable_hooks(trans),
- &nft_trans_flowtable(trans)->hook_list);
+ nft_trans_delhook_abort(&nft_trans_flowtable_hooks(trans));
} else {
nft_use_inc_restore(&table->use);
nft_clear(trans->net, nft_trans_flowtable(trans));
diff --git a/net/netfilter/nf_tables_core.c b/net/netfilter/nf_tables_core.c
index 5ddd5b6..8ab186f 100644
--- a/net/netfilter/nf_tables_core.c
+++ b/net/netfilter/nf_tables_core.c
@@ -153,7 +153,7 @@ static bool nft_payload_fast_eval(const struct nft_expr *expr,
if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
ptr = skb_network_header(skb) + pkt->nhoff;
else {
- if (!(pkt->flags & NFT_PKTINFO_L4PROTO))
+ if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
return false;
ptr = skb->data + nft_thoff(pkt);
}
diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c
index 58304fd..60ab88d 100644
--- a/net/netfilter/nfnetlink_queue.c
+++ b/net/netfilter/nfnetlink_queue.c
@@ -1141,6 +1141,9 @@ nfqnl_mangle(void *data, unsigned int data_len, struct nf_queue_entry *e, int di
{
struct sk_buff *nskb;
+ if (e->state.net->user_ns != &init_user_ns)
+ return -EPERM;
+
if (diff < 0) {
unsigned int min_len = skb_transport_offset(e->skb);
@@ -1212,6 +1215,8 @@ dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex)
if (physinif == ifindex || physoutif == ifindex)
return 1;
#endif
+ if (entry->skb_dev && entry->skb_dev->ifindex == ifindex)
+ return 1;
if (entry->state.in)
if (entry->state.in->ifindex == ifindex)
return 1;
@@ -1535,8 +1540,7 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info,
if (nfqnl_mangle(nla_data(nfqa[NFQA_PAYLOAD]),
payload_len, entry, diff) < 0)
verdict = NF_DROP;
-
- if (ct && diff)
+ else if (ct && diff)
nfnl_ct->seq_adjust(entry->skb, ct, ctinfo, diff);
}
diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c
index 13808e9..785b8e9 100644
--- a/net/netfilter/nft_bitwise.c
+++ b/net/netfilter/nft_bitwise.c
@@ -43,8 +43,10 @@ static void nft_bitwise_eval_lshift(u32 *dst, const u32 *src,
u32 carry = 0;
for (i = DIV_ROUND_UP(priv->len, sizeof(u32)); i > 0; i--) {
- dst[i - 1] = (src[i - 1] << shift) | carry;
- carry = src[i - 1] >> (BITS_PER_TYPE(u32) - shift);
+ u32 tmp_src = src[i - 1];
+
+ dst[i - 1] = (tmp_src << shift) | carry;
+ carry = tmp_src >> (BITS_PER_TYPE(u32) - shift);
}
}
@@ -56,8 +58,10 @@ static void nft_bitwise_eval_rshift(u32 *dst, const u32 *src,
u32 carry = 0;
for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++) {
- dst[i] = carry | (src[i] >> shift);
- carry = src[i] << (BITS_PER_TYPE(u32) - shift);
+ u32 tmp_src = src[i];
+
+ dst[i] = carry | (tmp_src >> shift);
+ carry = tmp_src << (BITS_PER_TYPE(u32) - shift);
}
}
@@ -196,7 +200,8 @@ static int nft_bitwise_init_shift(struct nft_bitwise *priv,
if (err < 0)
return err;
- if (priv->data.data[0] >= BITS_PER_TYPE(u32)) {
+ if (!priv->data.data[0] ||
+ priv->data.data[0] >= BITS_PER_TYPE(u32)) {
nft_data_release(&priv->data, desc.type);
return -EINVAL;
}
@@ -234,6 +239,9 @@ static int nft_bitwise_init_bool(const struct nft_ctx *ctx,
&priv->sreg2, priv->len);
if (err < 0)
return err;
+
+ if (nft_reg_overlap(priv->sreg2, priv->dreg, priv->len))
+ return -EINVAL;
}
return 0;
@@ -264,6 +272,9 @@ static int nft_bitwise_init(const struct nft_ctx *ctx,
if (err < 0)
return err;
+ if (nft_reg_overlap(priv->sreg, priv->dreg, priv->len))
+ return -EINVAL;
+
if (tb[NFTA_BITWISE_OP]) {
priv->op = ntohl(nla_get_be32(tb[NFTA_BITWISE_OP]));
switch (priv->op) {
diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c
index e00dddf..dfd41fc 100644
--- a/net/netfilter/nft_byteorder.c
+++ b/net/netfilter/nft_byteorder.c
@@ -19,7 +19,6 @@ struct nft_byteorder {
u8 sreg;
u8 dreg;
enum nft_byteorder_ops op:8;
- u8 len;
u8 size;
};
@@ -28,13 +27,8 @@ void nft_byteorder_eval(const struct nft_expr *expr,
const struct nft_pktinfo *pkt)
{
const struct nft_byteorder *priv = nft_expr_priv(expr);
- u32 *src = ®s->data[priv->sreg];
+ const u32 *src = ®s->data[priv->sreg];
u32 *dst = ®s->data[priv->dreg];
- u16 *s16, *d16;
- unsigned int i;
-
- s16 = (void *)src;
- d16 = (void *)dst;
switch (priv->size) {
case 8: {
@@ -43,18 +37,14 @@ void nft_byteorder_eval(const struct nft_expr *expr,
switch (priv->op) {
case NFT_BYTEORDER_NTOH:
- for (i = 0; i < priv->len / 8; i++) {
- src64 = nft_reg_load64(&src[i]);
- nft_reg_store64(&dst64[i],
- be64_to_cpu((__force __be64)src64));
- }
+ src64 = nft_reg_load64(src);
+
+ nft_reg_store64(dst64, be64_to_cpu((__force __be64)src64));
break;
case NFT_BYTEORDER_HTON:
- for (i = 0; i < priv->len / 8; i++) {
- src64 = (__force __u64)
- cpu_to_be64(nft_reg_load64(&src[i]));
- nft_reg_store64(&dst64[i], src64);
- }
+ src64 = (__force __u64)cpu_to_be64(nft_reg_load64(src));
+
+ nft_reg_store64(dst64, src64);
break;
}
break;
@@ -62,24 +52,20 @@ void nft_byteorder_eval(const struct nft_expr *expr,
case 4:
switch (priv->op) {
case NFT_BYTEORDER_NTOH:
- for (i = 0; i < priv->len / 4; i++)
- dst[i] = ntohl((__force __be32)src[i]);
+ *dst = ntohl((__force __be32)*src);
break;
case NFT_BYTEORDER_HTON:
- for (i = 0; i < priv->len / 4; i++)
- dst[i] = (__force __u32)htonl(src[i]);
+ *dst = (__force __u32)htonl(*src);
break;
}
break;
case 2:
switch (priv->op) {
case NFT_BYTEORDER_NTOH:
- for (i = 0; i < priv->len / 2; i++)
- d16[i] = ntohs((__force __be16)s16[i]);
+ nft_reg_store16(dst, ntohs(nft_reg_load_be16(src)));
break;
case NFT_BYTEORDER_HTON:
- for (i = 0; i < priv->len / 2; i++)
- d16[i] = (__force __u16)htons(s16[i]);
+ nft_reg_store_be16(dst, htons(nft_reg_load16(src)));
break;
}
break;
@@ -137,16 +123,25 @@ static int nft_byteorder_init(const struct nft_ctx *ctx,
if (err < 0)
return err;
- priv->len = len;
+ /* no longer support multi-reg conversions */
+ if (len != size)
+ return -EOPNOTSUPP;
err = nft_parse_register_load(ctx, tb[NFTA_BYTEORDER_SREG], &priv->sreg,
- priv->len);
+ len);
if (err < 0)
return err;
- return nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG],
- &priv->dreg, NULL, NFT_DATA_VALUE,
- priv->len);
+ err = nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG],
+ &priv->dreg, NULL, NFT_DATA_VALUE,
+ len);
+ if (err < 0)
+ return err;
+
+ if (nft_reg_overlap(priv->sreg, priv->dreg, len))
+ return -EINVAL;
+
+ return 0;
}
static int nft_byteorder_dump(struct sk_buff *skb,
@@ -160,10 +155,11 @@ static int nft_byteorder_dump(struct sk_buff *skb,
goto nla_put_failure;
if (nla_put_be32(skb, NFTA_BYTEORDER_OP, htonl(priv->op)))
goto nla_put_failure;
- if (nla_put_be32(skb, NFTA_BYTEORDER_LEN, htonl(priv->len)))
- goto nla_put_failure;
if (nla_put_be32(skb, NFTA_BYTEORDER_SIZE, htonl(priv->size)))
goto nla_put_failure;
+ /* compatibility for old userspace which permitted size != len */
+ if (nla_put_be32(skb, NFTA_BYTEORDER_LEN, htonl(priv->size)))
+ goto nla_put_failure;
return 0;
nla_put_failure:
diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c
index decc725..0caa930 100644
--- a/net/netfilter/nft_compat.c
+++ b/net/netfilter/nft_compat.c
@@ -261,10 +261,10 @@ nft_target_init(const struct nft_ctx *ctx, const struct nft_expr *expr,
return ret;
}
- nft_target_set_tgchk_param(&par, ctx, target, info, &e, proto, inv);
-
nft_compat_wait_for_destructors(ctx->net);
+ nft_target_set_tgchk_param(&par, ctx, target, info, &e, proto, inv);
+
ret = xt_check_target(&par, size, proto, inv);
if (ret < 0) {
if (ret == -ENOENT) {
@@ -353,8 +353,6 @@ static int nft_target_dump(struct sk_buff *skb,
static int nft_target_validate(const struct nft_ctx *ctx,
const struct nft_expr *expr)
{
- struct xt_target *target = expr->ops->data;
- unsigned int hook_mask = 0;
int ret;
if (ctx->family != NFPROTO_IPV4 &&
@@ -377,11 +375,21 @@ static int nft_target_validate(const struct nft_ctx *ctx,
const struct nft_base_chain *basechain =
nft_base_chain(ctx->chain);
const struct nf_hook_ops *ops = &basechain->ops;
+ unsigned int hook_mask = 1 << ops->hooknum;
+ struct xt_target *target = expr->ops->data;
+ void *info = nft_expr_priv(expr);
+ struct xt_tgchk_param par;
+ union nft_entry e = {};
- hook_mask = 1 << ops->hooknum;
if (target->hooks && !(hook_mask & target->hooks))
return -EINVAL;
+ nft_target_set_tgchk_param(&par, ctx, target, info, &e, 0, false);
+
+ ret = xt_check_hooks_target(&par);
+ if (ret < 0)
+ return ret;
+
ret = nft_compat_chain_validate_dependency(ctx, target->table);
if (ret < 0)
return ret;
@@ -515,10 +523,10 @@ __nft_match_init(const struct nft_ctx *ctx, const struct nft_expr *expr,
return ret;
}
- nft_match_set_mtchk_param(&par, ctx, match, info, &e, proto, inv);
-
nft_compat_wait_for_destructors(ctx->net);
+ nft_match_set_mtchk_param(&par, ctx, match, info, &e, proto, inv);
+
return xt_check_match(&par, size, proto, inv);
}
@@ -614,8 +622,6 @@ static int nft_match_large_dump(struct sk_buff *skb,
static int nft_match_validate(const struct nft_ctx *ctx,
const struct nft_expr *expr)
{
- struct xt_match *match = expr->ops->data;
- unsigned int hook_mask = 0;
int ret;
if (ctx->family != NFPROTO_IPV4 &&
@@ -638,11 +644,30 @@ static int nft_match_validate(const struct nft_ctx *ctx,
const struct nft_base_chain *basechain =
nft_base_chain(ctx->chain);
const struct nf_hook_ops *ops = &basechain->ops;
+ unsigned int hook_mask = 1 << ops->hooknum;
+ struct xt_match *match = expr->ops->data;
+ size_t size = XT_ALIGN(match->matchsize);
+ struct xt_mtchk_param par;
+ union nft_entry e = {};
+ void *info;
- hook_mask = 1 << ops->hooknum;
if (match->hooks && !(hook_mask & match->hooks))
return -EINVAL;
+ if (NFT_EXPR_SIZE(size) > NFT_MATCH_LARGE_THRESH) {
+ struct nft_xt_match_priv *priv = nft_expr_priv(expr);
+
+ info = priv->info;
+ } else {
+ info = nft_expr_priv(expr);
+ }
+
+ nft_match_set_mtchk_param(&par, ctx, match, info, &e, 0, false);
+
+ ret = xt_check_hooks_match(&par);
+ if (ret < 0)
+ return ret;
+
ret = nft_compat_chain_validate_dependency(ctx, match->table);
if (ret < 0)
return ret;
diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c
index 60ee8d9..357513c6 100644
--- a/net/netfilter/nft_ct.c
+++ b/net/netfilter/nft_ct.c
@@ -78,7 +78,7 @@ static void nft_ct_get_eval(const struct nft_expr *expr,
break;
}
- if (ct == NULL)
+ if (!ct || nf_ct_is_template(ct))
goto err;
switch (priv->key) {
@@ -180,12 +180,10 @@ static void nft_ct_get_eval(const struct nft_expr *expr,
tuple = &ct->tuplehash[priv->dir].tuple;
switch (priv->key) {
case NFT_CT_SRC:
- memcpy(dest, tuple->src.u3.all,
- nf_ct_l3num(ct) == NFPROTO_IPV4 ? 4 : 16);
+ memcpy(dest, tuple->src.u3.all, priv->len);
return;
case NFT_CT_DST:
- memcpy(dest, tuple->dst.u3.all,
- nf_ct_l3num(ct) == NFPROTO_IPV4 ? 4 : 16);
+ memcpy(dest, tuple->dst.u3.all, priv->len);
return;
case NFT_CT_PROTO_SRC:
nft_reg_store16(dest, (__force u16)tuple->src.u.all);
@@ -1334,6 +1332,8 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj,
if (nf_ct_expect_related(exp, 0) != 0)
regs->verdict.code = NF_DROP;
+
+ nf_ct_expect_put(exp);
}
static const struct nla_policy nft_ct_expect_policy[NFTA_CT_EXPECT_MAX + 1] = {
diff --git a/net/netfilter/nft_ct_fast.c b/net/netfilter/nft_ct_fast.c
index e684c8a..ecf7b3a 100644
--- a/net/netfilter/nft_ct_fast.c
+++ b/net/netfilter/nft_ct_fast.c
@@ -30,7 +30,7 @@ void nft_ct_get_fast_eval(const struct nft_expr *expr,
break;
}
- if (!ct) {
+ if (!ct || nf_ct_is_template(ct)) {
regs->verdict.code = NFT_BREAK;
return;
}
diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c
index 0407d6f..e6a07c0 100644
--- a/net/netfilter/nft_exthdr.c
+++ b/net/netfilter/nft_exthdr.c
@@ -376,7 +376,7 @@ static void nft_exthdr_sctp_eval(const struct nft_expr *expr,
const struct sctp_chunkhdr *sch;
struct sctp_chunkhdr _sch;
- if (pkt->tprot != IPPROTO_SCTP)
+ if (pkt->tprot != IPPROTO_SCTP || pkt->fragoff)
goto err;
do {
diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c
index 4bce36c..b9e88d7 100644
--- a/net/netfilter/nft_fwd_netdev.c
+++ b/net/netfilter/nft_fwd_netdev.c
@@ -95,12 +95,15 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr,
struct nft_regs *regs,
const struct nft_pktinfo *pkt)
{
+ u8 *nf_dup_skb_recursion = nf_get_nf_dup_skb_recursion();
struct nft_fwd_neigh *priv = nft_expr_priv(expr);
void *addr = ®s->data[priv->sreg_addr];
int oif = regs->data[priv->sreg_dev];
unsigned int verdict = NF_STOLEN;
struct sk_buff *skb = pkt->skb;
+ int nhoff = skb_network_offset(skb);
struct net_device *dev;
+ unsigned int hh_len;
int neigh_table;
switch (priv->nfproto) {
@@ -111,7 +114,7 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr,
verdict = NFT_BREAK;
goto out;
}
- if (skb_try_make_writable(skb, sizeof(*iph))) {
+ if (skb_ensure_writable(skb, nhoff + sizeof(*iph))) {
verdict = NF_DROP;
goto out;
}
@@ -132,7 +135,7 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr,
verdict = NFT_BREAK;
goto out;
}
- if (skb_try_make_writable(skb, sizeof(*ip6h))) {
+ if (skb_ensure_writable(skb, nhoff + sizeof(*ip6h))) {
verdict = NF_DROP;
goto out;
}
@@ -151,13 +154,31 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr,
goto out;
}
+ if (*nf_dup_skb_recursion > NF_RECURSION_LIMIT) {
+ verdict = NF_DROP;
+ goto out;
+ }
+
dev = dev_get_by_index_rcu(nft_net(pkt), oif);
- if (dev == NULL)
- return;
+ if (dev == NULL) {
+ verdict = NF_DROP;
+ goto out;
+ }
+
+ hh_len = LL_RESERVED_SPACE(dev);
+ if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
+ skb = skb_expand_head(skb, hh_len);
+ if (!skb) {
+ verdict = NF_STOLEN;
+ goto out;
+ }
+ }
skb->dev = dev;
skb_clear_tstamp(skb);
+ (*nf_dup_skb_recursion)++;
neigh_xmit(neigh_table, dev, addr, skb);
+ (*nf_dup_skb_recursion)--;
out:
regs->verdict.code = verdict;
}
diff --git a/net/netfilter/nft_inner.c b/net/netfilter/nft_inner.c
index 03ffb11..d14ca15 100644
--- a/net/netfilter/nft_inner.c
+++ b/net/netfilter/nft_inner.c
@@ -163,7 +163,6 @@ static int nft_inner_parse_l2l3(const struct nft_inner *priv,
return -1;
if (fragoff == 0) {
- thoff = nhoff + sizeof(_ip6h);
ctx->flags |= NFT_PAYLOAD_CTX_INNER_TH;
ctx->inner_thoff = thoff;
ctx->l4proto = l4proto;
@@ -247,8 +246,8 @@ static bool nft_inner_restore_tun_ctx(const struct nft_pktinfo *pkt,
local_lock_nested_bh(&nft_pcpu_tun_ctx.bh_lock);
this_cpu_tun_ctx = this_cpu_ptr(&nft_pcpu_tun_ctx.ctx);
if (this_cpu_tun_ctx->cookie != (unsigned long)pkt->skb) {
- local_bh_enable();
local_unlock_nested_bh(&nft_pcpu_tun_ctx.bh_lock);
+ local_bh_enable();
return false;
}
*tun_ctx = *this_cpu_tun_ctx;
diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c
index c02d5cb..45fe56d 100644
--- a/net/netfilter/nft_osf.c
+++ b/net/netfilter/nft_osf.c
@@ -33,7 +33,7 @@ static void nft_osf_eval(const struct nft_expr *expr, struct nft_regs *regs,
return;
}
- if (pkt->tprot != IPPROTO_TCP) {
+ if (pkt->tprot != IPPROTO_TCP || pkt->fragoff) {
regs->verdict.code = NFT_BREAK;
return;
}
diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c
index 01e13e5..484a549 100644
--- a/net/netfilter/nft_payload.c
+++ b/net/netfilter/nft_payload.c
@@ -917,6 +917,9 @@ static int nft_payload_set_init(const struct nft_ctx *ctx,
struct nft_payload_set *priv = nft_expr_priv(expr);
int err;
+ if (ctx->net->user_ns != &init_user_ns)
+ return -EPERM;
+
priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c
index f2101af..89be443 100644
--- a/net/netfilter/nft_tproxy.c
+++ b/net/netfilter/nft_tproxy.c
@@ -30,8 +30,8 @@ static void nft_tproxy_eval_v4(const struct nft_expr *expr,
__be16 tport = 0;
struct sock *sk;
- if (pkt->tprot != IPPROTO_TCP &&
- pkt->tprot != IPPROTO_UDP) {
+ if ((pkt->tprot != IPPROTO_TCP &&
+ pkt->tprot != IPPROTO_UDP) || pkt->fragoff) {
regs->verdict.code = NFT_BREAK;
return;
}
@@ -97,8 +97,8 @@ static void nft_tproxy_eval_v6(const struct nft_expr *expr,
memset(&taddr, 0, sizeof(taddr));
- if (pkt->tprot != IPPROTO_TCP &&
- pkt->tprot != IPPROTO_UDP) {
+ if ((pkt->tprot != IPPROTO_TCP &&
+ pkt->tprot != IPPROTO_UDP) || pkt->fragoff) {
regs->verdict.code = NFT_BREAK;
return;
}
diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c
index 0b987bc..68f7cfb 100644
--- a/net/netfilter/nft_tunnel.c
+++ b/net/netfilter/nft_tunnel.c
@@ -676,7 +676,7 @@ static void nft_tunnel_obj_destroy(const struct nft_ctx *ctx,
{
struct nft_tunnel_obj *priv = nft_obj_data(obj);
- metadata_dst_free(priv->md);
+ dst_release(&priv->md->dst);
}
static struct nft_object_type nft_tunnel_obj_type;
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 9f837fb..4e6708c 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -55,6 +55,9 @@ static struct list_head xt_templates[NFPROTO_NUMPROTO];
struct xt_pernet {
struct list_head tables[NFPROTO_NUMPROTO];
+
+ /* stash area used during netns exit */
+ struct list_head dead_tables[NFPROTO_NUMPROTO];
};
struct compat_delta {
@@ -477,11 +480,9 @@ int xt_check_proc_name(const char *name, unsigned int size)
}
EXPORT_SYMBOL(xt_check_proc_name);
-int xt_check_match(struct xt_mtchk_param *par,
- unsigned int size, u16 proto, bool inv_proto)
+static int xt_check_match_common(struct xt_mtchk_param *par,
+ unsigned int size, u16 proto, bool inv_proto)
{
- int ret;
-
if (XT_ALIGN(par->match->matchsize) != size &&
par->match->matchsize != -1) {
/*
@@ -530,6 +531,14 @@ int xt_check_match(struct xt_mtchk_param *par,
par->match->proto);
return -EINVAL;
}
+
+ return 0;
+}
+
+static int xt_checkentry_match(struct xt_mtchk_param *par)
+{
+ int ret;
+
if (par->match->checkentry != NULL) {
ret = par->match->checkentry(par);
if (ret < 0)
@@ -538,8 +547,34 @@ int xt_check_match(struct xt_mtchk_param *par,
/* Flag up potential errors. */
return -EIO;
}
+
return 0;
}
+
+int xt_check_hooks_match(struct xt_mtchk_param *par)
+{
+ if (par->match->check_hooks != NULL)
+ return par->match->check_hooks(par);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(xt_check_hooks_match);
+
+int xt_check_match(struct xt_mtchk_param *par,
+ unsigned int size, u16 proto, bool inv_proto)
+{
+ int ret;
+
+ ret = xt_check_match_common(par, size, proto, inv_proto);
+ if (ret < 0)
+ return ret;
+
+ ret = xt_check_hooks_match(par);
+ if (ret < 0)
+ return ret;
+
+ return xt_checkentry_match(par);
+}
EXPORT_SYMBOL_GPL(xt_check_match);
/** xt_check_entry_match - check that matches end before start of target
@@ -1012,11 +1047,9 @@ bool xt_find_jump_offset(const unsigned int *offsets,
}
EXPORT_SYMBOL(xt_find_jump_offset);
-int xt_check_target(struct xt_tgchk_param *par,
- unsigned int size, u16 proto, bool inv_proto)
+static int xt_check_target_common(struct xt_tgchk_param *par,
+ unsigned int size, u16 proto, bool inv_proto)
{
- int ret;
-
if (XT_ALIGN(par->target->targetsize) != size) {
pr_err_ratelimited("%s_tables: %s.%u target: invalid size %u (kernel) != (user) %u\n",
xt_prefix[par->family], par->target->name,
@@ -1061,6 +1094,23 @@ int xt_check_target(struct xt_tgchk_param *par,
par->target->proto);
return -EINVAL;
}
+
+ return 0;
+}
+
+int xt_check_hooks_target(struct xt_tgchk_param *par)
+{
+ if (par->target->check_hooks != NULL)
+ return par->target->check_hooks(par);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(xt_check_hooks_target);
+
+static int xt_checkentry_target(struct xt_tgchk_param *par)
+{
+ int ret;
+
if (par->target->checkentry != NULL) {
ret = par->target->checkentry(par);
if (ret < 0)
@@ -1071,6 +1121,22 @@ int xt_check_target(struct xt_tgchk_param *par,
}
return 0;
}
+
+int xt_check_target(struct xt_tgchk_param *par,
+ unsigned int size, u16 proto, bool inv_proto)
+{
+ int ret;
+
+ ret = xt_check_target_common(par, size, proto, inv_proto);
+ if (ret < 0)
+ return ret;
+
+ ret = xt_check_hooks_target(par);
+ if (ret < 0)
+ return ret;
+
+ return xt_checkentry_target(par);
+}
EXPORT_SYMBOL_GPL(xt_check_target);
/**
@@ -1409,11 +1475,9 @@ struct xt_counters *xt_counters_alloc(unsigned int counters)
}
EXPORT_SYMBOL(xt_counters_alloc);
-struct xt_table_info *
-xt_replace_table(struct xt_table *table,
- unsigned int num_counters,
- struct xt_table_info *newinfo,
- int *error)
+static struct xt_table_info *
+do_replace_table(struct xt_table *table, unsigned int num_counters,
+ struct xt_table_info *newinfo, int *error)
{
struct xt_table_info *private;
unsigned int cpu;
@@ -1468,30 +1532,54 @@ xt_replace_table(struct xt_table *table,
}
}
- audit_log_nfcfg(table->name, table->af, private->number,
- !private->number ? AUDIT_XT_OP_REGISTER :
- AUDIT_XT_OP_REPLACE,
- GFP_KERNEL);
+ return private;
+}
+
+struct xt_table_info *
+xt_replace_table(struct xt_table *table, unsigned int num_counters,
+ struct xt_table_info *newinfo,
+ int *error)
+{
+ struct xt_table_info *private;
+
+ private = do_replace_table(table, num_counters, newinfo, error);
+ if (private)
+ audit_log_nfcfg(table->name, table->af, private->number,
+ AUDIT_XT_OP_REPLACE,
+ GFP_KERNEL);
+
return private;
}
EXPORT_SYMBOL_GPL(xt_replace_table);
struct xt_table *xt_register_table(struct net *net,
const struct xt_table *input_table,
+ const struct nf_hook_ops *template_ops,
struct xt_table_info *bootstrap,
struct xt_table_info *newinfo)
{
struct xt_pernet *xt_net = net_generic(net, xt_pernet_id);
+ struct xt_table *t, *table = NULL;
+ struct nf_hook_ops *ops = NULL;
struct xt_table_info *private;
- struct xt_table *t, *table;
- int ret;
+ unsigned int num_ops;
+ int ret = -EINVAL;
+
+ num_ops = hweight32(input_table->valid_hooks);
+ if (num_ops == 0)
+ goto out;
+
+ ret = -ENOMEM;
+ if (template_ops) {
+ ops = kmemdup_array(template_ops, num_ops, sizeof(*ops), GFP_KERNEL);
+ if (!ops)
+ goto out;
+ }
/* Don't add one object to multiple lists. */
table = kmemdup(input_table, sizeof(struct xt_table), GFP_KERNEL);
- if (!table) {
- ret = -ENOMEM;
+ if (!table)
goto out;
- }
mutex_lock(&xt[table->af].mutex);
/* Don't autoload: we'd eat our tail... */
@@ -1505,7 +1593,7 @@ struct xt_table *xt_register_table(struct net *net,
/* Simplifies replace_table code. */
table->private = bootstrap;
- if (!xt_replace_table(table, 0, newinfo, &ret))
+ if (!do_replace_table(table, 0, newinfo, &ret))
goto unlock;
private = table->private;
@@ -1514,34 +1602,122 @@ struct xt_table *xt_register_table(struct net *net,
/* save number of initial entries */
private->initial_entries = private->number;
+ if (ops) {
+ int i;
+
+ for (i = 0; i < num_ops; i++)
+ ops[i].priv = table;
+
+ ret = nf_register_net_hooks(net, ops, num_ops);
+ if (ret != 0) {
+ mutex_unlock(&xt[table->af].mutex);
+ /* nf_register_net_hooks() might have published a
+ * base chain before internal error unwind.
+ */
+ synchronize_rcu();
+ goto out;
+ }
+
+ table->ops = ops;
+ }
+
+ audit_log_nfcfg(table->name, table->af, private->number,
+ AUDIT_XT_OP_REGISTER, GFP_KERNEL);
+
list_add(&table->list, &xt_net->tables[table->af]);
mutex_unlock(&xt[table->af].mutex);
return table;
unlock:
mutex_unlock(&xt[table->af].mutex);
- kfree(table);
out:
+ kfree(table);
+ kfree(ops);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(xt_register_table);
-void *xt_unregister_table(struct xt_table *table)
+/**
+ * xt_unregister_table_pre_exit - pre-shutdown unregister of a table
+ * @net: network namespace
+ * @af: address family (e.g., NFPROTO_IPV4, NFPROTO_IPV6)
+ * @name: name of the table to unregister
+ *
+ * Unregisters the specified netfilter table from the given network namespace
+ * and also unregisters the hooks from netfilter core: no new packets will be
+ * processed.
+ *
+ * This must be called prior to xt_unregister_table_exit() from the pernet
+ * .pre_exit callback. After this call, the table is no longer visible to
+ * the get/setsockopt path. In case of rmmod, module exit path must have
+ * called xt_unregister_template() prior to unregistering pernet ops to
+ * prevent re-instantiation of the table.
+ *
+ * See also: xt_unregister_table_exit()
+ */
+void xt_unregister_table_pre_exit(struct net *net, u8 af, const char *name)
{
- struct xt_table_info *private;
+ struct xt_pernet *xt_net = net_generic(net, xt_pernet_id);
+ struct xt_table *t;
- mutex_lock(&xt[table->af].mutex);
- private = table->private;
- list_del(&table->list);
- mutex_unlock(&xt[table->af].mutex);
- audit_log_nfcfg(table->name, table->af, private->number,
- AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
- kfree(table->ops);
- kfree(table);
+ mutex_lock(&xt[af].mutex);
+ list_for_each_entry(t, &xt_net->tables[af], list) {
+ if (strcmp(t->name, name) == 0) {
+ list_move(&t->list, &xt_net->dead_tables[af]);
+ mutex_unlock(&xt[af].mutex);
- return private;
+ if (t->ops) /* nat table registers with nat core, t->ops is NULL. */
+ nf_unregister_net_hooks(net, t->ops, hweight32(t->valid_hooks));
+ return;
+ }
+ }
+ mutex_unlock(&xt[af].mutex);
}
-EXPORT_SYMBOL_GPL(xt_unregister_table);
+EXPORT_SYMBOL(xt_unregister_table_pre_exit);
+
+/**
+ * xt_unregister_table_exit - remove a table during namespace teardown
+ * @net: the network namespace from which to unregister the table
+ * @af: address family (e.g., NFPROTO_IPV4, NFPROTO_IPV6)
+ * @name: name of the table to unregister
+ *
+ * Completes the unregister process for a table. This must be called from
+ * the pernet ops .exit callback. This is the second stage after
+ * xt_unregister_table_pre_exit().
+ *
+ * pair with xt_unregister_table_pre_exit() during namespace shutdown.
+ *
+ * Return: the unregistered table or NULL if the table was never
+ * instantiated. The caller needs to kfree() the table after it
+ * has removed the family specific matches/targets.
+ */
+struct xt_table *xt_unregister_table_exit(struct net *net, u8 af, const char *name)
+{
+ struct xt_pernet *xt_net = net_generic(net, xt_pernet_id);
+ struct xt_table *table;
+
+ mutex_lock(&xt[af].mutex);
+ list_for_each_entry(table, &xt_net->dead_tables[af], list) {
+ struct nf_hook_ops *ops = NULL;
+
+ if (strcmp(table->name, name) != 0)
+ continue;
+
+ list_del(&table->list);
+
+ audit_log_nfcfg(table->name, table->af, table->private->number,
+ AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
+ swap(table->ops, ops);
+ mutex_unlock(&xt[af].mutex);
+
+ kfree(ops);
+ return table;
+ }
+ mutex_unlock(&xt[af].mutex);
+
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(xt_unregister_table_exit);
#endif
#ifdef CONFIG_PROC_FS
@@ -1988,8 +2164,10 @@ static int __net_init xt_net_init(struct net *net)
struct xt_pernet *xt_net = net_generic(net, xt_pernet_id);
int i;
- for (i = 0; i < NFPROTO_NUMPROTO; i++)
+ for (i = 0; i < NFPROTO_NUMPROTO; i++) {
INIT_LIST_HEAD(&xt_net->tables[i]);
+ INIT_LIST_HEAD(&xt_net->dead_tables[i]);
+ }
return 0;
}
@@ -1998,8 +2176,10 @@ static void __net_exit xt_net_exit(struct net *net)
struct xt_pernet *xt_net = net_generic(net, xt_pernet_id);
int i;
- for (i = 0; i < NFPROTO_NUMPROTO; i++)
+ for (i = 0; i < NFPROTO_NUMPROTO; i++) {
WARN_ON_ONCE(!list_empty(&xt_net->tables[i]));
+ WARN_ON_ONCE(!list_empty(&xt_net->dead_tables[i]));
+ }
}
static struct pernet_operations xt_net_ops = {
diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c
index 498f587..d2aeacf 100644
--- a/net/netfilter/xt_CT.c
+++ b/net/netfilter/xt_CT.c
@@ -354,7 +354,7 @@ static struct xt_target xt_ct_tg_reg[] __read_mostly = {
.family = NFPROTO_IPV4,
.revision = 1,
.targetsize = sizeof(struct xt_ct_target_info_v1),
- .usersize = offsetof(struct xt_ct_target_info, ct),
+ .usersize = offsetof(struct xt_ct_target_info_v1, ct),
.checkentry = xt_ct_tg_check_v1,
.destroy = xt_ct_tg_destroy_v1,
.target = xt_ct_target_v1,
@@ -366,7 +366,7 @@ static struct xt_target xt_ct_tg_reg[] __read_mostly = {
.family = NFPROTO_IPV4,
.revision = 2,
.targetsize = sizeof(struct xt_ct_target_info_v1),
- .usersize = offsetof(struct xt_ct_target_info, ct),
+ .usersize = offsetof(struct xt_ct_target_info_v1, ct),
.checkentry = xt_ct_tg_check_v2,
.destroy = xt_ct_tg_destroy_v1,
.target = xt_ct_target_v1,
@@ -398,7 +398,7 @@ static struct xt_target xt_ct_tg_reg[] __read_mostly = {
.family = NFPROTO_IPV6,
.revision = 1,
.targetsize = sizeof(struct xt_ct_target_info_v1),
- .usersize = offsetof(struct xt_ct_target_info, ct),
+ .usersize = offsetof(struct xt_ct_target_info_v1, ct),
.checkentry = xt_ct_tg_check_v1,
.destroy = xt_ct_tg_destroy_v1,
.target = xt_ct_target_v1,
@@ -410,7 +410,7 @@ static struct xt_target xt_ct_tg_reg[] __read_mostly = {
.family = NFPROTO_IPV6,
.revision = 2,
.targetsize = sizeof(struct xt_ct_target_info_v1),
- .usersize = offsetof(struct xt_ct_target_info, ct),
+ .usersize = offsetof(struct xt_ct_target_info_v1, ct),
.checkentry = xt_ct_tg_check_v2,
.destroy = xt_ct_tg_destroy_v1,
.target = xt_ct_target_v1,
diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c
index 5171061..bfcf2d4 100644
--- a/net/netfilter/xt_IDLETIMER.c
+++ b/net/netfilter/xt_IDLETIMER.c
@@ -115,6 +115,21 @@ static void idletimer_tg_alarmproc(struct alarm *alarm, ktime_t now)
schedule_work(&timer->work);
}
+static void idletimer_start_alarm_ktime(struct idletimer_tg *timer, ktime_t timeout)
+{
+ /*
+ * The timer should always be queued as @tout it should be least one
+ * second, but handle it correctly in any case. Virt will manage!
+ */
+ if (!alarm_start_timer(&timer->alarm, timeout, true))
+ schedule_work(&timer->work);
+}
+
+static void idletimer_start_alarm_sec(struct idletimer_tg *timer, unsigned int seconds)
+{
+ idletimer_start_alarm_ktime(timer, ktime_set(seconds, 0));
+}
+
static int idletimer_check_sysfs_name(const char *name, unsigned int size)
{
int ret;
@@ -220,12 +235,10 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
INIT_WORK(&info->timer->work, idletimer_tg_work);
if (info->timer->timer_type & XT_IDLETIMER_ALARM) {
- ktime_t tout;
alarm_init(&info->timer->alarm, ALARM_BOOTTIME,
idletimer_tg_alarmproc);
info->timer->alarm.data = info->timer;
- tout = ktime_set(info->timeout, 0);
- alarm_start_relative(&info->timer->alarm, tout);
+ idletimer_start_alarm_sec(info->timer, info->timeout);
} else {
timer_setup(&info->timer->timer, idletimer_tg_expired, 0);
mod_timer(&info->timer->timer,
@@ -271,8 +284,7 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb,
info->label, info->timeout);
if (info->timer->timer_type & XT_IDLETIMER_ALARM) {
- ktime_t tout = ktime_set(info->timeout, 0);
- alarm_start_relative(&info->timer->alarm, tout);
+ idletimer_start_alarm_sec(info->timer, info->timeout);
} else {
mod_timer(&info->timer->timer,
secs_to_jiffies(info->timeout) + jiffies);
@@ -384,7 +396,7 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par)
if (ktimespec.tv_sec > 0) {
pr_debug("time_expiry_remaining %lld\n",
ktimespec.tv_sec);
- alarm_start_relative(&info->timer->alarm, tout);
+ idletimer_start_alarm_ktime(info->timer, tout);
}
} else {
mod_timer(&info->timer->timer,
diff --git a/net/netfilter/xt_NFQUEUE.c b/net/netfilter/xt_NFQUEUE.c
index 466da23..b32d153 100644
--- a/net/netfilter/xt_NFQUEUE.c
+++ b/net/netfilter/xt_NFQUEUE.c
@@ -91,7 +91,7 @@ nfqueue_tg_v3(struct sk_buff *skb, const struct xt_action_param *par)
if (info->queues_total > 1) {
if (info->flags & NFQ_FLAG_CPU_FANOUT) {
- int cpu = smp_processor_id();
+ int cpu = raw_smp_processor_id();
queue = info->queuenum + cpu % info->queues_total;
} else {
diff --git a/net/netfilter/xt_TCPMSS.c b/net/netfilter/xt_TCPMSS.c
index 116a885..80e1634 100644
--- a/net/netfilter/xt_TCPMSS.c
+++ b/net/netfilter/xt_TCPMSS.c
@@ -247,6 +247,21 @@ tcpmss_tg6(struct sk_buff *skb, const struct xt_action_param *par)
}
#endif
+static int tcpmss_tg4_check_hooks(const struct xt_tgchk_param *par)
+{
+ const struct xt_tcpmss_info *info = par->targinfo;
+
+ if (info->mss == XT_TCPMSS_CLAMP_PMTU &&
+ (par->hook_mask & ~((1 << NF_INET_FORWARD) |
+ (1 << NF_INET_LOCAL_OUT) |
+ (1 << NF_INET_POST_ROUTING))) != 0) {
+ pr_info_ratelimited("path-MTU clamping only supported in FORWARD, OUTPUT and POSTROUTING hooks\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
/* Must specify -p tcp --syn */
static inline bool find_syn_match(const struct xt_entry_match *m)
{
@@ -262,17 +277,9 @@ static inline bool find_syn_match(const struct xt_entry_match *m)
static int tcpmss_tg4_check(const struct xt_tgchk_param *par)
{
- const struct xt_tcpmss_info *info = par->targinfo;
const struct ipt_entry *e = par->entryinfo;
const struct xt_entry_match *ematch;
- if (info->mss == XT_TCPMSS_CLAMP_PMTU &&
- (par->hook_mask & ~((1 << NF_INET_FORWARD) |
- (1 << NF_INET_LOCAL_OUT) |
- (1 << NF_INET_POST_ROUTING))) != 0) {
- pr_info_ratelimited("path-MTU clamping only supported in FORWARD, OUTPUT and POSTROUTING hooks\n");
- return -EINVAL;
- }
if (par->nft_compat)
return 0;
@@ -286,17 +293,9 @@ static int tcpmss_tg4_check(const struct xt_tgchk_param *par)
#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
static int tcpmss_tg6_check(const struct xt_tgchk_param *par)
{
- const struct xt_tcpmss_info *info = par->targinfo;
const struct ip6t_entry *e = par->entryinfo;
const struct xt_entry_match *ematch;
- if (info->mss == XT_TCPMSS_CLAMP_PMTU &&
- (par->hook_mask & ~((1 << NF_INET_FORWARD) |
- (1 << NF_INET_LOCAL_OUT) |
- (1 << NF_INET_POST_ROUTING))) != 0) {
- pr_info_ratelimited("path-MTU clamping only supported in FORWARD, OUTPUT and POSTROUTING hooks\n");
- return -EINVAL;
- }
if (par->nft_compat)
return 0;
@@ -312,6 +311,7 @@ static struct xt_target tcpmss_tg_reg[] __read_mostly = {
{
.family = NFPROTO_IPV4,
.name = "TCPMSS",
+ .check_hooks = tcpmss_tg4_check_hooks,
.checkentry = tcpmss_tg4_check,
.target = tcpmss_tg4,
.targetsize = sizeof(struct xt_tcpmss_info),
@@ -322,6 +322,7 @@ static struct xt_target tcpmss_tg_reg[] __read_mostly = {
{
.family = NFPROTO_IPV6,
.name = "TCPMSS",
+ .check_hooks = tcpmss_tg4_check_hooks,
.checkentry = tcpmss_tg6_check,
.target = tcpmss_tg6,
.targetsize = sizeof(struct xt_tcpmss_info),
diff --git a/net/netfilter/xt_TPROXY.c b/net/netfilter/xt_TPROXY.c
index e4bea1d..5f60e72 100644
--- a/net/netfilter/xt_TPROXY.c
+++ b/net/netfilter/xt_TPROXY.c
@@ -86,6 +86,9 @@ tproxy_tg4_v0(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tproxy_target_info *tgi = par->targinfo;
+ if (par->fragoff)
+ return NF_DROP;
+
return tproxy_tg4(xt_net(par), skb, tgi->laddr, tgi->lport,
tgi->mark_mask, tgi->mark_value);
}
@@ -95,6 +98,9 @@ tproxy_tg4_v1(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tproxy_target_info_v1 *tgi = par->targinfo;
+ if (par->fragoff)
+ return NF_DROP;
+
return tproxy_tg4(xt_net(par), skb, tgi->laddr.ip, tgi->lport,
tgi->mark_mask, tgi->mark_value);
}
@@ -106,6 +112,7 @@ tproxy_tg6_v1(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct ipv6hdr *iph = ipv6_hdr(skb);
const struct xt_tproxy_target_info_v1 *tgi = par->targinfo;
+ unsigned short fragoff = 0;
struct udphdr _hdr, *hp;
struct sock *sk;
const struct in6_addr *laddr;
@@ -113,8 +120,8 @@ tproxy_tg6_v1(struct sk_buff *skb, const struct xt_action_param *par)
int thoff = 0;
int tproto;
- tproto = ipv6_find_hdr(skb, &thoff, -1, NULL, NULL);
- if (tproto < 0)
+ tproto = ipv6_find_hdr(skb, &thoff, -1, &fragoff, NULL);
+ if (tproto < 0 || fragoff)
return NF_DROP;
hp = skb_header_pointer(skb, thoff, sizeof(_hdr), &_hdr);
diff --git a/net/netfilter/xt_addrtype.c b/net/netfilter/xt_addrtype.c
index a770889..913dbe3 100644
--- a/net/netfilter/xt_addrtype.c
+++ b/net/netfilter/xt_addrtype.c
@@ -153,14 +153,10 @@ addrtype_mt_v1(const struct sk_buff *skb, struct xt_action_param *par)
return ret;
}
-static int addrtype_mt_checkentry_v1(const struct xt_mtchk_param *par)
+static int addrtype_mt_check_hooks(const struct xt_mtchk_param *par)
{
- const char *errmsg = "both incoming and outgoing interface limitation cannot be selected";
struct xt_addrtype_info_v1 *info = par->matchinfo;
-
- if (info->flags & XT_ADDRTYPE_LIMIT_IFACE_IN &&
- info->flags & XT_ADDRTYPE_LIMIT_IFACE_OUT)
- goto err;
+ const char *errmsg;
if (par->hook_mask & ((1 << NF_INET_PRE_ROUTING) |
(1 << NF_INET_LOCAL_IN)) &&
@@ -176,6 +172,21 @@ static int addrtype_mt_checkentry_v1(const struct xt_mtchk_param *par)
goto err;
}
+ return 0;
+err:
+ pr_info_ratelimited("%s\n", errmsg);
+ return -EINVAL;
+}
+
+static int addrtype_mt_checkentry_v1(const struct xt_mtchk_param *par)
+{
+ const char *errmsg = "both incoming and outgoing interface limitation cannot be selected";
+ struct xt_addrtype_info_v1 *info = par->matchinfo;
+
+ if (info->flags & XT_ADDRTYPE_LIMIT_IFACE_IN &&
+ info->flags & XT_ADDRTYPE_LIMIT_IFACE_OUT)
+ goto err;
+
#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
if (par->family == NFPROTO_IPV6) {
if ((info->source | info->dest) & XT_ADDRTYPE_BLACKHOLE) {
@@ -211,6 +222,7 @@ static struct xt_match addrtype_mt_reg[] __read_mostly = {
.family = NFPROTO_IPV4,
.revision = 1,
.match = addrtype_mt_v1,
+ .check_hooks = addrtype_mt_check_hooks,
.checkentry = addrtype_mt_checkentry_v1,
.matchsize = sizeof(struct xt_addrtype_info_v1),
.me = THIS_MODULE
@@ -221,6 +233,7 @@ static struct xt_match addrtype_mt_reg[] __read_mostly = {
.family = NFPROTO_IPV6,
.revision = 1,
.match = addrtype_mt_v1,
+ .check_hooks = addrtype_mt_check_hooks,
.checkentry = addrtype_mt_checkentry_v1,
.matchsize = sizeof(struct xt_addrtype_info_v1),
.me = THIS_MODULE
diff --git a/net/netfilter/xt_cpu.c b/net/netfilter/xt_cpu.c
index 3bdc302..9cb2599 100644
--- a/net/netfilter/xt_cpu.c
+++ b/net/netfilter/xt_cpu.c
@@ -34,7 +34,7 @@ static bool cpu_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
const struct xt_cpu_info *info = par->matchinfo;
- return (info->cpu == smp_processor_id()) ^ info->invert;
+ return (info->cpu == raw_smp_processor_id()) ^ info->invert;
}
static struct xt_match cpu_mt_reg __read_mostly = {
diff --git a/net/netfilter/xt_devgroup.c b/net/netfilter/xt_devgroup.c
index 9520dd0..6d1a44a 100644
--- a/net/netfilter/xt_devgroup.c
+++ b/net/netfilter/xt_devgroup.c
@@ -33,14 +33,10 @@ static bool devgroup_mt(const struct sk_buff *skb, struct xt_action_param *par)
return true;
}
-static int devgroup_mt_checkentry(const struct xt_mtchk_param *par)
+static int devgroup_mt_check_hooks(const struct xt_mtchk_param *par)
{
const struct xt_devgroup_info *info = par->matchinfo;
- if (info->flags & ~(XT_DEVGROUP_MATCH_SRC | XT_DEVGROUP_INVERT_SRC |
- XT_DEVGROUP_MATCH_DST | XT_DEVGROUP_INVERT_DST))
- return -EINVAL;
-
if (info->flags & XT_DEVGROUP_MATCH_SRC &&
par->hook_mask & ~((1 << NF_INET_PRE_ROUTING) |
(1 << NF_INET_LOCAL_IN) |
@@ -56,9 +52,21 @@ static int devgroup_mt_checkentry(const struct xt_mtchk_param *par)
return 0;
}
+static int devgroup_mt_checkentry(const struct xt_mtchk_param *par)
+{
+ const struct xt_devgroup_info *info = par->matchinfo;
+
+ if (info->flags & ~(XT_DEVGROUP_MATCH_SRC | XT_DEVGROUP_INVERT_SRC |
+ XT_DEVGROUP_MATCH_DST | XT_DEVGROUP_INVERT_DST))
+ return -EINVAL;
+
+ return 0;
+}
+
static struct xt_match devgroup_mt_reg __read_mostly = {
.name = "devgroup",
.match = devgroup_mt,
+ .check_hooks = devgroup_mt_check_hooks,
.checkentry = devgroup_mt_checkentry,
.matchsize = sizeof(struct xt_devgroup_info),
.family = NFPROTO_UNSPEC,
diff --git a/net/netfilter/xt_ecn.c b/net/netfilter/xt_ecn.c
index b96e820..a8503f5 100644
--- a/net/netfilter/xt_ecn.c
+++ b/net/netfilter/xt_ecn.c
@@ -30,6 +30,10 @@ static bool match_tcp(const struct sk_buff *skb, struct xt_action_param *par)
struct tcphdr _tcph;
const struct tcphdr *th;
+ /* this is fine for IPv6 as ecn_mt_check6() enforces -p tcp */
+ if (par->fragoff)
+ return false;
+
/* In practice, TCP match does this, so can't fail. But let's
* be good citizens.
*/
diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c
index 3bd127b..2704b4b 100644
--- a/net/netfilter/xt_hashlimit.c
+++ b/net/netfilter/xt_hashlimit.c
@@ -658,6 +658,8 @@ hashlimit_init_dst(const struct xt_hashlimit_htable *hinfo,
if (!(hinfo->cfg.mode &
(XT_HASHLIMIT_HASH_DPT | XT_HASHLIMIT_HASH_SPT)))
return 0;
+ if (ntohs(ip_hdr(skb)->frag_off) & IP_OFFSET)
+ return -1;
nexthdr = ip_hdr(skb)->protocol;
break;
#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
@@ -681,7 +683,7 @@ hashlimit_init_dst(const struct xt_hashlimit_htable *hinfo,
return 0;
nexthdr = ipv6_hdr(skb)->nexthdr;
protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &nexthdr, &frag_off);
- if ((int)protoff < 0)
+ if ((int)protoff < 0 || ntohs(frag_off) & IP6_OFFSET)
return -1;
break;
}
diff --git a/net/netfilter/xt_osf.c b/net/netfilter/xt_osf.c
index dc94858..e8807ca 100644
--- a/net/netfilter/xt_osf.c
+++ b/net/netfilter/xt_osf.c
@@ -27,6 +27,9 @@
static bool
xt_osf_match_packet(const struct sk_buff *skb, struct xt_action_param *p)
{
+ if (p->fragoff)
+ return false;
+
return nf_osf_match(skb, xt_family(p), xt_hooknum(p), xt_in(p),
xt_out(p), p->matchinfo, xt_net(p), nf_osf_fingers);
}
diff --git a/net/netfilter/xt_physdev.c b/net/netfilter/xt_physdev.c
index d2b0b52..dd98f75 100644
--- a/net/netfilter/xt_physdev.c
+++ b/net/netfilter/xt_physdev.c
@@ -91,6 +91,21 @@ physdev_mt(const struct sk_buff *skb, struct xt_action_param *par)
return (!!ret ^ !(info->invert & XT_PHYSDEV_OP_OUT));
}
+static int physdev_mt_check_hooks(const struct xt_mtchk_param *par)
+{
+ const struct xt_physdev_info *info = par->matchinfo;
+
+ if (info->bitmask & (XT_PHYSDEV_OP_OUT | XT_PHYSDEV_OP_ISOUT) &&
+ (!(info->bitmask & XT_PHYSDEV_OP_BRIDGED) ||
+ info->invert & XT_PHYSDEV_OP_BRIDGED) &&
+ par->hook_mask & (1 << NF_INET_LOCAL_OUT)) {
+ pr_info_ratelimited("--physdev-out and --physdev-is-out only supported in the FORWARD and POSTROUTING chains with bridged traffic\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int physdev_mt_check(const struct xt_mtchk_param *par)
{
const struct xt_physdev_info *info = par->matchinfo;
@@ -99,13 +114,6 @@ static int physdev_mt_check(const struct xt_mtchk_param *par)
if (!(info->bitmask & XT_PHYSDEV_OP_MASK) ||
info->bitmask & ~XT_PHYSDEV_OP_MASK)
return -EINVAL;
- if (info->bitmask & (XT_PHYSDEV_OP_OUT | XT_PHYSDEV_OP_ISOUT) &&
- (!(info->bitmask & XT_PHYSDEV_OP_BRIDGED) ||
- info->invert & XT_PHYSDEV_OP_BRIDGED) &&
- par->hook_mask & (1 << NF_INET_LOCAL_OUT)) {
- pr_info_ratelimited("--physdev-out and --physdev-is-out only supported in the FORWARD and POSTROUTING chains with bridged traffic\n");
- return -EINVAL;
- }
#define X(memb) strnlen(info->memb, sizeof(info->memb)) >= sizeof(info->memb)
if (info->bitmask & XT_PHYSDEV_OP_IN) {
@@ -141,6 +149,7 @@ static struct xt_match physdev_mt_reg[] __read_mostly = {
{
.name = "physdev",
.family = NFPROTO_IPV4,
+ .check_hooks = physdev_mt_check_hooks,
.checkentry = physdev_mt_check,
.match = physdev_mt,
.matchsize = sizeof(struct xt_physdev_info),
@@ -149,6 +158,7 @@ static struct xt_match physdev_mt_reg[] __read_mostly = {
{
.name = "physdev",
.family = NFPROTO_IPV6,
+ .check_hooks = physdev_mt_check_hooks,
.checkentry = physdev_mt_check,
.match = physdev_mt,
.matchsize = sizeof(struct xt_physdev_info),
diff --git a/net/netfilter/xt_policy.c b/net/netfilter/xt_policy.c
index cb6e827..ff54e3a 100644
--- a/net/netfilter/xt_policy.c
+++ b/net/netfilter/xt_policy.c
@@ -63,7 +63,7 @@ match_policy_in(const struct sk_buff *skb, const struct xt_policy_info *info,
return 0;
for (i = sp->len - 1; i >= 0; i--) {
- pos = strict ? i - sp->len + 1 : 0;
+ pos = strict ? sp->len - i - 1 : 0;
if (pos >= info->len)
return 0;
e = &info->pol[pos];
@@ -126,13 +126,10 @@ policy_mt(const struct sk_buff *skb, struct xt_action_param *par)
return ret;
}
-static int policy_mt_check(const struct xt_mtchk_param *par)
+static int policy_mt_check_hooks(const struct xt_mtchk_param *par)
{
const struct xt_policy_info *info = par->matchinfo;
- const char *errmsg = "neither incoming nor outgoing policy selected";
-
- if (!(info->flags & (XT_POLICY_MATCH_IN|XT_POLICY_MATCH_OUT)))
- goto err;
+ const char *errmsg;
if (par->hook_mask & ((1 << NF_INET_PRE_ROUTING) |
(1 << NF_INET_LOCAL_IN)) && info->flags & XT_POLICY_MATCH_OUT) {
@@ -144,6 +141,21 @@ static int policy_mt_check(const struct xt_mtchk_param *par)
errmsg = "input policy not valid in POSTROUTING and OUTPUT";
goto err;
}
+
+ return 0;
+err:
+ pr_info_ratelimited("%s\n", errmsg);
+ return -EINVAL;
+}
+
+static int policy_mt_check(const struct xt_mtchk_param *par)
+{
+ const struct xt_policy_info *info = par->matchinfo;
+ const char *errmsg = "neither incoming nor outgoing policy selected";
+
+ if (!(info->flags & (XT_POLICY_MATCH_IN|XT_POLICY_MATCH_OUT)))
+ goto err;
+
if (info->len > XT_POLICY_MAX_ELEM) {
errmsg = "too many policy elements";
goto err;
@@ -158,6 +170,7 @@ static struct xt_match policy_mt_reg[] __read_mostly = {
{
.name = "policy",
.family = NFPROTO_IPV4,
+ .check_hooks = policy_mt_check_hooks,
.checkentry = policy_mt_check,
.match = policy_mt,
.matchsize = sizeof(struct xt_policy_info),
@@ -166,6 +179,7 @@ static struct xt_match policy_mt_reg[] __read_mostly = {
{
.name = "policy",
.family = NFPROTO_IPV6,
+ .check_hooks = policy_mt_check_hooks,
.checkentry = policy_mt_check,
.match = policy_mt,
.matchsize = sizeof(struct xt_policy_info),
diff --git a/net/netfilter/xt_set.c b/net/netfilter/xt_set.c
index 731bc2c..4ae04bb 100644
--- a/net/netfilter/xt_set.c
+++ b/net/netfilter/xt_set.c
@@ -431,6 +431,29 @@ set_target_v3(struct sk_buff *skb, const struct xt_action_param *par)
}
static int
+set_target_v3_check_hooks(const struct xt_tgchk_param *par)
+{
+ const struct xt_set_info_target_v3 *info = par->targinfo;
+
+ if (info->map_set.index != IPSET_INVALID_ID) {
+ if (strncmp(par->table, "mangle", 7)) {
+ pr_info_ratelimited("--map-set only usable from mangle table\n");
+ return -EINVAL;
+ }
+ if (((info->flags & IPSET_FLAG_MAP_SKBPRIO) |
+ (info->flags & IPSET_FLAG_MAP_SKBQUEUE)) &&
+ (par->hook_mask & ~(1 << NF_INET_FORWARD |
+ 1 << NF_INET_LOCAL_OUT |
+ 1 << NF_INET_POST_ROUTING))) {
+ pr_info_ratelimited("mapping of prio or/and queue is allowed only from OUTPUT/FORWARD/POSTROUTING chains\n");
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int
set_target_v3_checkentry(const struct xt_tgchk_param *par)
{
const struct xt_set_info_target_v3 *info = par->targinfo;
@@ -459,20 +482,6 @@ set_target_v3_checkentry(const struct xt_tgchk_param *par)
}
if (info->map_set.index != IPSET_INVALID_ID) {
- if (strncmp(par->table, "mangle", 7)) {
- pr_info_ratelimited("--map-set only usable from mangle table\n");
- ret = -EINVAL;
- goto cleanup_del;
- }
- if (((info->flags & IPSET_FLAG_MAP_SKBPRIO) |
- (info->flags & IPSET_FLAG_MAP_SKBQUEUE)) &&
- (par->hook_mask & ~(1 << NF_INET_FORWARD |
- 1 << NF_INET_LOCAL_OUT |
- 1 << NF_INET_POST_ROUTING))) {
- pr_info_ratelimited("mapping of prio or/and queue is allowed only from OUTPUT/FORWARD/POSTROUTING chains\n");
- ret = -EINVAL;
- goto cleanup_del;
- }
index = ip_set_nfnl_get_byindex(par->net,
info->map_set.index);
if (index == IPSET_INVALID_ID) {
@@ -672,6 +681,7 @@ static struct xt_target set_targets[] __read_mostly = {
.family = NFPROTO_IPV4,
.target = set_target_v3,
.targetsize = sizeof(struct xt_set_info_target_v3),
+ .check_hooks = set_target_v3_check_hooks,
.checkentry = set_target_v3_checkentry,
.destroy = set_target_v3_destroy,
.me = THIS_MODULE
@@ -682,6 +692,7 @@ static struct xt_target set_targets[] __read_mostly = {
.family = NFPROTO_IPV6,
.target = set_target_v3,
.targetsize = sizeof(struct xt_set_info_target_v3),
+ .check_hooks = set_target_v3_check_hooks,
.checkentry = set_target_v3_checkentry,
.destroy = set_target_v3_destroy,
.me = THIS_MODULE
diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c
index 0d32d48..b9da826 100644
--- a/net/netfilter/xt_tcpmss.c
+++ b/net/netfilter/xt_tcpmss.c
@@ -32,6 +32,10 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par)
u8 _opt[15 * 4 - sizeof(_tcph)];
unsigned int i, optlen;
+ /* this is fine for IPv6 as xt_tcpmss enforces -p tcp */
+ if (par->fragoff)
+ return false;
+
/* If we don't have the whole header, drop packet. */
th = skb_header_pointer(skb, par->thoff, sizeof(_tcph), &_tcph);
if (th == NULL)
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 2aeb068..7269e23 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -1482,9 +1482,14 @@ static void do_one_broadcast(struct sock *sk,
p->skb2 = NULL;
goto out;
}
- NETLINK_CB(p->skb2).nsid = peernet2id(sock_net(sk), p->net);
- if (NETLINK_CB(p->skb2).nsid != NETNSA_NSID_NOT_ASSIGNED)
- NETLINK_CB(p->skb2).nsid_is_set = true;
+
+ NETLINK_CB(p->skb2).nsid_is_set = false;
+ if (!net_eq(sock_net(sk), p->net)) {
+ NETLINK_CB(p->skb2).nsid = peernet2id(sock_net(sk), p->net);
+ if (NETLINK_CB(p->skb2).nsid != NETNSA_NSID_NOT_ASSIGNED)
+ NETLINK_CB(p->skb2).nsid_is_set = true;
+ }
+
val = netlink_broadcast_deliver(sk, p->skb2);
if (val < 0) {
netlink_overrun(sk);
diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c
index d251d89..0da39ea 100644
--- a/net/netlink/genetlink.c
+++ b/net/netlink/genetlink.c
@@ -1972,8 +1972,10 @@ int genlmsg_multicast_allns(const struct genl_family *family,
struct sk_buff *skb, u32 portid,
unsigned int group)
{
- if (WARN_ON_ONCE(group >= family->n_mcgrps))
+ if (WARN_ON_ONCE(group >= family->n_mcgrps)) {
+ kfree_skb(skb);
return -EINVAL;
+ }
group = family->mcgrp_offset + group;
return genlmsg_mcast(skb, portid, group);
@@ -1986,8 +1988,10 @@ void genl_notify(const struct genl_family *family, struct sk_buff *skb,
struct net *net = genl_info_net(info);
struct sock *sk = net->genl_sock;
- if (WARN_ON_ONCE(group >= family->n_mcgrps))
+ if (WARN_ON_ONCE(group >= family->n_mcgrps)) {
+ kfree_skb(skb);
return;
+ }
group = family->mcgrp_offset + group;
nlmsg_notify(sk, skb, info->snd_portid, group,
diff --git a/net/nfc/hci/core.c b/net/nfc/hci/core.c
index 0d33c81..ba6f031 100644
--- a/net/nfc/hci/core.c
+++ b/net/nfc/hci/core.c
@@ -861,6 +861,11 @@ static void nfc_hci_recv_from_llc(struct nfc_hci_dev *hdev, struct sk_buff *skb)
struct sk_buff *frag_skb;
int msg_len;
+ if (!pskb_may_pull(skb, NFC_HCI_HCP_PACKET_HEADER_LEN)) {
+ kfree_skb(skb);
+ return;
+ }
+
packet = (struct hcp_packet *)skb->data;
if ((packet->header & ~NFC_HCI_FRAGMENT) == 0) {
skb_queue_tail(&hdev->rx_hcp_frags, skb);
@@ -904,6 +909,11 @@ static void nfc_hci_recv_from_llc(struct nfc_hci_dev *hdev, struct sk_buff *skb)
* unblock waiting cmd context. Otherwise, enqueue to dispatch
* in separate context where handler can also execute command.
*/
+ if (!pskb_may_pull(hcp_skb, NFC_HCI_HCP_HEADER_LEN)) {
+ kfree_skb(hcp_skb);
+ return;
+ }
+
packet = (struct hcp_packet *)hcp_skb->data;
type = HCP_MSG_GET_TYPE(packet->message.header);
if (type == NFC_HCI_HCP_RESPONSE) {
diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c
index db5bc6a..dc65c71 100644
--- a/net/nfc/llcp_core.c
+++ b/net/nfc/llcp_core.c
@@ -1218,6 +1218,15 @@ static void nfc_llcp_recv_cc(struct nfc_llcp_local *local,
sk = &llcp_sock->sk;
+ lock_sock(sk);
+
+ /* Check if socket was destroyed whilst waiting for the lock */
+ if (!sk_hashed(sk)) {
+ release_sock(sk);
+ nfc_llcp_sock_put(llcp_sock);
+ return;
+ }
+
/* Unlink from connecting and link to the client array */
nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
nfc_llcp_sock_link(&local->sockets, sk);
@@ -1229,6 +1238,8 @@ static void nfc_llcp_recv_cc(struct nfc_llcp_local *local,
sk->sk_state = LLCP_CONNECTED;
sk->sk_state_change(sk);
+ release_sock(sk);
+
nfc_llcp_sock_put(llcp_sock);
}
diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c
index f1be1e8..feab29f 100644
--- a/net/nfc/llcp_sock.c
+++ b/net/nfc/llcp_sock.c
@@ -633,6 +633,8 @@ static int llcp_sock_release(struct socket *sock)
if (sock->type == SOCK_RAW)
nfc_llcp_sock_unlink(&local->raw_sockets, sk);
+ else if (sk->sk_state == LLCP_CONNECTING)
+ nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
else
nfc_llcp_sock_unlink(&local->sockets, sk);
diff --git a/net/nfc/nci/hci.c b/net/nfc/nci/hci.c
index 40ae8e5..c03e8a0 100644
--- a/net/nfc/nci/hci.c
+++ b/net/nfc/nci/hci.c
@@ -439,6 +439,11 @@ void nci_hci_data_received_cb(void *context,
return;
}
+ if (!pskb_may_pull(skb, NCI_HCI_HCP_PACKET_HEADER_LEN)) {
+ kfree_skb(skb);
+ return;
+ }
+
packet = (struct nci_hcp_packet *)skb->data;
if ((packet->header & ~NCI_HCI_FRAGMENT) == 0) {
skb_queue_tail(&ndev->hci_dev->rx_hcp_frags, skb);
@@ -482,6 +487,11 @@ void nci_hci_data_received_cb(void *context,
* unblock waiting cmd context. Otherwise, enqueue to dispatch
* in separate context where handler can also execute command.
*/
+ if (!pskb_may_pull(hcp_skb, NCI_HCI_HCP_HEADER_LEN)) {
+ kfree_skb(hcp_skb);
+ return;
+ }
+
packet = (struct nci_hcp_packet *)hcp_skb->data;
type = NCI_HCP_MSG_GET_TYPE(packet->message.header);
if (type == NCI_HCI_HCP_RESPONSE) {
diff --git a/net/openvswitch/vport-geneve.c b/net/openvswitch/vport-geneve.c
index b10e160..cb5ea442 100644
--- a/net/openvswitch/vport-geneve.c
+++ b/net/openvswitch/vport-geneve.c
@@ -97,6 +97,9 @@ static struct vport *geneve_tnl_create(const struct vport_parms *parms)
goto error;
}
+ vport->dev = dev;
+ netdev_hold(vport->dev, &vport->dev_tracker, GFP_KERNEL);
+
rtnl_unlock();
return vport;
error:
@@ -111,7 +114,7 @@ static struct vport *geneve_create(const struct vport_parms *parms)
if (IS_ERR(vport))
return vport;
- return ovs_netdev_link(vport, parms->name);
+ return ovs_netdev_link(vport, true);
}
static struct vport_ops ovs_geneve_vport_ops = {
diff --git a/net/openvswitch/vport-gre.c b/net/openvswitch/vport-gre.c
index 4014c9b..6cb5a69 100644
--- a/net/openvswitch/vport-gre.c
+++ b/net/openvswitch/vport-gre.c
@@ -63,6 +63,9 @@ static struct vport *gre_tnl_create(const struct vport_parms *parms)
return ERR_PTR(err);
}
+ vport->dev = dev;
+ netdev_hold(vport->dev, &vport->dev_tracker, GFP_KERNEL);
+
rtnl_unlock();
return vport;
}
@@ -75,7 +78,7 @@ static struct vport *gre_create(const struct vport_parms *parms)
if (IS_ERR(vport))
return vport;
- return ovs_netdev_link(vport, parms->name);
+ return ovs_netdev_link(vport, true);
}
static struct vport_ops ovs_gre_vport_ops = {
diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c
index 12055af..e7e8490 100644
--- a/net/openvswitch/vport-netdev.c
+++ b/net/openvswitch/vport-netdev.c
@@ -73,37 +73,29 @@ static struct net_device *get_dpdev(const struct datapath *dp)
return local->dev;
}
-struct vport *ovs_netdev_link(struct vport *vport, const char *name)
+struct vport *ovs_netdev_link(struct vport *vport, bool tunnel)
{
int err;
- vport->dev = dev_get_by_name(ovs_dp_get_net(vport->dp), name);
- if (!vport->dev) {
+ if (WARN_ON_ONCE(!vport->dev)) {
err = -ENODEV;
goto error_free_vport;
}
- /* Ensure that the device exists and that the provided
- * name is not one of its aliases.
- */
- if (strcmp(name, ovs_vport_name(vport))) {
- err = -ENODEV;
- goto error_put;
- }
- netdev_tracker_alloc(vport->dev, &vport->dev_tracker, GFP_KERNEL);
- if (vport->dev->flags & IFF_LOOPBACK ||
- (vport->dev->type != ARPHRD_ETHER &&
- vport->dev->type != ARPHRD_NONE) ||
- ovs_is_internal_dev(vport->dev)) {
- err = -EINVAL;
- goto error_put;
- }
rtnl_lock();
+ /* Do not link devices that are not registered to avoid a potential
+ * race with the NETDEV_UNREGISTER notification in dp_device_event().
+ */
+ if (vport->dev->reg_state != NETREG_REGISTERED) {
+ err = -ENODEV;
+ goto error_put_unlock;
+ }
+
err = netdev_master_upper_dev_link(vport->dev,
get_dpdev(vport->dp),
NULL, NULL, NULL);
if (err)
- goto error_unlock;
+ goto error_put_unlock;
err = netdev_rx_handler_register(vport->dev, netdev_frame_hook,
vport);
@@ -119,10 +111,11 @@ struct vport *ovs_netdev_link(struct vport *vport, const char *name)
error_master_upper_dev_unlink:
netdev_upper_dev_unlink(vport->dev, get_dpdev(vport->dp));
-error_unlock:
- rtnl_unlock();
-error_put:
+error_put_unlock:
+ if (tunnel && vport->dev->reg_state == NETREG_REGISTERED)
+ rtnl_delete_link(vport->dev, 0, NULL);
netdev_put(vport->dev, &vport->dev_tracker);
+ rtnl_unlock();
error_free_vport:
ovs_vport_free(vport);
return ERR_PTR(err);
@@ -132,12 +125,39 @@ EXPORT_SYMBOL_GPL(ovs_netdev_link);
static struct vport *netdev_create(const struct vport_parms *parms)
{
struct vport *vport;
+ int err;
vport = ovs_vport_alloc(0, &ovs_netdev_vport_ops, parms);
if (IS_ERR(vport))
return vport;
- return ovs_netdev_link(vport, parms->name);
+ vport->dev = dev_get_by_name(ovs_dp_get_net(vport->dp), parms->name);
+ if (!vport->dev) {
+ err = -ENODEV;
+ goto error_free_vport;
+ }
+ netdev_tracker_alloc(vport->dev, &vport->dev_tracker, GFP_KERNEL);
+
+ /* Ensure that the provided name is not an alias. */
+ if (strcmp(parms->name, ovs_vport_name(vport))) {
+ err = -ENODEV;
+ goto error_put;
+ }
+
+ if (vport->dev->flags & IFF_LOOPBACK ||
+ (vport->dev->type != ARPHRD_ETHER &&
+ vport->dev->type != ARPHRD_NONE) ||
+ ovs_is_internal_dev(vport->dev)) {
+ err = -EINVAL;
+ goto error_put;
+ }
+
+ return ovs_netdev_link(vport, false);
+error_put:
+ netdev_put(vport->dev, &vport->dev_tracker);
+error_free_vport:
+ ovs_vport_free(vport);
+ return ERR_PTR(err);
}
static void vport_netdev_free(struct rcu_head *rcu)
@@ -196,9 +216,13 @@ void ovs_netdev_tunnel_destroy(struct vport *vport)
*/
if (vport->dev->reg_state == NETREG_REGISTERED)
rtnl_delete_link(vport->dev, 0, NULL);
- rtnl_unlock();
+ /* We can't put the device reference yet, since it can still be in
+ * use, but rtnl_unlock()->netdev_run_todo() will block until all
+ * the references are released, so the RCU call must be before it.
+ */
call_rcu(&vport->rcu, vport_netdev_free);
+ rtnl_unlock();
}
EXPORT_SYMBOL_GPL(ovs_netdev_tunnel_destroy);
diff --git a/net/openvswitch/vport-netdev.h b/net/openvswitch/vport-netdev.h
index c5d83a4..6c0d736 100644
--- a/net/openvswitch/vport-netdev.h
+++ b/net/openvswitch/vport-netdev.h
@@ -13,7 +13,7 @@
struct vport *ovs_netdev_get_vport(struct net_device *dev);
-struct vport *ovs_netdev_link(struct vport *vport, const char *name);
+struct vport *ovs_netdev_link(struct vport *vport, bool tunnel);
void ovs_netdev_detach_dev(struct vport *);
int __init ovs_netdev_init(void);
diff --git a/net/openvswitch/vport-vxlan.c b/net/openvswitch/vport-vxlan.c
index 0b881b0..c1b37b5 100644
--- a/net/openvswitch/vport-vxlan.c
+++ b/net/openvswitch/vport-vxlan.c
@@ -126,6 +126,9 @@ static struct vport *vxlan_tnl_create(const struct vport_parms *parms)
goto error;
}
+ vport->dev = dev;
+ netdev_hold(vport->dev, &vport->dev_tracker, GFP_KERNEL);
+
rtnl_unlock();
return vport;
error:
@@ -140,7 +143,7 @@ static struct vport *vxlan_create(const struct vport_parms *parms)
if (IS_ERR(vport))
return vport;
- return ovs_netdev_link(vport, parms->name);
+ return ovs_netdev_link(vport, true);
}
static struct vport_ops ovs_vxlan_netdev_vport_ops = {
diff --git a/net/phonet/pep.c b/net/phonet/pep.c
index 4dbf091..7069271 100644
--- a/net/phonet/pep.c
+++ b/net/phonet/pep.c
@@ -671,8 +671,23 @@ static int pep_do_rcv(struct sock *sk, struct sk_buff *skb)
/* Look for an existing pipe handle */
sknode = pep_find_pipe(&pn->hlist, &dst, pipe_handle);
- if (sknode)
- return sk_receive_skb(sknode, skb, 1);
+ if (sknode) {
+ int rc;
+
+ /* pep_do_rcv() runs from two contexts: from softirq via
+ * phonet_rcv() -> __sk_receive_skb() with BH disabled,
+ * and from process context via
+ * release_sock() -> __release_sock(), which drops
+ * the listener slock with spin_unlock_bh() before draining
+ * the backlog. The child pipe slock is taken below via
+ * bh_lock_sock_nested(), which does not itself disable BH, so
+ * disable BH here to keep both acquire contexts consistent.
+ */
+ local_bh_disable();
+ rc = sk_receive_skb(sknode, skb, 1);
+ local_bh_enable();
+ return rc;
+ }
switch (hdr->message_id) {
case PNS_PEP_CONNECT_REQ:
diff --git a/net/phonet/socket.c b/net/phonet/socket.c
index c4af263..631a99c 100644
--- a/net/phonet/socket.c
+++ b/net/phonet/socket.c
@@ -208,9 +208,15 @@ static int pn_socket_autobind(struct socket *sock)
sa.spn_family = AF_PHONET;
err = pn_socket_bind(sock, (struct sockaddr_unsized *)&sa,
sizeof(struct sockaddr_pn));
- if (err != -EINVAL)
+ /*
+ * pn_socket_bind() also returns -EINVAL when sk_state != TCP_CLOSE
+ * without a prior bind, so -EINVAL alone is not sufficient to infer
+ * that the socket was already bound. Only treat it as "already
+ * bound" when the port is non-zero; otherwise propagate the error
+ * instead of crashing the kernel.
+ */
+ if (err != -EINVAL || unlikely(!pn_port(pn_sk(sock->sk)->sobject)))
return err;
- BUG_ON(!pn_port(pn_sk(sock->sk)->sobject));
return 0; /* socket was already bound */
}
diff --git a/net/psp/psp-nl-gen.c b/net/psp/psp-nl-gen.c
index 22a48d0..9533099 100644
--- a/net/psp/psp-nl-gen.c
+++ b/net/psp/psp-nl-gen.c
@@ -76,7 +76,7 @@ static const struct genl_split_ops psp_nl_ops[] = {
.post_doit = psp_device_unlock,
.policy = psp_dev_set_nl_policy,
.maxattr = PSP_A_DEV_PSP_VERSIONS_ENA,
- .flags = GENL_CMD_CAP_DO,
+ .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
},
{
.cmd = PSP_CMD_KEY_ROTATE,
@@ -85,7 +85,7 @@ static const struct genl_split_ops psp_nl_ops[] = {
.post_doit = psp_device_unlock,
.policy = psp_key_rotate_nl_policy,
.maxattr = PSP_A_DEV_ID,
- .flags = GENL_CMD_CAP_DO,
+ .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
},
{
.cmd = PSP_CMD_RX_ASSOC,
diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c
index 9508b6c..e45549f 100644
--- a/net/psp/psp_main.c
+++ b/net/psp/psp_main.c
@@ -263,15 +263,16 @@ EXPORT_SYMBOL(psp_dev_encapsulate);
/* Receive handler for PSP packets.
*
- * Presently it accepts only already-authenticated packets and does not
- * support optional fields, such as virtualization cookies. The caller should
- * ensure that skb->data is pointing to the mac header, and that skb->mac_len
- * is set. This function does not currently adjust skb->csum (CHECKSUM_COMPLETE
- * is not supported).
+ * Accepts only already-authenticated packets. The full PSP header is
+ * stripped according to psph->hdrlen; any optional fields it advertises
+ * (virtualization cookies, etc.) are ignored and discarded along with the
+ * rest of the header. The caller should ensure that skb->data is pointing
+ * to the mac header, and that skb->mac_len is set. This function does not
+ * currently adjust skb->csum (CHECKSUM_COMPLETE is not supported).
*/
int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
{
- int l2_hlen = 0, l3_hlen, encap;
+ int l2_hlen = 0, l3_hlen, encap, psp_hlen;
struct psp_skb_ext *pse;
struct psphdr *psph;
struct ethhdr *eth;
@@ -312,18 +313,36 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
if (unlikely(uh->dest != htons(PSP_DEFAULT_UDP_PORT)))
return -EINVAL;
- pse = skb_ext_add(skb, SKB_EXT_PSP);
- if (!pse)
+ psph = (struct psphdr *)(skb->data + l2_hlen + l3_hlen +
+ sizeof(struct udphdr));
+
+ /* Strip the full PSP header per psph->hdrlen; VC/options are pulled
+ * into the linear region only so they can be discarded with the
+ * rest of the header.
+ */
+ psp_hlen = (psph->hdrlen + 1) * 8;
+
+ if (unlikely(psp_hlen < sizeof(struct psphdr)))
+ return -EINVAL;
+
+ if (psp_hlen > sizeof(struct psphdr) &&
+ !pskb_may_pull(skb, l2_hlen + l3_hlen +
+ sizeof(struct udphdr) + psp_hlen))
return -EINVAL;
psph = (struct psphdr *)(skb->data + l2_hlen + l3_hlen +
sizeof(struct udphdr));
+
+ pse = skb_ext_add(skb, SKB_EXT_PSP);
+ if (!pse)
+ return -EINVAL;
+
pse->spi = psph->spi;
pse->dev_id = dev_id;
pse->generation = generation;
pse->version = FIELD_GET(PSPHDR_VERFL_VERSION, psph->verfl);
- encap = PSP_ENCAP_HLEN;
+ encap = sizeof(struct udphdr) + psp_hlen;
encap += strip_icv ? PSP_TRL_SIZE : 0;
if (proto == htons(ETH_P_IP)) {
@@ -340,8 +359,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
ipv6h->payload_len = htons(ntohs(ipv6h->payload_len) - encap);
}
- memmove(skb->data + PSP_ENCAP_HLEN, skb->data, l2_hlen + l3_hlen);
- skb_pull(skb, PSP_ENCAP_HLEN);
+ memmove(skb->data + sizeof(struct udphdr) + psp_hlen,
+ skb->data, l2_hlen + l3_hlen);
+ skb_pull(skb, sizeof(struct udphdr) + psp_hlen);
if (strip_icv)
pskb_trim(skb, skb->len - PSP_TRL_SIZE);
diff --git a/net/psp/psp_nl.c b/net/psp/psp_nl.c
index 6afd770..0cc744a 100644
--- a/net/psp/psp_nl.c
+++ b/net/psp/psp_nl.c
@@ -305,8 +305,13 @@ int psp_assoc_device_get_locked(const struct genl_split_ops *ops,
psd = psp_dev_get_for_sock(socket->sk);
if (psd) {
- err = psp_dev_check_access(psd, genl_info_net(info));
- if (err) {
+ /* Extra care needed here, psp_dev_get_for_sock() only gives
+ * us access to struct psp_dev's memory, which is quite weak.
+ */
+ mutex_lock(&psd->lock);
+ if (!psp_dev_is_registered(psd) ||
+ psp_dev_check_access(psd, genl_info_net(info))) {
+ mutex_unlock(&psd->lock);
psp_dev_put(psd);
psd = NULL;
}
@@ -319,7 +324,6 @@ int psp_assoc_device_get_locked(const struct genl_split_ops *ops,
id = info->attrs[PSP_A_ASSOC_DEV_ID];
if (psd) {
- mutex_lock(&psd->lock);
if (id && psd->id != nla_get_u32(id)) {
mutex_unlock(&psd->lock);
NL_SET_ERR_MSG_ATTR(info->extack, id,
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 0c64c50..4001de0 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -656,6 +656,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
sends_out:
vfree(ic->i_sends);
+ ic->i_sends = NULL;
ack_dma_out:
rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
diff --git a/net/rds/message.c b/net/rds/message.c
index eaa6f22..7feb0eb 100644
--- a/net/rds/message.c
+++ b/net/rds/message.c
@@ -131,24 +131,34 @@ static void rds_rm_zerocopy_callback(struct rds_sock *rs,
*/
static void rds_message_purge(struct rds_message *rm)
{
+ struct rds_znotifier *znotifier;
unsigned long i, flags;
- bool zcopy = false;
+ bool zcopy;
if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))
return;
spin_lock_irqsave(&rm->m_rs_lock, flags);
+ znotifier = rm->data.op_mmp_znotifier;
+ rm->data.op_mmp_znotifier = NULL;
+ zcopy = !!znotifier;
+
if (rm->m_rs) {
struct rds_sock *rs = rm->m_rs;
- if (rm->data.op_mmp_znotifier) {
- zcopy = true;
- rds_rm_zerocopy_callback(rs, rm->data.op_mmp_znotifier);
+ if (znotifier) {
+ rds_rm_zerocopy_callback(rs, znotifier);
rds_wake_sk_sleep(rs);
- rm->data.op_mmp_znotifier = NULL;
}
sock_put(rds_rs_to_sk(rs));
rm->m_rs = NULL;
+ } else if (znotifier) {
+ /*
+ * Zerocopy can fail before the message is queued on the
+ * socket, so there is no rs to carry the notification.
+ */
+ mm_unaccount_pinned_pages(&znotifier->z_mmp);
+ kfree(rds_info_from_znotifier(znotifier));
}
spin_unlock_irqrestore(&rm->m_rs_lock, flags);
@@ -438,6 +448,7 @@ static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *
for (i = 0; i < rm->data.op_nents; i++)
put_page(sg_page(&rm->data.op_sg[i]));
+ rm->data.op_nents = 0;
mmp = &rm->data.op_mmp_znotifier->z_mmp;
mm_unaccount_pinned_pages(mmp);
ret = -EFAULT;
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 654e23d..5830b31 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -198,8 +198,13 @@ void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
write_lock_bh(&sock->sk->sk_callback_lock);
- /* done under the callback_lock to serialize with write_space */
+ /* done under the callback_lock to serialize with write_space.
+ * Set t_sock inside rds_tcp_tc_list_lock so readers walking
+ * rds_tcp_tc_list under the same lock cannot observe an
+ * entry whose t_sock is NULL.
+ */
spin_lock(&rds_tcp_tc_list_lock);
+ tc->t_sock = sock;
list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
#if IS_ENABLED(CONFIG_IPV6)
rds6_tcp_tc_count++;
@@ -211,8 +216,6 @@ void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
/* accepted sockets need our listen data ready undone */
if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
sock->sk->sk_data_ready = sock->sk->sk_user_data;
-
- tc->t_sock = sock;
if (!tc->t_rtn)
tc->t_rtn = net_generic(sock_net(sock->sk), rds_tcp_netid);
tc->t_cpath = cp;
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index 27c2aa2..98f2165 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -213,8 +213,6 @@ struct rxrpc_skb_priv {
struct {
u16 offset; /* Offset of data */
u16 len; /* Length of data */
- u8 flags;
-#define RXRPC_RX_VERIFIED 0x01
};
struct {
rxrpc_seq_t first_ack; /* First packet in acks table */
@@ -309,15 +307,16 @@ struct rxrpc_security {
struct sk_buff *challenge);
/* verify a response */
- int (*verify_response)(struct rxrpc_connection *,
- struct sk_buff *);
+ int (*verify_response)(struct rxrpc_connection *conn,
+ struct sk_buff *response_skb,
+ void *response, unsigned int len);
/* clear connection security */
void (*clear)(struct rxrpc_connection *);
/* Default ticket -> key decoder */
int (*default_decode_ticket)(struct rxrpc_connection *conn, struct sk_buff *skb,
- unsigned int ticket_offset, unsigned int ticket_len,
+ void *ticket, unsigned int ticket_len,
struct key **_key);
};
@@ -774,6 +773,11 @@ struct rxrpc_call {
struct sk_buff_head recvmsg_queue; /* Queue of packets ready for recvmsg() */
struct sk_buff_head rx_queue; /* Queue of packets for this call to receive */
struct sk_buff_head rx_oos_queue; /* Queue of out of sequence packets */
+ void *rx_dec_buffer; /* Decryption buffer */
+ unsigned short rx_dec_bsize; /* rx_dec_buffer size */
+ unsigned short rx_dec_offset; /* Decrypted packet data offset */
+ unsigned short rx_dec_len; /* Decrypted packet data len */
+ rxrpc_seq_t rx_dec_seq; /* Packet in decryption buffer */
rxrpc_seq_t rx_highest_seq; /* Higest sequence number received */
rxrpc_seq_t rx_consumed; /* Highest packet consumed */
diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c
index fdd6832..fec59d9 100644
--- a/net/rxrpc/call_event.c
+++ b/net/rxrpc/call_event.c
@@ -332,25 +332,7 @@ bool rxrpc_input_call_event(struct rxrpc_call *call)
saw_ack |= sp->hdr.type == RXRPC_PACKET_TYPE_ACK;
- if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
- sp->hdr.securityIndex != 0 &&
- skb_cloned(skb)) {
- /* Unshare the packet so that it can be
- * modified by in-place decryption.
- */
- struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
-
- if (nskb) {
- rxrpc_new_skb(nskb, rxrpc_skb_new_unshared);
- rxrpc_input_call_packet(call, nskb);
- rxrpc_free_skb(nskb, rxrpc_skb_put_call_rx);
- } else {
- /* OOM - Drop the packet. */
- rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem);
- }
- } else {
- rxrpc_input_call_packet(call, skb);
- }
+ rxrpc_input_call_packet(call, skb);
rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
did_receive = true;
}
diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c
index f035f48..fcb9d38 100644
--- a/net/rxrpc/call_object.c
+++ b/net/rxrpc/call_object.c
@@ -152,6 +152,7 @@ struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *rx, gfp_t gfp,
spin_lock_init(&call->notify_lock);
refcount_set(&call->ref, 1);
call->debug_id = debug_id;
+ call->rx_pkt_offset = USHRT_MAX;
call->tx_total_len = -1;
call->tx_jumbo_max = 1;
call->next_rx_timo = 20 * HZ;
@@ -553,6 +554,7 @@ static void rxrpc_cleanup_rx_buffers(struct rxrpc_call *call)
rxrpc_purge_queue(&call->recvmsg_queue);
rxrpc_purge_queue(&call->rx_queue);
rxrpc_purge_queue(&call->rx_oos_queue);
+ kfree(call->rx_dec_buffer);
}
/*
diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c
index a2130d2..c96ca61 100644
--- a/net/rxrpc/conn_event.c
+++ b/net/rxrpc/conn_event.c
@@ -243,27 +243,22 @@ static void rxrpc_call_is_secure(struct rxrpc_call *call)
static int rxrpc_verify_response(struct rxrpc_connection *conn,
struct sk_buff *skb)
{
+ unsigned int len = skb->len - sizeof(struct rxrpc_wire_header);
+ void *buffer;
int ret;
- if (skb_cloned(skb)) {
- /* Copy the packet if shared so that we can do in-place
- * decryption.
- */
- struct sk_buff *nskb = skb_copy(skb, GFP_NOFS);
+ buffer = kmalloc(len, GFP_NOFS);
+ if (!buffer)
+ return -ENOMEM;
- if (nskb) {
- rxrpc_new_skb(nskb, rxrpc_skb_new_unshared);
- ret = conn->security->verify_response(conn, nskb);
- rxrpc_free_skb(nskb, rxrpc_skb_put_response_copy);
- } else {
- /* OOM - Drop the packet. */
- rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem);
- ret = -ENOMEM;
- }
- } else {
- ret = conn->security->verify_response(conn, skb);
- }
+ ret = skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), buffer, len);
+ if (ret < 0)
+ goto out;
+ ret = conn->security->verify_response(conn, skb, buffer, len);
+
+out:
+ kfree(buffer);
return ret;
}
diff --git a/net/rxrpc/insecure.c b/net/rxrpc/insecure.c
index 0a260df..0b39046 100644
--- a/net/rxrpc/insecure.c
+++ b/net/rxrpc/insecure.c
@@ -32,9 +32,6 @@ static int none_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb)
static int none_verify_packet(struct rxrpc_call *call, struct sk_buff *skb)
{
- struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-
- sp->flags |= RXRPC_RX_VERIFIED;
return 0;
}
@@ -57,9 +54,10 @@ static int none_sendmsg_respond_to_challenge(struct sk_buff *challenge,
}
static int none_verify_response(struct rxrpc_connection *conn,
- struct sk_buff *skb)
+ struct sk_buff *response_skb,
+ void *response, unsigned int len)
{
- return rxrpc_abort_conn(conn, skb, RX_PROTOCOL_ERROR, -EPROTO,
+ return rxrpc_abort_conn(conn, response_skb, RX_PROTOCOL_ERROR, -EPROTO,
rxrpc_eproto_rxnull_response);
}
diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c
index e1f7513..c940600 100644
--- a/net/rxrpc/recvmsg.c
+++ b/net/rxrpc/recvmsg.c
@@ -147,15 +147,52 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
}
/*
- * Decrypt and verify a DATA packet.
+ * Decrypt and verify a DATA packet. The content of the packet is pulled out
+ * into a flat buffer rather than decrypting in place in the skbuff. This also
+ * has the advantage of aligning the buffer correctly for the crypto routines.
+ *
+ * We keep track of the sequence number of the packet currently decrypted into
+ * the buffer in ->rx_dec_seq. If MSG_PEEK is used and steps onto a new
+ * packet, subsequent recvmsg() calls will have to go back and re-decrypt the
+ * current packet.
*/
static int rxrpc_verify_data(struct rxrpc_call *call, struct sk_buff *skb)
{
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+ int ret;
- if (sp->flags & RXRPC_RX_VERIFIED)
- return 0;
- return call->security->verify_packet(call, skb);
+ if (sp->len > call->rx_dec_bsize) {
+ /* Make sure we can hold a 1412-byte jumbo subpacket and make
+ * sure that the buffer size is aligned to a crypto blocksize.
+ */
+ size_t size = clamp(round_up(sp->len, 32), 2048, 65535);
+ void *buffer = krealloc(call->rx_dec_buffer, size, GFP_NOFS);
+
+ if (!buffer)
+ return -ENOMEM;
+ call->rx_dec_buffer = buffer;
+ call->rx_dec_bsize = size;
+ }
+
+ ret = -EFAULT;
+ if (skb_copy_bits(skb, sp->offset, call->rx_dec_buffer, sp->len) < 0)
+ goto err;
+
+ call->rx_dec_offset = 0;
+ call->rx_dec_len = sp->len;
+ call->rx_dec_seq = sp->hdr.seq;
+ ret = call->security->verify_packet(call, skb);
+ if (ret < 0)
+ goto err;
+ return 0;
+
+err:
+ kfree(call->rx_dec_buffer);
+ call->rx_dec_buffer = NULL;
+ call->rx_dec_bsize = 0;
+ call->rx_dec_offset = 0;
+ call->rx_dec_len = 0;
+ return ret;
}
/*
@@ -283,16 +320,21 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
if (msg)
sock_recv_timestamp(msg, sock->sk, skb);
- if (rx_pkt_offset == 0) {
+ if (call->rx_dec_seq != sp->hdr.seq ||
+ !call->rx_dec_buffer) {
ret2 = rxrpc_verify_data(call, skb);
trace_rxrpc_recvdata(call, rxrpc_recvmsg_next, seq,
- sp->offset, sp->len, ret2);
+ call->rx_dec_offset,
+ call->rx_dec_len, ret2);
if (ret2 < 0) {
ret = ret2;
goto out;
}
- rx_pkt_offset = sp->offset;
- rx_pkt_len = sp->len;
+ }
+
+ if (rx_pkt_offset == USHRT_MAX) {
+ rx_pkt_offset = call->rx_dec_offset;
+ rx_pkt_len = call->rx_dec_len;
} else {
trace_rxrpc_recvdata(call, rxrpc_recvmsg_cont, seq,
rx_pkt_offset, rx_pkt_len, 0);
@@ -304,10 +346,10 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
if (copy > remain)
copy = remain;
if (copy > 0) {
- ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
- copy);
- if (ret2 < 0) {
- ret = ret2;
+ ret2 = copy_to_iter(call->rx_dec_buffer + rx_pkt_offset,
+ copy, iter);
+ if (ret2 != copy) {
+ ret = -EFAULT;
goto out;
}
@@ -328,7 +370,7 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
/* The whole packet has been transferred. */
if (sp->hdr.flags & RXRPC_LAST_PACKET)
ret = 1;
- rx_pkt_offset = 0;
+ rx_pkt_offset = USHRT_MAX;
rx_pkt_len = 0;
skb = skb_peek_next(skb, &call->recvmsg_queue);
diff --git a/net/rxrpc/rxgk.c b/net/rxrpc/rxgk.c
index 0d5e654..a1ee102 100644
--- a/net/rxrpc/rxgk.c
+++ b/net/rxrpc/rxgk.c
@@ -473,15 +473,20 @@ static int rxgk_verify_packet_integrity(struct rxrpc_call *call,
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
struct rxgk_header *hdr;
struct krb5_buffer metadata;
- unsigned int offset = sp->offset, len = sp->len;
+ unsigned int len = call->rx_dec_len;
size_t data_offset = 0, data_len = len;
+ void *data = call->rx_dec_buffer, *p = data;
u32 ac = 0;
int ret = -ENOMEM;
_enter("");
- crypto_krb5_where_is_the_data(gk->krb5, KRB5_CHECKSUM_MODE,
- &data_offset, &data_len);
+ if (crypto_krb5_where_is_the_data(gk->krb5, KRB5_CHECKSUM_MODE,
+ &data_offset, &data_len) < 0) {
+ ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
+ rxgk_abort_1_short_header);
+ goto put_gk;
+ }
hdr = kzalloc_obj(*hdr, GFP_NOFS);
if (!hdr)
@@ -496,16 +501,15 @@ static int rxgk_verify_packet_integrity(struct rxrpc_call *call,
metadata.len = sizeof(*hdr);
metadata.data = hdr;
- ret = rxgk_verify_mic_skb(gk->krb5, gk->rx_Kc, &metadata,
- skb, &offset, &len, &ac);
+ ret = rxgk_verify_mic(gk->krb5, gk->rx_Kc, &metadata, &p, &len, &ac);
kfree(hdr);
if (ret < 0) {
if (ret != -ENOMEM)
rxrpc_abort_eproto(call, skb, ac,
rxgk_abort_1_verify_mic_eproto);
} else {
- sp->offset = offset;
- sp->len = len;
+ call->rx_dec_offset = p - data;
+ call->rx_dec_len = len;
}
put_gk:
@@ -522,49 +526,53 @@ static int rxgk_verify_packet_encrypted(struct rxrpc_call *call,
struct sk_buff *skb)
{
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
- struct rxgk_header hdr;
- unsigned int offset = sp->offset, len = sp->len;
+ struct rxgk_header *hdr;
+ unsigned int offset = 0, len = call->rx_dec_len;
+ void *data = call->rx_dec_buffer, *p = data;
int ret;
u32 ac = 0;
_enter("");
- ret = rxgk_decrypt_skb(gk->krb5, gk->rx_enc, skb, &offset, &len, &ac);
+ if (crypto_krb5_check_data_len(gk->krb5, KRB5_ENCRYPT_MODE,
+ len, sizeof(*hdr)) < 0) {
+ ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
+ rxgk_abort_2_short_header);
+ goto error;
+ }
+
+ ret = rxgk_decrypt(gk->krb5, gk->rx_enc, &p, &len, &ac);
if (ret < 0) {
if (ret != -ENOMEM)
rxrpc_abort_eproto(call, skb, ac, rxgk_abort_2_decrypt_eproto);
goto error;
}
+ offset = p - data;
- if (len < sizeof(hdr)) {
+ if (len < sizeof(*hdr)) {
ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
rxgk_abort_2_short_header);
goto error;
}
/* Extract the header from the skb */
- ret = skb_copy_bits(skb, offset, &hdr, sizeof(hdr));
- if (ret < 0) {
- ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
- rxgk_abort_2_short_encdata);
- goto error;
- }
- offset += sizeof(hdr);
- len -= sizeof(hdr);
+ hdr = data + offset;
+ offset += sizeof(*hdr);
+ len -= sizeof(*hdr);
- if (ntohl(hdr.epoch) != call->conn->proto.epoch ||
- ntohl(hdr.cid) != call->cid ||
- ntohl(hdr.call_number) != call->call_id ||
- ntohl(hdr.seq) != sp->hdr.seq ||
- ntohl(hdr.sec_index) != call->security_ix ||
- ntohl(hdr.data_len) > len) {
+ if (ntohl(hdr->epoch) != call->conn->proto.epoch ||
+ ntohl(hdr->cid) != call->cid ||
+ ntohl(hdr->call_number) != call->call_id ||
+ ntohl(hdr->seq) != sp->hdr.seq ||
+ ntohl(hdr->sec_index) != call->security_ix ||
+ ntohl(hdr->data_len) > len) {
ret = rxrpc_abort_eproto(call, skb, RXGK_SEALEDINCON,
rxgk_abort_2_short_data);
goto error;
}
- sp->offset = offset;
- sp->len = ntohl(hdr.data_len);
+ call->rx_dec_offset = offset;
+ call->rx_dec_len = ntohl(hdr->data_len);
ret = 0;
error:
rxgk_put(gk);
@@ -1076,11 +1084,12 @@ static int rxgk_sendmsg_respond_to_challenge(struct sk_buff *challenge,
* unsigned int call_numbers<>;
* };
*/
-static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn,
- const struct krb5_enctype *krb5,
- struct sk_buff *skb,
- __be32 *p, __be32 *end)
+static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
+ const struct krb5_enctype *krb5,
+ struct sk_buff *skb,
+ void *auth, unsigned int auth_len)
{
+ __be32 *p = auth, *end = auth + auth_len;
u32 app_len, call_count, level, epoch, cid, i;
_enter("");
@@ -1144,37 +1153,6 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn,
}
/*
- * Extract the authenticator and verify it.
- */
-static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
- const struct krb5_enctype *krb5,
- struct sk_buff *skb,
- unsigned int auth_offset, unsigned int auth_len)
-{
- void *auth;
- __be32 *p;
- int ret;
-
- auth = kmalloc(auth_len, GFP_NOFS);
- if (!auth)
- return -ENOMEM;
-
- ret = skb_copy_bits(skb, auth_offset, auth, auth_len);
- if (ret < 0) {
- ret = rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO,
- rxgk_abort_resp_short_auth);
- goto error;
- }
-
- p = auth;
- ret = rxgk_do_verify_authenticator(conn, krb5, skb, p,
- p + auth_len / sizeof(*p));
-error:
- kfree(auth);
- return ret;
-}
-
-/*
* Verify a response.
*
* struct RXGK_Response {
@@ -1184,49 +1162,45 @@ static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
* };
*/
static int rxgk_verify_response(struct rxrpc_connection *conn,
- struct sk_buff *skb)
+ struct sk_buff *skb,
+ void *buffer, unsigned int len)
{
const struct krb5_enctype *krb5;
struct rxrpc_key_token *token;
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
- struct rxgk_response rhdr;
+ struct rxgk_response *rhdr;
struct rxgk_context *gk;
struct key *key = NULL;
- unsigned int offset = sizeof(struct rxrpc_wire_header);
- unsigned int len = skb->len - sizeof(struct rxrpc_wire_header);
- unsigned int token_offset, token_len;
- unsigned int auth_offset, auth_len;
+ unsigned int resp_token_len, auth_len;
+ void *resp_token, *auth;
__be32 xauth_len;
int ret, ec;
_enter("{%d}", conn->debug_id);
/* Parse the RXGK_Response object */
- if (sizeof(rhdr) + sizeof(__be32) > len)
+ if (len < sizeof(*rhdr) + sizeof(__be32))
+ goto short_packet;
+ rhdr = buffer;
+ buffer += sizeof(*rhdr);
+ len -= sizeof(*rhdr);
+
+ resp_token = buffer;
+ resp_token_len = ntohl(rhdr->token_len);
+ if (resp_token_len > len ||
+ xdr_round_up(resp_token_len) + sizeof(__be32) > len)
goto short_packet;
- if (skb_copy_bits(skb, offset, &rhdr, sizeof(rhdr)) < 0)
- goto short_packet;
- offset += sizeof(rhdr);
- len -= sizeof(rhdr);
+ trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, resp_token_len);
- token_offset = offset;
- token_len = ntohl(rhdr.token_len);
- if (token_len > len ||
- xdr_round_up(token_len) + sizeof(__be32) > len)
- goto short_packet;
+ buffer += xdr_round_up(resp_token_len);
+ len -= xdr_round_up(resp_token_len);
- trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, token_len);
-
- offset += xdr_round_up(token_len);
- len -= xdr_round_up(token_len);
-
- if (skb_copy_bits(skb, offset, &xauth_len, sizeof(xauth_len)) < 0)
- goto short_packet;
- offset += sizeof(xauth_len);
+ xauth_len = *(__be32 *)buffer;
+ buffer += sizeof(xauth_len);
len -= sizeof(xauth_len);
- auth_offset = offset;
+ auth = buffer;
auth_len = ntohl(xauth_len);
if (auth_len > len)
goto short_packet;
@@ -1241,7 +1215,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
* to the app to deal with - which might mean a round trip to
* userspace.
*/
- ret = rxgk_extract_token(conn, skb, token_offset, token_len, &key);
+ ret = rxgk_extract_token(conn, skb, resp_token, resp_token_len, &key);
if (ret < 0)
goto out;
@@ -1255,7 +1229,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
*/
token = key->payload.data[0];
conn->security_level = token->rxgk->level;
- conn->rxgk.start_time = __be64_to_cpu(rhdr.start_time);
+ conn->rxgk.start_time = __be64_to_cpu(rhdr->start_time);
gk = rxgk_generate_transport_key(conn, token->rxgk, sp->hdr.cksum, GFP_NOFS);
if (IS_ERR(gk)) {
@@ -1265,18 +1239,18 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
krb5 = gk->krb5;
- trace_rxrpc_rx_response(conn, sp->hdr.serial, krb5->etype, sp->hdr.cksum, token_len);
+ trace_rxrpc_rx_response(conn, sp->hdr.serial, krb5->etype, sp->hdr.cksum,
+ resp_token_len);
/* Decrypt, parse and verify the authenticator. */
- ret = rxgk_decrypt_skb(krb5, gk->resp_enc, skb,
- &auth_offset, &auth_len, &ec);
+ ret = rxgk_decrypt(krb5, gk->resp_enc, &auth, &auth_len, &ec);
if (ret < 0) {
rxrpc_abort_conn(conn, skb, RXGK_SEALEDINCON, ret,
rxgk_abort_resp_auth_dec);
goto out_gk;
}
- ret = rxgk_verify_authenticator(conn, krb5, skb, auth_offset, auth_len);
+ ret = rxgk_verify_authenticator(conn, krb5, skb, auth, auth_len);
if (ret < 0)
goto out_gk;
diff --git a/net/rxrpc/rxgk_app.c b/net/rxrpc/rxgk_app.c
index 0ef2a29..200a300 100644
--- a/net/rxrpc/rxgk_app.c
+++ b/net/rxrpc/rxgk_app.c
@@ -40,7 +40,7 @@
* };
*/
int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
- unsigned int ticket_offset, unsigned int ticket_len,
+ void *buffer, unsigned int ticket_len,
struct key **_key)
{
struct rxrpc_key_token *token;
@@ -49,7 +49,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
size_t pre_ticket_len, payload_len;
unsigned int klen, enctype;
void *payload, *ticket;
- __be32 *t, *p, *q, tmp[2];
+ __be32 *t, *p, *q, *tmp;
int ret;
_enter("");
@@ -59,10 +59,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
rxgk_abort_resp_short_yfs_tkt);
/* Get the session key length */
- ret = skb_copy_bits(skb, ticket_offset, tmp, sizeof(tmp));
- if (ret < 0)
- return rxrpc_abort_conn(conn, skb, RXGK_INCONSISTENCY, -EPROTO,
- rxgk_abort_resp_short_yfs_klen);
+ tmp = buffer;
enctype = ntohl(tmp[0]);
klen = ntohl(tmp[1]);
@@ -84,12 +81,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
* it.
*/
ticket = payload + pre_ticket_len;
- ret = skb_copy_bits(skb, ticket_offset, ticket, ticket_len);
- if (ret < 0) {
- ret = rxrpc_abort_conn(conn, skb, RXGK_INCONSISTENCY, -EPROTO,
- rxgk_abort_resp_short_yfs_tkt);
- goto error;
- }
+ memcpy(ticket, buffer, ticket_len);
/* Fill out the form header. */
p = payload;
@@ -131,7 +123,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
goto error;
}
- /* Ticket read in with skb_copy_bits above */
+ /* Ticket appended above. */
q += xdr_round_up(ticket_len) / 4;
if (WARN_ON((unsigned long)q - (unsigned long)payload != payload_len)) {
ret = -EIO;
@@ -182,14 +174,15 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
* [tools.ietf.org/html/draft-wilkinson-afs3-rxgk-afs-08 sec 6.1]
*/
int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
- unsigned int token_offset, unsigned int token_len,
+ void *token, unsigned int token_len,
struct key **_key)
{
const struct krb5_enctype *krb5;
const struct krb5_buffer *server_secret;
struct crypto_aead *token_enc = NULL;
struct key *server_key;
- unsigned int ticket_offset, ticket_len;
+ unsigned int ticket_len;
+ void *ticket;
u32 kvno, enctype;
int ret, ec = 0;
@@ -197,24 +190,23 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
__be32 kvno;
__be32 enctype;
__be32 token_len;
- } container;
+ } *container;
- if (token_len < sizeof(container))
+ if (token_len < sizeof(*container))
goto short_packet;
/* Decode the RXGK_TokenContainer object. This tells us which server
* key we should be using. We can then fetch the key, get the secret
* and set up the crypto to extract the token.
*/
- if (skb_copy_bits(skb, token_offset, &container, sizeof(container)) < 0)
- goto short_packet;
+ container = token;
+ token += sizeof(*container);
- kvno = ntohl(container.kvno);
- enctype = ntohl(container.enctype);
- ticket_len = ntohl(container.token_len);
- ticket_offset = token_offset + sizeof(container);
+ kvno = ntohl(container->kvno);
+ enctype = ntohl(container->enctype);
+ ticket_len = ntohl(container->token_len);
- if (ticket_len > xdr_round_down(token_len - sizeof(container)))
+ if (ticket_len > xdr_round_down(token_len - sizeof(*container)))
goto short_packet;
_debug("KVNO %u", kvno);
@@ -237,8 +229,8 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
* gain access to K0, from which we can derive the transport key and
* thence decode the authenticator.
*/
- ret = rxgk_decrypt_skb(krb5, token_enc, skb,
- &ticket_offset, &ticket_len, &ec);
+ ticket = token;
+ ret = rxgk_decrypt(krb5, token_enc, &ticket, &ticket_len, &ec);
crypto_free_aead(token_enc);
token_enc = NULL;
if (ret < 0) {
@@ -248,7 +240,7 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
return ret;
}
- ret = conn->security->default_decode_ticket(conn, skb, ticket_offset,
+ ret = conn->security->default_decode_ticket(conn, skb, ticket,
ticket_len, _key);
if (ret < 0)
goto cant_get_token;
diff --git a/net/rxrpc/rxgk_common.h b/net/rxrpc/rxgk_common.h
index 1e257d7..3deed5863 100644
--- a/net/rxrpc/rxgk_common.h
+++ b/net/rxrpc/rxgk_common.h
@@ -41,10 +41,10 @@ struct rxgk_context {
* rxgk_app.c
*/
int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
- unsigned int ticket_offset, unsigned int ticket_len,
+ void *ticket, unsigned int ticket_len,
struct key **_key);
int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
- unsigned int token_offset, unsigned int token_len,
+ void *token, unsigned int token_len,
struct key **_key);
/*
@@ -62,31 +62,30 @@ int rxgk_set_up_token_cipher(const struct krb5_buffer *server_key,
gfp_t gfp);
/*
- * Apply decryption and checksumming functions to part of an skbuff. The
- * offset and length are updated to reflect the actual content of the encrypted
+ * Apply decryption and checksumming functions a flat data buffer. The data
+ * point and length are updated to reflect the actual content of the encrypted
* region.
*/
-static inline
-int rxgk_decrypt_skb(const struct krb5_enctype *krb5,
- struct crypto_aead *aead,
- struct sk_buff *skb,
- unsigned int *_offset, unsigned int *_len,
- int *_error_code)
+static inline int rxgk_decrypt(const struct krb5_enctype *krb5,
+ struct crypto_aead *aead,
+ void **_data, unsigned int *_len,
+ int *_error_code)
{
- struct scatterlist sg[16];
+ struct scatterlist sg[1];
size_t offset = 0, len = *_len;
- int nr_sg, ret;
+ int ret;
- sg_init_table(sg, ARRAY_SIZE(sg));
- nr_sg = skb_to_sgvec(skb, sg, *_offset, len);
- if (unlikely(nr_sg < 0))
- return nr_sg;
+ sg_init_one(sg, *_data, len);
- ret = crypto_krb5_decrypt(krb5, aead, sg, nr_sg,
- &offset, &len);
+ ret = crypto_krb5_decrypt(krb5, aead, sg, 1, &offset, &len);
switch (ret) {
case 0:
- *_offset += offset;
+ if (offset & 3) {
+ *_error_code = RXGK_INCONSISTENCY;
+ ret = -EPROTO;
+ break;
+ }
+ *_data += offset;
*_len = len;
break;
case -EBADMSG: /* Checksum mismatch. */
@@ -106,31 +105,26 @@ int rxgk_decrypt_skb(const struct krb5_enctype *krb5,
}
/*
- * Check the MIC on a region of an skbuff. The offset and length are updated
- * to reflect the actual content of the secure region.
+ * Check the MIC on a flat buffer. The data pointer and length are updated to
+ * reflect the actual content of the secure region.
*/
static inline
-int rxgk_verify_mic_skb(const struct krb5_enctype *krb5,
- struct crypto_shash *shash,
- const struct krb5_buffer *metadata,
- struct sk_buff *skb,
- unsigned int *_offset, unsigned int *_len,
- u32 *_error_code)
+int rxgk_verify_mic(const struct krb5_enctype *krb5,
+ struct crypto_shash *shash,
+ const struct krb5_buffer *metadata,
+ void **_data, unsigned int *_len,
+ u32 *_error_code)
{
- struct scatterlist sg[16];
+ struct scatterlist sg[1];
size_t offset = 0, len = *_len;
- int nr_sg, ret;
+ int ret;
- sg_init_table(sg, ARRAY_SIZE(sg));
- nr_sg = skb_to_sgvec(skb, sg, *_offset, len);
- if (unlikely(nr_sg < 0))
- return nr_sg;
+ sg_init_one(sg, *_data, len);
- ret = crypto_krb5_verify_mic(krb5, shash, metadata, sg, nr_sg,
- &offset, &len);
+ ret = crypto_krb5_verify_mic(krb5, shash, metadata, sg, 1, &offset, &len);
switch (ret) {
case 0:
- *_offset += offset;
+ *_data += offset;
*_len = len;
break;
case -EBADMSG: /* Checksum mismatch */
diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c
index cba7935..6fbd883 100644
--- a/net/rxrpc/rxkad.c
+++ b/net/rxrpc/rxkad.c
@@ -430,27 +430,25 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
rxrpc_seq_t seq,
struct skcipher_request *req)
{
- struct rxkad_level1_hdr sechdr;
+ struct rxkad_level1_hdr *sechdr;
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
struct rxrpc_crypt iv;
- struct scatterlist sg[16];
- u32 data_size, buf;
+ struct scatterlist sg[1];
+ void *data = call->rx_dec_buffer;
+ u32 len = sp->len, data_size, buf;
u16 check;
int ret;
_enter("");
- if (sp->len < 8)
+ if (len < 8)
return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
rxkad_abort_1_short_header);
/* Decrypt the skbuff in-place. TODO: We really want to decrypt
* directly into the target buffer.
*/
- sg_init_table(sg, ARRAY_SIZE(sg));
- ret = skb_to_sgvec(skb, sg, sp->offset, 8);
- if (unlikely(ret < 0))
- return ret;
+ sg_init_one(sg, data, len);
/* start the decryption afresh */
memset(&iv, 0, sizeof(iv));
@@ -464,13 +462,11 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
return ret;
/* Extract the decrypted packet length */
- if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0)
- return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
- rxkad_abort_1_short_encdata);
- sp->offset += sizeof(sechdr);
- sp->len -= sizeof(sechdr);
+ sechdr = data;
+ call->rx_dec_offset = sizeof(*sechdr);
+ len -= sizeof(*sechdr);
- buf = ntohl(sechdr.data_size);
+ buf = ntohl(sechdr->data_size);
data_size = buf & 0xffff;
check = buf >> 16;
@@ -479,10 +475,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
if (check != 0)
return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
rxkad_abort_1_short_check);
- if (data_size > sp->len)
+ if (data_size > len)
return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
rxkad_abort_1_short_data);
- sp->len = data_size;
+ call->rx_dec_len = data_size;
_leave(" = 0 [dlen=%x]", data_size);
return 0;
@@ -496,43 +492,28 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
struct skcipher_request *req)
{
const struct rxrpc_key_token *token;
- struct rxkad_level2_hdr sechdr;
+ struct rxkad_level2_hdr *sechdr;
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
struct rxrpc_crypt iv;
- struct scatterlist _sg[4], *sg;
- u32 data_size, buf;
+ struct scatterlist sg[1];
+ void *data = call->rx_dec_buffer;
+ u32 len = sp->len, data_size, buf;
u16 check;
- int nsg, ret;
+ int ret;
- _enter(",{%d}", sp->len);
+ _enter(",{%d}", len);
- if (sp->len < 8)
+ if (len < 8)
return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
rxkad_abort_2_short_header);
/* Don't let the crypto algo see a misaligned length. */
- sp->len = round_down(sp->len, 8);
+ len = round_down(len, 8);
- /* Decrypt the skbuff in-place. TODO: We really want to decrypt
- * directly into the target buffer.
+ /* Decrypt in place in the call's decryption buffer. TODO: We really
+ * want to decrypt directly into the target buffer.
*/
- sg = _sg;
- nsg = skb_shinfo(skb)->nr_frags + 1;
- if (nsg <= 4) {
- nsg = 4;
- } else {
- sg = kmalloc_objs(*sg, nsg, GFP_NOIO);
- if (!sg)
- return -ENOMEM;
- }
-
- sg_init_table(sg, nsg);
- ret = skb_to_sgvec(skb, sg, sp->offset, sp->len);
- if (unlikely(ret < 0)) {
- if (sg != _sg)
- kfree(sg);
- return ret;
- }
+ sg_init_one(sg, data, len);
/* decrypt from the session key */
token = call->conn->key->payload.data[0];
@@ -540,11 +521,9 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher);
skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, sg, sg, sp->len, iv.x);
+ skcipher_request_set_crypt(req, sg, sg, len, iv.x);
ret = crypto_skcipher_decrypt(req);
skcipher_request_zero(req);
- if (sg != _sg)
- kfree(sg);
if (ret < 0) {
if (ret == -ENOMEM)
return ret;
@@ -553,13 +532,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
}
/* Extract the decrypted packet length */
- if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0)
- return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
- rxkad_abort_2_short_len);
- sp->offset += sizeof(sechdr);
- sp->len -= sizeof(sechdr);
+ sechdr = data;
+ call->rx_dec_offset = sizeof(*sechdr);
+ len -= sizeof(*sechdr);
- buf = ntohl(sechdr.data_size);
+ buf = ntohl(sechdr->data_size);
data_size = buf & 0xffff;
check = buf >> 16;
@@ -569,17 +546,18 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
rxkad_abort_2_short_check);
- if (data_size > sp->len)
+ if (data_size > len)
return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
rxkad_abort_2_short_data);
- sp->len = data_size;
+ call->rx_dec_len = data_size;
_leave(" = 0 [dlen=%x]", data_size);
return 0;
}
/*
- * Verify the security on a received packet and the subpackets therein.
+ * Verify the security on a received (sub)packet. If the packet needs
+ * modifying (e.g. decrypting), it must be copied.
*/
static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb)
{
@@ -985,7 +963,6 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn,
*_expiry = 0;
ASSERT(server_key->payload.data[0] != NULL);
- ASSERTCMP((unsigned long) ticket & 7UL, ==, 0);
memcpy(&iv, &server_key->payload.data[2], sizeof(iv));
@@ -1134,14 +1111,15 @@ static int rxkad_decrypt_response(struct rxrpc_connection *conn,
* verify a response
*/
static int rxkad_verify_response(struct rxrpc_connection *conn,
- struct sk_buff *skb)
+ struct sk_buff *skb,
+ void *buffer, unsigned int len)
{
struct rxkad_response *response;
struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
struct rxrpc_crypt session_key;
struct key *server_key;
time64_t expiry;
- void *ticket = NULL;
+ void *ticket;
u32 version, kvno, ticket_len, level;
__be32 csum;
int ret, i;
@@ -1164,13 +1142,8 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
}
}
- ret = -ENOMEM;
- response = kzalloc_obj(struct rxkad_response, GFP_NOFS);
- if (!response)
- goto error;
-
- if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header),
- response, sizeof(*response)) < 0) {
+ response = buffer;
+ if (len < sizeof(*response)) {
ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO,
rxkad_abort_resp_short);
goto error;
@@ -1182,6 +1155,9 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
trace_rxrpc_rx_response(conn, sp->hdr.serial, version, kvno, ticket_len);
+ buffer += sizeof(*response);
+ len -= sizeof(*response);
+
if (version != RXKAD_VERSION) {
ret = rxrpc_abort_conn(conn, skb, RXKADINCONSISTENCY, -EPROTO,
rxkad_abort_resp_version);
@@ -1201,13 +1177,8 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
}
/* extract the kerberos ticket and decrypt and decode it */
- ret = -ENOMEM;
- ticket = kmalloc(ticket_len, GFP_NOFS);
- if (!ticket)
- goto error;
-
- if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header) + sizeof(*response),
- ticket, ticket_len) < 0) {
+ ticket = buffer;
+ if (ticket_len > len) {
ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO,
rxkad_abort_resp_short_tkt);
goto error;
@@ -1287,8 +1258,6 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
ret = rxrpc_get_server_data_key(conn, &session_key, expiry, kvno);
error:
- kfree(ticket);
- kfree(response);
key_put(server_key);
_leave(" = %d", ret);
return ret;
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 332fd96..04ea11c 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -112,11 +112,6 @@ struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action,
}
EXPORT_SYMBOL(tcf_action_set_ctrlact);
-/* XXX: For standalone actions, we don't need a RCU grace period either, because
- * actions are always connected to filters and filters are already destroyed in
- * RCU callbacks, so after a RCU grace period actions are already disconnected
- * from filters. Readers later can not find us.
- */
static void free_tcf(struct tc_action *p)
{
struct tcf_chain *chain = rcu_dereference_protected(p->goto_chain, 1);
@@ -129,7 +124,7 @@ static void free_tcf(struct tc_action *p)
if (chain)
tcf_chain_put_by_act(chain);
- kfree(p);
+ kfree_rcu(p, tcfa_rcu);
}
static void offload_action_hw_count_set(struct tc_action *act,
diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
index 2c5a7a3..553342c 100644
--- a/net/sched/act_mirred.c
+++ b/net/sched/act_mirred.c
@@ -26,6 +26,10 @@
#include <net/tc_act/tc_mirred.h>
#include <net/tc_wrapper.h>
+#define MIRRED_DEFER_LIMIT 3
+_Static_assert(MIRRED_DEFER_LIMIT <= 3,
+ "MIRRED_DEFER_LIMIT exceeds tc_depth bitfield width");
+
static LIST_HEAD(mirred_list);
static DEFINE_SPINLOCK(mirred_list_lock);
@@ -234,12 +238,15 @@ tcf_mirred_forward(bool at_ingress, bool want_ingress, struct sk_buff *skb)
{
int err;
- if (!want_ingress)
+ if (!want_ingress) {
err = tcf_dev_queue_xmit(skb, dev_queue_xmit);
- else if (!at_ingress)
- err = netif_rx(skb);
- else
- err = netif_receive_skb(skb);
+ } else {
+ skb->tc_depth++;
+ if (!at_ingress)
+ err = netif_rx(skb);
+ else
+ err = netif_receive_skb(skb);
+ }
return err;
}
@@ -365,7 +372,8 @@ static int tcf_blockcast_redir(struct sk_buff *skb, struct tcf_mirred *m,
dev_is_mac_header_xmit(dev_prev),
m_eaction, retval);
- return retval;
+ /* If the packet wasn't redirected, we have to register as a drop */
+ return TC_ACT_SHOT;
}
static int tcf_blockcast_mirror(struct sk_buff *skb, struct tcf_mirred *m,
@@ -389,14 +397,12 @@ static int tcf_blockcast_mirror(struct sk_buff *skb, struct tcf_mirred *m,
static int tcf_blockcast(struct sk_buff *skb, struct tcf_mirred *m,
const u32 blockid, struct tcf_result *res,
- int retval)
+ int m_eaction, int retval)
{
const u32 exception_ifindex = skb->dev->ifindex;
struct tcf_block *block;
bool is_redirect;
- int m_eaction;
- m_eaction = READ_ONCE(m->tcfm_eaction);
is_redirect = tcf_mirred_is_act_redirect(m_eaction);
/* we are already under rcu protection, so can call block lookup
@@ -405,7 +411,7 @@ static int tcf_blockcast(struct sk_buff *skb, struct tcf_mirred *m,
block = tcf_block_lookup(dev_net(skb->dev), blockid);
if (!block || xa_empty(&block->ports)) {
tcf_action_inc_overlimit_qstats(&m->common);
- return retval;
+ return is_redirect ? TC_ACT_SHOT : retval;
}
if (is_redirect)
@@ -423,9 +429,10 @@ TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
{
struct tcf_mirred *m = to_mirred(a);
int retval = READ_ONCE(m->tcf_action);
+ bool m_mac_header_xmit, is_redirect;
struct netdev_xmit *xmit;
- bool m_mac_header_xmit;
struct net_device *dev;
+ bool want_ingress;
int i, m_eaction;
u32 blockid;
@@ -434,7 +441,8 @@ TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
#else
xmit = this_cpu_ptr(&softnet_data.xmit);
#endif
- if (unlikely(xmit->sched_mirred_nest >= MIRRED_NEST_LIMIT)) {
+ if (unlikely(xmit->sched_mirred_nest >= MIRRED_NEST_LIMIT ||
+ skb->tc_depth >= MIRRED_DEFER_LIMIT)) {
net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n",
netdev_name(skb->dev));
return TC_ACT_SHOT;
@@ -444,34 +452,51 @@ TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
tcf_action_update_bstats(&m->common, skb);
blockid = READ_ONCE(m->tcfm_blockid);
- if (blockid)
- return tcf_blockcast(skb, m, blockid, res, retval);
+ m_eaction = READ_ONCE(m->tcfm_eaction);
+ want_ingress = tcf_mirred_act_wants_ingress(m_eaction);
+ if (blockid) {
+ if (!want_ingress)
+ xmit->sched_mirred_dev[xmit->sched_mirred_nest++] = NULL;
+ retval = tcf_blockcast(skb, m, blockid, res, m_eaction, retval);
+ if (!want_ingress)
+ xmit->sched_mirred_nest--;
+ return retval;
+ }
+
+ is_redirect = tcf_mirred_is_act_redirect(m_eaction);
dev = rcu_dereference_bh(m->tcfm_dev);
if (unlikely(!dev)) {
pr_notice_once("tc mirred: target device is gone\n");
tcf_action_inc_overlimit_qstats(&m->common);
- return retval;
- }
- for (i = 0; i < xmit->sched_mirred_nest; i++) {
- if (xmit->sched_mirred_dev[i] != dev)
- continue;
- pr_notice_once("tc mirred: loop on device %s\n",
- netdev_name(dev));
- tcf_action_inc_overlimit_qstats(&m->common);
- return retval;
+ goto err_out;
}
- xmit->sched_mirred_dev[xmit->sched_mirred_nest++] = dev;
+ if (!want_ingress) {
+ for (i = 0; i < xmit->sched_mirred_nest; i++) {
+ if (xmit->sched_mirred_dev[i] != dev)
+ continue;
+ pr_notice_once("tc mirred: loop on device %s\n",
+ netdev_name(dev));
+ tcf_action_inc_overlimit_qstats(&m->common);
+ goto err_out;
+ }
+ xmit->sched_mirred_dev[xmit->sched_mirred_nest++] = dev;
+ }
m_mac_header_xmit = READ_ONCE(m->tcfm_mac_header_xmit);
- m_eaction = READ_ONCE(m->tcfm_eaction);
retval = tcf_mirred_to_dev(skb, m, dev, m_mac_header_xmit, m_eaction,
retval);
- xmit->sched_mirred_nest--;
+ if (!want_ingress)
+ xmit->sched_mirred_nest--;
return retval;
+
+err_out:
+ if (is_redirect)
+ retval = TC_ACT_SHOT;
+ return retval;
}
static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c
index bc20f08..bd3b1da 100644
--- a/net/sched/act_pedit.c
+++ b/net/sched/act_pedit.c
@@ -16,6 +16,8 @@
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/slab.h>
+#include <linux/overflow.h>
+#include <linux/unaligned.h>
#include <net/ipv6.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
@@ -242,7 +244,6 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla,
goto out_free_ex;
}
- nparms->tcfp_off_max_hint = 0;
nparms->tcfp_flags = parm->flags;
nparms->tcfp_nkeys = parm->nkeys;
@@ -268,14 +269,6 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla,
BITS_PER_TYPE(int) - 1,
nparms->tcfp_keys[i].shift);
- /* The AT option can read a single byte, we can bound the actual
- * value with uchar max.
- */
- cur += (0xff & offmask) >> nparms->tcfp_keys[i].shift;
-
- /* Each key touches 4 bytes starting from the computed offset */
- nparms->tcfp_off_max_hint =
- max(nparms->tcfp_off_max_hint, cur + 4);
}
p = to_pedit(*a);
@@ -318,15 +311,12 @@ static void tcf_pedit_cleanup(struct tc_action *a)
call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu);
}
-static bool offset_valid(struct sk_buff *skb, int offset)
+static bool offset_valid(struct sk_buff *skb, int offset, int len)
{
- if (offset > 0 && offset > skb->len)
+ if (offset < -(int)skb_headroom(skb))
return false;
- if (offset < 0 && -offset > skb_headroom(skb))
- return false;
-
- return true;
+ return offset <= (int)skb->len - len;
}
static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type)
@@ -393,18 +383,10 @@ TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
struct tcf_pedit_key_ex *tkey_ex;
struct tcf_pedit_parms *parms;
struct tc_pedit_key *tkey;
- u32 max_offset;
int i;
parms = rcu_dereference_bh(p->parms);
- max_offset = (skb_transport_header_was_set(skb) ?
- skb_transport_offset(skb) :
- skb_network_offset(skb)) +
- parms->tcfp_off_max_hint;
- if (skb_ensure_writable(skb, min(skb->len, max_offset)))
- goto done;
-
tcf_lastuse_update(&p->tcf_tm);
tcf_action_update_bstats(&p->common, skb);
@@ -412,10 +394,11 @@ TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
tkey_ex = parms->tcfp_keys_ex;
for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) {
+ int write_offset, write_len;
int offset = tkey->off;
int hoffset = 0;
- u32 *ptr, hdata;
- u32 val;
+ u32 cur_val, val;
+ u32 *ptr;
int rc;
if (tkey_ex) {
@@ -433,13 +416,15 @@ TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
if (tkey->offmask) {
u8 *d, _d;
+ int at_offset;
- if (!offset_valid(skb, hoffset + tkey->at)) {
+ if (check_add_overflow(hoffset, (int)tkey->at, &at_offset) ||
+ !offset_valid(skb, at_offset, sizeof(_d))) {
pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n",
hoffset + tkey->at);
goto bad;
}
- d = skb_header_pointer(skb, hoffset + tkey->at,
+ d = skb_header_pointer(skb, at_offset,
sizeof(_d), &_d);
if (!d)
goto bad;
@@ -451,31 +436,51 @@ TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
}
}
- if (!offset_valid(skb, hoffset + offset)) {
- pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset + offset);
+ if (check_add_overflow(hoffset, offset, &write_offset)) {
+ pr_info_ratelimited("tc action pedit offset overflow\n");
goto bad;
}
- ptr = skb_header_pointer(skb, hoffset + offset,
- sizeof(hdata), &hdata);
- if (!ptr)
+ if (!offset_valid(skb, write_offset, sizeof(*ptr))) {
+ pr_info_ratelimited("tc action pedit offset %d out of bounds\n",
+ write_offset);
goto bad;
+ }
+
+ if (write_offset < 0) {
+ if (skb_cow(skb, -write_offset))
+ goto bad;
+ if (write_offset + (int)sizeof(*ptr) > 0) {
+ if (skb_ensure_writable(skb,
+ min_t(int, skb->len,
+ write_offset + (int)sizeof(*ptr))))
+ goto bad;
+ }
+ } else {
+ if (check_add_overflow(write_offset, (int)sizeof(*ptr),
+ &write_len))
+ goto bad;
+ if (skb_ensure_writable(skb, min_t(int, skb->len,
+ write_len)))
+ goto bad;
+ }
+
+ ptr = (u32 *)(skb->data + write_offset);
+ cur_val = get_unaligned(ptr);
/* just do it, baby */
switch (cmd) {
case TCA_PEDIT_KEY_EX_CMD_SET:
val = tkey->val;
break;
case TCA_PEDIT_KEY_EX_CMD_ADD:
- val = (*ptr + tkey->val) & ~tkey->mask;
+ val = (cur_val + tkey->val) & ~tkey->mask;
break;
default:
pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd);
goto bad;
}
- *ptr = ((*ptr & tkey->mask) ^ val);
- if (ptr == &hdata)
- skb_store_bits(skb, hoffset + offset, ptr, 4);
+ put_unaligned((cur_val & tkey->mask) ^ val, ptr);
}
goto done;
diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index 02e1fa4..5862933 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -399,14 +399,14 @@ static void cake_configure_rates(struct Qdisc *sch, u64 rate, bool rate_adjust);
* Here, invsqrt is a fixed point number (< 1.0), 32bit mantissa, aka Q0.32
*/
-static void cobalt_newton_step(struct cobalt_vars *vars)
+static void cobalt_newton_step(struct cobalt_vars *vars, u32 count)
{
u32 invsqrt, invsqrt2;
u64 val;
invsqrt = vars->rec_inv_sqrt;
invsqrt2 = ((u64)invsqrt * invsqrt) >> 32;
- val = (3LL << 32) - ((u64)vars->count * invsqrt2);
+ val = (3LL << 32) - ((u64)count * invsqrt2);
val >>= 2; /* avoid overflow in following multiply */
val = (val * invsqrt) >> (32 - 2 + 1);
@@ -414,12 +414,12 @@ static void cobalt_newton_step(struct cobalt_vars *vars)
vars->rec_inv_sqrt = val;
}
-static void cobalt_invsqrt(struct cobalt_vars *vars)
+static void cobalt_invsqrt(struct cobalt_vars *vars, u32 count)
{
- if (vars->count < REC_INV_SQRT_CACHE)
- vars->rec_inv_sqrt = inv_sqrt_cache[vars->count];
+ if (count < REC_INV_SQRT_CACHE)
+ vars->rec_inv_sqrt = inv_sqrt_cache[count];
else
- cobalt_newton_step(vars);
+ cobalt_newton_step(vars, count);
}
static void cobalt_vars_init(struct cobalt_vars *vars)
@@ -449,16 +449,19 @@ static bool cobalt_queue_full(struct cobalt_vars *vars,
bool up = false;
if (ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {
- up = !vars->p_drop;
- vars->p_drop += p->p_inc;
- if (vars->p_drop < p->p_inc)
- vars->p_drop = ~0;
- vars->blue_timer = now;
+ u32 p_drop = vars->p_drop;
+
+ up = !p_drop;
+ p_drop += p->p_inc;
+ if (p_drop < p->p_inc)
+ p_drop = ~0;
+ WRITE_ONCE(vars->p_drop, p_drop);
+ WRITE_ONCE(vars->blue_timer, now);
}
- vars->dropping = true;
- vars->drop_next = now;
+ WRITE_ONCE(vars->dropping, true);
+ WRITE_ONCE(vars->drop_next, now);
if (!vars->count)
- vars->count = 1;
+ WRITE_ONCE(vars->count, 1);
return up;
}
@@ -475,20 +478,20 @@ static bool cobalt_queue_empty(struct cobalt_vars *vars,
if (vars->p_drop &&
ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {
if (vars->p_drop < p->p_dec)
- vars->p_drop = 0;
+ WRITE_ONCE(vars->p_drop, 0);
else
- vars->p_drop -= p->p_dec;
- vars->blue_timer = now;
+ WRITE_ONCE(vars->p_drop, vars->p_drop - p->p_dec);
+ WRITE_ONCE(vars->blue_timer, now);
down = !vars->p_drop;
}
- vars->dropping = false;
+ WRITE_ONCE(vars->dropping, false);
if (vars->count && ktime_to_ns(ktime_sub(now, vars->drop_next)) >= 0) {
- vars->count--;
- cobalt_invsqrt(vars);
- vars->drop_next = cobalt_control(vars->drop_next,
- p->interval,
- vars->rec_inv_sqrt);
+ WRITE_ONCE(vars->count, vars->count - 1);
+ cobalt_invsqrt(vars, vars->count);
+ WRITE_ONCE(vars->drop_next,
+ cobalt_control(vars->drop_next, p->interval,
+ vars->rec_inv_sqrt));
}
return down;
@@ -507,6 +510,7 @@ static enum qdisc_drop_reason cobalt_should_drop(struct cobalt_vars *vars,
bool next_due, over_target;
ktime_t schedule;
u64 sojourn;
+ u32 count;
/* The 'schedule' variable records, in its sign, whether 'now' is before or
* after 'drop_next'. This allows 'drop_next' to be updated before the next
@@ -528,21 +532,22 @@ static enum qdisc_drop_reason cobalt_should_drop(struct cobalt_vars *vars,
over_target = sojourn > p->target &&
sojourn > p->mtu_time * bulk_flows * 2 &&
sojourn > p->mtu_time * 4;
- next_due = vars->count && ktime_to_ns(schedule) >= 0;
+ count = vars->count;
+ next_due = count && ktime_to_ns(schedule) >= 0;
vars->ecn_marked = false;
if (over_target) {
if (!vars->dropping) {
- vars->dropping = true;
- vars->drop_next = cobalt_control(now,
- p->interval,
- vars->rec_inv_sqrt);
+ WRITE_ONCE(vars->dropping, true);
+ WRITE_ONCE(vars->drop_next,
+ cobalt_control(now, p->interval,
+ vars->rec_inv_sqrt));
}
- if (!vars->count)
- vars->count = 1;
+ if (!count)
+ count = 1;
} else if (vars->dropping) {
- vars->dropping = false;
+ WRITE_ONCE(vars->dropping, false);
}
if (next_due && vars->dropping) {
@@ -550,23 +555,23 @@ static enum qdisc_drop_reason cobalt_should_drop(struct cobalt_vars *vars,
if (!(vars->ecn_marked = INET_ECN_set_ce(skb)))
reason = QDISC_DROP_CONGESTED;
- vars->count++;
- if (!vars->count)
- vars->count--;
- cobalt_invsqrt(vars);
- vars->drop_next = cobalt_control(vars->drop_next,
- p->interval,
- vars->rec_inv_sqrt);
+ count++;
+ if (!count)
+ count--;
+ cobalt_invsqrt(vars, count);
+ WRITE_ONCE(vars->drop_next,
+ cobalt_control(vars->drop_next, p->interval,
+ vars->rec_inv_sqrt));
schedule = ktime_sub(now, vars->drop_next);
} else {
while (next_due) {
- vars->count--;
- cobalt_invsqrt(vars);
- vars->drop_next = cobalt_control(vars->drop_next,
- p->interval,
- vars->rec_inv_sqrt);
+ count--;
+ cobalt_invsqrt(vars, count);
+ WRITE_ONCE(vars->drop_next,
+ cobalt_control(vars->drop_next, p->interval,
+ vars->rec_inv_sqrt));
schedule = ktime_sub(now, vars->drop_next);
- next_due = vars->count && ktime_to_ns(schedule) >= 0;
+ next_due = count && ktime_to_ns(schedule) >= 0;
}
}
@@ -575,11 +580,12 @@ static enum qdisc_drop_reason cobalt_should_drop(struct cobalt_vars *vars,
get_random_u32() < vars->p_drop)
reason = QDISC_DROP_FLOOD_PROTECTION;
+ WRITE_ONCE(vars->count, count);
/* Overload the drop_next field as an activity timeout */
- if (!vars->count)
- vars->drop_next = ktime_add_ns(now, p->interval);
+ if (!count)
+ WRITE_ONCE(vars->drop_next, ktime_add_ns(now, p->interval));
else if (ktime_to_ns(schedule) > 0 && reason == QDISC_DROP_UNSPEC)
- vars->drop_next = now;
+ WRITE_ONCE(vars->drop_next, now);
return reason;
}
@@ -813,7 +819,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb,
i++, k = (k + 1) % CAKE_SET_WAYS) {
if (q->tags[outer_hash + k] == flow_hash) {
if (i)
- q->way_hits++;
+ WRITE_ONCE(q->way_hits, q->way_hits + 1);
if (!q->flows[outer_hash + k].set) {
/* need to increment host refcnts */
@@ -831,7 +837,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb,
for (i = 0; i < CAKE_SET_WAYS;
i++, k = (k + 1) % CAKE_SET_WAYS) {
if (!q->flows[outer_hash + k].set) {
- q->way_misses++;
+ WRITE_ONCE(q->way_misses, q->way_misses + 1);
allocate_src = cake_dsrc(flow_mode);
allocate_dst = cake_ddst(flow_mode);
goto found;
@@ -841,7 +847,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb,
/* With no empty queues, default to the original
* queue, accept the collision, update the host tags.
*/
- q->way_collisions++;
+ WRITE_ONCE(q->way_collisions, q->way_collisions + 1);
allocate_src = cake_dsrc(flow_mode);
allocate_dst = cake_ddst(flow_mode);
@@ -914,7 +920,7 @@ static struct sk_buff *dequeue_head(struct cake_flow *flow)
struct sk_buff *skb = flow->head;
if (skb) {
- flow->head = skb->next;
+ WRITE_ONCE(flow->head, skb->next);
skb_mark_not_on_list(skb);
}
@@ -926,7 +932,7 @@ static struct sk_buff *dequeue_head(struct cake_flow *flow)
static void flow_queue_add(struct cake_flow *flow, struct sk_buff *skb)
{
if (!flow->head)
- flow->head = skb;
+ WRITE_ONCE(flow->head, skb);
else
flow->tail->next = skb;
flow->tail = skb;
@@ -1357,7 +1363,7 @@ static struct sk_buff *cake_ack_filter(struct cake_sched_data *q,
if (elig_ack_prev)
elig_ack_prev->next = elig_ack->next;
else
- flow->head = elig_ack->next;
+ WRITE_ONCE(flow->head, elig_ack->next);
skb_mark_not_on_list(elig_ack);
@@ -1379,9 +1385,9 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off)
len -= off;
if (qd->max_netlen < len)
- qd->max_netlen = len;
+ WRITE_ONCE(qd->max_netlen, len);
if (qd->min_netlen > len)
- qd->min_netlen = len;
+ WRITE_ONCE(qd->min_netlen, len);
len += q->rate_overhead;
@@ -1401,9 +1407,9 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off)
}
if (qd->max_adjlen < len)
- qd->max_adjlen = len;
+ WRITE_ONCE(qd->max_adjlen, len);
if (qd->min_adjlen > len)
- qd->min_adjlen = len;
+ WRITE_ONCE(qd->min_adjlen, len);
return len;
}
@@ -1416,7 +1422,7 @@ static u32 cake_overhead(struct cake_sched_data *q, const struct sk_buff *skb)
u16 segs = qdisc_pkt_segs(skb);
u32 len = qdisc_pkt_len(skb);
- q->avg_netoff = cake_ewma(q->avg_netoff, off << 16, 8);
+ WRITE_ONCE(q->avg_netoff, cake_ewma(q->avg_netoff, off << 16, 8));
if (segs == 1)
return cake_calc_overhead(q, len, off);
@@ -1590,16 +1596,17 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
}
if (cobalt_queue_full(&flow->cvars, &b->cparams, now))
- b->unresponsive_flow_count++;
+ WRITE_ONCE(b->unresponsive_flow_count,
+ b->unresponsive_flow_count + 1);
len = qdisc_pkt_len(skb);
q->buffer_used -= skb->truesize;
- b->backlogs[idx] -= len;
- b->tin_backlog -= len;
+ WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
+ WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] - len);
sch->qstats.backlog -= len;
- flow->dropped++;
- b->tin_dropped++;
+ WRITE_ONCE(flow->dropped, flow->dropped + 1);
+ WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1);
if (q->config->rate_flags & CAKE_FLAG_INGRESS)
cake_advance_shaper(q, b, skb, now, true);
@@ -1795,7 +1802,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
if (unlikely(len > b->max_skblen))
- b->max_skblen = len;
+ WRITE_ONCE(b->max_skblen, len);
if (qdisc_pkt_segs(skb) > 1 && q->config->rate_flags & CAKE_FLAG_SPLIT_GSO) {
struct sk_buff *segs, *nskb;
@@ -1819,15 +1826,15 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
numsegs++;
slen += segs->len;
q->buffer_used += segs->truesize;
- b->packets++;
+ WRITE_ONCE(b->packets, b->packets + 1);
}
/* stats */
- b->bytes += slen;
- b->backlogs[idx] += slen;
- b->tin_backlog += slen;
sch->qstats.backlog += slen;
q->avg_window_bytes += slen;
+ WRITE_ONCE(b->bytes, b->bytes + slen);
+ WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen);
+ WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] + slen);
qdisc_tree_reduce_backlog(sch, 1-numsegs, len-slen);
consume_skb(skb);
@@ -1843,10 +1850,10 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
ack = cake_ack_filter(q, flow);
if (ack) {
- b->ack_drops++;
+ WRITE_ONCE(b->ack_drops, b->ack_drops + 1);
sch->qstats.drops++;
ack_pkt_len = qdisc_pkt_len(ack);
- b->bytes += ack_pkt_len;
+ WRITE_ONCE(b->bytes, b->bytes + ack_pkt_len);
q->buffer_used += skb->truesize - ack->truesize;
if (q->config->rate_flags & CAKE_FLAG_INGRESS)
cake_advance_shaper(q, b, ack, now, true);
@@ -1859,12 +1866,12 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
/* stats */
- b->packets++;
- b->bytes += len - ack_pkt_len;
- b->backlogs[idx] += len - ack_pkt_len;
- b->tin_backlog += len - ack_pkt_len;
+ WRITE_ONCE(b->packets, b->packets + 1);
sch->qstats.backlog += len - ack_pkt_len;
q->avg_window_bytes += len - ack_pkt_len;
+ WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len);
+ WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len);
+ WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] + len - ack_pkt_len);
}
if (q->overflow_timeout)
@@ -1894,9 +1901,9 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
u64 b = q->avg_window_bytes * (u64)NSEC_PER_SEC;
b = div64_u64(b, window_interval);
- q->avg_peak_bandwidth =
- cake_ewma(q->avg_peak_bandwidth, b,
- b > q->avg_peak_bandwidth ? 2 : 8);
+ WRITE_ONCE(q->avg_peak_bandwidth,
+ cake_ewma(q->avg_peak_bandwidth, b,
+ b > q->avg_peak_bandwidth ? 2 : 8));
q->avg_window_bytes = 0;
q->avg_window_begin = now;
@@ -1917,27 +1924,27 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (!flow->set) {
list_add_tail(&flow->flowchain, &b->new_flows);
} else {
- b->decaying_flow_count--;
+ WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count - 1);
list_move_tail(&flow->flowchain, &b->new_flows);
}
flow->set = CAKE_SET_SPARSE;
- b->sparse_flow_count++;
+ WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count + 1);
- flow->deficit = cake_get_flow_quantum(b, flow, q->config->flow_mode);
+ WRITE_ONCE(flow->deficit, cake_get_flow_quantum(b, flow, q->config->flow_mode));
} else if (flow->set == CAKE_SET_SPARSE_WAIT) {
/* this flow was empty, accounted as a sparse flow, but actually
* in the bulk rotation.
*/
flow->set = CAKE_SET_BULK;
- b->sparse_flow_count--;
- b->bulk_flow_count++;
+ WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1);
+ WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count + 1);
cake_inc_srchost_bulk_flow_count(b, flow, q->config->flow_mode);
cake_inc_dsthost_bulk_flow_count(b, flow, q->config->flow_mode);
}
if (q->buffer_used > q->buffer_max_used)
- q->buffer_max_used = q->buffer_used;
+ WRITE_ONCE(q->buffer_max_used, q->buffer_used);
if (q->buffer_used <= q->buffer_limit)
return NET_XMIT_SUCCESS;
@@ -1976,8 +1983,8 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch)
if (flow->head) {
skb = dequeue_head(flow);
len = qdisc_pkt_len(skb);
- b->backlogs[q->cur_flow] -= len;
- b->tin_backlog -= len;
+ WRITE_ONCE(b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len);
+ WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
sch->qstats.backlog -= len;
q->buffer_used -= skb->truesize;
sch->q.qlen--;
@@ -2042,7 +2049,7 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
cake_configure_rates(sch, new_rate, true);
q->last_checked_active = now;
- q->active_queues = num_active_qs;
+ WRITE_ONCE(q->active_queues, num_active_qs);
}
begin:
@@ -2149,8 +2156,8 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
*/
if (flow->set == CAKE_SET_SPARSE) {
if (flow->head) {
- b->sparse_flow_count--;
- b->bulk_flow_count++;
+ WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1);
+ WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count + 1);
cake_inc_srchost_bulk_flow_count(b, flow, q->config->flow_mode);
cake_inc_dsthost_bulk_flow_count(b, flow, q->config->flow_mode);
@@ -2165,7 +2172,8 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
}
}
- flow->deficit += cake_get_flow_quantum(b, flow, q->config->flow_mode);
+ WRITE_ONCE(flow->deficit,
+ flow->deficit + cake_get_flow_quantum(b, flow, q->config->flow_mode));
list_move_tail(&flow->flowchain, &b->old_flows);
goto retry;
@@ -2177,7 +2185,8 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
if (!skb) {
/* this queue was actually empty */
if (cobalt_queue_empty(&flow->cvars, &b->cparams, now))
- b->unresponsive_flow_count--;
+ WRITE_ONCE(b->unresponsive_flow_count,
+ b->unresponsive_flow_count - 1);
if (flow->cvars.p_drop || flow->cvars.count ||
ktime_before(now, flow->cvars.drop_next)) {
@@ -2187,32 +2196,32 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
list_move_tail(&flow->flowchain,
&b->decaying_flows);
if (flow->set == CAKE_SET_BULK) {
- b->bulk_flow_count--;
+ WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count - 1);
cake_dec_srchost_bulk_flow_count(b, flow, q->config->flow_mode);
cake_dec_dsthost_bulk_flow_count(b, flow, q->config->flow_mode);
- b->decaying_flow_count++;
+ WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count + 1);
} else if (flow->set == CAKE_SET_SPARSE ||
flow->set == CAKE_SET_SPARSE_WAIT) {
- b->sparse_flow_count--;
- b->decaying_flow_count++;
+ WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1);
+ WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count + 1);
}
flow->set = CAKE_SET_DECAYING;
} else {
/* remove empty queue from the flowchain */
list_del_init(&flow->flowchain);
if (flow->set == CAKE_SET_SPARSE ||
- flow->set == CAKE_SET_SPARSE_WAIT)
- b->sparse_flow_count--;
- else if (flow->set == CAKE_SET_BULK) {
- b->bulk_flow_count--;
+ flow->set == CAKE_SET_SPARSE_WAIT) {
+ WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1);
+ } else if (flow->set == CAKE_SET_BULK) {
+ WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count - 1);
cake_dec_srchost_bulk_flow_count(b, flow, q->config->flow_mode);
cake_dec_dsthost_bulk_flow_count(b, flow, q->config->flow_mode);
- } else
- b->decaying_flow_count--;
-
+ } else {
+ WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count - 1);
+ }
flow->set = CAKE_SET_NONE;
}
goto begin;
@@ -2230,11 +2239,11 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
if (q->config->rate_flags & CAKE_FLAG_INGRESS) {
len = cake_advance_shaper(q, b, skb,
now, true);
- flow->deficit -= len;
+ WRITE_ONCE(flow->deficit, flow->deficit - len);
b->tin_deficit -= len;
}
- flow->dropped++;
- b->tin_dropped++;
+ WRITE_ONCE(flow->dropped, flow->dropped + 1);
+ WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1);
qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
qdisc_qstats_drop(sch);
qdisc_dequeue_drop(sch, skb, reason);
@@ -2242,20 +2251,22 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
goto retry;
}
- b->tin_ecn_mark += !!flow->cvars.ecn_marked;
+ WRITE_ONCE(b->tin_ecn_mark, b->tin_ecn_mark + !!flow->cvars.ecn_marked);
qdisc_bstats_update(sch, skb);
WRITE_ONCE(q->last_active, now);
/* collect delay stats */
delay = ktime_to_ns(ktime_sub(now, cobalt_get_enqueue_time(skb)));
- b->avge_delay = cake_ewma(b->avge_delay, delay, 8);
- b->peak_delay = cake_ewma(b->peak_delay, delay,
- delay > b->peak_delay ? 2 : 8);
- b->base_delay = cake_ewma(b->base_delay, delay,
- delay < b->base_delay ? 2 : 8);
+ WRITE_ONCE(b->avge_delay, cake_ewma(b->avge_delay, delay, 8));
+ WRITE_ONCE(b->peak_delay,
+ cake_ewma(b->peak_delay, delay,
+ delay > b->peak_delay ? 2 : 8));
+ WRITE_ONCE(b->base_delay,
+ cake_ewma(b->base_delay, delay,
+ delay < b->base_delay ? 2 : 8));
len = cake_advance_shaper(q, b, skb, now, false);
- flow->deficit -= len;
+ WRITE_ONCE(flow->deficit, flow->deficit - len);
b->tin_deficit -= len;
if (ktime_after(q->time_next_packet, now) && sch->q.qlen) {
@@ -2329,9 +2340,9 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu,
u8 rate_shft = 0;
u64 rate_ns = 0;
- b->flow_quantum = 1514;
if (rate) {
- b->flow_quantum = max(min(rate >> 12, 1514ULL), 300ULL);
+ WRITE_ONCE(b->flow_quantum,
+ max(min(rate >> 12, 1514ULL), 300ULL));
rate_shft = 34;
rate_ns = ((u64)NSEC_PER_SEC) << rate_shft;
rate_ns = div64_u64(rate_ns, max(MIN_RATE, rate));
@@ -2339,9 +2350,11 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu,
rate_ns >>= 1;
rate_shft--;
}
- } /* else unlimited, ie. zero delay */
-
- b->tin_rate_bps = rate;
+ } else {
+ /* else unlimited, ie. zero delay */
+ WRITE_ONCE(b->flow_quantum, 1514);
+ }
+ WRITE_ONCE(b->tin_rate_bps, rate);
b->tin_rate_ns = rate_ns;
b->tin_rate_shft = rate_shft;
@@ -2350,10 +2363,11 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu,
byte_target_ns = (byte_target * rate_ns) >> rate_shft;
- b->cparams.target = max((byte_target_ns * 3) / 2, target_ns);
- b->cparams.interval = max(rtt_est_ns +
- b->cparams.target - target_ns,
- b->cparams.target * 2);
+ WRITE_ONCE(b->cparams.target,
+ max((byte_target_ns * 3) / 2, target_ns));
+ WRITE_ONCE(b->cparams.interval,
+ max(rtt_est_ns + b->cparams.target - target_ns,
+ b->cparams.target * 2));
b->cparams.mtu_time = byte_target_ns;
b->cparams.p_inc = 1 << 24; /* 1/256 */
b->cparams.p_dec = 1 << 20; /* 1/4096 */
@@ -2611,25 +2625,27 @@ static void cake_reconfigure(struct Qdisc *sch)
{
struct cake_sched_data *qd = qdisc_priv(sch);
struct cake_sched_config *q = qd->config;
+ u32 buffer_limit;
cake_configure_rates(sch, qd->config->rate_bps, false);
if (q->buffer_config_limit) {
- qd->buffer_limit = q->buffer_config_limit;
+ buffer_limit = q->buffer_config_limit;
} else if (q->rate_bps) {
u64 t = q->rate_bps * q->interval;
do_div(t, USEC_PER_SEC / 4);
- qd->buffer_limit = max_t(u32, t, 4U << 20);
+ buffer_limit = max_t(u32, t, 4U << 20);
} else {
- qd->buffer_limit = ~0;
+ buffer_limit = ~0;
}
sch->flags &= ~TCQ_F_CAN_BYPASS;
- qd->buffer_limit = min(qd->buffer_limit,
- max(sch->limit * psched_mtu(qdisc_dev(sch)),
- q->buffer_config_limit));
+ WRITE_ONCE(qd->buffer_limit,
+ min(buffer_limit,
+ max(sch->limit * psched_mtu(qdisc_dev(sch)),
+ q->buffer_config_limit)));
}
static int cake_config_change(struct cake_sched_config *q, struct nlattr *opt,
@@ -2774,10 +2790,10 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
return ret;
if (overhead_changed) {
- qd->max_netlen = 0;
- qd->max_adjlen = 0;
- qd->min_netlen = ~0;
- qd->min_adjlen = ~0;
+ WRITE_ONCE(qd->max_netlen, 0);
+ WRITE_ONCE(qd->max_adjlen, 0);
+ WRITE_ONCE(qd->min_netlen, ~0);
+ WRITE_ONCE(qd->min_adjlen, ~0);
}
if (qd->tins) {
@@ -2995,15 +3011,15 @@ static int cake_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
goto nla_put_failure; \
} while (0)
- PUT_STAT_U64(CAPACITY_ESTIMATE64, q->avg_peak_bandwidth);
- PUT_STAT_U32(MEMORY_LIMIT, q->buffer_limit);
- PUT_STAT_U32(MEMORY_USED, q->buffer_max_used);
- PUT_STAT_U32(AVG_NETOFF, ((q->avg_netoff + 0x8000) >> 16));
- PUT_STAT_U32(MAX_NETLEN, q->max_netlen);
- PUT_STAT_U32(MAX_ADJLEN, q->max_adjlen);
- PUT_STAT_U32(MIN_NETLEN, q->min_netlen);
- PUT_STAT_U32(MIN_ADJLEN, q->min_adjlen);
- PUT_STAT_U32(ACTIVE_QUEUES, q->active_queues);
+ PUT_STAT_U64(CAPACITY_ESTIMATE64, READ_ONCE(q->avg_peak_bandwidth));
+ PUT_STAT_U32(MEMORY_LIMIT, READ_ONCE(q->buffer_limit));
+ PUT_STAT_U32(MEMORY_USED, READ_ONCE(q->buffer_max_used));
+ PUT_STAT_U32(AVG_NETOFF, ((READ_ONCE(q->avg_netoff) + 0x8000) >> 16));
+ PUT_STAT_U32(MAX_NETLEN, READ_ONCE(q->max_netlen));
+ PUT_STAT_U32(MAX_ADJLEN, READ_ONCE(q->max_adjlen));
+ PUT_STAT_U32(MIN_NETLEN, READ_ONCE(q->min_netlen));
+ PUT_STAT_U32(MIN_ADJLEN, READ_ONCE(q->min_adjlen));
+ PUT_STAT_U32(ACTIVE_QUEUES, READ_ONCE(q->active_queues));
#undef PUT_STAT_U32
#undef PUT_STAT_U64
@@ -3029,38 +3045,38 @@ static int cake_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
if (!ts)
goto nla_put_failure;
- PUT_TSTAT_U64(THRESHOLD_RATE64, b->tin_rate_bps);
- PUT_TSTAT_U64(SENT_BYTES64, b->bytes);
- PUT_TSTAT_U32(BACKLOG_BYTES, b->tin_backlog);
+ PUT_TSTAT_U64(THRESHOLD_RATE64, READ_ONCE(b->tin_rate_bps));
+ PUT_TSTAT_U64(SENT_BYTES64, READ_ONCE(b->bytes));
+ PUT_TSTAT_U32(BACKLOG_BYTES, READ_ONCE(b->tin_backlog));
PUT_TSTAT_U32(TARGET_US,
- ktime_to_us(ns_to_ktime(b->cparams.target)));
+ ktime_to_us(ns_to_ktime(READ_ONCE(b->cparams.target))));
PUT_TSTAT_U32(INTERVAL_US,
- ktime_to_us(ns_to_ktime(b->cparams.interval)));
+ ktime_to_us(ns_to_ktime(READ_ONCE(b->cparams.interval))));
- PUT_TSTAT_U32(SENT_PACKETS, b->packets);
- PUT_TSTAT_U32(DROPPED_PACKETS, b->tin_dropped);
- PUT_TSTAT_U32(ECN_MARKED_PACKETS, b->tin_ecn_mark);
- PUT_TSTAT_U32(ACKS_DROPPED_PACKETS, b->ack_drops);
+ PUT_TSTAT_U32(SENT_PACKETS, READ_ONCE(b->packets));
+ PUT_TSTAT_U32(DROPPED_PACKETS, READ_ONCE(b->tin_dropped));
+ PUT_TSTAT_U32(ECN_MARKED_PACKETS, READ_ONCE(b->tin_ecn_mark));
+ PUT_TSTAT_U32(ACKS_DROPPED_PACKETS, READ_ONCE(b->ack_drops));
PUT_TSTAT_U32(PEAK_DELAY_US,
- ktime_to_us(ns_to_ktime(b->peak_delay)));
+ ktime_to_us(ns_to_ktime(READ_ONCE(b->peak_delay))));
PUT_TSTAT_U32(AVG_DELAY_US,
- ktime_to_us(ns_to_ktime(b->avge_delay)));
+ ktime_to_us(ns_to_ktime(READ_ONCE(b->avge_delay))));
PUT_TSTAT_U32(BASE_DELAY_US,
- ktime_to_us(ns_to_ktime(b->base_delay)));
+ ktime_to_us(ns_to_ktime(READ_ONCE(b->base_delay))));
- PUT_TSTAT_U32(WAY_INDIRECT_HITS, b->way_hits);
- PUT_TSTAT_U32(WAY_MISSES, b->way_misses);
- PUT_TSTAT_U32(WAY_COLLISIONS, b->way_collisions);
+ PUT_TSTAT_U32(WAY_INDIRECT_HITS, READ_ONCE(b->way_hits));
+ PUT_TSTAT_U32(WAY_MISSES, READ_ONCE(b->way_misses));
+ PUT_TSTAT_U32(WAY_COLLISIONS, READ_ONCE(b->way_collisions));
- PUT_TSTAT_U32(SPARSE_FLOWS, b->sparse_flow_count +
- b->decaying_flow_count);
- PUT_TSTAT_U32(BULK_FLOWS, b->bulk_flow_count);
- PUT_TSTAT_U32(UNRESPONSIVE_FLOWS, b->unresponsive_flow_count);
- PUT_TSTAT_U32(MAX_SKBLEN, b->max_skblen);
+ PUT_TSTAT_U32(SPARSE_FLOWS, READ_ONCE(b->sparse_flow_count) +
+ READ_ONCE(b->decaying_flow_count));
+ PUT_TSTAT_U32(BULK_FLOWS, READ_ONCE(b->bulk_flow_count));
+ PUT_TSTAT_U32(UNRESPONSIVE_FLOWS, READ_ONCE(b->unresponsive_flow_count));
+ PUT_TSTAT_U32(MAX_SKBLEN, READ_ONCE(b->max_skblen));
- PUT_TSTAT_U32(FLOW_QUANTUM, b->flow_quantum);
+ PUT_TSTAT_U32(FLOW_QUANTUM, READ_ONCE(b->flow_quantum));
nla_nest_end(d->skb, ts);
}
@@ -3128,7 +3144,7 @@ static int cake_dump_class_stats(struct Qdisc *sch, unsigned long cl,
flow = &b->flows[idx % CAKE_QUEUES];
- if (flow->head) {
+ if (READ_ONCE(flow->head)) {
sch_tree_lock(sch);
skb = flow->head;
while (skb) {
@@ -3137,13 +3153,15 @@ static int cake_dump_class_stats(struct Qdisc *sch, unsigned long cl,
}
sch_tree_unlock(sch);
}
- qs.backlog = b->backlogs[idx % CAKE_QUEUES];
- qs.drops = flow->dropped;
+ qs.backlog = READ_ONCE(b->backlogs[idx % CAKE_QUEUES]);
+ qs.drops = READ_ONCE(flow->dropped);
}
if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
return -1;
if (flow) {
ktime_t now = ktime_get();
+ bool dropping;
+ u32 p_drop;
stats = nla_nest_start_noflag(d->skb, TCA_STATS_APP);
if (!stats)
@@ -3158,21 +3176,23 @@ static int cake_dump_class_stats(struct Qdisc *sch, unsigned long cl,
goto nla_put_failure; \
} while (0)
- PUT_STAT_S32(DEFICIT, flow->deficit);
- PUT_STAT_U32(DROPPING, flow->cvars.dropping);
- PUT_STAT_U32(COBALT_COUNT, flow->cvars.count);
- PUT_STAT_U32(P_DROP, flow->cvars.p_drop);
- if (flow->cvars.p_drop) {
+ PUT_STAT_S32(DEFICIT, READ_ONCE(flow->deficit));
+ dropping = READ_ONCE(flow->cvars.dropping);
+ PUT_STAT_U32(DROPPING, dropping);
+ PUT_STAT_U32(COBALT_COUNT, READ_ONCE(flow->cvars.count));
+ p_drop = READ_ONCE(flow->cvars.p_drop);
+ PUT_STAT_U32(P_DROP, p_drop);
+ if (p_drop) {
PUT_STAT_S32(BLUE_TIMER_US,
ktime_to_us(
ktime_sub(now,
- flow->cvars.blue_timer)));
+ READ_ONCE(flow->cvars.blue_timer))));
}
- if (flow->cvars.dropping) {
+ if (dropping) {
PUT_STAT_S32(DROP_NEXT_US,
ktime_to_us(
ktime_sub(now,
- flow->cvars.drop_next)));
+ READ_ONCE(flow->cvars.drop_next))));
}
if (nla_nest_end(d->skb, stats) < 0)
@@ -3298,10 +3318,10 @@ static int cake_mq_change(struct Qdisc *sch, struct nlattr *opt,
struct cake_sched_data *qd = qdisc_priv(chld);
if (overhead_changed) {
- qd->max_netlen = 0;
- qd->max_adjlen = 0;
- qd->min_netlen = ~0;
- qd->min_adjlen = ~0;
+ WRITE_ONCE(qd->max_netlen, 0);
+ WRITE_ONCE(qd->max_adjlen, 0);
+ WRITE_ONCE(qd->min_netlen, ~0);
+ WRITE_ONCE(qd->min_adjlen, ~0);
}
if (qd->tins) {
diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c
index 8c9a040..0f953bd 100644
--- a/net/sched/sch_cbs.c
+++ b/net/sched/sch_cbs.c
@@ -243,6 +243,20 @@ static struct sk_buff *cbs_dequeue(struct Qdisc *sch)
return q->dequeue(sch);
}
+static void cbs_reset(struct Qdisc *sch)
+{
+ struct cbs_sched_data *q = qdisc_priv(sch);
+
+ /* Nothing to do if we couldn't create the underlying qdisc */
+ if (!q->qdisc)
+ return;
+
+ qdisc_reset(q->qdisc);
+ qdisc_watchdog_cancel(&q->watchdog);
+ q->credits = 0;
+ q->last = 0;
+}
+
static const struct nla_policy cbs_policy[TCA_CBS_MAX + 1] = {
[TCA_CBS_PARMS] = { .len = sizeof(struct tc_cbs_qopt) },
};
@@ -540,7 +554,7 @@ static struct Qdisc_ops cbs_qdisc_ops __read_mostly = {
.dequeue = cbs_dequeue,
.peek = qdisc_peek_dequeued,
.init = cbs_init,
- .reset = qdisc_reset_queue,
+ .reset = cbs_reset,
.destroy = cbs_destroy,
.change = cbs_change,
.dump = cbs_dump,
diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c
index 94df8e7..2875bcd 100644
--- a/net/sched/sch_choke.c
+++ b/net/sched/sch_choke.c
@@ -229,7 +229,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
/* Draw a packet at random from queue and compare flow */
if (choke_match_random(q, skb, &idx)) {
- q->stats.matched++;
+ WRITE_ONCE(q->stats.matched, q->stats.matched + 1);
choke_drop_by_idx(sch, idx, to_free);
goto congestion_drop;
}
@@ -241,11 +241,13 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
qdisc_qstats_overlimit(sch);
if (use_harddrop(q) || !use_ecn(q) ||
!INET_ECN_set_ce(skb)) {
- q->stats.forced_drop++;
+ WRITE_ONCE(q->stats.forced_drop,
+ q->stats.forced_drop + 1);
goto congestion_drop;
}
- q->stats.forced_mark++;
+ WRITE_ONCE(q->stats.forced_mark,
+ q->stats.forced_mark + 1);
} else if (++q->vars.qcount) {
if (red_mark_probability(p, &q->vars, q->vars.qavg)) {
q->vars.qcount = 0;
@@ -253,11 +255,13 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
qdisc_qstats_overlimit(sch);
if (!use_ecn(q) || !INET_ECN_set_ce(skb)) {
- q->stats.prob_drop++;
+ WRITE_ONCE(q->stats.prob_drop,
+ q->stats.prob_drop + 1);
goto congestion_drop;
}
- q->stats.prob_mark++;
+ WRITE_ONCE(q->stats.prob_mark,
+ q->stats.prob_mark + 1);
}
} else
q->vars.qR = red_random(p);
@@ -272,7 +276,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
return NET_XMIT_SUCCESS;
}
- q->stats.pdrop++;
+ WRITE_ONCE(q->stats.pdrop, q->stats.pdrop + 1);
return qdisc_drop(skb, sch, to_free);
congestion_drop:
@@ -461,10 +465,12 @@ static int choke_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
struct choke_sched_data *q = qdisc_priv(sch);
struct tc_choke_xstats st = {
- .early = q->stats.prob_drop + q->stats.forced_drop,
- .marked = q->stats.prob_mark + q->stats.forced_mark,
- .pdrop = q->stats.pdrop,
- .matched = q->stats.matched,
+ .early = READ_ONCE(q->stats.prob_drop) +
+ READ_ONCE(q->stats.forced_drop),
+ .marked = READ_ONCE(q->stats.prob_mark) +
+ READ_ONCE(q->stats.forced_mark),
+ .pdrop = READ_ONCE(q->stats.pdrop),
+ .matched = READ_ONCE(q->stats.matched),
};
return gnet_stats_copy_app(d, &st, sizeof(st));
diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c
index 241e6a4..a22489c 100644
--- a/net/sched/sch_dualpi2.c
+++ b/net/sched/sch_dualpi2.c
@@ -938,6 +938,8 @@ static int dualpi2_init(struct Qdisc *sch, struct nlattr *opt,
int err;
sch->flags |= TCQ_F_DEQUEUE_DROPS;
+ hrtimer_setup(&q->pi2_timer, dualpi2_timer, CLOCK_MONOTONIC,
+ HRTIMER_MODE_ABS_PINNED_SOFT);
q->l_queue = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
TC_H_MAKE(sch->handle, 1), extack);
@@ -950,8 +952,6 @@ static int dualpi2_init(struct Qdisc *sch, struct nlattr *opt,
q->sch = sch;
dualpi2_reset_default(sch);
- hrtimer_setup(&q->pi2_timer, dualpi2_timer, CLOCK_MONOTONIC,
- HRTIMER_MODE_ABS_PINNED_SOFT);
if (opt && nla_len(opt)) {
err = dualpi2_change(sch, opt, extack);
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index 0664b2f..24db546 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -117,7 +117,7 @@ static inline struct sk_buff *dequeue_head(struct fq_codel_flow *flow)
{
struct sk_buff *skb = flow->head;
- flow->head = skb->next;
+ WRITE_ONCE(flow->head, skb->next);
skb_mark_not_on_list(skb);
return skb;
}
@@ -127,7 +127,7 @@ static inline void flow_queue_add(struct fq_codel_flow *flow,
struct sk_buff *skb)
{
if (flow->head == NULL)
- flow->head = skb;
+ WRITE_ONCE(flow->head, skb);
else
flow->tail->next = skb;
flow->tail = skb;
@@ -173,8 +173,8 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
} while (++i < max_packets && len < threshold);
/* Tell codel to increase its signal strength also */
- flow->cvars.count += i;
- q->backlogs[idx] -= len;
+ WRITE_ONCE(flow->cvars.count, flow->cvars.count + i);
+ WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] - len);
q->memory_usage -= mem;
sch->qstats.drops += i;
sch->qstats.backlog -= len;
@@ -204,13 +204,13 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
codel_set_enqueue_time(skb);
flow = &q->flows[idx];
flow_queue_add(flow, skb);
- q->backlogs[idx] += qdisc_pkt_len(skb);
+ WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] + qdisc_pkt_len(skb));
qdisc_qstats_backlog_inc(sch, skb);
if (list_empty(&flow->flowchain)) {
list_add_tail(&flow->flowchain, &q->new_flows);
q->new_flow_count++;
- flow->deficit = q->quantum;
+ WRITE_ONCE(flow->deficit, q->quantum);
}
get_codel_cb(skb)->mem_usage = skb->truesize;
q->memory_usage += get_codel_cb(skb)->mem_usage;
@@ -263,7 +263,8 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
flow = container_of(vars, struct fq_codel_flow, cvars);
if (flow->head) {
skb = dequeue_head(flow);
- q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);
+ WRITE_ONCE(q->backlogs[flow - q->flows],
+ q->backlogs[flow - q->flows] - qdisc_pkt_len(skb));
q->memory_usage -= get_codel_cb(skb)->mem_usage;
sch->q.qlen--;
sch->qstats.backlog -= qdisc_pkt_len(skb);
@@ -296,7 +297,7 @@ static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch)
flow = list_first_entry(head, struct fq_codel_flow, flowchain);
if (flow->deficit <= 0) {
- flow->deficit += q->quantum;
+ WRITE_ONCE(flow->deficit, flow->deficit + q->quantum);
list_move_tail(&flow->flowchain, &q->old_flows);
goto begin;
}
@@ -314,7 +315,7 @@ static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch)
goto begin;
}
qdisc_bstats_update(sch, skb);
- flow->deficit -= qdisc_pkt_len(skb);
+ WRITE_ONCE(flow->deficit, flow->deficit - qdisc_pkt_len(skb));
if (q->cstats.drop_count) {
qdisc_tree_reduce_backlog(sch, q->cstats.drop_count,
@@ -328,7 +329,7 @@ static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch)
static void fq_codel_flow_purge(struct fq_codel_flow *flow)
{
rtnl_kfree_skbs(flow->head, flow->tail);
- flow->head = NULL;
+ WRITE_ONCE(flow->head, NULL);
}
static void fq_codel_reset(struct Qdisc *sch)
@@ -656,21 +657,21 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
memset(&xstats, 0, sizeof(xstats));
xstats.type = TCA_FQ_CODEL_XSTATS_CLASS;
- xstats.class_stats.deficit = flow->deficit;
+ xstats.class_stats.deficit = READ_ONCE(flow->deficit);
xstats.class_stats.ldelay =
- codel_time_to_us(flow->cvars.ldelay);
- xstats.class_stats.count = flow->cvars.count;
- xstats.class_stats.lastcount = flow->cvars.lastcount;
- xstats.class_stats.dropping = flow->cvars.dropping;
- if (flow->cvars.dropping) {
- codel_tdiff_t delta = flow->cvars.drop_next -
+ codel_time_to_us(READ_ONCE(flow->cvars.ldelay));
+ xstats.class_stats.count = READ_ONCE(flow->cvars.count);
+ xstats.class_stats.lastcount = READ_ONCE(flow->cvars.lastcount);
+ xstats.class_stats.dropping = READ_ONCE(flow->cvars.dropping);
+ if (xstats.class_stats.dropping) {
+ codel_tdiff_t delta = READ_ONCE(flow->cvars.drop_next) -
codel_get_time();
xstats.class_stats.drop_next = (delta >= 0) ?
codel_time_to_us(delta) :
-codel_time_to_us(-delta);
}
- if (flow->head) {
+ if (READ_ONCE(flow->head)) {
sch_tree_lock(sch);
skb = flow->head;
while (skb) {
@@ -679,7 +680,7 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
}
sch_tree_unlock(sch);
}
- qs.backlog = q->backlogs[idx];
+ qs.backlog = READ_ONCE(q->backlogs[idx]);
qs.drops = 0;
}
if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
index 154c70f..7becbf5 100644
--- a/net/sched/sch_fq_pie.c
+++ b/net/sched/sch_fq_pie.c
@@ -509,18 +509,19 @@ static int fq_pie_dump(struct Qdisc *sch, struct sk_buff *skb)
static int fq_pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
- struct tc_fq_pie_xstats st = {
- .packets_in = q->stats.packets_in,
- .overlimit = q->stats.overlimit,
- .overmemory = q->overmemory,
- .dropped = q->stats.dropped,
- .ecn_mark = q->stats.ecn_mark,
- .new_flow_count = q->new_flow_count,
- .memory_usage = q->memory_usage,
- };
+ struct tc_fq_pie_xstats st = { 0 };
struct list_head *pos;
sch_tree_lock(sch);
+
+ st.packets_in = q->stats.packets_in;
+ st.overlimit = q->stats.overlimit;
+ st.overmemory = q->overmemory;
+ st.dropped = q->stats.dropped;
+ st.ecn_mark = q->stats.ecn_mark;
+ st.new_flow_count = q->new_flow_count;
+ st.memory_usage = q->memory_usage;
+
list_for_each(pos, &q->new_flows)
st.new_flows_len++;
diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c
index 20df1c0..17a79fe 100644
--- a/net/sched/sch_netem.c
+++ b/net/sched/sch_netem.c
@@ -227,10 +227,10 @@ static bool loss_4state(struct netem_sched_data *q)
if (rnd < clg->a4) {
clg->state = LOST_IN_GAP_PERIOD;
return true;
- } else if (clg->a4 < rnd && rnd < clg->a1 + clg->a4) {
+ } else if (rnd < clg->a1 + clg->a4) {
clg->state = LOST_IN_BURST_PERIOD;
return true;
- } else if (clg->a1 + clg->a4 < rnd) {
+ } else {
clg->state = TX_IN_GAP_PERIOD;
}
@@ -247,9 +247,9 @@ static bool loss_4state(struct netem_sched_data *q)
case LOST_IN_BURST_PERIOD:
if (rnd < clg->a3)
clg->state = TX_IN_BURST_PERIOD;
- else if (clg->a3 < rnd && rnd < clg->a2 + clg->a3) {
+ else if (rnd < clg->a2 + clg->a3) {
clg->state = TX_IN_GAP_PERIOD;
- } else if (clg->a2 + clg->a3 < rnd) {
+ } else {
clg->state = LOST_IN_BURST_PERIOD;
return true;
}
@@ -461,7 +461,8 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
skb->prev = NULL;
/* Random duplication */
- if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng))
+ if (q->duplicate && skb->tc_depth == 0 &&
+ q->duplicate >= get_crandom(&q->dup_cor, &q->prng))
++count;
/* Drop packet? */
@@ -524,7 +525,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1 << get_random_u32_below(8);
}
- if (unlikely(q->t_len >= sch->limit)) {
+ if (unlikely(sch->q.qlen >= sch->limit)) {
/* re-link segs, so that qdisc_drop_all() frees them all */
skb->next = segs;
qdisc_drop_all(skb, sch, to_free);
@@ -540,11 +541,9 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
*/
if (skb2) {
struct Qdisc *rootq = qdisc_root_bh(sch);
- u32 dupsave = q->duplicate; /* prevent duplicating a dup... */
- q->duplicate = 0;
+ skb2->tc_depth++; /* prevent duplicating a dup... */
rootq->enqueue(skb2, rootq, to_free);
- q->duplicate = dupsave;
skb2 = NULL;
}
@@ -659,9 +658,8 @@ static void get_slot_next(struct netem_sched_data *q, u64 now)
if (!q->slot_dist)
next_delay = q->slot_config.min_delay +
- (get_random_u32() *
- (q->slot_config.max_delay -
- q->slot_config.min_delay) >> 32);
+ mul_u64_u32_shr(q->slot_config.max_delay - q->slot_config.min_delay,
+ get_random_u32(), 32);
else
next_delay = tabledist(q->slot_config.dist_delay,
(s32)(q->slot_config.dist_jitter),
@@ -827,6 +825,39 @@ static int get_dist_table(struct disttable **tbl, const struct nlattr *attr)
return 0;
}
+static int validate_time(const struct nlattr *attr, const char *name,
+ struct netlink_ext_ack *extack)
+{
+ if (nla_get_s64(attr) < 0) {
+ NL_SET_ERR_MSG_ATTR_FMT(extack, attr, "negative %s", name);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int validate_slot(const struct nlattr *attr, struct netlink_ext_ack *extack)
+{
+ const struct tc_netem_slot *c = nla_data(attr);
+
+ if (c->min_delay < 0 || c->max_delay < 0) {
+ NL_SET_ERR_MSG_ATTR(extack, attr, "negative slot delay");
+ return -EINVAL;
+ }
+ if (c->min_delay > c->max_delay) {
+ NL_SET_ERR_MSG_ATTR(extack, attr, "slot min delay greater than max delay");
+ return -EINVAL;
+ }
+ if (c->dist_delay < 0 || c->dist_jitter < 0) {
+ NL_SET_ERR_MSG_ATTR(extack, attr, "negative dist delay");
+ return -EINVAL;
+ }
+ if (c->max_packets < 0 || c->max_bytes < 0) {
+ NL_SET_ERR_MSG_ATTR(extack, attr, "negative slot limit");
+ return -EINVAL;
+ }
+ return 0;
+}
+
static void get_slot(struct netem_sched_data *q, const struct nlattr *attr)
{
const struct tc_netem_slot *c = nla_data(attr);
@@ -975,41 +1006,6 @@ static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla,
return 0;
}
-static const struct Qdisc_class_ops netem_class_ops;
-
-static int check_netem_in_tree(struct Qdisc *sch, bool duplicates,
- struct netlink_ext_ack *extack)
-{
- struct Qdisc *root, *q;
- unsigned int i;
-
- root = qdisc_root_sleeping(sch);
-
- if (sch != root && root->ops->cl_ops == &netem_class_ops) {
- if (duplicates ||
- ((struct netem_sched_data *)qdisc_priv(root))->duplicate)
- goto err;
- }
-
- if (!qdisc_dev(root))
- return 0;
-
- hash_for_each(qdisc_dev(root)->qdisc_hash, i, q, hash) {
- if (sch != q && q->ops->cl_ops == &netem_class_ops) {
- if (duplicates ||
- ((struct netem_sched_data *)qdisc_priv(q))->duplicate)
- goto err;
- }
- }
-
- return 0;
-
-err:
- NL_SET_ERR_MSG(extack,
- "netem: cannot mix duplicating netems with other netems in tree");
- return -EINVAL;
-}
-
/* Parse netlink message to set options */
static int netem_change(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
@@ -1040,6 +1036,24 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
goto table_free;
}
+ if (tb[TCA_NETEM_SLOT]) {
+ ret = validate_slot(tb[TCA_NETEM_SLOT], extack);
+ if (ret)
+ goto table_free;
+ }
+
+ if (tb[TCA_NETEM_LATENCY64]) {
+ ret = validate_time(tb[TCA_NETEM_LATENCY64], "latency", extack);
+ if (ret)
+ goto table_free;
+ }
+
+ if (tb[TCA_NETEM_JITTER64]) {
+ ret = validate_time(tb[TCA_NETEM_JITTER64], "jitter", extack);
+ if (ret)
+ goto table_free;
+ }
+
sch_tree_lock(sch);
/* backup q->clg and q->loss_model */
old_clg = q->clg;
@@ -1068,11 +1082,6 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
q->gap = qopt->gap;
q->counter = 0;
q->loss = qopt->loss;
-
- ret = check_netem_in_tree(sch, qopt->duplicate, extack);
- if (ret)
- goto unlock;
-
q->duplicate = qopt->duplicate;
/* for compatibility with earlier versions.
@@ -1112,11 +1121,10 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
/* capping jitter to the range acceptable by tabledist() */
q->jitter = min_t(s64, abs(q->jitter), INT_MAX);
- if (tb[TCA_NETEM_PRNG_SEED])
+ if (tb[TCA_NETEM_PRNG_SEED]) {
q->prng.seed = nla_get_u64(tb[TCA_NETEM_PRNG_SEED]);
- else
- q->prng.seed = get_random_u64();
- prandom_seed_state(&q->prng.prng_state, q->prng.seed);
+ prandom_seed_state(&q->prng.prng_state, q->prng.seed);
+ }
unlock:
sch_tree_unlock(sch);
@@ -1139,6 +1147,9 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt,
return -EINVAL;
q->loss_model = CLG_RANDOM;
+ q->prng.seed = get_random_u64();
+ prandom_seed_state(&q->prng.prng_state, q->prng.seed);
+
ret = netem_change(sch, opt, extack);
if (ret)
pr_info("netem: change failed\n");
diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c
index fb53fbf..b41f2de 100644
--- a/net/sched/sch_pie.c
+++ b/net/sched/sch_pie.c
@@ -219,16 +219,14 @@ void pie_process_dequeue(struct sk_buff *skb, struct pie_params *params,
* packet timestamp.
*/
if (!params->dq_rate_estimator) {
- vars->qdelay = now - pie_get_enqueue_time(skb);
+ WRITE_ONCE(vars->qdelay,
+ backlog ? now - pie_get_enqueue_time(skb) : 0);
if (vars->dq_tstamp != DTIME_INVALID)
dtime = now - vars->dq_tstamp;
vars->dq_tstamp = now;
- if (backlog == 0)
- vars->qdelay = 0;
-
if (dtime == 0)
return;
@@ -376,7 +374,7 @@ void pie_calculate_probability(struct pie_params *params, struct pie_vars *vars,
if (qdelay > (PSCHED_NS2TICKS(250 * NSEC_PER_MSEC)))
delta += MAX_PROB / (100 / 2);
- vars->prob += delta;
+ WRITE_ONCE(vars->prob, vars->prob + delta);
if (delta > 0) {
/* prevent overflow */
@@ -401,7 +399,7 @@ void pie_calculate_probability(struct pie_params *params, struct pie_vars *vars,
if (qdelay == 0 && qdelay_old == 0 && update_prob)
/* Reduce drop probability to 98.4% */
- vars->prob -= vars->prob / 64;
+ WRITE_ONCE(vars->prob, vars->prob - vars->prob / 64);
WRITE_ONCE(vars->qdelay, qdelay);
vars->backlog_old = backlog;
@@ -501,7 +499,7 @@ static int pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
struct pie_sched_data *q = qdisc_priv(sch);
struct tc_pie_xstats st = {
- .prob = q->vars.prob << BITS_PER_BYTE,
+ .prob = READ_ONCE(q->vars.prob) << BITS_PER_BYTE,
.delay = ((u32)PSCHED_TICKS2NS(READ_ONCE(q->vars.qdelay))) /
NSEC_PER_USEC,
.packets_in = READ_ONCE(q->stats.packets_in),
@@ -512,7 +510,7 @@ static int pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
};
/* avg_dq_rate is only valid if dq_rate_estimator is enabled */
- st.dq_rate_estimating = q->params.dq_rate_estimator;
+ st.dq_rate_estimating = READ_ONCE(q->params.dq_rate_estimator);
/* unscale and return dq_rate in bytes per sec */
if (st.dq_rate_estimating)
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index 432b8a30..4d0e44a 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -162,7 +162,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch)
struct red_sched_data *q = qdisc_priv(sch);
struct Qdisc *child = q->qdisc;
- skb = child->dequeue(child);
+ skb = qdisc_dequeue_peeked(child);
if (skb) {
qdisc_bstats_update(sch, skb);
qdisc_qstats_backlog_dec(sch, skb);
diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c
index bd5ef56..d3ee8e5 100644
--- a/net/sched/sch_sfb.c
+++ b/net/sched/sch_sfb.c
@@ -441,7 +441,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch)
struct Qdisc *child = q->qdisc;
struct sk_buff *skb;
- skb = child->dequeue(q->qdisc);
+ skb = qdisc_dequeue_peeked(child);
if (skb) {
qdisc_bstats_update(sch, skb);
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index c3f3181..f39822b 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -225,7 +225,8 @@ static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
sfq_unlink(q, x, n, p);
- d = q->slots[x].qlen--;
+ d = q->slots[x].qlen;
+ WRITE_ONCE(q->slots[x].qlen, d - 1);
if (n == p && q->cur_depth == d)
q->cur_depth--;
sfq_link(q, x);
@@ -238,7 +239,8 @@ static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
sfq_unlink(q, x, n, p);
- d = ++q->slots[x].qlen;
+ d = q->slots[x].qlen + 1;
+ WRITE_ONCE(q->slots[x].qlen, d);
if (q->cur_depth < d)
q->cur_depth = d;
sfq_link(q, x);
@@ -298,7 +300,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
drop:
skb = q->headdrop ? slot_dequeue_head(slot) : slot_dequeue_tail(slot);
len = qdisc_pkt_len(skb);
- slot->backlog -= len;
+ WRITE_ONCE(slot->backlog, slot->backlog - len);
sfq_dec(q, x);
sch->q.qlen--;
qdisc_qstats_backlog_dec(sch, skb);
@@ -314,7 +316,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
q->tail = NULL; /* no more active slots */
else
q->tail->next = slot->next;
- q->ht[slot->hash] = SFQ_EMPTY_SLOT;
+ WRITE_ONCE(q->ht[slot->hash], SFQ_EMPTY_SLOT);
goto drop;
}
@@ -364,10 +366,10 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
x = q->dep[0].next; /* get a free slot */
if (x >= SFQ_MAX_FLOWS)
return qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_MAXFLOWS);
- q->ht[hash] = x;
+ WRITE_ONCE(q->ht[hash], x);
slot = &q->slots[x];
slot->hash = hash;
- slot->backlog = 0; /* should already be 0 anyway... */
+ WRITE_ONCE(slot->backlog, 0); /* should already be 0 anyway... */
red_set_vars(&slot->vars);
goto enqueue;
}
@@ -426,7 +428,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
head = slot_dequeue_head(slot);
delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
sch->qstats.backlog -= delta;
- slot->backlog -= delta;
+ WRITE_ONCE(slot->backlog, slot->backlog - delta);
qdisc_drop_reason(head, sch, to_free, QDISC_DROP_FLOW_LIMIT);
slot_queue_add(slot, skb);
@@ -436,7 +438,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
enqueue:
qdisc_qstats_backlog_inc(sch, skb);
- slot->backlog += qdisc_pkt_len(skb);
+ WRITE_ONCE(slot->backlog, slot->backlog + qdisc_pkt_len(skb));
slot_queue_add(slot, skb);
sfq_inc(q, x);
if (slot->qlen == 1) { /* The flow is new */
@@ -452,7 +454,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
*/
q->tail = slot;
/* We could use a bigger initial quantum for new flows */
- slot->allot = q->quantum;
+ WRITE_ONCE(slot->allot, q->quantum);
}
if (++sch->q.qlen <= q->limit)
return NET_XMIT_SUCCESS;
@@ -489,7 +491,7 @@ sfq_dequeue(struct Qdisc *sch)
slot = &q->slots[a];
if (slot->allot <= 0) {
q->tail = slot;
- slot->allot += q->quantum;
+ WRITE_ONCE(slot->allot, slot->allot + q->quantum);
goto next_slot;
}
skb = slot_dequeue_head(slot);
@@ -497,10 +499,10 @@ sfq_dequeue(struct Qdisc *sch)
qdisc_bstats_update(sch, skb);
sch->q.qlen--;
qdisc_qstats_backlog_dec(sch, skb);
- slot->backlog -= qdisc_pkt_len(skb);
+ WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb));
/* Is the slot empty? */
if (slot->qlen == 0) {
- q->ht[slot->hash] = SFQ_EMPTY_SLOT;
+ WRITE_ONCE(q->ht[slot->hash], SFQ_EMPTY_SLOT);
next_a = slot->next;
if (a == next_a) {
q->tail = NULL; /* no more active slots */
@@ -508,7 +510,7 @@ sfq_dequeue(struct Qdisc *sch)
}
q->tail->next = next_a;
} else {
- slot->allot -= qdisc_pkt_len(skb);
+ WRITE_ONCE(slot->allot, slot->allot - qdisc_pkt_len(skb));
}
return skb;
}
@@ -549,9 +551,9 @@ static void sfq_rehash(struct Qdisc *sch)
sfq_dec(q, i);
__skb_queue_tail(&list, skb);
}
- slot->backlog = 0;
+ WRITE_ONCE(slot->backlog, 0);
red_set_vars(&slot->vars);
- q->ht[slot->hash] = SFQ_EMPTY_SLOT;
+ WRITE_ONCE(q->ht[slot->hash], SFQ_EMPTY_SLOT);
}
q->tail = NULL;
@@ -570,7 +572,7 @@ static void sfq_rehash(struct Qdisc *sch)
dropped++;
continue;
}
- q->ht[hash] = x;
+ WRITE_ONCE(q->ht[hash], x);
slot = &q->slots[x];
slot->hash = hash;
}
@@ -581,7 +583,7 @@ static void sfq_rehash(struct Qdisc *sch)
slot->vars.qavg = red_calc_qavg(q->red_parms,
&slot->vars,
slot->backlog);
- slot->backlog += qdisc_pkt_len(skb);
+ WRITE_ONCE(slot->backlog, slot->backlog + qdisc_pkt_len(skb));
sfq_inc(q, x);
if (slot->qlen == 1) { /* The flow is new */
if (q->tail == NULL) { /* It is the first flow */
@@ -591,7 +593,7 @@ static void sfq_rehash(struct Qdisc *sch)
q->tail->next = x;
}
q->tail = slot;
- slot->allot = q->quantum;
+ WRITE_ONCE(slot->allot, q->quantum);
}
}
sch->q.qlen -= dropped;
@@ -905,16 +907,16 @@ static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
struct gnet_dump *d)
{
struct sfq_sched_data *q = qdisc_priv(sch);
- sfq_index idx = q->ht[cl - 1];
+ sfq_index idx = READ_ONCE(q->ht[cl - 1]);
struct gnet_stats_queue qs = { 0 };
struct tc_sfq_xstats xstats = { 0 };
if (idx != SFQ_EMPTY_SLOT) {
const struct sfq_slot *slot = &q->slots[idx];
- xstats.allot = slot->allot;
- qs.qlen = slot->qlen;
- qs.backlog = slot->backlog;
+ xstats.allot = READ_ONCE(slot->allot);
+ qs.qlen = READ_ONCE(slot->qlen);
+ qs.backlog = READ_ONCE(slot->backlog);
}
if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
return -1;
@@ -930,7 +932,7 @@ static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
return;
for (i = 0; i < q->divisor; i++) {
- if (q->ht[i] == SFQ_EMPTY_SLOT) {
+ if (READ_ONCE(q->ht[i]) == SFQ_EMPTY_SLOT) {
arg->count++;
continue;
}
diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
index a47a09d..4524515 100644
--- a/net/sched/sch_taprio.c
+++ b/net/sched/sch_taprio.c
@@ -634,7 +634,7 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
queue = skb_get_queue_mapping(skb);
child = q->qdiscs[queue];
- if (unlikely(!child))
+ if (unlikely(child == &noop_qdisc))
return qdisc_drop(skb, sch, to_free);
if (taprio_skb_exceeds_queue_max_sdu(sch, skb)) {
@@ -717,7 +717,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,
int len;
u8 tc;
- if (unlikely(!child))
+ if (unlikely(child == &noop_qdisc))
return NULL;
if (TXTIME_ASSIST_IS_ENABLED(q->flags))
@@ -2184,6 +2184,9 @@ static int taprio_graft(struct Qdisc *sch, unsigned long cl,
if (!dev_queue)
return -EINVAL;
+ if (!new)
+ new = &noop_qdisc;
+
if (dev->flags & IFF_UP)
dev_deactivate(dev, false);
@@ -2197,14 +2200,14 @@ static int taprio_graft(struct Qdisc *sch, unsigned long cl,
*old = q->qdiscs[cl - 1];
if (FULL_OFFLOAD_IS_ENABLED(q->flags)) {
WARN_ON_ONCE(dev_graft_qdisc(dev_queue, new) != *old);
- if (new)
+ if (new != &noop_qdisc)
qdisc_refcount_inc(new);
- if (*old)
+ if (*old && *old != &noop_qdisc)
qdisc_put(*old);
}
q->qdiscs[cl - 1] = new;
- if (new)
+ if (new != &noop_qdisc)
new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
if (dev->flags & IFF_UP)
diff --git a/net/sctp/diag.c b/net/sctp/diag.c
index 2afb376..d758f5c 100644
--- a/net/sctp/diag.c
+++ b/net/sctp/diag.c
@@ -266,15 +266,15 @@ static int sctp_sock_dump_one(struct sctp_endpoint *ep, struct sctp_transport *t
lock_sock(sk);
- rep = nlmsg_new(inet_assoc_attr_size(sk, assoc), GFP_KERNEL);
- if (!rep) {
- release_sock(sk);
- return -ENOMEM;
+ if (ep != assoc->ep || assoc->base.dead) {
+ err = -ESTALE;
+ goto out_unlock;
}
- if (ep != assoc->ep) {
- err = -EAGAIN;
- goto out;
+ rep = nlmsg_new(inet_assoc_attr_size(sk, assoc), GFP_KERNEL);
+ if (!rep) {
+ err = -ENOMEM;
+ goto out_unlock;
}
err = inet_sctp_diag_fill(sk, assoc, rep, req, sk_user_ns(NETLINK_CB(skb).sk),
@@ -289,8 +289,9 @@ static int sctp_sock_dump_one(struct sctp_endpoint *ep, struct sctp_transport *t
return nlmsg_unicast(sock_net(skb->sk)->diag_nlsk, rep, NETLINK_CB(skb).portid);
out:
- release_sock(sk);
kfree_skb(rep);
+out_unlock:
+ release_sock(sk);
return err;
}
diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c
index de86ac0..8526486 100644
--- a/net/sctp/sm_make_chunk.c
+++ b/net/sctp/sm_make_chunk.c
@@ -1730,6 +1730,7 @@ struct sctp_association *sctp_unpack_cookie(
struct sctp_signed_cookie *cookie;
struct sk_buff *skb = chunk->skb;
struct sctp_cookie *bear_cookie;
+ struct sctp_chunkhdr *ch;
enum sctp_scope scope;
unsigned int len;
ktime_t kt;
@@ -1759,6 +1760,10 @@ struct sctp_association *sctp_unpack_cookie(
cookie = chunk->subh.cookie_hdr;
bear_cookie = &cookie->c;
+ ch = (struct sctp_chunkhdr *)(bear_cookie + 1);
+ if (ntohs(ch->length) > len - fixed_size)
+ goto malformed;
+
/* Verify the cookie's MAC, if cookie authentication is enabled. */
if (sctp_sk(ep->base.sk)->cookie_auth_enable) {
u8 mac[SHA256_DIGEST_SIZE];
diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c
index 7b823d7..9b23c11c 100644
--- a/net/sctp/sm_statefuns.c
+++ b/net/sctp/sm_statefuns.c
@@ -1556,6 +1556,12 @@ static enum sctp_disposition sctp_sf_do_unexpected_init(
/* Tag the variable length parameters. */
chunk->param_hdr.v = skb_pull(chunk->skb, sizeof(struct sctp_inithdr));
+ if (asoc->state >= SCTP_STATE_ESTABLISHED) {
+ /* Discard INIT matching peer vtag after handshake completion (stale INIT). */
+ if (ntohl(chunk->subh.init_hdr->init_tag) == asoc->peer.i.init_tag)
+ return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands);
+ }
+
/* Verify the INIT chunk before processing it. */
err_chunk = NULL;
if (!sctp_verify_init(net, ep, asoc, chunk->chunk_hdr->type,
@@ -2592,11 +2598,7 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale(
*/
sctp_add_cmd_sf(commands, SCTP_CMD_DEL_NON_PRIMARY, SCTP_NULL());
- /* If we've sent any data bundled with COOKIE-ECHO we will need to
- * resend
- */
- sctp_add_cmd_sf(commands, SCTP_CMD_T1_RETRAN,
- SCTP_TRANSPORT(asoc->peer.primary_path));
+ sctp_add_cmd_sf(commands, SCTP_CMD_PURGE_OUTQUEUE, SCTP_NULL());
/* Cast away the const modifier, as we want to just
* rerun it through as a sideffect.
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index 58d0d97..66e12fb 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -1986,6 +1986,15 @@ static int sctp_sendmsg(struct sock *sk, struct msghdr *msg, size_t msg_len)
goto out_unlock;
iov_iter_revert(&msg->msg_iter, err);
+
+ /* sctp_sendmsg_to_asoc() may have released the socket
+ * lock (sctp_wait_for_sndbuf), during which other
+ * associations on ep->asocs could have been peeled
+ * off or freed. @asoc itself is revalidated by the
+ * base.dead and base.sk checks in sctp_wait_for_sndbuf,
+ * so re-derive the cached cursor from it.
+ */
+ tmp = list_next_entry(asoc, asocs);
}
goto out_unlock;
@@ -9394,6 +9403,8 @@ static int sctp_wait_for_connect(struct sctp_association *asoc, long *timeo_p)
release_sock(sk);
current_timeo = schedule_timeout(current_timeo);
lock_sock(sk);
+ if (sk != asoc->base.sk)
+ goto do_error;
*timeo_p = current_timeo;
}
diff --git a/net/shaper/shaper.c b/net/shaper/shaper.c
index 94bc9c7..dea9270 100644
--- a/net/shaper/shaper.c
+++ b/net/shaper/shaper.c
@@ -21,6 +21,8 @@
#define NET_SHAPER_ID_UNSPEC NET_SHAPER_ID_MASK
+static_assert(NET_SHAPER_ID_UNSPEC == NET_SHAPER_MAX_HANDLE_ID + 1);
+
struct net_shaper_hierarchy {
struct xarray shapers;
};
@@ -90,6 +92,12 @@ static int net_shaper_handle_size(void)
nla_total_size(sizeof(u32)));
}
+static int net_shaper_group_reply_size(void)
+{
+ return nla_total_size(sizeof(u32)) + /* NET_SHAPER_A_IFINDEX */
+ net_shaper_handle_size(); /* NET_SHAPER_A_HANDLE */
+}
+
static int net_shaper_fill_binding(struct sk_buff *msg,
const struct net_shaper_binding *binding,
u32 type)
@@ -130,35 +138,58 @@ static int net_shaper_fill_handle(struct sk_buff *msg,
return -EMSGSIZE;
}
+static void net_shaper_copy(struct net_shaper *dst,
+ const struct net_shaper *src)
+{
+ WRITE_ONCE(dst->parent.scope, READ_ONCE(src->parent.scope));
+ WRITE_ONCE(dst->parent.id, READ_ONCE(src->parent.id));
+ WRITE_ONCE(dst->handle.scope, READ_ONCE(src->handle.scope));
+ WRITE_ONCE(dst->handle.id, READ_ONCE(src->handle.id));
+
+ WRITE_ONCE(dst->metric, READ_ONCE(src->metric));
+ WRITE_ONCE(dst->bw_min, READ_ONCE(src->bw_min));
+ WRITE_ONCE(dst->bw_max, READ_ONCE(src->bw_max));
+ WRITE_ONCE(dst->burst, READ_ONCE(src->burst));
+ WRITE_ONCE(dst->priority, READ_ONCE(src->priority));
+ WRITE_ONCE(dst->weight, READ_ONCE(src->weight));
+
+ /* private fields are only used on the write path under the lock */
+ data_race(dst->leaves = src->leaves);
+}
+
static int
net_shaper_fill_one(struct sk_buff *msg,
const struct net_shaper_binding *binding,
const struct net_shaper *shaper,
const struct genl_info *info)
{
+ struct net_shaper cur;
void *hdr;
hdr = genlmsg_iput(msg, info);
if (!hdr)
return -EMSGSIZE;
+ /* Make a copy to avoid data races */
+ net_shaper_copy(&cur, shaper);
+
if (net_shaper_fill_binding(msg, binding, NET_SHAPER_A_IFINDEX) ||
- net_shaper_fill_handle(msg, &shaper->parent,
+ net_shaper_fill_handle(msg, &cur.parent,
NET_SHAPER_A_PARENT) ||
- net_shaper_fill_handle(msg, &shaper->handle,
+ net_shaper_fill_handle(msg, &cur.handle,
NET_SHAPER_A_HANDLE) ||
- ((shaper->bw_min || shaper->bw_max || shaper->burst) &&
- nla_put_u32(msg, NET_SHAPER_A_METRIC, shaper->metric)) ||
- (shaper->bw_min &&
- nla_put_uint(msg, NET_SHAPER_A_BW_MIN, shaper->bw_min)) ||
- (shaper->bw_max &&
- nla_put_uint(msg, NET_SHAPER_A_BW_MAX, shaper->bw_max)) ||
- (shaper->burst &&
- nla_put_uint(msg, NET_SHAPER_A_BURST, shaper->burst)) ||
- (shaper->priority &&
- nla_put_u32(msg, NET_SHAPER_A_PRIORITY, shaper->priority)) ||
- (shaper->weight &&
- nla_put_u32(msg, NET_SHAPER_A_WEIGHT, shaper->weight)))
+ ((cur.bw_min || cur.bw_max || cur.burst) &&
+ nla_put_u32(msg, NET_SHAPER_A_METRIC, cur.metric)) ||
+ (cur.bw_min &&
+ nla_put_uint(msg, NET_SHAPER_A_BW_MIN, cur.bw_min)) ||
+ (cur.bw_max &&
+ nla_put_uint(msg, NET_SHAPER_A_BW_MAX, cur.bw_max)) ||
+ (cur.burst &&
+ nla_put_uint(msg, NET_SHAPER_A_BURST, cur.burst)) ||
+ (cur.priority &&
+ nla_put_u32(msg, NET_SHAPER_A_PRIORITY, cur.priority)) ||
+ (cur.weight &&
+ nla_put_u32(msg, NET_SHAPER_A_WEIGHT, cur.weight)))
goto nla_put_failure;
genlmsg_end(msg, hdr);
@@ -275,25 +306,24 @@ static void net_shaper_default_parent(const struct net_shaper_handle *handle,
parent->id = 0;
}
-/*
- * MARK_0 is already in use due to XA_FLAGS_ALLOC, can't reuse such flag as
- * it's cleared by xa_store().
- */
-#define NET_SHAPER_NOT_VALID XA_MARK_1
-
static struct net_shaper *
net_shaper_lookup(struct net_shaper_binding *binding,
const struct net_shaper_handle *handle)
{
u32 index = net_shaper_handle_to_index(handle);
struct net_shaper_hierarchy *hierarchy;
+ struct net_shaper *cur;
hierarchy = net_shaper_hierarchy_rcu(binding);
- if (!hierarchy || xa_get_mark(&hierarchy->shapers, index,
- NET_SHAPER_NOT_VALID))
+ if (!hierarchy)
return NULL;
- return xa_load(&hierarchy->shapers, index);
+ cur = xa_load(&hierarchy->shapers, index);
+ /* Check valid before reading fields */
+ if (!cur || !smp_load_acquire(&cur->valid))
+ return NULL;
+
+ return cur;
}
/* Allocate on demand the per device shaper's hierarchy container.
@@ -348,7 +378,7 @@ static int net_shaper_pre_insert(struct net_shaper_binding *binding,
handle->id == NET_SHAPER_ID_UNSPEC) {
u32 min, max;
- handle->id = NET_SHAPER_ID_MASK - 1;
+ handle->id = NET_SHAPER_MAX_HANDLE_ID;
max = net_shaper_handle_to_index(handle);
handle->id = 0;
min = net_shaper_handle_to_index(handle);
@@ -370,13 +400,10 @@ static int net_shaper_pre_insert(struct net_shaper_binding *binding,
goto free_id;
}
- /* Mark 'tentative' shaper inside the hierarchy container.
- * xa_set_mark is a no-op if the previous store fails.
+ /* Insert as 'tentative' (no VALID mark). The mark will be set by
+ * net_shaper_commit() once the driver-side configuration succeeds.
*/
- xa_lock(&hierarchy->shapers);
- prev = __xa_store(&hierarchy->shapers, index, cur, GFP_KERNEL);
- __xa_set_mark(&hierarchy->shapers, index, NET_SHAPER_NOT_VALID);
- xa_unlock(&hierarchy->shapers);
+ prev = xa_store(&hierarchy->shapers, index, cur, GFP_KERNEL);
if (xa_err(prev)) {
NL_SET_ERR_MSG(extack, "Can't insert shaper into device store");
kfree_rcu(cur, rcu);
@@ -410,12 +437,10 @@ static void net_shaper_commit(struct net_shaper_binding *binding,
if (WARN_ON_ONCE(!cur))
continue;
- /* Successful update: drop the tentative mark
- * and update the hierarchy container.
- */
- __xa_clear_mark(&hierarchy->shapers, index,
- NET_SHAPER_NOT_VALID);
- *cur = shapers[i];
+ /* Successful update: update the hierarchy container... */
+ net_shaper_copy(cur, &shapers[i]);
+ /* ... publish to lockless readers. */
+ smp_store_release(&cur->valid, true);
}
xa_unlock(&hierarchy->shapers);
}
@@ -431,10 +456,11 @@ static void net_shaper_rollback(struct net_shaper_binding *binding)
return;
xa_lock(&hierarchy->shapers);
- xa_for_each_marked(&hierarchy->shapers, index, cur,
- NET_SHAPER_NOT_VALID) {
+ xa_for_each(&hierarchy->shapers, index, cur) {
+ if (cur->valid)
+ continue;
__xa_erase(&hierarchy->shapers, index);
- kfree(cur);
+ kfree_rcu(cur, rcu);
}
xa_unlock(&hierarchy->shapers);
}
@@ -465,10 +491,21 @@ static int net_shaper_parse_handle(const struct nlattr *attr,
* shaper (any other value).
*/
id_attr = tb[NET_SHAPER_A_HANDLE_ID];
- if (id_attr)
+ if (id_attr) {
id = nla_get_u32(id_attr);
- else if (handle->scope == NET_SHAPER_SCOPE_NODE)
+ } else if (handle->scope == NET_SHAPER_SCOPE_NODE) {
id = NET_SHAPER_ID_UNSPEC;
+ } else if (handle->scope == NET_SHAPER_SCOPE_QUEUE) {
+ NL_SET_ERR_ATTR_MISS(info->extack, attr,
+ NET_SHAPER_A_HANDLE_ID);
+ return -EINVAL;
+ }
+
+ if (id && handle->scope == NET_SHAPER_SCOPE_NETDEV) {
+ NL_SET_ERR_MSG_ATTR(info->extack, id_attr,
+ "Netdev scope is a singleton, must use ID 0");
+ return -EINVAL;
+ }
handle->id = id;
return 0;
@@ -836,7 +873,12 @@ int net_shaper_nl_get_dumpit(struct sk_buff *skb,
goto out_unlock;
for (; (shaper = xa_find(&hierarchy->shapers, &ctx->start_index,
- U32_MAX, XA_PRESENT)); ctx->start_index++) {
+ U32_MAX, XA_PRESENT));
+ ctx->start_index++) {
+ /* Check valid before reading fields */
+ if (!smp_load_acquire(&shaper->valid))
+ continue;
+
ret = net_shaper_fill_one(skb, binding, shaper, info);
if (ret)
break;
@@ -932,6 +974,46 @@ static int net_shaper_handle_cmp(const struct net_shaper_handle *a,
return memcmp(a, b, sizeof(*a));
}
+static int net_shaper_parse_leaves(struct net_shaper_binding *binding,
+ struct genl_info *info,
+ const struct net_shaper *node,
+ struct net_shaper *leaves,
+ int leaves_count)
+{
+ struct nlattr *attr;
+ int i, j, ret, rem;
+
+ i = 0;
+ nla_for_each_attr_type(attr, NET_SHAPER_A_LEAVES,
+ genlmsg_data(info->genlhdr),
+ genlmsg_len(info->genlhdr), rem) {
+ if (WARN_ON_ONCE(i >= leaves_count))
+ return -EINVAL;
+
+ ret = net_shaper_parse_leaf(binding, attr, info,
+ node, &leaves[i]);
+ if (ret)
+ return ret;
+
+ /* Reject duplicates */
+ for (j = 0; j < i; j++) {
+ if (net_shaper_handle_cmp(&leaves[i].handle,
+ &leaves[j].handle))
+ continue;
+
+ NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr,
+ "Duplicate leaf shaper %d:%d",
+ leaves[i].handle.scope,
+ leaves[i].handle.id);
+ return -EINVAL;
+ }
+
+ i++;
+ }
+
+ return 0;
+}
+
static int net_shaper_parent_from_leaves(int leaves_count,
const struct net_shaper *leaves,
struct net_shaper *node,
@@ -964,15 +1046,22 @@ static int __net_shaper_group(struct net_shaper_binding *binding,
int i, ret;
if (node->handle.scope == NET_SHAPER_SCOPE_NODE) {
+ struct net_shaper *cur = NULL;
+
new_node = node->handle.id == NET_SHAPER_ID_UNSPEC;
- if (!new_node && !net_shaper_lookup(binding, &node->handle)) {
- /* The related attribute is not available when
- * reaching here from the delete() op.
- */
- NL_SET_ERR_MSG_FMT(extack, "Node shaper %d:%d does not exists",
- node->handle.scope, node->handle.id);
- return -ENOENT;
+ if (!new_node) {
+ cur = net_shaper_lookup(binding, &node->handle);
+ if (!cur) {
+ /* The related attribute is not available
+ * when reaching here from the delete() op.
+ */
+ NL_SET_ERR_MSG_FMT(extack,
+ "Node shaper %d:%d does not exist",
+ node->handle.scope,
+ node->handle.id);
+ return -ENOENT;
+ }
}
/* When unspecified, the node parent scope is inherited from
@@ -986,6 +1075,15 @@ static int __net_shaper_group(struct net_shaper_binding *binding,
return ret;
}
+ if (cur && net_shaper_handle_cmp(&cur->parent,
+ &node->parent)) {
+ NL_SET_ERR_MSG_FMT(extack,
+ "Cannot reparent node shaper %d:%d",
+ node->handle.scope,
+ node->handle.id);
+ return -EOPNOTSUPP;
+ }
+
} else {
net_shaper_default_parent(&node->handle, &node->parent);
}
@@ -1162,7 +1260,7 @@ static int net_shaper_group_send_reply(struct net_shaper_binding *binding,
free_msg:
/* Should never happen as msg is pre-allocated with enough space. */
WARN_ONCE(true, "calculated message payload length (%d)",
- net_shaper_handle_size());
+ net_shaper_group_reply_size());
nlmsg_free(msg);
return -EMSGSIZE;
}
@@ -1172,10 +1270,9 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info)
struct net_shaper **old_nodes, *leaves, node = {};
struct net_shaper_hierarchy *hierarchy;
struct net_shaper_binding *binding;
- int i, ret, rem, leaves_count;
+ int i, ret, leaves_count;
int old_nodes_count = 0;
struct sk_buff *msg;
- struct nlattr *attr;
if (GENL_REQ_ATTR_CHECK(info, NET_SHAPER_A_LEAVES))
return -EINVAL;
@@ -1203,26 +1300,19 @@ int net_shaper_nl_group_doit(struct sk_buff *skb, struct genl_info *info)
if (ret)
goto free_leaves;
- i = 0;
- nla_for_each_attr_type(attr, NET_SHAPER_A_LEAVES,
- genlmsg_data(info->genlhdr),
- genlmsg_len(info->genlhdr), rem) {
- if (WARN_ON_ONCE(i >= leaves_count))
- goto free_leaves;
-
- ret = net_shaper_parse_leaf(binding, attr, info,
- &node, &leaves[i]);
- if (ret)
- goto free_leaves;
- i++;
- }
+ ret = net_shaper_parse_leaves(binding, info, &node,
+ leaves, leaves_count);
+ if (ret)
+ goto free_leaves;
/* Prepare the msg reply in advance, to avoid device operation
* rollback on allocation failure.
*/
- msg = genlmsg_new(net_shaper_handle_size(), GFP_KERNEL);
- if (!msg)
+ msg = genlmsg_new(net_shaper_group_reply_size(), GFP_KERNEL);
+ if (!msg) {
+ ret = -ENOMEM;
goto free_leaves;
+ }
hierarchy = net_shaper_hierarchy_setup(binding);
if (!hierarchy) {
diff --git a/net/shaper/shaper_nl_gen.c b/net/shaper/shaper_nl_gen.c
index 9b29be3..76eff85 100644
--- a/net/shaper/shaper_nl_gen.c
+++ b/net/shaper/shaper_nl_gen.c
@@ -11,10 +11,15 @@
#include <uapi/linux/net_shaper.h>
+/* Integer value ranges */
+static const struct netlink_range_validation net_shaper_a_handle_id_range = {
+ .max = NET_SHAPER_MAX_HANDLE_ID,
+};
+
/* Common nested types */
const struct nla_policy net_shaper_handle_nl_policy[NET_SHAPER_A_HANDLE_ID + 1] = {
[NET_SHAPER_A_HANDLE_SCOPE] = NLA_POLICY_MAX(NLA_U32, 3),
- [NET_SHAPER_A_HANDLE_ID] = { .type = NLA_U32, },
+ [NET_SHAPER_A_HANDLE_ID] = NLA_POLICY_FULL_RANGE(NLA_U32, &net_shaper_a_handle_id_range),
};
const struct nla_policy net_shaper_leaf_info_nl_policy[NET_SHAPER_A_WEIGHT + 1] = {
diff --git a/net/shaper/shaper_nl_gen.h b/net/shaper/shaper_nl_gen.h
index 42c46c5..2406652 100644
--- a/net/shaper/shaper_nl_gen.h
+++ b/net/shaper/shaper_nl_gen.h
@@ -12,6 +12,8 @@
#include <uapi/linux/net_shaper.h>
+#define NET_SHAPER_MAX_HANDLE_ID 67108862
+
/* Common nested types */
extern const struct nla_policy net_shaper_handle_nl_policy[NET_SHAPER_A_HANDLE_ID + 1];
extern const struct nla_policy net_shaper_leaf_info_nl_policy[NET_SHAPER_A_WEIGHT + 1];
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 1a56509..b5db690 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -188,10 +188,12 @@ static bool smc_hs_congested(const struct sock *sk)
struct smc_hashinfo smc_v4_hashinfo = {
.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
+ .ht = HLIST_HEAD_INIT,
};
struct smc_hashinfo smc_v6_hashinfo = {
.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
+ .ht = HLIST_HEAD_INIT,
};
int smc_hash_sk(struct sock *sk)
@@ -1400,7 +1402,8 @@ smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm *aclc,
int i;
for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
- if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
+ if (ini->ism_dev[i] &&
+ ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
ini->ism_selected = i;
return 0;
}
@@ -1628,12 +1631,8 @@ static void smc_connect_work(struct work_struct *work)
lock_sock(&smc->sk);
if (rc != 0 || smc->sk.sk_err) {
smc->sk.sk_state = SMC_CLOSED;
- if (rc == -EPIPE || rc == -EAGAIN)
- smc->sk.sk_err = EPIPE;
- else if (rc == -ECONNREFUSED)
- smc->sk.sk_err = ECONNREFUSED;
- else if (signal_pending(current))
- smc->sk.sk_err = -sock_intr_errno(timeo);
+ if (!smc->sk.sk_err)
+ smc->sk.sk_err = (rc == -EAGAIN) ? EPIPE : -rc;
sock_put(&smc->sk); /* passive closing */
goto out;
}
@@ -3058,18 +3057,17 @@ static int __smc_setsockopt(struct socket *sock, int level, int optname,
smc = smc_sk(sk);
+ /* pre-fetch user data outside the lock */
+ if (optname == SMC_LIMIT_HS) {
+ if (optlen < sizeof(int))
+ return -EINVAL;
+ if (copy_from_sockptr(&val, optval, sizeof(int)))
+ return -EFAULT;
+ }
+
lock_sock(sk);
switch (optname) {
case SMC_LIMIT_HS:
- if (optlen < sizeof(int)) {
- rc = -EINVAL;
- break;
- }
- if (copy_from_sockptr(&val, optval, sizeof(int))) {
- rc = -EFAULT;
- break;
- }
-
smc->limit_smc_hs = !!val;
rc = 0;
break;
@@ -3521,8 +3519,6 @@ static int __init smc_init(void)
pr_err("%s: sock_register fails with %d\n", __func__, rc);
goto out_proto6;
}
- INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
- INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
rc = smc_ib_register_client();
if (rc) {
diff --git a/net/smc/smc_tracepoint.h b/net/smc/smc_tracepoint.h
index a9a6e3c..53da84f 100644
--- a/net/smc/smc_tracepoint.h
+++ b/net/smc/smc_tracepoint.h
@@ -51,7 +51,7 @@ DECLARE_EVENT_CLASS(smc_msg_event,
__field(const void *, smc)
__field(u64, net_cookie)
__field(size_t, len)
- __string(name, smc->conn.lnk->ibname)
+ __string(name, smc->conn.lnk ? smc->conn.lnk->ibname : "")
),
TP_fast_assign(
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 7081c12..27dd6b5 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -403,7 +403,7 @@ void sunrpc_init_cache_detail(struct cache_detail *cd)
INIT_LIST_HEAD(&cd->readers);
spin_lock_init(&cd->queue_lock);
init_waitqueue_head(&cd->queue_wait);
- cd->next_seqno = 0;
+ cd->next_seqno = 1;
spin_lock(&cache_list_lock);
cd->nextcheck = 0;
cd->entries = 0;
@@ -1348,6 +1348,9 @@ static void *__cache_seq_start(struct seq_file *m, loff_t *pos)
hash = n >> 32;
entry = n & ((1LL<<32) - 1);
+ if (hash >= cd->hash_size)
+ return NULL;
+
hlist_for_each_entry_rcu(ch, &cd->hash_table[hash], cache_list)
if (!entry--)
return ch;
diff --git a/net/tls/tls.h b/net/tls/tls.h
index e8f81a0..12f44cb 100644
--- a/net/tls/tls.h
+++ b/net/tls/tls.h
@@ -188,6 +188,7 @@ int tls_strp_dev_init(void);
void tls_strp_dev_exit(void);
void tls_strp_done(struct tls_strparser *strp);
+void __tls_strp_done(struct tls_strparser *strp);
void tls_strp_stop(struct tls_strparser *strp);
int tls_strp_init(struct tls_strparser *strp, struct sock *sk);
void tls_strp_data_ready(struct tls_strparser *strp);
diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c
index 98e12f0..c72e883 100644
--- a/net/tls/tls_strp.c
+++ b/net/tls/tls_strp.c
@@ -624,6 +624,12 @@ void tls_strp_done(struct tls_strparser *strp)
WARN_ON(!strp->stopped);
cancel_work_sync(&strp->work);
+ __tls_strp_done(strp);
+}
+
+/* For setup error paths where the strparser was initialized but never armed. */
+void __tls_strp_done(struct tls_strparser *strp)
+{
tls_strp_anchor_free(strp);
}
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 94d2ae0..964ebc2 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -789,23 +789,33 @@ static int tls_push_record(struct sock *sk, int flags,
i = msg_pl->sg.end;
sk_msg_iter_var_prev(i);
+ /* msg_pl->sg.data is a ring; data[MAX+1] is reserved for the wrap
+ * link (frags won't use it). 'i' is now the last filled entry:
+ *
+ * i end start
+ * v v v [ rsv ]
+ * [ d ][ d ][ ][ ]...[ ][ d ][ d ][ d ][chain]
+ * ^ END v
+ * `-----------------------------------------'
+ *
+ * Note that SGL does not allow chain-after-chain, so for TLS 1.3,
+ * we must make sure we don't create the wrap entry and then chain
+ * link to content_type immediately at index 0.
+ */
+ if (i < msg_pl->sg.start)
+ sg_chain(msg_pl->sg.data, ARRAY_SIZE(msg_pl->sg.data),
+ msg_pl->sg.data);
+
rec->content_type = record_type;
if (prot->version == TLS_1_3_VERSION) {
/* Add content type to end of message. No padding added */
sg_set_buf(&rec->sg_content_type, &rec->content_type, 1);
sg_mark_end(&rec->sg_content_type);
- sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1,
- &rec->sg_content_type);
+ sg_chain(msg_pl->sg.data, i + 2, &rec->sg_content_type);
} else {
sg_mark_end(sk_msg_elem(msg_pl, i));
}
- if (msg_pl->sg.end < msg_pl->sg.start) {
- sg_chain(&msg_pl->sg.data[msg_pl->sg.start],
- MAX_SKB_FRAGS - msg_pl->sg.start + 1,
- msg_pl->sg.data);
- }
-
i = msg_pl->sg.start;
sg_chain(rec->sg_aead_in, 2, &msg_pl->sg.data[i]);
@@ -1356,9 +1366,14 @@ void tls_sw_splice_eof(struct socket *sock)
mutex_unlock(&tls_ctx->tx_lock);
}
+/* When has_copied is true the caller has already moved bytes to
+ * userspace. Report sk_err but leave it set so the next read
+ * surfaces it instead of a spurious EOF, otherwise sk_err is
+ * consumed via sock_error().
+ */
static int
tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
- bool released)
+ bool released, bool has_copied)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
@@ -1376,8 +1391,11 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
if (!sk_psock_queue_empty(psock))
return 0;
- if (sk->sk_err)
+ if (sk->sk_err) {
+ if (has_copied)
+ return -READ_ONCE(sk->sk_err);
return sock_error(sk);
+ }
if (ret < 0)
return ret;
@@ -1413,7 +1431,7 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
}
if (unlikely(!tls_strp_msg_load(&ctx->strp, released)))
- return tls_rx_rec_wait(sk, psock, nonblock, false);
+ return tls_rx_rec_wait(sk, psock, nonblock, false, has_copied);
return 1;
}
@@ -2100,7 +2118,7 @@ int tls_sw_recvmsg(struct sock *sk,
int to_decrypt, chunk;
err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT,
- released);
+ released, !!(decrypted + copied));
if (err <= 0) {
if (psock) {
chunk = sk_msg_recvmsg(sk, psock, msg, len,
@@ -2287,7 +2305,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
struct tls_decrypt_arg darg;
err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK,
- true);
+ true, false);
if (err <= 0)
goto splice_read_end;
@@ -2317,9 +2335,9 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
if (copied < 0)
goto splice_requeue;
- if (chunk < rxm->full_len) {
- rxm->offset += len;
- rxm->full_len -= len;
+ if (copied < rxm->full_len) {
+ rxm->offset += copied;
+ rxm->full_len -= copied;
goto splice_requeue;
}
@@ -2373,7 +2391,7 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
} else {
struct tls_decrypt_arg darg;
- err = tls_rx_rec_wait(sk, NULL, true, released);
+ err = tls_rx_rec_wait(sk, NULL, true, released, !!copied);
if (err <= 0)
goto read_sock_end;
@@ -2624,8 +2642,12 @@ void tls_sw_free_ctx_rx(struct tls_context *tls_ctx)
void tls_sw_free_resources_rx(struct sock *sk)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
+ struct tls_sw_context_rx *ctx;
+
+ ctx = tls_sw_ctx_rx(tls_ctx);
tls_sw_release_resources_rx(sk);
+ __tls_strp_done(&ctx->strp);
tls_sw_free_ctx_rx(tls_ctx);
}
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index e2d787c..0d9cd97 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -2711,8 +2711,7 @@ static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
* Sleep until more data has arrived. But check for races..
*/
static long unix_stream_data_wait(struct sock *sk, long timeo,
- struct sk_buff *last, unsigned int last_len,
- bool freezable)
+ struct sk_buff *last, bool freezable)
{
unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE;
struct sk_buff *tail;
@@ -2725,7 +2724,6 @@ static long unix_stream_data_wait(struct sock *sk, long timeo,
tail = skb_peek_tail(&sk->sk_receive_queue);
if (tail != last ||
- (tail && tail->len != last_len) ||
sk->sk_err ||
(sk->sk_shutdown & RCV_SHUTDOWN) ||
signal_pending(current) ||
@@ -2888,7 +2886,7 @@ static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
return -EAGAIN;
}
- WRITE_ONCE(u->inq_len, u->inq_len - skb->len);
+ WRITE_ONCE(u->inq_len, u->inq_len - unix_skb_len(skb));
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
if (skb == u->oob_skb) {
@@ -2921,7 +2919,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
int flags = state->flags;
bool check_creds = false;
struct scm_cookie scm;
- unsigned int last_len;
struct unix_sock *u;
int copied = 0;
int err = 0;
@@ -2967,7 +2964,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
goto unlock;
}
last = skb = skb_peek(&sk->sk_receive_queue);
- last_len = last ? last->len : 0;
again:
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
@@ -3001,8 +2997,7 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
mutex_unlock(&u->iolock);
- timeo = unix_stream_data_wait(sk, timeo, last,
- last_len, freezable);
+ timeo = unix_stream_data_wait(sk, timeo, last, freezable);
if (signal_pending(current)) {
err = sock_intr_errno(timeo);
@@ -3019,7 +3014,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
while (skip >= unix_skb_len(skb)) {
skip -= unix_skb_len(skb);
last = skb;
- last_len = skb->len;
skb = skb_peek_next(skb, &sk->sk_receive_queue);
if (!skb)
goto again;
@@ -3069,11 +3063,12 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
unix_detach_fds(&scm, skb);
}
- if (unix_skb_len(skb))
- break;
-
spin_lock(&sk->sk_receive_queue.lock);
- WRITE_ONCE(u->inq_len, u->inq_len - skb->len);
+ WRITE_ONCE(u->inq_len, u->inq_len - chunk);
+ if (unix_skb_len(skb)) {
+ spin_unlock(&sk->sk_receive_queue.lock);
+ break;
+ }
__skb_unlink(skb, &sk->sk_receive_queue);
spin_unlock(&sk->sk_receive_queue.lock);
@@ -3094,7 +3089,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
skip = 0;
last = skb;
- last_len = skb->len;
unix_state_lock(sk);
skb = skb_peek_next(skb, &sk->sk_receive_queue);
if (skb)
@@ -3323,6 +3317,9 @@ static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
struct sk_buff *skb;
int answ = 0;
+ if (sk->sk_type != SOCK_STREAM)
+ return -EOPNOTSUPP;
+
mutex_lock(&u->iolock);
skb = skb_peek(&sk->sk_receive_queue);
diff --git a/net/unix/garbage.c b/net/unix/garbage.c
index a7967a3..0783555 100644
--- a/net/unix/garbage.c
+++ b/net/unix/garbage.c
@@ -607,6 +607,8 @@ static void unix_gc(struct work_struct *work)
struct sk_buff_head hitlist;
struct sk_buff *skb;
+ WRITE_ONCE(gc_in_progress, true);
+
spin_lock(&unix_gc_lock);
if (unix_graph_state == UNIX_GRAPH_NOT_CYCLIC) {
@@ -649,10 +651,8 @@ void unix_schedule_gc(struct user_struct *user)
READ_ONCE(user->unix_inflight) < UNIX_INFLIGHT_SANE_USER)
return;
- if (!READ_ONCE(gc_in_progress)) {
- WRITE_ONCE(gc_in_progress, true);
+ if (!READ_ONCE(gc_in_progress))
queue_work(system_dfl_wq, &unix_gc_work);
- }
if (user && READ_ONCE(unix_graph_cyclic_sccs))
flush_work(&unix_gc_work);
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 44037b0..2ce1063 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -642,7 +642,7 @@ int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk)
*/
sock_reset_flag(sk, SOCK_DONE);
sk->sk_state = TCP_CLOSE;
- vsk->peer_shutdown = 0;
+ WRITE_ONCE(vsk->peer_shutdown, 0);
}
if (sk->sk_type == SOCK_SEQPACKET) {
@@ -933,7 +933,7 @@ static struct sock *__vsock_create(struct net *net,
vsk->rejected = false;
vsk->sent_request = false;
vsk->ignore_connecting_rst = false;
- vsk->peer_shutdown = 0;
+ WRITE_ONCE(vsk->peer_shutdown, 0);
INIT_DELAYED_WORK(&vsk->connect_work, vsock_connect_timeout);
INIT_DELAYED_WORK(&vsk->pending_work, vsock_pending_work);
@@ -1241,6 +1241,25 @@ static int vsock_shutdown(struct socket *sock, int mode)
return err;
}
+static __poll_t vsock_poll_shutdown(struct sock *sk, u32 peer_shutdown)
+{
+ __poll_t mask = 0;
+
+ /* INET sockets treat local write shutdown and peer write shutdown as a
+ * case of EPOLLHUP set.
+ */
+ if (sk->sk_shutdown == SHUTDOWN_MASK ||
+ ((sk->sk_shutdown & SEND_SHUTDOWN) &&
+ (peer_shutdown & SEND_SHUTDOWN)))
+ mask |= EPOLLHUP;
+
+ if (sk->sk_shutdown & RCV_SHUTDOWN ||
+ peer_shutdown & SEND_SHUTDOWN)
+ mask |= EPOLLRDHUP;
+
+ return mask;
+}
+
static __poll_t vsock_poll(struct file *file, struct socket *sock,
poll_table *wait)
{
@@ -1258,24 +1277,17 @@ static __poll_t vsock_poll(struct file *file, struct socket *sock,
/* Signify that there has been an error on this socket. */
mask |= EPOLLERR;
- /* INET sockets treat local write shutdown and peer write shutdown as a
- * case of EPOLLHUP set.
- */
- if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
- ((sk->sk_shutdown & SEND_SHUTDOWN) &&
- (vsk->peer_shutdown & SEND_SHUTDOWN))) {
- mask |= EPOLLHUP;
- }
-
- if (sk->sk_shutdown & RCV_SHUTDOWN ||
- vsk->peer_shutdown & SEND_SHUTDOWN) {
- mask |= EPOLLRDHUP;
- }
-
if (sk_is_readable(sk))
mask |= EPOLLIN | EPOLLRDNORM;
if (sock->type == SOCK_DGRAM) {
+ u32 peer_shutdown = READ_ONCE(vsk->peer_shutdown);
+
+ /* DGRAM sockets do not take lock_sock() in poll(), so use one
+ * lockless snapshot for all shutdown-derived mask bits.
+ */
+ mask |= vsock_poll_shutdown(sk, peer_shutdown);
+
/* For datagram sockets we can read if there is something in
* the queue and write as long as the socket isn't shutdown for
* sending.
@@ -1290,6 +1302,7 @@ static __poll_t vsock_poll(struct file *file, struct socket *sock,
} else if (sock_type_connectible(sk->sk_type)) {
const struct vsock_transport *transport;
+ u32 peer_shutdown;
lock_sock(sk);
@@ -1322,8 +1335,10 @@ static __poll_t vsock_poll(struct file *file, struct socket *sock,
* terminated should also be considered read, and we check the
* shutdown flag for that.
*/
+ peer_shutdown = READ_ONCE(vsk->peer_shutdown);
+ mask |= vsock_poll_shutdown(sk, peer_shutdown);
if (sk->sk_shutdown & RCV_SHUTDOWN ||
- vsk->peer_shutdown & SEND_SHUTDOWN) {
+ peer_shutdown & SEND_SHUTDOWN) {
mask |= EPOLLIN | EPOLLRDNORM;
}
diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c
index f862988..b339494 100644
--- a/net/vmw_vsock/hyperv_transport.c
+++ b/net/vmw_vsock/hyperv_transport.c
@@ -264,7 +264,7 @@ static void hvs_do_close_lock_held(struct vsock_sock *vsk,
struct sock *sk = sk_vsock(vsk);
sock_set_flag(sk, SOCK_DONE);
- vsk->peer_shutdown = SHUTDOWN_MASK;
+ WRITE_ONCE(vsk->peer_shutdown, SHUTDOWN_MASK);
if (vsock_stream_has_data(vsk) <= 0)
sk->sk_state = TCP_CLOSING;
sk->sk_state_change(sk);
@@ -375,10 +375,10 @@ static void hvs_open_connection(struct vmbus_channel *chan)
} else {
sndbuf = max_t(int, sk->sk_sndbuf, RINGBUFFER_HVS_SND_SIZE);
sndbuf = min_t(int, sndbuf, RINGBUFFER_HVS_MAX_SIZE);
- sndbuf = ALIGN(sndbuf, HV_HYP_PAGE_SIZE);
+ sndbuf = VMBUS_RING_SIZE(sndbuf);
rcvbuf = max_t(int, sk->sk_rcvbuf, RINGBUFFER_HVS_RCV_SIZE);
rcvbuf = min_t(int, rcvbuf, RINGBUFFER_HVS_MAX_SIZE);
- rcvbuf = ALIGN(rcvbuf, HV_HYP_PAGE_SIZE);
+ rcvbuf = VMBUS_RING_SIZE(rcvbuf);
}
chan->max_pkt_size = HVS_MAX_PKT_SIZE;
@@ -593,7 +593,9 @@ static int hvs_update_recv_data(struct hvsock *hvs)
return -EIO;
if (payload_len == 0)
- hvs->vsk->peer_shutdown |= SEND_SHUTDOWN;
+ WRITE_ONCE(hvs->vsk->peer_shutdown,
+ READ_ONCE(hvs->vsk->peer_shutdown) |
+ SEND_SHUTDOWN);
hvs->recv_data_len = payload_len;
hvs->recv_data_off = 0;
@@ -736,7 +738,8 @@ static s64 hvs_stream_has_data(struct vsock_sock *vsk)
return ret;
return hvs->recv_data_len;
case 0:
- vsk->peer_shutdown |= SEND_SHUTDOWN;
+ WRITE_ONCE(vsk->peer_shutdown,
+ READ_ONCE(vsk->peer_shutdown) | SEND_SHUTDOWN);
ret = 0;
break;
default: /* -1 */
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index 416d533..b106669 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -70,34 +70,6 @@ static bool virtio_transport_can_zcopy(const struct virtio_transport *t_ops,
return true;
}
-static int virtio_transport_init_zcopy_skb(struct vsock_sock *vsk,
- struct sk_buff *skb,
- struct msghdr *msg,
- size_t pkt_len,
- bool zerocopy)
-{
- struct ubuf_info *uarg;
-
- if (msg->msg_ubuf) {
- uarg = msg->msg_ubuf;
- net_zcopy_get(uarg);
- } else {
- struct ubuf_info_msgzc *uarg_zc;
-
- uarg = msg_zerocopy_realloc(sk_vsock(vsk),
- pkt_len, NULL, false);
- if (!uarg)
- return -1;
-
- uarg_zc = uarg_to_msgzc(uarg);
- uarg_zc->zerocopy = zerocopy ? 1 : 0;
- }
-
- skb_zcopy_init(skb, uarg);
-
- return 0;
-}
-
static int virtio_transport_fill_skb(struct sk_buff *skb,
struct virtio_vsock_pkt_info *info,
size_t len,
@@ -136,27 +108,6 @@ static void virtio_transport_init_hdr(struct sk_buff *skb,
hdr->fwd_cnt = cpu_to_le32(0);
}
-static void virtio_transport_copy_nonlinear_skb(const struct sk_buff *skb,
- void *dst,
- size_t len)
-{
- struct iov_iter iov_iter = { 0 };
- struct kvec kvec;
- size_t to_copy;
-
- kvec.iov_base = dst;
- kvec.iov_len = len;
-
- iov_iter.iter_type = ITER_KVEC;
- iov_iter.kvec = &kvec;
- iov_iter.nr_segs = 1;
-
- to_copy = min_t(size_t, len, skb->len);
-
- skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
- &iov_iter, to_copy);
-}
-
/* Packet capture */
static struct sk_buff *virtio_transport_build_skb(void *opaque)
{
@@ -166,12 +117,12 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque)
struct sk_buff *skb;
size_t payload_len;
- /* A packet could be split to fit the RX buffer, so we can retrieve
- * the payload length from the header and the buffer pointer taking
- * care of the offset in the original packet.
+ /* A packet could be split to fit the RX buffer, so we use
+ * the payload length from the header, which has been updated
+ * by the sender to reflect the fragment size.
*/
pkt_hdr = virtio_vsock_hdr(pkt);
- payload_len = pkt->len;
+ payload_len = le32_to_cpu(pkt_hdr->len);
skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len,
GFP_ATOMIC);
@@ -214,12 +165,18 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque)
skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr));
if (payload_len) {
- if (skb_is_nonlinear(pkt)) {
- void *data = skb_put(skb, payload_len);
+ struct iov_iter iov_iter;
+ struct kvec kvec;
+ void *data = skb_put(skb, payload_len);
- virtio_transport_copy_nonlinear_skb(pkt, data, payload_len);
- } else {
- skb_put_data(skb, pkt->data, payload_len);
+ kvec.iov_base = data;
+ kvec.iov_len = payload_len;
+ iov_iter_kvec(&iov_iter, ITER_DEST, &kvec, 1, payload_len);
+
+ if (skb_copy_datagram_iter(pkt, VIRTIO_VSOCK_SKB_CB(pkt)->offset,
+ &iov_iter, payload_len)) {
+ kfree_skb(skb);
+ return NULL;
}
}
@@ -248,6 +205,7 @@ static u16 virtio_transport_get_type(struct sock *sk)
static struct sk_buff *virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info,
size_t payload_len,
bool zcopy,
+ struct ubuf_info *uarg,
u32 src_cid,
u32 src_port,
u32 dst_cid,
@@ -288,6 +246,12 @@ static struct sk_buff *virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *
if (info->msg && payload_len > 0) {
int err;
+ /* Bind the zerocopy lifetime before filling frags so error
+ * rollback frees managed fixed-buffer pages through
+ * the uarg-aware path.
+ */
+ skb_zcopy_set(skb, uarg, NULL);
+
err = virtio_transport_fill_skb(skb, info, payload_len, zcopy);
if (err)
goto out;
@@ -332,8 +296,10 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
u32 src_cid, src_port, dst_cid, dst_port;
const struct virtio_transport *t_ops;
struct virtio_vsock_sock *vvs;
+ struct ubuf_info *uarg = NULL;
u32 pkt_len = info->pkt_len;
bool can_zcopy = false;
+ bool have_uref = false;
u32 rest_len;
int ret;
@@ -375,6 +341,25 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
if (can_zcopy)
max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE,
(MAX_SKB_FRAGS * PAGE_SIZE));
+
+ if (info->msg->msg_flags & MSG_ZEROCOPY &&
+ info->op == VIRTIO_VSOCK_OP_RW) {
+ uarg = info->msg->msg_ubuf;
+
+ if (!uarg) {
+ uarg = msg_zerocopy_realloc(sk_vsock(vsk),
+ pkt_len, NULL, false);
+ if (!uarg) {
+ virtio_transport_put_credit(vvs, pkt_len);
+ return -ENOMEM;
+ }
+
+ if (!can_zcopy)
+ uarg_to_msgzc(uarg)->zerocopy = 0;
+
+ have_uref = true;
+ }
+ }
}
rest_len = pkt_len;
@@ -386,6 +371,7 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
skb_len = min(max_skb_len, rest_len);
skb = virtio_transport_alloc_skb(info, skb_len, can_zcopy,
+ uarg,
src_cid, src_port,
dst_cid, dst_port);
if (!skb) {
@@ -393,28 +379,6 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
break;
}
- /* We process buffer part by part, allocating skb on
- * each iteration. If this is last skb for this buffer
- * and MSG_ZEROCOPY mode is in use - we must allocate
- * completion for the current syscall.
- *
- * Pass pkt_len because msg iter is already consumed
- * by virtio_transport_fill_skb(), so iter->count
- * can not be used for RLIMIT_MEMLOCK pinned-pages
- * accounting done by msg_zerocopy_realloc().
- */
- if (info->msg && info->msg->msg_flags & MSG_ZEROCOPY &&
- skb_len == rest_len && info->op == VIRTIO_VSOCK_OP_RW) {
- if (virtio_transport_init_zcopy_skb(vsk, skb,
- info->msg,
- pkt_len,
- can_zcopy)) {
- kfree_skb(skb);
- ret = -ENOMEM;
- break;
- }
- }
-
virtio_transport_inc_tx_pkt(vvs, skb);
ret = t_ops->send_pkt(skb, info->net);
@@ -437,6 +401,18 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
virtio_transport_put_credit(vvs, rest_len);
+ /* msg_zerocopy_realloc() initializes the ubuf_info refcnt to 1.
+ * skb_zcopy_set() increases it for each skb, so we can drop that
+ * initial reference to keep it balanced.
+ */
+ if (have_uref) {
+ if (rest_len == pkt_len)
+ /* No data sent, abort the notification. */
+ net_zcopy_put_abort(uarg, true);
+ else
+ net_zcopy_put(uarg);
+ }
+
/* Return number of bytes, if any data has been sent. */
if (rest_len != pkt_len)
ret = pkt_len - rest_len;
@@ -447,7 +423,16 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs,
u32 len)
{
- if (vvs->buf_used + len > vvs->buf_alloc)
+ u64 skb_overhead = ((u64)skb_queue_len(&vvs->rx_queue) + 1) * SKB_TRUESIZE(0);
+
+ /* Allow at most buf_alloc * 2 total budget (payload + overhead),
+ * similar to how SO_RCVBUF is doubled to reserve space for sk_buff
+ * metadata. Check payload against buf_alloc to be sure the other
+ * peer is respecting the credit, and sk_buff overhead to bound
+ * queue growth.
+ */
+ if ((u64)vvs->buf_used + len > vvs->buf_alloc ||
+ skb_overhead > vvs->buf_alloc)
return false;
vvs->rx_bytes += len;
@@ -1204,7 +1189,7 @@ static int virtio_transport_reset_no_sock(const struct virtio_transport *t,
if (!t)
return -ENOTCONN;
- reply = virtio_transport_alloc_skb(&info, 0, false,
+ reply = virtio_transport_alloc_skb(&info, 0, false, NULL,
le64_to_cpu(hdr->dst_cid),
le32_to_cpu(hdr->dst_port),
le64_to_cpu(hdr->src_cid),
@@ -1249,7 +1234,7 @@ static void virtio_transport_do_close(struct vsock_sock *vsk,
struct sock *sk = sk_vsock(vsk);
sock_set_flag(sk, SOCK_DONE);
- vsk->peer_shutdown = SHUTDOWN_MASK;
+ WRITE_ONCE(vsk->peer_shutdown, SHUTDOWN_MASK);
if (vsock_stream_has_data(vsk) <= 0)
sk->sk_state = TCP_CLOSING;
sk->sk_state_change(sk);
@@ -1363,7 +1348,7 @@ virtio_transport_recv_connecting(struct sock *sk,
return err;
}
-static void
+static bool
virtio_transport_recv_enqueue(struct vsock_sock *vsk,
struct sk_buff *skb)
{
@@ -1378,10 +1363,8 @@ virtio_transport_recv_enqueue(struct vsock_sock *vsk,
spin_lock_bh(&vvs->rx_lock);
can_enqueue = virtio_transport_inc_rx_pkt(vvs, len);
- if (!can_enqueue) {
- free_pkt = true;
+ if (!can_enqueue)
goto out;
- }
if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)
vvs->msg_count++;
@@ -1421,6 +1404,8 @@ virtio_transport_recv_enqueue(struct vsock_sock *vsk,
spin_unlock_bh(&vvs->rx_lock);
if (free_pkt)
kfree_skb(skb);
+
+ return can_enqueue;
}
static int
@@ -1433,7 +1418,17 @@ virtio_transport_recv_connected(struct sock *sk,
switch (le16_to_cpu(hdr->op)) {
case VIRTIO_VSOCK_OP_RW:
- virtio_transport_recv_enqueue(vsk, skb);
+ if (!virtio_transport_recv_enqueue(vsk, skb)) {
+ /* There is no more space to queue the packet, so let's
+ * close the connection; otherwise, we'll lose data.
+ */
+ (void)virtio_transport_reset(vsk, skb);
+ virtio_transport_do_close(vsk, true);
+ sk->sk_err = ENOBUFS;
+ sk_error_report(sk);
+ vsock_remove_sock(vsk);
+ break;
+ }
vsock_data_ready(sk);
return err;
case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
@@ -1442,12 +1437,15 @@ virtio_transport_recv_connected(struct sock *sk,
case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
sk->sk_write_space(sk);
break;
- case VIRTIO_VSOCK_OP_SHUTDOWN:
+ case VIRTIO_VSOCK_OP_SHUTDOWN: {
+ u32 peer_shutdown = READ_ONCE(vsk->peer_shutdown);
+
if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV)
- vsk->peer_shutdown |= RCV_SHUTDOWN;
+ peer_shutdown |= RCV_SHUTDOWN;
if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND)
- vsk->peer_shutdown |= SEND_SHUTDOWN;
- if (vsk->peer_shutdown == SHUTDOWN_MASK) {
+ peer_shutdown |= SEND_SHUTDOWN;
+ WRITE_ONCE(vsk->peer_shutdown, peer_shutdown);
+ if (peer_shutdown == SHUTDOWN_MASK) {
if (vsock_stream_has_data(vsk) <= 0 && !sock_flag(sk, SOCK_DONE)) {
(void)virtio_transport_reset(vsk, NULL);
virtio_transport_do_close(vsk, true);
@@ -1462,6 +1460,7 @@ virtio_transport_recv_connected(struct sock *sk,
if (le32_to_cpu(virtio_vsock_hdr(skb)->flags))
sk->sk_state_change(sk);
break;
+ }
case VIRTIO_VSOCK_OP_RST:
virtio_transport_do_close(vsk, true);
break;
diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c
index 4296ca1..9151648 100644
--- a/net/vmw_vsock/vmci_transport.c
+++ b/net/vmw_vsock/vmci_transport.c
@@ -819,7 +819,7 @@ static void vmci_transport_handle_detach(struct sock *sk)
/* On a detach the peer will not be sending or receiving
* anymore.
*/
- vsk->peer_shutdown = SHUTDOWN_MASK;
+ WRITE_ONCE(vsk->peer_shutdown, SHUTDOWN_MASK);
/* We should not be sending anymore since the peer won't be
* there to receive, but we can still receive if there is data
@@ -980,8 +980,10 @@ static int vmci_transport_recv_listen(struct sock *sk,
err = -EINVAL;
}
- if (err < 0)
+ if (err < 0) {
vsock_remove_pending(sk, pending);
+ sk_acceptq_removed(sk);
+ }
release_sock(pending);
vmci_transport_release_pending(pending);
@@ -1164,7 +1166,7 @@ vmci_transport_recv_connecting_server(struct sock *listener,
/* Close and cleanup the connection. */
vmci_transport_send_reset(pending, pkt);
skerr = EPROTO;
- err = pkt->type == VMCI_TRANSPORT_PACKET_TYPE_RST ? 0 : -EINVAL;
+ err = -EINVAL;
goto destroy;
}
@@ -1542,7 +1544,9 @@ static int vmci_transport_recv_connected(struct sock *sk,
if (pkt->u.mode) {
vsk = vsock_sk(sk);
- vsk->peer_shutdown |= pkt->u.mode;
+ WRITE_ONCE(vsk->peer_shutdown,
+ READ_ONCE(vsk->peer_shutdown) |
+ pkt->u.mode);
sk->sk_state_change(sk);
}
break;
@@ -1559,7 +1563,7 @@ static int vmci_transport_recv_connected(struct sock *sk,
* a clean shutdown.
*/
sock_set_flag(sk, SOCK_DONE);
- vsk->peer_shutdown = SHUTDOWN_MASK;
+ WRITE_ONCE(vsk->peer_shutdown, SHUTDOWN_MASK);
if (vsock_stream_has_data(vsk) <= 0)
sk->sk_state = TCP_CLOSING;
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index f334cdef..76c537a 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -1276,6 +1276,18 @@ static int nl80211_prepare_wdev_dump(struct netlink_callback *cb,
rtnl_unlock();
return -ENODEV;
}
+
+ /*
+ * The first invocation validated the wdev's netns against
+ * the caller via __cfg80211_wdev_from_attrs(). The wiphy
+ * may have moved netns between dumpit invocations (via
+ * NL80211_CMD_SET_WIPHY_NETNS), so re-check here.
+ */
+ if (!net_eq(wiphy_net(wiphy), sock_net(cb->skb->sk))) {
+ rtnl_unlock();
+ return -ENODEV;
+ }
+
*rdev = wiphy_to_rdev(wiphy);
*wdev = NULL;
@@ -6354,6 +6366,9 @@ nl80211_parse_rnr_elems(struct wiphy *wiphy, struct nlattr *attrs,
if (ret)
return ERR_PTR(ret);
+ if (num_elems >= 255)
+ return ERR_PTR(-EINVAL);
+
num_elems++;
}
@@ -6699,6 +6714,12 @@ static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params)
return -EINVAL;
}
+ if (!!params->he_cap != !!params->he_oper)
+ return -EINVAL;
+
+ if (!!params->eht_cap != !!params->eht_oper)
+ return -EINVAL;
+
return 0;
}
@@ -13867,6 +13888,19 @@ static int nl80211_wiphy_netns(struct sk_buff *skb, struct genl_info *info)
if (IS_ERR(net))
return PTR_ERR(net);
+ /*
+ * The caller already has CAP_NET_ADMIN over the source netns
+ * (enforced by GENL_UNS_ADMIN_PERM on the genl op). Mirror the
+ * convention used by net/core/rtnetlink.c::rtnl_get_net_ns_capable()
+ * and require CAP_NET_ADMIN over the target netns as well, so that
+ * a caller that is privileged in their own user namespace cannot
+ * push a wiphy into a netns where they have no privilege.
+ */
+ if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) {
+ put_net(net);
+ return -EPERM;
+ }
+
err = 0;
/* check if anything to do */
@@ -19828,6 +19862,7 @@ static const struct genl_small_ops nl80211_small_ops[] = {
.cmd = NL80211_CMD_SET_PMK,
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
.doit = nl80211_set_pmk,
+ .flags = GENL_UNS_ADMIN_PERM,
.internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP |
NL80211_FLAG_CLEAR_SKB),
},
@@ -19835,6 +19870,7 @@ static const struct genl_small_ops nl80211_small_ops[] = {
.cmd = NL80211_CMD_DEL_PMK,
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
.doit = nl80211_del_pmk,
+ .flags = GENL_UNS_ADMIN_PERM,
.internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP),
},
{
diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c
index 4c8ea05..d6cd0de 100644
--- a/net/wireless/pmsr.c
+++ b/net/wireless/pmsr.c
@@ -88,7 +88,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
out->ftm.ftms_per_burst = 0;
if (tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST])
out->ftm.ftms_per_burst =
- nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST]);
+ nla_get_u8(tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST]);
if (capa->ftm.max_ftms_per_burst &&
(out->ftm.ftms_per_burst > capa->ftm.max_ftms_per_burst ||
diff --git a/net/wireless/scan.c b/net/wireless/scan.c
index 328af43..27a56ee 100644
--- a/net/wireless/scan.c
+++ b/net/wireless/scan.c
@@ -1071,6 +1071,7 @@ int cfg80211_scan(struct cfg80211_registered_device *rdev)
struct cfg80211_scan_request_int *request;
struct cfg80211_scan_request_int *rdev_req = rdev->scan_req;
u32 n_channels = 0, idx, i;
+ int err;
if (!(rdev->wiphy.flags & WIPHY_FLAG_SPLIT_SCAN_6GHZ)) {
rdev_req->req.first_part = true;
@@ -1100,8 +1101,14 @@ int cfg80211_scan(struct cfg80211_registered_device *rdev)
rdev_req->req.scan_6ghz = false;
rdev_req->req.first_part = true;
+ err = rdev_scan(rdev, request);
+ if (err) {
+ kfree(request);
+ return err;
+ }
+
rdev->int_scan_req = request;
- return rdev_scan(rdev, request);
+ return 0;
}
void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev,
@@ -2462,6 +2469,9 @@ size_t cfg80211_merge_profile(const u8 *ie, size_t ielen,
memcpy(merged_ie + copied_len, next_sub->data,
next_sub->datalen);
copied_len += next_sub->datalen;
+
+ mbssid_elem = next_mbssid;
+ sub_elem = next_sub;
}
return copied_len;
diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c
index 22d9d9b..63d145b 100644
--- a/net/wireless/wext-compat.c
+++ b/net/wireless/wext-compat.c
@@ -789,6 +789,8 @@ static int cfg80211_wext_siwfreq(struct net_device *dev,
chandef.chan = ieee80211_get_channel(&rdev->wiphy, freq);
if (!chandef.chan)
return -EINVAL;
+ if (!cfg80211_chandef_valid(&chandef))
+ return -EINVAL;
return cfg80211_set_monitor_channel(rdev, dev, &chandef);
case NL80211_IFTYPE_MESH_POINT:
freq = cfg80211_wext_freq(wextfreq);
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 887abed..f8c8a8c 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -646,9 +646,42 @@ static u64 xsk_skb_destructor_get_addr(struct sk_buff *skb)
return (u64)((uintptr_t)skb_shinfo(skb)->destructor_arg & ~0x1UL);
}
-static void xsk_skb_destructor_set_addr(struct sk_buff *skb, u64 addr)
+static struct xsk_addrs *__xsk_addrs_alloc(struct sk_buff *skb, u64 addr)
{
- skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t)addr | 0x1UL);
+ struct xsk_addrs *xsk_addr;
+
+ xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL);
+ if (unlikely(!xsk_addr))
+ return NULL;
+
+ xsk_addr->addrs[0] = addr;
+ skb_shinfo(skb)->destructor_arg = (void *)xsk_addr;
+ return xsk_addr;
+}
+
+static struct xsk_addrs *xsk_addrs_alloc(struct sk_buff *skb)
+{
+ struct xsk_addrs *xsk_addr;
+
+ if (!xsk_skb_destructor_is_addr(skb))
+ return (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
+
+ xsk_addr = __xsk_addrs_alloc(skb, xsk_skb_destructor_get_addr(skb));
+ if (likely(xsk_addr))
+ xsk_addr->num_descs = 1;
+ return xsk_addr;
+}
+
+static int xsk_skb_destructor_set_addr(struct sk_buff *skb, u64 addr)
+{
+ if (IS_ENABLED(CONFIG_64BIT)) {
+ skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t)addr | 0x1UL);
+ return 0;
+ }
+
+ if (unlikely(!__xsk_addrs_alloc(skb, addr)))
+ return -ENOMEM;
+ return 0;
}
static void xsk_inc_num_desc(struct sk_buff *skb)
@@ -685,7 +718,7 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool,
spin_lock_irqsave(&pool->cq_prod_lock, flags);
idx = xskq_get_prod(pool->cq);
- if (unlikely(num_descs > 1)) {
+ if (unlikely(!xsk_skb_destructor_is_addr(skb))) {
xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
for (i = 0; i < num_descs; i++) {
@@ -724,14 +757,20 @@ void xsk_destruct_skb(struct sk_buff *skb)
sock_wfree(skb);
}
-static void xsk_skb_init_misc(struct sk_buff *skb, struct xdp_sock *xs,
- u64 addr)
+static int xsk_skb_init_misc(struct sk_buff *skb, struct xdp_sock *xs,
+ u64 addr)
{
+ int err;
+
+ err = xsk_skb_destructor_set_addr(skb, addr);
+ if (unlikely(err))
+ return err;
+
skb->dev = xs->dev;
skb->priority = READ_ONCE(xs->sk.sk_priority);
skb->mark = READ_ONCE(xs->sk.sk_mark);
skb->destructor = xsk_destruct_skb;
- xsk_skb_destructor_set_addr(skb, addr);
+ return 0;
}
static void xsk_consume_skb(struct sk_buff *skb)
@@ -740,7 +779,7 @@ static void xsk_consume_skb(struct sk_buff *skb)
u32 num_descs = xsk_get_num_desc(skb);
struct xsk_addrs *xsk_addr;
- if (unlikely(num_descs > 1)) {
+ if (unlikely(!xsk_skb_destructor_is_addr(skb))) {
xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
kmem_cache_free(xsk_tx_generic_cache, xsk_addr);
}
@@ -763,6 +802,7 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
u32 hr)
{
struct xsk_tx_metadata *meta = NULL;
+ u16 csum_start, csum_offset;
if (unlikely(pool->tx_metadata_len == 0))
return -EINVAL;
@@ -772,13 +812,15 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
return -EINVAL;
if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
- if (unlikely(meta->request.csum_start +
- meta->request.csum_offset +
+ csum_start = READ_ONCE(meta->request.csum_start);
+ csum_offset = READ_ONCE(meta->request.csum_offset);
+
+ if (unlikely(csum_start + csum_offset +
sizeof(__sum16) > desc->len))
return -EINVAL;
- skb->csum_start = hr + meta->request.csum_start;
- skb->csum_offset = meta->request.csum_offset;
+ skb->csum_start = hr + csum_start;
+ skb->csum_offset = csum_offset;
skb->ip_summed = CHECKSUM_PARTIAL;
if (unlikely(pool->tx_sw_csum)) {
@@ -819,28 +861,19 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
return ERR_PTR(err);
skb_reserve(skb, hr);
-
- xsk_skb_init_misc(skb, xs, desc->addr);
if (desc->options & XDP_TX_METADATA) {
err = xsk_skb_metadata(skb, buffer, desc, pool, hr);
- if (unlikely(err))
+ if (unlikely(err)) {
+ kfree_skb(skb);
return ERR_PTR(err);
+ }
}
} else {
struct xsk_addrs *xsk_addr;
- if (xsk_skb_destructor_is_addr(skb)) {
- xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache,
- GFP_KERNEL);
- if (!xsk_addr)
- return ERR_PTR(-ENOMEM);
-
- xsk_addr->num_descs = 1;
- xsk_addr->addrs[0] = xsk_skb_destructor_get_addr(skb);
- skb_shinfo(skb)->destructor_arg = (void *)xsk_addr;
- } else {
- xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
- }
+ xsk_addr = xsk_addrs_alloc(skb);
+ if (!xsk_addr)
+ return ERR_PTR(-ENOMEM);
/* in case of -EOVERFLOW that could happen below,
* xsk_consume_skb() will release this node as whole skb
@@ -856,8 +889,11 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
addr = buffer - pool->addrs;
for (copied = 0, i = skb_shinfo(skb)->nr_frags; copied < len; i++) {
- if (unlikely(i >= MAX_SKB_FRAGS))
+ if (unlikely(i >= MAX_SKB_FRAGS)) {
+ if (!xs->skb)
+ kfree_skb(skb);
return ERR_PTR(-EOVERFLOW);
+ }
page = pool->umem->pgs[addr >> PAGE_SHIFT];
get_page(page);
@@ -914,7 +950,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
if (unlikely(err))
goto free_err;
- xsk_skb_init_misc(skb, xs, desc->addr);
if (desc->options & XDP_TX_METADATA) {
err = xsk_skb_metadata(skb, buffer, desc,
xs->pool, hr);
@@ -927,19 +962,10 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
struct page *page;
u8 *vaddr;
- if (xsk_skb_destructor_is_addr(skb)) {
- xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache,
- GFP_KERNEL);
- if (!xsk_addr) {
- err = -ENOMEM;
- goto free_err;
- }
-
- xsk_addr->num_descs = 1;
- xsk_addr->addrs[0] = xsk_skb_destructor_get_addr(skb);
- skb_shinfo(skb)->destructor_arg = (void *)xsk_addr;
- } else {
- xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg;
+ xsk_addr = xsk_addrs_alloc(skb);
+ if (!xsk_addr) {
+ err = -ENOMEM;
+ goto free_err;
}
if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) {
@@ -964,18 +990,28 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
}
}
+ if (!xs->skb) {
+ err = xsk_skb_init_misc(skb, xs, desc->addr);
+ if (unlikely(err))
+ goto free_err;
+ }
xsk_inc_num_desc(skb);
return skb;
free_err:
- if (skb && !skb_shinfo(skb)->nr_frags)
+ if (skb && !xs->skb)
kfree_skb(skb);
if (err == -EOVERFLOW) {
- /* Drop the packet */
- xsk_inc_num_desc(xs->skb);
- xsk_drop_skb(xs->skb);
+ if (xs->skb) {
+ /* Drop the packet */
+ xsk_inc_num_desc(xs->skb);
+ xsk_drop_skb(xs->skb);
+ } else {
+ xsk_cq_cancel_locked(xs->pool, 1);
+ xs->tx->invalid_descs++;
+ }
xskq_cons_release(xs->tx);
} else {
/* Let application retry */
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index cd7bc50..d981cfd 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -175,6 +175,9 @@ int xp_assign_dev(struct xsk_buff_pool *pool,
if (force_zc && force_copy)
return -EINVAL;
+ if (pool->tx_sw_csum && (netdev->priv_flags & IFF_TX_SKB_NO_LINEAR))
+ return -EOPNOTSUPP;
+
if (xsk_get_pool_from_qid(netdev, queue_id))
return -EBUSY;
diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c
index afa4575..3bff346 100644
--- a/net/xdp/xskmap.c
+++ b/net/xdp/xskmap.c
@@ -184,6 +184,10 @@ static long xsk_map_update_elem(struct bpf_map *map, void *key, void *value,
}
xs = (struct xdp_sock *)sock->sk;
+ if (!READ_ONCE(xs->rx)) {
+ sockfd_put(sock);
+ return -ENOBUFS;
+ }
map_entry = &m->xsk_map[i];
node = xsk_map_node_alloc(m, map_entry);
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index f65291e..e4c2cd2 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -797,9 +797,12 @@ static void xfrm_trans_reinject(struct work_struct *work)
spin_unlock_bh(&trans->queue_lock);
local_bh_disable();
- while ((skb = __skb_dequeue(&queue)))
- XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
- NULL, skb);
+ while ((skb = __skb_dequeue(&queue))) {
+ struct net *net = XFRM_TRANS_SKB_CB(skb)->net;
+
+ XFRM_TRANS_SKB_CB(skb)->finish(net, NULL, skb);
+ put_net(net);
+ }
local_bh_enable();
}
@@ -808,6 +811,7 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
struct sk_buff *))
{
struct xfrm_trans_tasklet *trans;
+ struct net *hold_net;
trans = this_cpu_ptr(&xfrm_trans_tasklet);
@@ -816,8 +820,12 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
+ hold_net = maybe_get_net(net);
+ if (!hold_net)
+ return -ENODEV;
+
XFRM_TRANS_SKB_CB(skb)->finish = finish;
- XFRM_TRANS_SKB_CB(skb)->net = net;
+ XFRM_TRANS_SKB_CB(skb)->net = hold_net;
spin_lock_bh(&trans->queue_lock);
__skb_queue_tail(&trans->queue, skb);
spin_unlock_bh(&trans->queue_lock);
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
index 5f38dff..671d48f 100644
--- a/net/xfrm/xfrm_ipcomp.c
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -51,11 +51,15 @@ static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
struct scatterlist *dsg;
int len, dlen;
- if (unlikely(err))
- goto out_free_req;
+ if (unlikely(!req))
+ return err;
extra = acomp_request_extra(req);
dsg = extra->sg;
+
+ if (unlikely(err))
+ goto out_free_req;
+
dlen = req->dlen;
pskb_trim_unique(skb, 0);
@@ -84,10 +88,10 @@ static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
skb_shinfo(skb)->nr_frags++;
} while ((dlen -= len));
- for (; dsg; dsg = sg_next(dsg))
+out_free_req:
+ for (; dsg && sg_page(dsg); dsg = sg_next(dsg))
__free_page(sg_page(dsg));
-out_free_req:
acomp_request_free(req);
return err;
}
diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c
index 97bc979e5..6c6bbc0 100644
--- a/net/xfrm/xfrm_iptfs.c
+++ b/net/xfrm/xfrm_iptfs.c
@@ -2650,7 +2650,8 @@ static void __iptfs_init_state(struct xfrm_state *x,
x->props.enc_hdr_len = sizeof(struct ip_iptfs_hdr);
/* Always keep a module reference when x->mode_data is set */
- __module_get(x->mode_cbs->owner);
+ if (x->mode_data != xtfs)
+ __module_get(x->mode_cbs->owner);
x->mode_data = xtfs;
xtfs->x = x;
@@ -2658,22 +2659,39 @@ static void __iptfs_init_state(struct xfrm_state *x,
static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig)
{
+ struct skb_wseq *w_saved = NULL;
struct xfrm_iptfs_data *xtfs;
xtfs = kmemdup(orig->mode_data, sizeof(*xtfs), GFP_KERNEL);
if (!xtfs)
return -ENOMEM;
- xtfs->ra_newskb = NULL;
if (xtfs->cfg.reorder_win_size) {
- xtfs->w_saved = kzalloc_objs(*xtfs->w_saved,
- xtfs->cfg.reorder_win_size);
- if (!xtfs->w_saved) {
+ w_saved = kzalloc_objs(*w_saved, xtfs->cfg.reorder_win_size);
+ if (!w_saved) {
kfree_sensitive(xtfs);
return -ENOMEM;
}
}
+ xtfs->w_saved = w_saved;
+ __skb_queue_head_init(&xtfs->queue);
+ xtfs->queue_size = 0;
+ hrtimer_setup(&xtfs->iptfs_timer, iptfs_delay_timer, CLOCK_MONOTONIC,
+ IPTFS_HRTIMER_MODE);
+
+ spin_lock_init(&xtfs->drop_lock);
+ hrtimer_setup(&xtfs->drop_timer, iptfs_drop_timer, CLOCK_MONOTONIC,
+ IPTFS_HRTIMER_MODE);
+
+ xtfs->w_seq_set = false;
+ xtfs->w_wantseq = 0;
+ xtfs->w_savedlen = 0;
+ xtfs->ra_newskb = NULL;
+ xtfs->ra_wantseq = 0;
+ xtfs->ra_runtlen = 0;
+
+ __module_get(x->mode_cbs->owner);
x->mode_data = xtfs;
xtfs->x = x;
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
index a9652b4..cc35c2f 100644
--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -66,7 +66,9 @@ static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
struct iphdr *iph = ip_hdr(skb);
int ihl = iph->ihl * 4;
- skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ if (!skb->inner_protocol)
+ skb_set_inner_transport_header(skb,
+ skb_transport_offset(skb));
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
@@ -167,7 +169,9 @@ static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
int hdr_len;
iph = ipv6_hdr(skb);
- skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ if (!skb->inner_protocol)
+ skb_set_inner_transport_header(skb,
+ skb_transport_offset(skb));
hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
if (hdr_len < 0)
@@ -276,8 +280,10 @@ static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
struct iphdr *top_iph;
int flags;
- skb_set_inner_network_header(skb, skb_network_offset(skb));
- skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ if (!skb->inner_protocol) {
+ skb_set_inner_network_header(skb, skb_network_offset(skb));
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ }
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
@@ -321,8 +327,10 @@ static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
struct ipv6hdr *top_iph;
int dsfield;
- skb_set_inner_network_header(skb, skb_network_offset(skb));
- skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ if (!skb->inner_protocol) {
+ skb_set_inner_network_header(skb, skb_network_offset(skb));
+ skb_set_inner_transport_header(skb, skb_transport_offset(skb));
+ }
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index c944327..dd09d20 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -4276,21 +4276,21 @@ static int __net_init xfrm_policy_init(struct net *net)
return -ENOMEM;
}
-static void xfrm_policy_fini(struct net *net)
+static void __net_exit xfrm_net_pre_exit(struct net *net)
{
- struct xfrm_pol_inexact_bin *b, *t;
- unsigned int sz;
- int dir;
-
disable_work_sync(&net->xfrm.policy_hthresh.work);
-
flush_work(&net->xfrm.policy_hash_work);
#ifdef CONFIG_XFRM_SUB_POLICY
xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
#endif
xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
+}
- synchronize_rcu();
+static void xfrm_policy_fini(struct net *net)
+{
+ struct xfrm_pol_inexact_bin *b, *t;
+ unsigned int sz;
+ int dir;
WARN_ON(!list_empty(&net->xfrm.policy_all));
@@ -4368,6 +4368,7 @@ static void __net_exit xfrm_net_exit(struct net *net)
static struct pernet_operations __net_initdata xfrm_net_ops = {
.init = xfrm_net_init,
+ .pre_exit = xfrm_net_pre_exit,
.exit = xfrm_net_exit,
};
@@ -4703,7 +4704,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
}
/* Stage 5 - announce */
- km_migrate(sel, dir, type, m, num_migrate, k, encap);
+ km_migrate(sel, dir, type, m, num_migrate, k, net, encap);
xfrm_pol_put(pol);
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 1748d37..589c3b6 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -818,17 +818,17 @@ int __xfrm_state_delete(struct xfrm_state *x)
spin_lock(&net->xfrm.xfrm_state_lock);
list_del(&x->km.all);
- hlist_del_rcu(&x->bydst);
- hlist_del_rcu(&x->bysrc);
- if (x->km.seq)
- hlist_del_rcu(&x->byseq);
+ hlist_del_init_rcu(&x->bydst);
+ hlist_del_init_rcu(&x->bysrc);
+ if (!hlist_unhashed(&x->byseq))
+ hlist_del_init_rcu(&x->byseq);
if (!hlist_unhashed(&x->state_cache))
hlist_del_rcu(&x->state_cache);
if (!hlist_unhashed(&x->state_cache_input))
hlist_del_rcu(&x->state_cache_input);
- if (x->id.spi)
- hlist_del_rcu(&x->byspi);
+ if (!hlist_unhashed(&x->byspi))
+ hlist_del_init_rcu(&x->byspi);
net->xfrm.state_num--;
xfrm_nat_keepalive_state_updated(x);
spin_unlock(&net->xfrm.xfrm_state_lock);
@@ -2837,7 +2837,7 @@ EXPORT_SYMBOL(km_policy_expired);
#ifdef CONFIG_XFRM_MIGRATE
int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_migrate,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap)
{
int err = -EINVAL;
@@ -2848,7 +2848,7 @@ int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
list_for_each_entry_rcu(km, &xfrm_km_list, list) {
if (km->migrate) {
ret = km->migrate(sel, dir, type, m, num_migrate, k,
- encap);
+ net, encap);
if (!ret)
err = ret;
}
@@ -3114,10 +3114,14 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
const struct xfrm_type *type = READ_ONCE(x->type);
struct crypto_aead *aead;
u32 blksize, net_adj = 0;
+ u32 overhead, payload_mtu;
if (x->km.state != XFRM_STATE_VALID ||
- !type || type->proto != IPPROTO_ESP)
+ !type || type->proto != IPPROTO_ESP) {
+ if (mtu <= x->props.header_len)
+ return 1;
return mtu - x->props.header_len;
+ }
aead = x->data;
blksize = ALIGN(crypto_aead_blocksize(aead), 4);
@@ -3140,8 +3144,17 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
break;
}
- return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
- net_adj) & ~(blksize - 1)) + net_adj - 2;
+ overhead = x->props.header_len + crypto_aead_authsize(aead) + net_adj;
+ if (mtu <= overhead)
+ return 1;
+
+ payload_mtu = mtu - overhead;
+ payload_mtu &= ~(blksize - 1);
+ if (payload_mtu <= 2)
+ return 1;
+
+ return payload_mtu + net_adj - 2;
+
}
EXPORT_SYMBOL_GPL(xfrm_state_mtu);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index d56450f..71a4b72 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -3271,10 +3271,9 @@ static int build_migrate(struct sk_buff *skb, const struct xfrm_migrate *m,
static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_migrate,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap)
{
- struct net *net = &init_net;
struct sk_buff *skb;
int err;
@@ -3292,7 +3291,7 @@ static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
#else
static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
const struct xfrm_migrate *m, int num_migrate,
- const struct xfrm_kmaddress *k,
+ const struct xfrm_kmaddress *k, struct net *net,
const struct xfrm_encap_tmpl *encap)
{
return -ENOPROTOOPT;
@@ -3323,6 +3322,7 @@ const int xfrm_msg_min[XFRM_NR_MSGTYPES] = {
[XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = sizeof(u32),
[XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
[XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32),
+ [XFRM_MSG_MAPPING - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_mapping),
[XFRM_MSG_SETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default),
[XFRM_MSG_GETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default),
};
diff --git a/rust/Makefile b/rust/Makefile
index b361bfe..b9e9f51 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -403,6 +403,8 @@
BINDGEN_TARGET_arm64 := aarch64-linux-gnu
BINDGEN_TARGET_arm := arm-linux-gnueabi
BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf
+# This is only for i386 UM builds, which need the 32-bit target not -m32
+BINDGEN_TARGET_i386 := i386-linux-gnu
BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH))
BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH))
diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs
index adbafe8..403fc35 100644
--- a/rust/kernel/drm/device.rs
+++ b/rust/kernel/drm/device.rs
@@ -119,13 +119,20 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<A
// compatible `Layout`.
let layout = Kmalloc::aligned_layout(Layout::new::<Self>());
+ // Use a temporary vtable without a `release` callback until `data` is initialized, so
+ // init failure can release the DRM device without dropping uninitialized fields.
+ let alloc_vtable = bindings::drm_driver {
+ release: None,
+ ..Self::VTABLE
+ };
+
// SAFETY:
- // - `VTABLE`, as a `const` is pinned to the read-only section of the compilation,
+ // - `alloc_vtable` reference remains valid until no longer used,
// - `dev` is valid by its type invarants,
let raw_drm: *mut Self = unsafe {
bindings::__drm_dev_alloc(
dev.as_raw(),
- &Self::VTABLE,
+ &alloc_vtable,
layout.size(),
mem::offset_of!(Self, dev),
)
@@ -133,6 +140,10 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<A
.cast();
let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?;
+ // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was
+ // successful.
+ let drm_dev = unsafe { Self::into_drm_device(raw_drm) };
+
// SAFETY: `raw_drm` is a valid pointer to `Self`.
let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) };
@@ -140,15 +151,14 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<A
// - `raw_data` is a valid pointer to uninitialized memory.
// - `raw_data` will not move until it is dropped.
unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| {
- // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was
- // successful.
- let drm_dev = unsafe { Self::into_drm_device(raw_drm) };
-
// SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the
// refcount must be non-zero.
unsafe { bindings::drm_dev_put(drm_dev) };
})?;
+ // SAFETY: `drm_dev` is still private to this function.
+ unsafe { (*drm_dev).driver = const { &Self::VTABLE } };
+
// SAFETY: The reference count is one, and now we take ownership of that reference as a
// `drm::Device`.
Ok(unsafe { ARef::from_raw(raw_drm) })
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index 75acda7..01b5bd4 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -277,8 +277,17 @@ pub fn new(dev: &drm::Device<T::Driver>, size: usize, args: T::Args) -> Result<A
// SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above.
unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS };
- // SAFETY: The arguments are all valid per the type invariants.
- to_result(unsafe { bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) })?;
+ if let Err(err) =
+ // SAFETY: The arguments are all valid per the type invariants.
+ to_result(unsafe {
+ bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size)
+ })
+ {
+ // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before
+ // failing, so `drm_gem_private_object_fini()` is the matching cleanup.
+ unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) };
+ return Err(err);
+ }
// SAFETY: We will never move out of `Self` as `ARef<Self>` is always treated as pinned.
let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) });
diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs
index d025fb0..e1b6489 100644
--- a/rust/kernel/drm/gem/shmem.rs
+++ b/rust/kernel/drm/gem/shmem.rs
@@ -19,10 +19,8 @@
},
error::to_result,
prelude::*,
- types::{
- ARef,
- Opaque, //
- }, //
+ sync::aref::ARef,
+ types::Opaque, //
};
use core::{
ops::{
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
index daa3f1c..487ee00 100644
--- a/rust/pin-init/internal/src/init.rs
+++ b/rust/pin-init/internal/src/init.rs
@@ -249,22 +249,6 @@ fn init_fields(
});
// Again span for better diagnostics
let write = quote_spanned!(ident.span()=> ::core::ptr::write);
- // NOTE: the field accessor ensures that the initialized field is properly aligned.
- // Unaligned fields will cause the compiler to emit E0793. We do not support
- // unaligned fields since `Init::__init` requires an aligned pointer; the call to
- // `ptr::write` below has the same requirement.
- let accessor = if pinned {
- let project_ident = format_ident!("__project_{ident}");
- quote! {
- // SAFETY: TODO
- unsafe { #data.#project_ident(&mut (*#slot).#ident) }
- }
- } else {
- quote! {
- // SAFETY: TODO
- unsafe { &mut (*#slot).#ident }
- }
- };
quote! {
#(#attrs)*
{
@@ -272,51 +256,31 @@ fn init_fields(
// SAFETY: TODO
unsafe { #write(&raw mut (*#slot).#ident, #value_ident) };
}
- #(#cfgs)*
- #[allow(unused_variables)]
- let #ident = #accessor;
}
}
InitializerKind::Init { ident, value, .. } => {
// Again span for better diagnostics
let init = format_ident!("init", span = value.span());
- // NOTE: the field accessor ensures that the initialized field is properly aligned.
- // Unaligned fields will cause the compiler to emit E0793. We do not support
- // unaligned fields since `Init::__init` requires an aligned pointer; the call to
- // `ptr::write` below has the same requirement.
- let (value_init, accessor) = if pinned {
- let project_ident = format_ident!("__project_{ident}");
- (
- quote! {
- // SAFETY:
- // - `slot` is valid, because we are inside of an initializer closure, we
- // return when an error/panic occurs.
- // - We also use `#data` to require the correct trait (`Init` or `PinInit`)
- // for `#ident`.
- unsafe { #data.#ident(&raw mut (*#slot).#ident, #init)? };
- },
- quote! {
- // SAFETY: TODO
- unsafe { #data.#project_ident(&mut (*#slot).#ident) }
- },
- )
+ let value_init = if pinned {
+ quote! {
+ // SAFETY:
+ // - `slot` is valid, because we are inside of an initializer closure, we
+ // return when an error/panic occurs.
+ // - We also use `#data` to require the correct trait (`Init` or `PinInit`)
+ // for `#ident`.
+ unsafe { #data.#ident(&raw mut (*#slot).#ident, #init)? };
+ }
} else {
- (
- quote! {
- // SAFETY: `slot` is valid, because we are inside of an initializer
- // closure, we return when an error/panic occurs.
- unsafe {
- ::pin_init::Init::__init(
- #init,
- &raw mut (*#slot).#ident,
- )?
- };
- },
- quote! {
- // SAFETY: TODO
- unsafe { &mut (*#slot).#ident }
- },
- )
+ quote! {
+ // SAFETY: `slot` is valid, because we are inside of an initializer
+ // closure, we return when an error/panic occurs.
+ unsafe {
+ ::pin_init::Init::__init(
+ #init,
+ &raw mut (*#slot).#ident,
+ )?
+ };
+ }
};
quote! {
#(#attrs)*
@@ -324,9 +288,6 @@ fn init_fields(
let #init = #value;
#value_init
}
- #(#cfgs)*
- #[allow(unused_variables)]
- let #ident = #accessor;
}
}
InitializerKind::Code { block: value, .. } => quote! {
@@ -339,18 +300,41 @@ fn init_fields(
if let Some(ident) = kind.ident() {
// `mixed_site` ensures that the guard is not accessible to the user-controlled code.
let guard = format_ident!("__{ident}_guard", span = Span::mixed_site());
+
+ // NOTE: The reference is derived from the guard so that it only lives as long as the
+ // guard does and cannot escape the scope. If it's created via `&mut (*#slot).#ident`
+ // like the unaligned field guard, it will become effectively `'static`.
+ let accessor = if pinned {
+ let project_ident = format_ident!("__project_{ident}");
+ quote! {
+ // SAFETY: the initialization is pinned.
+ unsafe { #data.#project_ident(#guard.let_binding()) }
+ }
+ } else {
+ quote! {
+ #guard.let_binding()
+ }
+ };
+
res.extend(quote! {
#(#cfgs)*
- // Create the drop guard:
+ // Create the drop guard.
//
- // We rely on macro hygiene to make it impossible for users to access this local
- // variable.
- // SAFETY: We forget the guard later when initialization has succeeded.
- let #guard = unsafe {
+ // SAFETY:
+ // - `&raw mut (*slot).#ident` is valid.
+ // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
+ // - `(*slot).#ident` has been initialized above.
+ // - We only need the ownership to the pointee back when initialization has
+ // succeeded, where we `forget` the guard.
+ let mut #guard = unsafe {
::pin_init::__internal::DropGuard::new(
&raw mut (*slot).#ident
)
};
+
+ #(#cfgs)*
+ #[allow(unused_variables)]
+ let #ident = #accessor;
});
guards.push(guard);
guard_attrs.push(cfgs);
@@ -367,49 +351,49 @@ fn init_fields(
}
}
-/// Generate the check for ensuring that every field has been initialized.
+/// Generate the check for ensuring that every field has been initialized and aligned.
fn make_field_check(
fields: &Punctuated<InitializerField, Token![,]>,
init_kind: InitKind,
path: &Path,
) -> TokenStream {
- let field_attrs = fields
+ let field_attrs: Vec<_> = fields
.iter()
- .filter_map(|f| f.kind.ident().map(|_| &f.attrs));
- let field_name = fields.iter().filter_map(|f| f.kind.ident());
- match init_kind {
- InitKind::Normal => quote! {
- // We use unreachable code to ensure that all fields have been mentioned exactly once,
- // this struct initializer will still be type-checked and complain with a very natural
- // error message if a field is forgotten/mentioned more than once.
- #[allow(unreachable_code, clippy::diverging_sub_expression)]
- // SAFETY: this code is never executed.
- let _ = || unsafe {
- ::core::ptr::write(slot, #path {
- #(
- #(#field_attrs)*
- #field_name: ::core::panic!(),
- )*
- })
- };
- },
- InitKind::Zeroing => quote! {
- // We use unreachable code to ensure that all fields have been mentioned at most once.
- // Since the user specified `..Zeroable::zeroed()` at the end, all missing fields will
- // be zeroed. This struct initializer will still be type-checked and complain with a
- // very natural error message if a field is mentioned more than once, or doesn't exist.
- #[allow(unreachable_code, clippy::diverging_sub_expression, unused_assignments)]
- // SAFETY: this code is never executed.
- let _ = || unsafe {
- ::core::ptr::write(slot, #path {
- #(
- #(#field_attrs)*
- #field_name: ::core::panic!(),
- )*
- ..::core::mem::zeroed()
- })
- };
- },
+ .filter_map(|f| f.kind.ident().map(|_| &f.attrs))
+ .collect();
+ let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect();
+ let zeroing_trailer = match init_kind {
+ InitKind::Normal => None,
+ InitKind::Zeroing => Some(quote! {
+ ..::core::mem::zeroed()
+ }),
+ };
+ quote! {
+ #[allow(unreachable_code, clippy::diverging_sub_expression)]
+ // We use unreachable code to perform field checks. They're still checked by the compiler.
+ // SAFETY: this code is never executed.
+ let _ = || unsafe {
+ // Create references to ensure that the initialized field is properly aligned.
+ // Unaligned fields will cause the compiler to emit E0793. We do not support
+ // unaligned fields since `Init::__init` requires an aligned pointer; the call to
+ // `ptr::write` for value-initialization case has the same requirement.
+ #(
+ #(#field_attrs)*
+ let _ = &(*slot).#field_name;
+ )*
+
+ // If the zeroing trailer is not present, this checks that all fields have been
+ // mentioned exactly once. If the zeroing trailer is present, all missing fields will be
+ // zeroed, so this checks that all fields have been mentioned at most once. The use of
+ // struct initializer will still generate very natural error messages for any misuse.
+ ::core::ptr::write(slot, #path {
+ #(
+ #(#field_attrs)*
+ #field_name: ::core::panic!(),
+ )*
+ #zeroing_trailer
+ })
+ };
}
}
diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs
index 90adbdc..5720a62 100644
--- a/rust/pin-init/src/__internal.rs
+++ b/rust/pin-init/src/__internal.rs
@@ -238,32 +238,42 @@ struct Foo {
/// When a value of this type is dropped, it drops a `T`.
///
/// Can be forgotten to prevent the drop.
+///
+/// # Invariants
+///
+/// - `ptr` is valid and properly aligned.
+/// - `*ptr` is initialized and owned by this guard.
pub struct DropGuard<T: ?Sized> {
ptr: *mut T,
}
impl<T: ?Sized> DropGuard<T> {
- /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
+ /// Creates a drop guard and transfer the ownership of the pointer content.
+ ///
+ /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`].
///
/// # Safety
///
- /// `ptr` must be a valid pointer.
- ///
- /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
- /// - has not been dropped,
- /// - is not accessible by any other means,
- /// - will not be dropped by any other means.
+ /// - `ptr` is valid and properly aligned.
+ /// - `*ptr` is initialized, and the ownership is transferred to this guard.
#[inline]
pub unsafe fn new(ptr: *mut T) -> Self {
+ // INVARIANT: By safety requirement.
Self { ptr }
}
+
+ /// Create a let binding for accessor use.
+ #[inline]
+ pub fn let_binding(&mut self) -> &mut T {
+ // SAFETY: Per type invariant.
+ unsafe { &mut *self.ptr }
+ }
}
impl<T: ?Sized> Drop for DropGuard<T> {
#[inline]
fn drop(&mut self) {
- // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
- // ensuring that this operation is safe.
+ // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard.
unsafe { ptr::drop_in_place(self.ptr) }
}
}
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 0718e39c..7e216d8 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -187,7 +187,11 @@
objtool-args-$(CONFIG_HAVE_NOINSTR_HACK) += --hacks=noinstr
objtool-args-$(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) += --hacks=skylake
objtool-args-$(CONFIG_X86_KERNEL_IBT) += --ibt
-objtool-args-$(CONFIG_FINEIBT) += --cfi
+objtool-args-$(CONFIG_CALL_PADDING) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
+ifdef CONFIG_CALL_PADDING
+objtool-args-$(CONFIG_CFI) += --cfi
+objtool-args-$(CONFIG_FINEIBT) += --fineibt
+endif
objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL) += --mcount
ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL
objtool-args-$(CONFIG_HAVE_OBJTOOL_NOP_MCOUNT) += --mnop
@@ -200,7 +204,6 @@
objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE) += --static-call
objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess
objtool-args-$(or $(CONFIG_GCOV_KERNEL),$(CONFIG_KCOV)) += --no-unreachable
-objtool-args-$(CONFIG_PREFIX_SYMBOLS) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
objtool-args-$(CONFIG_OBJTOOL_WERROR) += --werror
objtool-args = $(objtool-args-y) \
diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h
index 8f1b350..abb1964 100644
--- a/scripts/gcc-plugins/gcc-common.h
+++ b/scripts/gcc-plugins/gcc-common.h
@@ -309,7 +309,9 @@ typedef const gimple *const_gimple_ptr;
#define gimple gimple_ptr
#define const_gimple const_gimple_ptr
#undef CONST_CAST_GIMPLE
-#define CONST_CAST_GIMPLE(X) CONST_CAST(gimple, (X))
+#define CONST_CAST_GIMPLE(X) const_cast<gimple>((X))
+#undef CONST_CAST_TREE
+#define CONST_CAST_TREE(X) const_cast<tree>((X))
/* gimple related */
static inline gimple gimple_build_assign_with_ops(enum tree_code subcode, tree lhs, tree op1, tree op2 MEM_STAT_DECL)
diff --git a/scripts/gdb/linux/interrupts.py b/scripts/gdb/linux/interrupts.py
index f4f715a..a68ae91 100644
--- a/scripts/gdb/linux/interrupts.py
+++ b/scripts/gdb/linux/interrupts.py
@@ -20,7 +20,7 @@
def irqd_is_level(desc):
return desc['irq_data']['common']['state_use_accessors'] & constants.LX_IRQD_LEVEL
-def show_irq_desc(prec, irq):
+def show_irq_desc(prec, chip_width, irq):
text = ""
desc = mapletree.mtree_load(gdb.parse_and_eval("&sparse_irqs"), irq)
@@ -48,7 +48,7 @@
count = cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt']
else:
count = 0
- text += "%10u" % (count)
+ text += "%10u " % (count)
name = "None"
if desc['irq_data']['chip']:
@@ -58,7 +58,7 @@
else:
name = "-"
- text += " %8s" % (name)
+ text += " %-*s" % (chip_width, name)
if desc['irq_data']['domain']:
text += " %*lu" % (prec, desc['irq_data']['hwirq'])
@@ -97,64 +97,29 @@
text += "%*s: %10u\n" % (prec, "ERR", cnt['counter'])
return text
-def x86_show_irqstat(prec, pfx, field, desc):
- irq_stat = gdb.parse_and_eval("&irq_stat")
+def x86_show_irqstat(prec, pfx, idx, desc):
+ irq_stat = gdb.parse_and_eval("&irq_stat.counts[%d]" %idx)
text = "%*s: " % (prec, pfx)
for cpu in cpus.each_online_cpu():
stat = cpus.per_cpu(irq_stat, cpu)
- text += "%10u " % (stat[field])
- text += " %s\n" % (desc)
- return text
-
-def x86_show_mce(prec, var, pfx, desc):
- pvar = gdb.parse_and_eval(var)
- text = "%*s: " % (prec, pfx)
- for cpu in cpus.each_online_cpu():
- text += "%10u " % (cpus.per_cpu(pvar, cpu).dereference())
- text += " %s\n" % (desc)
+ text += "%10u " % (stat.dereference())
+ text += desc
return text
def x86_show_interupts(prec):
- text = x86_show_irqstat(prec, "NMI", '__nmi_count', 'Non-maskable interrupts')
+ info_type = gdb.lookup_type('struct irq_stat_info')
+ info = gdb.parse_and_eval('irq_stat_info')
+ bitmap = gdb.parse_and_eval('irq_stat_count_show')
+ bitsperlong = 8 * int(bitmap.type.target().sizeof)
- if constants.LX_CONFIG_X86_LOCAL_APIC:
- text += x86_show_irqstat(prec, "LOC", 'apic_timer_irqs', "Local timer interrupts")
- text += x86_show_irqstat(prec, "SPU", 'irq_spurious_count', "Spurious interrupts")
- text += x86_show_irqstat(prec, "PMI", 'apic_perf_irqs', "Performance monitoring interrupts")
- text += x86_show_irqstat(prec, "IWI", 'apic_irq_work_irqs', "IRQ work interrupts")
- text += x86_show_irqstat(prec, "RTR", 'icr_read_retry_count', "APIC ICR read retries")
- if utils.gdb_eval_or_none("x86_platform_ipi_callback") is not None:
- text += x86_show_irqstat(prec, "PLT", 'x86_platform_ipis', "Platform interrupts")
-
- if constants.LX_CONFIG_SMP:
- text += x86_show_irqstat(prec, "RES", 'irq_resched_count', "Rescheduling interrupts")
- text += x86_show_irqstat(prec, "CAL", 'irq_call_count', "Function call interrupts")
- text += x86_show_irqstat(prec, "TLB", 'irq_tlb_count', "TLB shootdowns")
-
- if constants.LX_CONFIG_X86_THERMAL_VECTOR:
- text += x86_show_irqstat(prec, "TRM", 'irq_thermal_count', "Thermal events interrupts")
-
- if constants.LX_CONFIG_X86_MCE_THRESHOLD:
- text += x86_show_irqstat(prec, "THR", 'irq_threshold_count', "Threshold APIC interrupts")
-
- if constants.LX_CONFIG_X86_MCE_AMD:
- text += x86_show_irqstat(prec, "DFR", 'irq_deferred_error_count', "Deferred Error APIC interrupts")
-
- if constants.LX_CONFIG_X86_MCE:
- text += x86_show_mce(prec, "&mce_exception_count", "MCE", "Machine check exceptions")
- text += x86_show_mce(prec, "&mce_poll_count", "MCP", "Machine check polls")
-
- text += show_irq_err_count(prec)
-
- if constants.LX_CONFIG_X86_IO_APIC:
- cnt = utils.gdb_eval_or_none("irq_mis_count")
- if cnt is not None:
- text += "%*s: %10u\n" % (prec, "MIS", cnt['counter'])
-
- if constants.LX_CONFIG_KVM:
- text += x86_show_irqstat(prec, "PIN", 'kvm_posted_intr_ipis', 'Posted-interrupt notification event')
- text += x86_show_irqstat(prec, "NPI", 'kvm_posted_intr_nested_ipis', 'Nested posted-interrupt event')
- text += x86_show_irqstat(prec, "PIW", 'kvm_posted_intr_wakeup_ipis', 'Posted-interrupt wakeup event')
+ text = ""
+ for idx in range(int(info.type.sizeof / info_type.sizeof)):
+ show = bitmap[int(idx / bitsperlong)]
+ if not show & 1 << int(idx % bitsperlong):
+ continue
+ pfx = info[idx]['symbol'].string()
+ desc = info[idx]['text'].string()
+ text += x86_show_irqstat(prec, pfx, idx, desc)
return text
@@ -166,23 +131,19 @@
if nr_ipi is None or ipi_desc is None or ipi_types is None:
return text
- if prec >= 4:
- sep = " "
- else:
- sep = ""
-
for ipi in range(nr_ipi):
- text += "%*s%u:%s" % (prec - 1, "IPI", ipi, sep)
+ text += "%*s%u: " % (prec - 1, "IPI", ipi)
desc = ipi_desc[ipi].cast(irq_desc_type.get_type().pointer())
if desc == 0:
continue
for cpu in cpus.each_online_cpu():
- text += "%10u" % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt'])
- text += " %s" % (ipi_types[ipi].string())
+ text += "%10u " % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt'])
+ text += "%s" % (ipi_types[ipi].string())
text += "\n"
return text
def aarch64_show_interrupts(prec):
+ # Does not work for ARM64 as "ipi_desc" is not available there
text = arm_common_show_interrupts(prec)
text += "%*s: %10lu\n" % (prec, "ERR", gdb.parse_and_eval("irq_err_count"))
return text
@@ -209,12 +170,19 @@
super(LxInterruptList, self).__init__("lx-interruptlist", gdb.COMMAND_DATA)
def invoke(self, arg, from_tty):
- nr_irqs = gdb.parse_and_eval("nr_irqs")
- prec = 3
- j = 1000
- while prec < 10 and j <= nr_irqs:
- prec += 1
- j *= 10
+ nr_irqs = gdb.parse_and_eval("total_nr_irqs")
+ constr = utils.gdb_eval_or_none('irq_proc_constraints')
+
+ if constr:
+ prec = int(constr['num_prec'])
+ chip_width = int(constr['chip_width'])
+ else:
+ prec = 4
+ j = 10000
+ while prec < 10 and j <= nr_irqs:
+ prec += 1
+ j *= 10
+ chip_width = 8
gdb.write("%*s" % (prec + 8, ""))
for cpu in cpus.each_online_cpu():
@@ -225,7 +193,7 @@
raise gdb.GdbError("Unable to find the sparse IRQ tree, is CONFIG_SPARSE_IRQ enabled?")
for irq in range(nr_irqs):
- gdb.write(show_irq_desc(prec, irq))
+ gdb.write(show_irq_desc(prec, chip_width, irq))
gdb.write(arch_show_interrupts(prec))
diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py
index d78908f..dffadcc 100644
--- a/scripts/gdb/linux/mm.py
+++ b/scripts/gdb/linux/mm.py
@@ -40,11 +40,11 @@
self.PAGE_OFFSET = int(gdb.parse_and_eval("page_offset_base"))
self.VMEMMAP_START = int(gdb.parse_and_eval("vmemmap_base"))
- self.PHYS_BASE = int(gdb.parse_and_eval("phys_base"))
+ self.PHYS_BASE = int(gdb.parse_and_eval("(unsigned long) phys_base"))
self.START_KERNEL_map = 0xffffffff80000000
- self.KERNEL_START = gdb.parse_and_eval("_text")
- self.KERNEL_END = gdb.parse_and_eval("_end")
+ self.KERNEL_START = gdb.parse_and_eval("(unsigned long) &_text")
+ self.KERNEL_END = gdb.parse_and_eval("(unsigned long) &_end")
self.VMALLOC_START = int(gdb.parse_and_eval("vmalloc_base"))
if self.VMALLOC_START == 0xffffc90000000000:
diff --git a/scripts/gdb/linux/slab.py b/scripts/gdb/linux/slab.py
index 0e2d938..ddde25a 100644
--- a/scripts/gdb/linux/slab.py
+++ b/scripts/gdb/linux/slab.py
@@ -196,7 +196,7 @@
if target_cache['flags'] & SLAB_STORE_USER:
for i in range(0, nr_node_ids):
- cache_node = target_cache['node'][i]
+ cache_node = target_cache['per_node']['node'][i]
if cache_node['nr_slabs']['counter'] == 0:
continue
process_slab(loc_track, cache_node['partial'], alloc, target_cache)
@@ -300,7 +300,7 @@
nr_free = 0
nr_slabs = 0
for i in range(0, nr_node_ids):
- cache_node = cache['node'][i]
+ cache_node = cache['per_node']['node'][i]
try:
nr_slabs += cache_node['nr_slabs']['counter']
nr_objs = int(cache_node['total_objects']['counter'])
diff --git a/scripts/gdb/linux/timerlist.py b/scripts/gdb/linux/timerlist.py
index 9fb3436..744b032 100644
--- a/scripts/gdb/linux/timerlist.py
+++ b/scripts/gdb/linux/timerlist.py
@@ -90,14 +90,10 @@
text += f" .{'nohz':15s}: {int(bool(ts['flags'] & TS_FLAG_NOHZ))}\n"
text += f" .{'last_tick':15s}: {ts['last_tick']}\n"
text += f" .{'tick_stopped':15s}: {int(bool(ts['flags'] & TS_FLAG_STOPPED))}\n"
- text += f" .{'idle_jiffies':15s}: {ts['idle_jiffies']}\n"
text += f" .{'idle_calls':15s}: {ts['idle_calls']}\n"
text += f" .{'idle_sleeps':15s}: {ts['idle_sleeps']}\n"
text += f" .{'idle_entrytime':15s}: {ts['idle_entrytime']} nsecs\n"
text += f" .{'idle_waketime':15s}: {ts['idle_waketime']} nsecs\n"
- text += f" .{'idle_exittime':15s}: {ts['idle_exittime']} nsecs\n"
- text += f" .{'idle_sleeptime':15s}: {ts['idle_sleeptime']} nsecs\n"
- text += f" .{'iowait_sleeptime':15s}: {ts['iowait_sleeptime']} nsecs\n"
text += f" .{'last_jiffies':15s}: {ts['last_jiffies']}\n"
text += f" .{'next_timer':15s}: {ts['next_timer']}\n"
text += f" .{'idle_expires':15s}: {ts['idle_expires']} nsecs\n"
diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build
index 0ad7e66..c4a7acf 100755
--- a/scripts/livepatch/klp-build
+++ b/scripts/livepatch/klp-build
@@ -3,7 +3,7 @@
#
# Build a livepatch module
-# shellcheck disable=SC1090,SC2155
+# shellcheck disable=SC1090,SC2155,SC2164
if (( BASH_VERSINFO[0] < 4 || \
(BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 4) )); then
@@ -11,21 +11,19 @@
exit 1
fi
-set -o errexit
set -o errtrace
set -o pipefail
set -o nounset
# Allow doing 'cmd | mapfile -t array' instead of 'mapfile -t array < <(cmd)'.
-# This helps keep execution in pipes so pipefail+errexit can catch errors.
+# This helps keep execution in pipes so pipefail+ERR trap can catch errors.
shopt -s lastpipe
-unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP XTRACE
+unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP VERBOSE XTRACE
REPLACE=1
SHORT_CIRCUIT=0
JOBS="$(getconf _NPROCESSORS_ONLN)"
-VERBOSE="-s"
shopt -o xtrace | grep -q 'on' && XTRACE=1
# Avoid removing the previous $TMP_DIR until args have been fully processed.
@@ -35,16 +33,16 @@
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FIX_PATCH_LINES="$SCRIPT_DIR/fix-patch-lines"
-SRC="$(pwd)"
-OBJ="$(pwd)"
+OBJTOOL="$PWD/tools/objtool/objtool"
+CONFIG="$PWD/.config"
+TMP_DIR="$PWD/klp-tmp"
-CONFIG="$OBJ/.config"
-TMP_DIR="$OBJ/klp-tmp"
-
-ORIG_DIR="$TMP_DIR/orig"
-PATCHED_DIR="$TMP_DIR/patched"
-DIFF_DIR="$TMP_DIR/diff"
-KMOD_DIR="$TMP_DIR/kmod"
+ORIG_DIR="$TMP_DIR/1-orig"
+PATCHED_DIR="$TMP_DIR/2-patched"
+ORIG_CSUM_DIR="$TMP_DIR/3-checksum-orig"
+PATCHED_CSUM_DIR="$TMP_DIR/3-checksum-patched"
+DIFF_DIR="$TMP_DIR/4-diff"
+KMOD_DIR="$TMP_DIR/5-kmod"
STASH_DIR="$TMP_DIR/stash"
TIMESTAMP="$TMP_DIR/timestamp"
@@ -90,7 +88,7 @@
stash_file() {
local file="$1"
- local rel_file="${file#"$SRC"/}"
+ local rel_file="${file#"$PWD"/}"
[[ ! -e "$file" ]] && die "no file to stash: $file"
@@ -104,7 +102,7 @@
local file
for file in "${STASHED_FILES[@]}"; do
- mv -f "$STASH_DIR/$file" "$SRC/$file" || warn "can't restore file: $file"
+ mv -f "$STASH_DIR/$file" "$PWD/$file" || warn "can't restore file: $file"
done
STASHED_FILES=()
@@ -140,10 +138,11 @@
Advanced Options:
-d, --debug Show symbol/reloc cloning decisions
-S, --short-circuit=STEP Start at build step (requires prior --keep-tmp)
- 1|orig Build original kernel (default)
- 2|patched Build patched kernel
- 3|diff Diff objects
- 4|kmod Build patch module
+ 1|orig Build original kernel (default)
+ 2|patched Build patched kernel
+ 3|checksum Generate checksums
+ 4|diff Diff objects
+ 5|kmod Build patch module
-T, --keep-tmp Preserve tmp dir on exit
EOF
@@ -158,6 +157,7 @@
local short
local long
local args
+ local patch
short="hfj:o:vdS:T"
long="help,show-first-changed,jobs:,output:,no-replace,verbose,debug,short-circuit:,keep-tmp"
@@ -194,7 +194,7 @@
shift
;;
-v | --verbose)
- VERBOSE="V=1"
+ VERBOSE=1
shift
;;
-d | --debug)
@@ -206,10 +206,11 @@
[[ ! -d "$TMP_DIR" ]] && die "--short-circuit requires preserved klp-tmp dir"
keep_tmp=1
case "$2" in
- 1 | orig) SHORT_CIRCUIT=1; ;;
- 2 | patched) SHORT_CIRCUIT=2; ;;
- 3 | diff) SHORT_CIRCUIT=3; ;;
- 4 | mod) SHORT_CIRCUIT=4; ;;
+ 1 | orig) SHORT_CIRCUIT=1; ;;
+ 2 | patched) SHORT_CIRCUIT=2; ;;
+ 3 | checksum) SHORT_CIRCUIT=3; ;;
+ 4 | diff) SHORT_CIRCUIT=4; ;;
+ 5 | kmod) SHORT_CIRCUIT=5; ;;
*) die "invalid short-circuit step '$2'" ;;
esac
shift 2
@@ -236,6 +237,10 @@
KEEP_TMP="$keep_tmp"
PATCHES=("$@")
+
+ for patch in "${PATCHES[@]}"; do
+ [[ -f "$patch" ]] || die "$patch doesn't exist"
+ done
}
# temporarily disable xtrace for especially verbose code
@@ -270,6 +275,9 @@
[[ "$CONFIG_AS_VERSION" -lt 200000 ]] && \
die "Clang assembler version < 20 not supported"
+ [[ -x "$OBJTOOL" ]] && "$OBJTOOL" klp 2>&1 | command grep -q "not implemented" && \
+ die "objtool not built with KLP support; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile"
+
return 0
}
@@ -301,12 +309,17 @@
# Hardcode the value printed by the localversion script to prevent patch
# application from appending it with '+' due to a dirty working tree.
set_kernelversion() {
- local file="$SRC/scripts/setlocalversion"
+ local file="$PWD/scripts/setlocalversion"
local kernelrelease
stash_file "$file"
- kernelrelease="$(cd "$SRC" && make syncconfig &>/dev/null && make -s kernelrelease)"
+ if [[ -n "$(make -s listnewconfig 2>/dev/null)" ]]; then
+ die ".config mismatch, check your .config or run 'make olddefconfig'"
+ fi
+ make syncconfig &>/dev/null || die "make syncconfig failed"
+
+ kernelrelease="$(make -s kernelrelease)"
[[ -z "$kernelrelease" ]] && die "failed to get kernel version"
sed -i "2i echo $kernelrelease; exit 0" scripts/setlocalversion
@@ -349,7 +362,7 @@
for file in "${files[@]}"; do
case "$file" in
- lib/*|*.S)
+ lib/*|*/vdso/*|*/realmode/rm/*|*.S)
die "${patch}: unsupported patch to $file"
;;
esac
@@ -367,24 +380,24 @@
[[ ! -f "$patch" ]] && die "$patch doesn't exist"
status=0
- output=$(patch -d "$SRC" -p1 --dry-run --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" < "$patch" 2>&1) || status=$?
+ output=$(patch -p1 --dry-run --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" < "$patch" 2>&1) || status=$?
if [[ "$status" -ne 0 ]]; then
echo "$output" >&2
die "$patch did not apply"
elif [[ "$output" =~ $drift_regex ]]; then
- echo "$output" >&2
+ [[ -v VERBOSE ]] && echo "$output" >&2
warn "${patch} applied with fuzz"
fi
- patch -d "$SRC" -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch"
APPLIED_PATCHES+=("$patch")
+ patch -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch"
}
revert_patch() {
local patch="$1"
local tmp=()
- patch -d "$SRC" -p1 -R --silent --no-backup-if-mismatch -r /dev/null < "$patch"
+ patch -p1 -R --force --no-backup-if-mismatch -r /dev/null &> /dev/null < "$patch" || true
for p in "${APPLIED_PATCHES[@]}"; do
[[ "$p" == "$patch" ]] && continue
@@ -422,8 +435,21 @@
do_init() {
# We're not yet smart enough to handle anything other than in-tree
# builds in pwd.
- [[ ! "$SRC" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
- [[ ! "$OBJ" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
+ [[ ! "$PWD" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
+
+ if (( SHORT_CIRCUIT >= 2 )); then
+ [[ -f "$ORIG_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_DIR"
+ fi
+ if (( SHORT_CIRCUIT >= 3 )); then
+ [[ -f "$PATCHED_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_DIR"
+ fi
+ if (( SHORT_CIRCUIT >= 4 )); then
+ [[ -f "$ORIG_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_CSUM_DIR"
+ [[ -f "$PATCHED_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_CSUM_DIR"
+ fi
+ if (( SHORT_CIRCUIT >= 5 )); then
+ [[ -f "$DIFF_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $DIFF_DIR"
+ fi
(( SHORT_CIRCUIT <= 1 )) && rm -rf "$TMP_DIR"
mkdir -p "$TMP_DIR"
@@ -454,11 +480,11 @@
get_patch_output_files "$patch" | mapfile -t output_files
# Copy orig source files to 'a'
- ( cd "$SRC" && echo "${input_files[@]}" | xargs cp --parents --target-directory="$tmpdir/a" )
+ echo "${input_files[@]}" | xargs cp --parents --target-directory="$tmpdir/a"
# Copy patched source files to 'b'
apply_patch "$patch" "--silent"
- ( cd "$SRC" && echo "${output_files[@]}" | xargs cp --parents --target-directory="$tmpdir/b" )
+ echo "${output_files[@]}" | xargs cp --parents --target-directory="$tmpdir/b"
revert_patch "$patch"
# Diff 'a' and 'b' to make a clean patch
@@ -502,20 +528,14 @@
cmd+=("-j$JOBS")
cmd+=("clean")
- (
- cd "$SRC"
- "${cmd[@]}"
- )
+ "${cmd[@]}"
}
build_kernel() {
local build="$1"
local log="$TMP_DIR/build.log"
- local objtool_args=()
local cmd=()
- objtool_args=("--checksum")
-
cmd=("make")
# When a patch to a kernel module references a newly created unexported
@@ -535,19 +555,20 @@
#
cmd+=("KBUILD_MODPOST_WARN=1")
- cmd+=("$VERBOSE")
+ if [[ -v VERBOSE ]]; then
+ cmd+=("V=1")
+ else
+ cmd+=("-s")
+ fi
cmd+=("-j$JOBS")
cmd+=("KCFLAGS=-ffunction-sections -fdata-sections")
- cmd+=("OBJTOOL_ARGS=${objtool_args[*]}")
cmd+=("vmlinux")
cmd+=("modules")
- (
- cd "$SRC"
- "${cmd[@]}" \
- 1> >(tee -a "$log") \
- 2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2)
- ) || die "$build kernel build failed"
+ "${cmd[@]}" \
+ 1> >(tee -a "$log") \
+ 2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2) \
+ || die "$build kernel build failed"
}
find_objects() {
@@ -555,9 +576,9 @@
# Find root-level vmlinux.o and non-root-level .ko files,
# excluding klp-tmp/ and .git/
- find "$OBJ" \( -path "$TMP_DIR" -o -path "$OBJ/.git" -o -regex "$OBJ/[^/][^/]*\.ko" \) -prune -o \
+ find "$PWD" \( -path "$TMP_DIR" -o -path "$PWD/.git" -o -regex "$PWD/[^/][^/]*\.ko" \) -prune -o \
-type f "${opts[@]}" \
- \( -name "*.ko" -o -path "$OBJ/vmlinux.o" \) \
+ \( -name "*.ko" -o -path "$PWD/vmlinux.o" \) \
-printf '%P\n'
}
@@ -570,10 +591,10 @@
find_objects | mapfile -t files
- xtrace_save "copying orig objects"
+ xtrace_save "copying original objects"
for _file in "${files[@]}"; do
local rel_file="${_file/.ko/.o}"
- local file="$OBJ/$rel_file"
+ local file="$PWD/$rel_file"
local orig_file="$ORIG_DIR/$rel_file"
local orig_dir="$(dirname "$orig_file")"
@@ -586,6 +607,7 @@
mv -f "$TMP_DIR/build.log" "$ORIG_DIR"
touch "$TIMESTAMP"
+ touch "$ORIG_DIR/.complete"
}
# Copy all changed objects to $PATCHED_DIR
@@ -606,7 +628,7 @@
xtrace_save "copying changed objects"
for _file in "${files[@]}"; do
local rel_file="${_file/.ko/.o}"
- local file="$OBJ/$rel_file"
+ local file="$PWD/$rel_file"
local orig_file="$ORIG_DIR/$rel_file"
local patched_file="$PATCHED_DIR/$rel_file"
local patched_dir="$(dirname "$patched_file")"
@@ -624,6 +646,36 @@
(( found == 0 )) && die "no changes detected"
mv -f "$TMP_DIR/build.log" "$PATCHED_DIR"
+ touch "$PATCHED_DIR/.complete"
+}
+
+# Copy .o files to a separate directory and run "objtool klp checksum" on each
+# copy. The checksums are written to a .discard.sym_checksum section.
+#
+# If match_dir is given, only process files which also exist there.
+generate_checksums() {
+ local src_dir="$1"
+ local dest_dir="$2"
+ local match_dir="${3:-}"
+ local files=()
+ local file
+
+ rm -rf "$dest_dir"
+ mkdir -p "$dest_dir"
+
+ find "$src_dir" -type f -name "*.o" | mapfile -t files
+ for file in "${files[@]}"; do
+ local rel="${file#"$src_dir"/}"
+ local dest="$dest_dir/$rel"
+
+ [[ -n "$match_dir" && ! -f "$match_dir/$rel" ]] && continue
+
+ mkdir -p "$(dirname "$dest")"
+ cp -f "$file" "$dest"
+ "$OBJTOOL" klp checksum "$dest"
+ done
+
+ touch "$dest_dir/.complete"
}
# Diff changed objects, writing output object to $DIFF_DIR
@@ -635,23 +687,23 @@
rm -rf "$DIFF_DIR"
mkdir -p "$DIFF_DIR"
- find "$PATCHED_DIR" -type f -name "*.o" | mapfile -t files
+ find "$PATCHED_CSUM_DIR" -type f -name "*.o" | mapfile -t files
[[ ${#files[@]} -eq 0 ]] && die "no changes detected"
[[ -v DEBUG_CLONE ]] && opts=("--debug")
# Diff all changed objects
for file in "${files[@]}"; do
- local rel_file="${file#"$PATCHED_DIR"/}"
+ local rel_file="${file#"$PATCHED_CSUM_DIR"/}"
local orig_file="$rel_file"
- local patched_file="$PATCHED_DIR/$rel_file"
+ local patched_file="$PATCHED_CSUM_DIR/$rel_file"
local out_file="$DIFF_DIR/$rel_file"
local filter=()
local cmd=()
mkdir -p "$(dirname "$out_file")"
- cmd=("$SRC/tools/objtool/objtool")
+ cmd=("$OBJTOOL")
cmd+=("klp")
cmd+=("diff")
(( ${#opts[@]} > 0 )) && cmd+=("${opts[@]}")
@@ -668,18 +720,21 @@
fi
(
- cd "$ORIG_DIR"
+ cd "$ORIG_CSUM_DIR"
+ [[ -v VERBOSE ]] && echo "cd $ORIG_CSUM_DIR && ${cmd[*]}"
"${cmd[@]}" \
1> >(tee -a "$log") \
2> >(tee -a "$log" | "${filter[@]}" >&2) || \
die "objtool klp diff failed"
)
done
+
+ touch "$DIFF_DIR/.complete"
}
-# For each changed object, run objtool with --debug-checksum to get the
-# per-instruction checksums, and then diff those to find the first changed
-# instruction for each function.
+# For each changed object, run "objtool klp checksum" with --debug-checksum to
+# get the per-instruction checksums, and then diff those to find the first
+# changed instruction for each function.
diff_checksums() {
local orig_log="$ORIG_DIR/checksum.log"
local patched_log="$PATCHED_DIR/checksum.log"
@@ -703,9 +758,8 @@
fi
done
- cmd=("$SRC/tools/objtool/objtool")
- cmd+=("--checksum")
- cmd+=("--link")
+ cmd=("$OBJTOOL")
+ cmd+=("klp" "checksum")
cmd+=("--dry-run")
for file in "${!funcs[@]}"; do
@@ -714,21 +768,37 @@
(
cd "$ORIG_DIR"
"${cmd[@]}" "$opt" "$file" &> "$orig_log" || \
- ( cat "$orig_log" >&2; die "objtool --debug-checksum failed" )
+ ( cat "$orig_log" >&2; die "objtool klp checksum failed" )
cd "$PATCHED_DIR"
"${cmd[@]}" "$opt" "$file" &> "$patched_log" || \
- ( cat "$patched_log" >&2; die "objtool --debug-checksum failed" )
+ ( cat "$patched_log" >&2; die "objtool klp checksum failed" )
)
for func in ${funcs[$file]}; do
- diff <( grep0 -E "^DEBUG: .*checksum: $func " "$orig_log" | sed "s|$ORIG_DIR/||") \
- <( grep0 -E "^DEBUG: .*checksum: $func " "$patched_log" | sed "s|$PATCHED_DIR/||") \
- | gawk '/^< DEBUG: / {
- gsub(/:/, "")
- printf "%s: %s: %s\n", $3, $5, $6
- exit
- }' || true
+ local -a orig patched
+ paste <(grep0 -E "^DEBUG: .*checksum: $func " "$orig_log") \
+ <(grep0 -E "^DEBUG: .*checksum: $func " "$patched_log") |
+ while IFS= read -r line; do
+ read -ra orig <<< "${line%%$'\t'*}"
+ read -ra patched <<< "${line#*$'\t'}"
+
+ if [[ ${#patched[@]} -eq 0 ]]; then
+ printf "%s: %s: %s (removed)\n" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}"
+ break
+ elif [[ ${#orig[@]} -eq 0 ]]; then
+ printf "%s: %s: %s (added)\n" "${patched[1]%:}" "${patched[3]}" "${patched[-2]}"
+ break
+ fi
+
+ [[ "${orig[-1]}" == "${patched[-1]}" ]] && continue
+
+ printf "%s: %s: %s" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}"
+ [[ "${orig[-2]}" != "${patched[-2]}" ]] && \
+ printf " (patched: %s)" "${patched[-2]}"
+ printf "\n"
+ break
+ done || true
done
done
}
@@ -745,7 +815,7 @@
rm -rf "$KMOD_DIR"
mkdir -p "$KMOD_DIR"
- cp -f "$SRC/scripts/livepatch/init.c" "$KMOD_DIR"
+ cp -f "$SCRIPT_DIR/init.c" "$KMOD_DIR"
echo "obj-m := $NAME.o" > "$makefile"
echo -n "$NAME-y := init.o" >> "$makefile"
@@ -780,19 +850,20 @@
[[ $REPLACE -eq 0 ]] && cflags+=("-DKLP_NO_REPLACE")
cmd=("make")
- cmd+=("$VERBOSE")
+ if [[ -v VERBOSE ]]; then
+ cmd+=("V=1")
+ else
+ cmd+=("-s")
+ fi
cmd+=("-j$JOBS")
cmd+=("--directory=.")
cmd+=("M=$KMOD_DIR")
cmd+=("KCFLAGS=${cflags[*]}")
# Build a "normal" kernel module with init.c and the diffed objects
- (
- cd "$SRC"
- "${cmd[@]}" \
- 1> >(tee -a "$log") \
- 2> >(tee -a "$log" >&2)
- )
+ "${cmd[@]}" \
+ 1> >(tee -a "$log") \
+ 2> >(tee -a "$log" >&2)
kmod_file="$KMOD_DIR/$NAME.ko"
@@ -803,7 +874,7 @@
objcopy --remove-section=.BTF "$kmod_file"
# Fix (and work around) linker wreckage for klp syms / relocs
- "$SRC/tools/objtool/objtool" klp post-link "$kmod_file" || die "objtool klp post-link failed"
+ "$OBJTOOL" klp post-link "$kmod_file" || die "objtool klp post-link failed"
cp -f "$kmod_file" "$OUTFILE"
}
@@ -839,6 +910,13 @@
fi
if (( SHORT_CIRCUIT <= 3 )); then
+ status "Generating original checksums"
+ generate_checksums "$ORIG_DIR" "$ORIG_CSUM_DIR" "$PATCHED_DIR"
+ status "Generating patched checksums"
+ generate_checksums "$PATCHED_DIR" "$PATCHED_CSUM_DIR"
+fi
+
+if (( SHORT_CIRCUIT <= 4 )); then
status "Diffing objects"
diff_objects
if [[ -v DIFF_CHECKSUM ]]; then
@@ -847,7 +925,7 @@
fi
fi
-if (( SHORT_CIRCUIT <= 4 )); then
+if (( SHORT_CIRCUIT <= 5 )); then
status "Building patch module: $OUTFILE"
build_patch_module
fi
diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
index 4e99393..2ad87a7 100644
--- a/scripts/mod/file2alias.c
+++ b/scripts/mod/file2alias.c
@@ -651,7 +651,26 @@ static void do_vio_entry(struct module *mod, void *symval)
module_alias_printf(mod, true, "%s", alias);
}
-static void do_input(char *alias,
+static void __attribute__((format(printf, 3, 4)))
+alias_append(char *alias, size_t size, const char *fmt, ...)
+{
+ size_t len = strlen(alias);
+ va_list args;
+ int n;
+
+ if (len >= size)
+ fatal("alias buffer (%zu) overflow before append\n", size);
+
+ va_start(args, fmt);
+ n = vsnprintf(alias + len, size - len, fmt, args);
+ va_end(args);
+
+ if (n < 0 || (size_t)n >= size - len)
+ fatal("alias buffer (%zu) overflow on append (need %d, have %zu)\n",
+ size, n, size - len);
+}
+
+static void do_input(char *alias, size_t size,
kernel_ulong_t *arr, unsigned int min, unsigned int max)
{
unsigned int i;
@@ -659,13 +678,14 @@ static void do_input(char *alias,
for (i = min; i <= max; i++)
if (get_unaligned_native(arr + i / BITS_PER_LONG) &
(1ULL << (i % BITS_PER_LONG)))
- sprintf(alias + strlen(alias), "%X,*", i);
+ alias_append(alias, size, "%X,*", i);
}
/* input:b0v0p0e0-eXkXrXaXmXlXsXfXwX where X is comma-separated %02X. */
static void do_input_entry(struct module *mod, void *symval)
{
char alias[256] = {};
+ const size_t sizeof_alias = sizeof(alias);
DEF_FIELD(symval, input_device_id, flags);
DEF_FIELD(symval, input_device_id, bustype);
@@ -687,35 +707,35 @@ static void do_input_entry(struct module *mod, void *symval)
ADD(alias, "p", flags & INPUT_DEVICE_ID_MATCH_PRODUCT, product);
ADD(alias, "e", flags & INPUT_DEVICE_ID_MATCH_VERSION, version);
- sprintf(alias + strlen(alias), "-e*");
+ alias_append(alias, sizeof_alias, "-e*");
if (flags & INPUT_DEVICE_ID_MATCH_EVBIT)
- do_input(alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX);
- sprintf(alias + strlen(alias), "k*");
+ do_input(alias, sizeof_alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX);
+ alias_append(alias, sizeof_alias, "k*");
if (flags & INPUT_DEVICE_ID_MATCH_KEYBIT)
- do_input(alias, *keybit,
+ do_input(alias, sizeof_alias, *keybit,
INPUT_DEVICE_ID_KEY_MIN_INTERESTING,
INPUT_DEVICE_ID_KEY_MAX);
- sprintf(alias + strlen(alias), "r*");
+ alias_append(alias, sizeof_alias, "r*");
if (flags & INPUT_DEVICE_ID_MATCH_RELBIT)
- do_input(alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX);
- sprintf(alias + strlen(alias), "a*");
+ do_input(alias, sizeof_alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX);
+ alias_append(alias, sizeof_alias, "a*");
if (flags & INPUT_DEVICE_ID_MATCH_ABSBIT)
- do_input(alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX);
- sprintf(alias + strlen(alias), "m*");
+ do_input(alias, sizeof_alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX);
+ alias_append(alias, sizeof_alias, "m*");
if (flags & INPUT_DEVICE_ID_MATCH_MSCIT)
- do_input(alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX);
- sprintf(alias + strlen(alias), "l*");
+ do_input(alias, sizeof_alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX);
+ alias_append(alias, sizeof_alias, "l*");
if (flags & INPUT_DEVICE_ID_MATCH_LEDBIT)
- do_input(alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX);
- sprintf(alias + strlen(alias), "s*");
+ do_input(alias, sizeof_alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX);
+ alias_append(alias, sizeof_alias, "s*");
if (flags & INPUT_DEVICE_ID_MATCH_SNDBIT)
- do_input(alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX);
- sprintf(alias + strlen(alias), "f*");
+ do_input(alias, sizeof_alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX);
+ alias_append(alias, sizeof_alias, "f*");
if (flags & INPUT_DEVICE_ID_MATCH_FFBIT)
- do_input(alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX);
- sprintf(alias + strlen(alias), "w*");
+ do_input(alias, sizeof_alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX);
+ alias_append(alias, sizeof_alias, "w*");
if (flags & INPUT_DEVICE_ID_MATCH_SWBIT)
- do_input(alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX);
+ do_input(alias, sizeof_alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX);
module_alias_printf(mod, false, "input:%s", alias);
}
@@ -895,12 +915,16 @@ static const struct dmifield {
{ NULL, DMI_NONE }
};
-static void dmi_ascii_filter(char *d, const char *s)
+static void dmi_ascii_filter(char *d, size_t avail, const char *s)
{
/* Filter out characters we don't want to see in the modalias string */
for (; *s; s++)
- if (*s > ' ' && *s < 127 && *s != ':')
+ if (*s > ' ' && *s < 127 && *s != ':') {
+ if (avail <= 1)
+ fatal("%s: alias buffer overflow\n", __func__);
*(d++) = *s;
+ avail--;
+ }
*d = 0;
}
@@ -909,6 +933,8 @@ static void dmi_ascii_filter(char *d, const char *s)
static void do_dmi_entry(struct module *mod, void *symval)
{
char alias[256] = {};
+ const size_t sizeof_alias = sizeof(alias);
+ size_t len;
int i, j;
DEF_FIELD_ADDR(symval, dmi_system_id, matches);
@@ -916,11 +942,12 @@ static void do_dmi_entry(struct module *mod, void *symval)
for (j = 0; j < 4; j++) {
if ((*matches)[j].slot &&
(*matches)[j].slot == dmi_fields[i].field) {
- sprintf(alias + strlen(alias), ":%s*",
- dmi_fields[i].prefix);
- dmi_ascii_filter(alias + strlen(alias),
+ alias_append(alias, sizeof_alias, ":%s*",
+ dmi_fields[i].prefix);
+ len = strlen(alias);
+ dmi_ascii_filter(alias + len, sizeof_alias - len,
(*matches)[j].substr);
- strcat(alias, "*");
+ alias_append(alias, sizeof_alias, "*");
}
}
}
diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD
index 452374d..1213c8e0 100644
--- a/scripts/package/PKGBUILD
+++ b/scripts/package/PKGBUILD
@@ -10,7 +10,7 @@
pkgname+=("${pkgbase}-${pkg}")
done
-pkgver="${KERNELRELEASE//-/_}"
+pkgver="$(echo "${KERNELRELEASE}" | sed 's/-\(rc[0-9]\+\)/\1/;s/-/_/g')"
# The PKGBUILD is evaluated multiple times.
# Running scripts/build-version from here would introduce inconsistencies.
pkgrel="${KBUILD_REVISION}"
diff --git a/scripts/timer_migration_tree.py b/scripts/timer_migration_tree.py
new file mode 100755
index 0000000..faac9de
--- /dev/null
+++ b/scripts/timer_migration_tree.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""
+Draw the timer migration tree.
+
+1) Boot with trace_event==tmigr_connect_cpu_parent,tmigr_connect_child_parent
+2) ./timer_migration_tree.py < /sys/kernel/tracing/trace
+"""
+
+import re, sys
+from ete3 import Tree
+
+class Node:
+ def __init__(self, group):
+ self.group = group
+ self.children = []
+ self.parent = None
+ self.num_children = 0
+ self.groupmask = 0
+ self.lvl = -1
+
+ def set_groupmask(self, groupmask):
+ self.groupmask = groupmask
+
+ def set_parent(self, parent):
+ self.parent = parent
+
+ def add_child(self, child):
+ self.children.append(child)
+
+ def set_lvl(self, lvl):
+ self.lvl = lvl
+
+ def set_numa(self, numa):
+ self.numa = numa
+
+ def set_num_children(self, num_children):
+ self.num_children = num_children
+
+ def __repr__(self):
+ if self.parent:
+ parent_grp = self.parent.group
+ else:
+ parent_grp = "-"
+ return "Group: %s mask: %s parent: %s lvl: %d numa: %d num_children: %d" % (self.group, self.groupmask, parent_grp, self.lvl, self.numa, self.num_children)
+
+hierarchies = { }
+
+def get_hierarchy(capacity):
+ if capacity not in hierarchies:
+ hierarchies[capacity] = {}
+ return hierarchies[capacity]
+
+def get_node(capacity, group):
+ hier = get_hierarchy(capacity)
+ if group in hier:
+ return hier[group]
+ else:
+ n = Node(group)
+ hier[group] = n
+ return n
+
+def tmigr_connect_cpu_parent(ts, line):
+ s = re.search("tmigr_connect_cpu_parent: cpu=([0-9]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line)
+ if s is None:
+ return False
+ (cpu, groupmask, parent, lvl, numa, capacity, num_children) = (int(s.group(1)), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7)))
+ n = get_node(capacity, cpu)
+ p = get_node(capacity, parent)
+ n.set_parent(p)
+ n.set_groupmask(groupmask)
+ n.set_lvl(-1)
+ p.set_lvl(lvl)
+ p.set_numa(numa)
+ n.set_numa(numa)
+ p.set_num_children(num_children)
+ p.add_child(n)
+
+def tmigr_connect_child_parent(ts, line):
+ s = re.search("tmigr_connect_child_parent: group=([0-9a-zA-Z]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line)
+ if s is None:
+ return False
+ (group, groupmask, parent, lvl, numa, capacity, num_children) = (s.group(1), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7)))
+ n = get_node(capacity, group)
+ p = get_node(capacity, parent)
+ n.set_parent(p)
+ n.set_groupmask(groupmask)
+ p.set_lvl(lvl)
+ p.set_numa(numa)
+ p.set_num_children(num_children)
+ p.add_child(n)
+
+def populate(enode, node):
+ enode = enode.add_child(name = node.group)
+ enode.add_feature("groupmask", "m:%s" % node.groupmask)
+ enode.add_feature("lvl", "lvl:%d" % node.lvl)
+ enode.add_feature("numa", "node %d" % node.numa)
+ enode.add_feature("num_children", "c=%d" % node.num_children)
+ for child in node.children:
+ populate(enode, child)
+
+if __name__ == "__main__":
+ for line in sys.stdin:
+ s = re.search("([0-9]+[.][0-9]{6}): (.+?)$", line, re.S)
+ if s is not None:
+ if tmigr_connect_cpu_parent(float(s.group(1)), s.group(2)):
+ continue
+ if tmigr_connect_child_parent(float(s.group(1)), s.group(2)):
+ continue
+
+ for cap in hierarchies:
+ h = hierarchies[cap]
+ print("Tree for capacity %d" % cap)
+ for k in h:
+ n = h[k]
+ while n.parent != None:
+ n = n.parent
+ root = Tree()
+ populate(root, n)
+ print(root.get_ascii(show_internal=True, attributes=["name", "numa", "lvl"]))
+ break
diff --git a/scripts/update-intel-ucode-defs.py b/scripts/update-intel-ucode-defs.py
new file mode 100755
index 0000000..9d6cc2c
--- /dev/null
+++ b/scripts/update-intel-ucode-defs.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import re
+import shutil
+import subprocess
+import sys
+import os
+
+script = os.path.relpath(__file__)
+
+DESCRIPTION = f"""
+For Intel CPUs, update the microcode revisions that determine
+X86_BUG_OLD_MICROCODE.
+
+This script is intended to be run in response to releases of the
+official Intel microcode GitHub repository:
+https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files.git
+
+It takes the Intel microcode files as input and uses iucode-tool to
+extract the revision information. It prints the output in the format
+expected by intel-ucode-defs.h.
+
+Usage:
+ ./{script} /path/to/microcode/files > /path/to/intel-ucode-defs.h
+
+Typically, someone at Intel would see a new public release, wait for at
+least three months to ensure the update is stable, run this script to
+refresh the intel-ucode-defs.h file, and send a patch upstream to update
+the mainline and stable versions.
+
+Any exception to this process should be supported with an appropriate
+justification.
+"""
+
+SIG_RE = re.compile(r'sig (0x[0-9a-fA-F]+)')
+PFM_RE = re.compile(r'pf_mask (0x[0-9a-fA-F]+)')
+REV_RE = re.compile(r'rev (0x[0-9a-fA-F]+)')
+
+# Functions to extract family, model, and stepping
+def bits(val, bottom, top):
+ mask = (1 << (top + 1 - bottom)) - 1
+ return (val >> bottom) & mask
+
+def family(sig):
+ if bits(sig, 8, 11) == 0xf:
+ return bits(sig, 8, 11) + bits(sig, 20, 27)
+ return bits(sig, 8, 11)
+
+def model(sig):
+ return bits(sig, 4, 7) | (bits(sig, 16, 19) << 4)
+
+def step(sig):
+ return bits(sig, 0, 3)
+
+class Ucode:
+ def __init__(self, sig, pfm, rev):
+ self.family = family(sig)
+ self.model = model(sig)
+ self.steppings = 1 << step(sig)
+ self.platforms = pfm
+ self.rev = rev
+
+ self.key = (self.family, self.model, self.steppings, self.platforms)
+
+ def __eq__(self, other):
+ return self.key == other.key
+
+ def __hash__(self):
+ return hash(self.key)
+
+ def __str__(self):
+ return "{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x%x, .model = 0x%02x, .steppings = 0x%04x, .platform_mask = 0x%02x, .driver_data = 0x%x }," % \
+ (self.family, self.model, self.steppings, self.platforms, self.rev)
+
+def main():
+ parser = argparse.ArgumentParser(description=DESCRIPTION,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('ucode_files', nargs='+', help='Path(s) to the microcode files')
+
+ args = parser.parse_args()
+
+ # Process the microcode files using iucode-tool
+ iucode_tool = shutil.which("iucode-tool") or shutil.which("iucode_tool")
+ if iucode_tool is None:
+ print("Error: iucode-tool not found, please install it", file=sys.stderr)
+ sys.exit(1)
+
+ cmd = [iucode_tool, '--list-all'] + args.ucode_files
+
+ result = subprocess.run(cmd, capture_output=True, text=True)
+ if result.returncode != 0:
+ print("Error: iucode-tool ran into an error, exiting", file=sys.stderr)
+ if result.stderr:
+ print(result.stderr, file=sys.stderr, end='')
+ sys.exit(1)
+
+ ucodes = set()
+
+ # Parse the output of iucode-tool
+ for line in result.stdout.splitlines():
+ sig_match = SIG_RE.search(line)
+ pfm_match = PFM_RE.search(line)
+ rev_match = REV_RE.search(line)
+
+ if not (sig_match and pfm_match and rev_match):
+ continue
+
+ sig = int(sig_match.group(1), 16)
+ pfm = int(pfm_match.group(1), 16)
+ rev = int(rev_match.group(1), 16)
+ debug_rev = bits(rev, 31, 31)
+ if debug_rev != 0:
+ print("Error: Debug ucode file found, exiting", file=sys.stderr)
+ sys.exit(1)
+
+ ucodes.add(Ucode(sig, pfm, rev))
+
+ if not ucodes:
+ print("Error: No valid microcode files found, exiting", file=sys.stderr)
+ sys.exit(1)
+
+ # Sort and print the microcode entries
+ print("/* SPDX-License-Identifier: GPL-2.0 */")
+ print("/* Auto-generated by scripts/update-intel-ucode-defs.py */")
+ for u in sorted(ucodes, key=lambda x: x.key):
+ print(u)
+
+if __name__ == "__main__":
+ main()
diff --git a/security/keys/keyring.c b/security/keys/keyring.c
index b39038f..5a9887d 100644
--- a/security/keys/keyring.c
+++ b/security/keys/keyring.c
@@ -1109,6 +1109,7 @@ key_ref_t find_key_to_update(key_ref_t keyring_ref,
kenter("{%d},{%s,%s}",
keyring->serial, index_key->type->name, index_key->description);
+ guard(rcu)();
object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops,
index_key);
diff --git a/security/lsm_syscalls.c b/security/lsm_syscalls.c
index 5648b1f..08a0176 100644
--- a/security/lsm_syscalls.c
+++ b/security/lsm_syscalls.c
@@ -57,7 +57,14 @@ u64 lsm_name_to_attr(const char *name)
SYSCALL_DEFINE4(lsm_set_self_attr, unsigned int, attr, struct lsm_ctx __user *,
ctx, u32, size, u32, flags)
{
- return security_setselfattr(attr, ctx, size, flags);
+ int rc;
+
+ rc = mutex_lock_interruptible(¤t->signal->cred_guard_mutex);
+ if (rc < 0)
+ return rc;
+ rc = security_setselfattr(attr, ctx, size, flags);
+ mutex_unlock(¤t->signal->cred_guard_mutex);
+ return rc;
}
/**
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 9780196..0f70438 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -2966,7 +2966,7 @@ static int selinux_inode_init_security(struct inode *inode, struct inode *dir,
{
const struct cred_security_struct *crsec = selinux_cred(current_cred());
struct superblock_security_struct *sbsec;
- struct xattr *xattr = lsm_get_xattr_slot(xattrs, xattr_count);
+ struct xattr *xattr;
u32 newsid, clen;
u16 newsclass;
int rc;
@@ -2992,6 +2992,7 @@ static int selinux_inode_init_security(struct inode *inode, struct inode *dir,
!(sbsec->flags & SBLABEL_MNT))
return -EOPNOTSUPP;
+ xattr = lsm_get_xattr_slot(xattrs, xattr_count);
if (xattr) {
rc = security_sid_to_context_force(newsid,
&context, &clen);
@@ -3208,15 +3209,13 @@ static inline int task_avdcache_search(struct task_security_struct *tsec,
* @tsec: the task's security state
* @isec: the inode associated with the cache entry
* @avd: the AVD to cache
- * @audited: the permission audit bitmask to cache
*
- * Update the AVD cache in @tsec with the @avdc and @audited info associated
+ * Update the AVD cache in @tsec with the @avd info associated
* with @isec.
*/
static inline void task_avdcache_update(struct task_security_struct *tsec,
struct inode_security_struct *isec,
- struct av_decision *avd,
- u32 audited)
+ struct av_decision *avd)
{
int spot;
@@ -3228,9 +3227,7 @@ static inline void task_avdcache_update(struct task_security_struct *tsec,
spot = (tsec->avdcache.dir_spot + 1) & (TSEC_AVDC_DIR_SIZE - 1);
tsec->avdcache.dir_spot = spot;
tsec->avdcache.dir[spot].isid = isec->sid;
- tsec->avdcache.dir[spot].audited = audited;
- tsec->avdcache.dir[spot].allowed = avd->allowed;
- tsec->avdcache.dir[spot].permissive = avd->flags & AVD_FLAGS_PERMISSIVE;
+ tsec->avdcache.dir[spot].avd = *avd;
tsec->avdcache.permissive_neveraudit =
(avd->flags == (AVD_FLAGS_PERMISSIVE|AVD_FLAGS_NEVERAUDIT));
}
@@ -3251,6 +3248,7 @@ static int selinux_inode_permission(struct inode *inode, int requested)
struct task_security_struct *tsec;
struct inode_security_struct *isec;
struct avdc_entry *avdc;
+ struct av_decision avd, *avdp = &avd;
int rc, rc2;
u32 audited, denied;
@@ -3272,23 +3270,21 @@ static int selinux_inode_permission(struct inode *inode, int requested)
rc = task_avdcache_search(tsec, isec, &avdc);
if (likely(!rc)) {
/* Cache hit. */
- audited = perms & avdc->audited;
- denied = perms & ~avdc->allowed;
- if (unlikely(denied && enforcing_enabled() &&
- !avdc->permissive))
+ avdp = &avdc->avd;
+ denied = perms & ~avdp->allowed;
+ if (unlikely(denied) && enforcing_enabled() &&
+ !(avdp->flags & AVD_FLAGS_PERMISSIVE))
rc = -EACCES;
} else {
- struct av_decision avd;
-
/* Cache miss. */
rc = avc_has_perm_noaudit(sid, isec->sid, isec->sclass,
- perms, 0, &avd);
- audited = avc_audit_required(perms, &avd, rc,
- (requested & MAY_ACCESS) ? FILE__AUDIT_ACCESS : 0,
- &denied);
- task_avdcache_update(tsec, isec, &avd, audited);
+ perms, 0, avdp);
+ task_avdcache_update(tsec, isec, avdp);
}
+ audited = avc_audit_required(perms, avdp, rc,
+ (requested & MAY_ACCESS) ?
+ FILE__AUDIT_ACCESS : 0, &denied);
if (likely(!audited))
return rc;
@@ -4920,7 +4916,7 @@ static bool sock_skip_has_perm(u32 sid)
static int sock_has_perm(struct sock *sk, u32 perms)
{
- struct sk_security_struct *sksec = sk->sk_security;
+ struct sk_security_struct *sksec = selinux_sock(sk);
struct common_audit_data ad;
struct lsm_network_audit net;
@@ -6227,7 +6223,7 @@ static unsigned int selinux_ip_postroute(void *priv,
static int nlmsg_sock_has_extended_perms(struct sock *sk, u32 perms, u16 nlmsg_type)
{
- struct sk_security_struct *sksec = sk->sk_security;
+ struct sk_security_struct *sksec = selinux_sock(sk);
struct common_audit_data ad;
u8 driver;
u8 xperm;
diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h
index b19e5d9..3c0a16e 100644
--- a/security/selinux/include/objsec.h
+++ b/security/selinux/include/objsec.h
@@ -32,9 +32,7 @@
struct avdc_entry {
u32 isid; /* inode SID */
- u32 allowed; /* allowed permission bitmask */
- u32 audited; /* audited permission bitmask */
- bool permissive; /* AVC permissive flag */
+ struct av_decision avd; /* av decision */
};
struct cred_security_struct {
diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h
index d1f16d7..0babb89 100644
--- a/security/selinux/include/security.h
+++ b/security/selinux/include/security.h
@@ -312,8 +312,6 @@ int security_context_to_sid_default(const char *scontext, u32 scontext_len,
int security_context_to_sid_force(const char *scontext, u32 scontext_len,
u32 *sid);
-int security_get_user_sids(u32 fromsid, const char *username, u32 **sids, u32 *nel);
-
int security_port_sid(u8 protocol, u16 port, u32 *out_sid);
int security_ib_pkey_sid(u64 subnet_prefix, u16 pkey_num, u32 *out_sid);
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 83aa765..25ca7d7 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -76,7 +76,6 @@ struct selinux_fs_info {
int *bool_pending_values;
struct dentry *class_dir;
unsigned long last_class_ino;
- bool policy_opened;
unsigned long last_ino;
struct super_block *sb;
};
@@ -272,35 +271,13 @@ static ssize_t sel_write_disable(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
- char *page;
- ssize_t length;
- int new_value;
-
- if (count >= PAGE_SIZE)
- return -ENOMEM;
-
- /* No partial writes. */
- if (*ppos != 0)
- return -EINVAL;
-
- page = memdup_user_nul(buf, count);
- if (IS_ERR(page))
- return PTR_ERR(page);
-
- if (sscanf(page, "%d", &new_value) != 1) {
- length = -EINVAL;
- goto out;
- }
- length = count;
-
- if (new_value) {
- pr_err("SELinux: https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable\n");
- pr_err("SELinux: Runtime disable is not supported, use selinux=0 on the kernel cmdline.\n");
- }
-
-out:
- kfree(page);
- return length;
+ /*
+ * Setting disable is no longer supported, see
+ * https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable
+ */
+ pr_err_once("SELinux: %s (%d) wrote to disable. This is no longer supported.\n",
+ current->comm, current->pid);
+ return count;
}
static const struct file_operations sel_disable_ops = {
@@ -362,44 +339,31 @@ struct policy_load_memory {
static int sel_open_policy(struct inode *inode, struct file *filp)
{
- struct selinux_fs_info *fsi = inode->i_sb->s_fs_info;
struct policy_load_memory *plm = NULL;
int rc;
- BUG_ON(filp->private_data);
-
- mutex_lock(&selinux_state.policy_mutex);
-
rc = avc_has_perm(current_sid(), SECINITSID_SECURITY,
SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL);
if (rc)
- goto err;
+ return rc;
- rc = -EBUSY;
- if (fsi->policy_opened)
- goto err;
-
- rc = -ENOMEM;
plm = kzalloc_obj(*plm);
if (!plm)
- goto err;
+ return -ENOMEM;
+ mutex_lock(&selinux_state.policy_mutex);
rc = security_read_policy(&plm->data, &plm->len);
if (rc)
goto err;
-
if ((size_t)i_size_read(inode) != plm->len) {
inode_lock(inode);
i_size_write(inode, plm->len);
inode_unlock(inode);
}
-
- fsi->policy_opened = 1;
+ mutex_unlock(&selinux_state.policy_mutex);
filp->private_data = plm;
- mutex_unlock(&selinux_state.policy_mutex);
-
return 0;
err:
mutex_unlock(&selinux_state.policy_mutex);
@@ -412,13 +376,8 @@ static int sel_open_policy(struct inode *inode, struct file *filp)
static int sel_release_policy(struct inode *inode, struct file *filp)
{
- struct selinux_fs_info *fsi = inode->i_sb->s_fs_info;
struct policy_load_memory *plm = filp->private_data;
- BUG_ON(!plm);
-
- fsi->policy_opened = 0;
-
vfree(plm->data);
kfree(plm);
@@ -594,34 +553,31 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf,
if (!count)
return -EINVAL;
- mutex_lock(&selinux_state.policy_mutex);
-
length = avc_has_perm(current_sid(), SECINITSID_SECURITY,
SECCLASS_SECURITY, SECURITY__LOAD_POLICY, NULL);
if (length)
- goto out;
+ return length;
data = vmalloc(count);
- if (!data) {
- length = -ENOMEM;
- goto out;
- }
+ if (!data)
+ return -ENOMEM;
if (copy_from_user(data, buf, count) != 0) {
length = -EFAULT;
goto out;
}
+ mutex_lock(&selinux_state.policy_mutex);
length = security_load_policy(data, count, &load_state);
if (length) {
pr_warn_ratelimited("SELinux: failed to load policy\n");
- goto out;
+ goto out_unlock;
}
fsi = file_inode(file)->i_sb->s_fs_info;
length = sel_make_policy_nodes(fsi, load_state.policy);
if (length) {
pr_warn_ratelimited("SELinux: failed to initialize selinuxfs\n");
selinux_policy_cancel(&load_state);
- goto out;
+ goto out_unlock;
}
selinux_policy_commit(&load_state);
@@ -631,8 +587,9 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf,
from_kuid(&init_user_ns, audit_get_loginuid(current)),
audit_get_sessionid(current));
-out:
+out_unlock:
mutex_unlock(&selinux_state.policy_mutex);
+out:
vfree(data);
return length;
}
@@ -689,46 +646,13 @@ static ssize_t sel_read_checkreqprot(struct file *filp, char __user *buf,
static ssize_t sel_write_checkreqprot(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
- char *page;
- ssize_t length;
- unsigned int new_value;
-
- length = avc_has_perm(current_sid(), SECINITSID_SECURITY,
- SECCLASS_SECURITY, SECURITY__SETCHECKREQPROT,
- NULL);
- if (length)
- return length;
-
- if (count >= PAGE_SIZE)
- return -ENOMEM;
-
- /* No partial writes. */
- if (*ppos != 0)
- return -EINVAL;
-
- page = memdup_user_nul(buf, count);
- if (IS_ERR(page))
- return PTR_ERR(page);
-
- if (sscanf(page, "%u", &new_value) != 1) {
- length = -EINVAL;
- goto out;
- }
- length = count;
-
- if (new_value) {
- char comm[sizeof(current->comm)];
-
- strscpy(comm, current->comm);
- pr_err("SELinux: %s (%d) set checkreqprot to 1. This is no longer supported.\n",
- comm, current->pid);
- }
-
- selinux_ima_measure_state();
-
-out:
- kfree(page);
- return length;
+ /*
+ * Setting checkreqprot is no longer supported, see
+ * https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-checkreqprot
+ */
+ pr_err_once("SELinux: %s (%d) wrote to checkreqprot. This is no longer supported.\n",
+ current->comm, current->pid);
+ return count;
}
static const struct file_operations sel_checkreqprot_ops = {
.read = sel_read_checkreqprot,
@@ -1073,69 +997,11 @@ static ssize_t sel_write_relabel(struct file *file, char *buf, size_t size)
static ssize_t sel_write_user(struct file *file, char *buf, size_t size)
{
- char *con = NULL, *user = NULL, *ptr;
- u32 sid, *sids = NULL;
- ssize_t length;
- char *newcon;
- int rc;
- u32 i, len, nsids;
-
- pr_warn_ratelimited("SELinux: %s (%d) wrote to /sys/fs/selinux/user!"
- " This will not be supported in the future; please update your"
- " userspace.\n", current->comm, current->pid);
- ssleep(5);
-
- length = avc_has_perm(current_sid(), SECINITSID_SECURITY,
- SECCLASS_SECURITY, SECURITY__COMPUTE_USER,
- NULL);
- if (length)
- goto out;
-
- length = -ENOMEM;
- con = kzalloc(size + 1, GFP_KERNEL);
- if (!con)
- goto out;
-
- length = -ENOMEM;
- user = kzalloc(size + 1, GFP_KERNEL);
- if (!user)
- goto out;
-
- length = -EINVAL;
- if (sscanf(buf, "%s %s", con, user) != 2)
- goto out;
-
- length = security_context_str_to_sid(con, &sid, GFP_KERNEL);
- if (length)
- goto out;
-
- length = security_get_user_sids(sid, user, &sids, &nsids);
- if (length)
- goto out;
-
- length = sprintf(buf, "%u", nsids) + 1;
- ptr = buf + length;
- for (i = 0; i < nsids; i++) {
- rc = security_sid_to_context(sids[i], &newcon, &len);
- if (rc) {
- length = rc;
- goto out;
- }
- if ((length + len) >= SIMPLE_TRANSACTION_LIMIT) {
- kfree(newcon);
- length = -ERANGE;
- goto out;
- }
- memcpy(ptr, newcon, len);
- kfree(newcon);
- ptr += len;
- length += len;
- }
-out:
- kfree(sids);
- kfree(user);
- kfree(con);
- return length;
+ pr_err_once("SELinux: %s (%d) wrote to user. This is no longer supported.\n",
+ current->comm, current->pid);
+ buf[0] = '0';
+ buf[1] = 0;
+ return 2;
}
static ssize_t sel_write_member(struct file *file, char *buf, size_t size)
diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
index e8e7ccb..143021c 100644
--- a/security/selinux/ss/services.c
+++ b/security/selinux/ss/services.c
@@ -2746,131 +2746,6 @@ int security_node_sid(u16 domain,
return rc;
}
-#define SIDS_NEL 25
-
-/**
- * security_get_user_sids - Obtain reachable SIDs for a user.
- * @fromsid: starting SID
- * @username: username
- * @sids: array of reachable SIDs for user
- * @nel: number of elements in @sids
- *
- * Generate the set of SIDs for legal security contexts
- * for a given user that can be reached by @fromsid.
- * Set *@sids to point to a dynamically allocated
- * array containing the set of SIDs. Set *@nel to the
- * number of elements in the array.
- */
-
-int security_get_user_sids(u32 fromsid,
- const char *username,
- u32 **sids,
- u32 *nel)
-{
- struct selinux_policy *policy;
- struct policydb *policydb;
- struct sidtab *sidtab;
- struct context *fromcon, usercon;
- u32 *mysids = NULL, *mysids2, sid;
- u32 i, j, mynel, maxnel = SIDS_NEL;
- struct user_datum *user;
- struct role_datum *role;
- struct ebitmap_node *rnode, *tnode;
- int rc;
-
- *sids = NULL;
- *nel = 0;
-
- if (!selinux_initialized())
- return 0;
-
- mysids = kcalloc(maxnel, sizeof(*mysids), GFP_KERNEL);
- if (!mysids)
- return -ENOMEM;
-
-retry:
- mynel = 0;
- rcu_read_lock();
- policy = rcu_dereference(selinux_state.policy);
- policydb = &policy->policydb;
- sidtab = policy->sidtab;
-
- context_init(&usercon);
-
- rc = -EINVAL;
- fromcon = sidtab_search(sidtab, fromsid);
- if (!fromcon)
- goto out_unlock;
-
- rc = -EINVAL;
- user = symtab_search(&policydb->p_users, username);
- if (!user)
- goto out_unlock;
-
- usercon.user = user->value;
-
- ebitmap_for_each_positive_bit(&user->roles, rnode, i) {
- role = policydb->role_val_to_struct[i];
- usercon.role = i + 1;
- ebitmap_for_each_positive_bit(&role->types, tnode, j) {
- usercon.type = j + 1;
-
- if (mls_setup_user_range(policydb, fromcon, user,
- &usercon))
- continue;
-
- rc = sidtab_context_to_sid(sidtab, &usercon, &sid);
- if (rc == -ESTALE) {
- rcu_read_unlock();
- goto retry;
- }
- if (rc)
- goto out_unlock;
- if (mynel < maxnel) {
- mysids[mynel++] = sid;
- } else {
- rc = -ENOMEM;
- maxnel += SIDS_NEL;
- mysids2 = kcalloc(maxnel, sizeof(*mysids2), GFP_ATOMIC);
- if (!mysids2)
- goto out_unlock;
- memcpy(mysids2, mysids, mynel * sizeof(*mysids2));
- kfree(mysids);
- mysids = mysids2;
- mysids[mynel++] = sid;
- }
- }
- }
- rc = 0;
-out_unlock:
- rcu_read_unlock();
- if (rc || !mynel) {
- kfree(mysids);
- return rc;
- }
-
- rc = -ENOMEM;
- mysids2 = kcalloc(mynel, sizeof(*mysids2), GFP_KERNEL);
- if (!mysids2) {
- kfree(mysids);
- return rc;
- }
- for (i = 0, j = 0; i < mynel; i++) {
- struct av_decision dummy_avd;
- rc = avc_has_perm_noaudit(fromsid, mysids[i],
- SECCLASS_PROCESS, /* kernel value */
- PROCESS__TRANSITION, AVC_STRICT,
- &dummy_avd);
- if (!rc)
- mysids2[j++] = mysids[i];
- cond_resched();
- }
- kfree(mysids);
- *sids = mysids2;
- *nel = j;
- return 0;
-}
-
/**
* __security_genfs_sid - Helper to obtain a SID for a file in a filesystem
* @policy: policy
diff --git a/sound/core/misc.c b/sound/core/misc.c
index 5aca09e..833124c 100644
--- a/sound/core/misc.c
+++ b/sound/core/misc.c
@@ -148,9 +148,11 @@ EXPORT_SYMBOL_GPL(snd_fasync_helper);
void snd_kill_fasync(struct snd_fasync *fasync, int signal, int poll)
{
- if (!fasync || !fasync->on)
+ if (!fasync)
return;
guard(spinlock_irqsave)(&snd_fasync_lock);
+ if (!fasync->on)
+ return;
fasync->signal = signal;
fasync->poll = poll;
list_move(&fasync->list, &snd_fasync_list);
@@ -163,8 +165,10 @@ void snd_fasync_free(struct snd_fasync *fasync)
if (!fasync)
return;
- scoped_guard(spinlock_irq, &snd_fasync_lock)
+ scoped_guard(spinlock_irq, &snd_fasync_lock) {
+ fasync->on = 0;
list_del_init(&fasync->list);
+ }
flush_work(&snd_fasync_work);
kfree(fasync);
diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index a140a0d..746eaf9 100644
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -2155,10 +2155,16 @@ static int snd_pcm_oss_get_trigger(struct snd_pcm_oss_file *pcm_oss_file)
psubstream = pcm_oss_file->streams[SNDRV_PCM_STREAM_PLAYBACK];
csubstream = pcm_oss_file->streams[SNDRV_PCM_STREAM_CAPTURE];
- if (psubstream && psubstream->runtime && psubstream->runtime->oss.trigger)
- result |= PCM_ENABLE_OUTPUT;
- if (csubstream && csubstream->runtime && csubstream->runtime->oss.trigger)
- result |= PCM_ENABLE_INPUT;
+ if (psubstream && psubstream->runtime) {
+ guard(mutex)(&psubstream->runtime->oss.params_lock);
+ if (psubstream->runtime->oss.trigger)
+ result |= PCM_ENABLE_OUTPUT;
+ }
+ if (csubstream && csubstream->runtime) {
+ guard(mutex)(&csubstream->runtime->oss.params_lock);
+ if (csubstream->runtime->oss.trigger)
+ result |= PCM_ENABLE_INPUT;
+ }
return result;
}
@@ -2832,6 +2838,17 @@ static int snd_pcm_oss_capture_ready(struct snd_pcm_substream *substream)
runtime->oss.period_frames;
}
+static bool need_input_retrigger(struct snd_pcm_runtime *runtime)
+{
+ bool ret;
+
+ guard(mutex)(&runtime->oss.params_lock);
+ ret = runtime->oss.trigger;
+ if (ret)
+ runtime->oss.trigger = 0;
+ return ret;
+}
+
static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait)
{
struct snd_pcm_oss_file *pcm_oss_file;
@@ -2864,11 +2881,11 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait)
snd_pcm_oss_capture_ready(csubstream))
mask |= EPOLLIN | EPOLLRDNORM;
}
- if (ostate != SNDRV_PCM_STATE_RUNNING && runtime->oss.trigger) {
+ if (ostate != SNDRV_PCM_STATE_RUNNING &&
+ need_input_retrigger(runtime)) {
struct snd_pcm_oss_file ofile;
memset(&ofile, 0, sizeof(ofile));
ofile.streams[SNDRV_PCM_STREAM_CAPTURE] = pcm_oss_file->streams[SNDRV_PCM_STREAM_CAPTURE];
- runtime->oss.trigger = 0;
snd_pcm_oss_set_trigger(&ofile, PCM_ENABLE_INPUT);
}
}
@@ -2957,8 +2974,10 @@ static void snd_pcm_oss_proc_read(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
struct snd_pcm_str *pstr = entry->private_data;
- struct snd_pcm_oss_setup *setup = pstr->oss.setup_list;
+ struct snd_pcm_oss_setup *setup;
+
guard(mutex)(&pstr->oss.setup_mutex);
+ setup = pstr->oss.setup_list;
while (setup) {
snd_iprintf(buffer, "%s %u %u%s%s%s%s%s%s\n",
setup->task_name,
@@ -3043,6 +3062,13 @@ static void snd_pcm_oss_proc_write(struct snd_info_entry *entry,
buffer->error = -ENOMEM;
return;
}
+ template.task_name = kstrdup(task_name, GFP_KERNEL);
+ if (!template.task_name) {
+ kfree(setup);
+ buffer->error = -ENOMEM;
+ return;
+ }
+ *setup = template;
if (pstr->oss.setup_list == NULL)
pstr->oss.setup_list = setup;
else {
@@ -3050,12 +3076,7 @@ static void snd_pcm_oss_proc_write(struct snd_info_entry *entry,
setup1->next; setup1 = setup1->next);
setup1->next = setup;
}
- template.task_name = kstrdup(task_name, GFP_KERNEL);
- if (! template.task_name) {
- kfree(setup);
- buffer->error = -ENOMEM;
- return;
- }
+ continue;
}
*setup = template;
}
diff --git a/sound/core/pcm_drm_eld.c b/sound/core/pcm_drm_eld.c
index cb2eebaa..1941ee5 100644
--- a/sound/core/pcm_drm_eld.c
+++ b/sound/core/pcm_drm_eld.c
@@ -334,7 +334,7 @@ int snd_parse_eld(struct device *dev, struct snd_parsed_hdmi_eld *e,
e->eld_ver = GRAB_BITS(buf, 0, 3, 5);
if (e->eld_ver != ELD_VER_CEA_861D &&
e->eld_ver != ELD_VER_PARTIAL) {
- dev_info(dev, "HDMI: Unknown ELD version %d\n", e->eld_ver);
+ dev_info_ratelimited(dev, "HDMI: Unknown ELD version %d\n", e->eld_ver);
goto out_fail;
}
@@ -357,7 +357,7 @@ int snd_parse_eld(struct device *dev, struct snd_parsed_hdmi_eld *e,
e->product_id = get_unaligned_le16(buf + 18);
if (mnl > ELD_MAX_MNL) {
- dev_info(dev, "HDMI: MNL is reserved value %d\n", mnl);
+ dev_info_ratelimited(dev, "HDMI: MNL is reserved value %d\n", mnl);
goto out_fail;
} else if (ELD_FIXED_BYTES + mnl > size) {
dev_info(dev, "HDMI: out of range MNL %d\n", mnl);
diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c
index 09c421cd..fe597f7 100644
--- a/sound/core/pcm_lib.c
+++ b/sound/core/pcm_lib.c
@@ -2138,6 +2138,9 @@ static int interleaved_copy(struct snd_pcm_substream *substream,
off = frames_to_bytes(runtime, off);
frames = frames_to_bytes(runtime, frames);
+ if (!data)
+ return fill_silence(substream, 0, hwoff, NULL, frames);
+
return do_transfer(substream, 0, hwoff, data + off, frames, transfer,
in_kernel);
}
diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c
index 75a7a2a..5719637 100644
--- a/sound/core/seq/seq_clientmgr.c
+++ b/sound/core/seq/seq_clientmgr.c
@@ -1253,7 +1253,7 @@ static int snd_seq_ioctl_set_client_info(struct snd_seq_client *client,
if (client->user_pversion >= SNDRV_PROTOCOL_VERSION(1, 0, 3))
client->midi_version = client_info->midi_version;
memcpy(client->event_filter, client_info->event_filter, 32);
- client->group_filter = client_info->group_filter;
+ client->group_filter = client_info->group_filter & SND_SEQ_GROUP_FILTER_MASK;
/* notify the change */
snd_seq_system_client_ev_client_change(client->number);
diff --git a/sound/core/seq/seq_clientmgr.h b/sound/core/seq/seq_clientmgr.h
index ece02c5..feea8bb 100644
--- a/sound/core/seq/seq_clientmgr.h
+++ b/sound/core/seq/seq_clientmgr.h
@@ -14,6 +14,9 @@
/* client manager */
+#define SND_SEQ_GROUP_FILTER_MASK GENMASK(SNDRV_UMP_MAX_GROUPS, 0)
+#define SND_SEQ_GROUP_FILTER_GROUPS GENMASK(SNDRV_UMP_MAX_GROUPS, 1)
+
struct snd_seq_user_client {
struct file *file; /* file struct of client */
/* ... */
@@ -40,7 +43,7 @@ struct snd_seq_client {
int number; /* client number */
unsigned int filter; /* filter flags */
DECLARE_BITMAP(event_filter, 256);
- unsigned short group_filter;
+ unsigned int group_filter;
snd_use_lock_t use_lock;
int event_lost;
/* ports */
diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c
index da8d358..31ab468 100644
--- a/sound/core/seq/seq_ports.c
+++ b/sound/core/seq/seq_ports.c
@@ -144,18 +144,21 @@ int snd_seq_create_port(struct snd_seq_client *client, int port,
num = max(port, 0);
guard(mutex)(&client->ports_mutex);
guard(write_lock_irq)(&client->ports_lock);
+ struct list_head *insert_before = &client->ports_list_head;
list_for_each_entry(p, &client->ports_list_head, list) {
if (p->addr.port == port) {
kfree(new_port);
return -EBUSY;
}
- if (p->addr.port > num)
+ if (p->addr.port > num) {
+ insert_before = &p->list;
break;
+ }
if (port < 0) /* auto-probe mode */
num = p->addr.port + 1;
}
/* insert the new port */
- list_add_tail(&new_port->list, &p->list);
+ list_add_tail(&new_port->list, insert_before);
client->num_ports++;
new_port->addr.port = num; /* store the port number in the port */
sprintf(new_port->name, "port-%d", num);
diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c
index fdc76f2..ccd9359 100644
--- a/sound/core/seq/seq_ump_client.c
+++ b/sound/core/seq/seq_ump_client.c
@@ -37,6 +37,7 @@ struct seq_ump_client {
struct snd_ump_endpoint *ump; /* assigned endpoint */
int seq_client; /* sequencer client id */
int opened[2]; /* current opens for each direction */
+ rwlock_t output_lock; /* protects out_rfile output access */
struct snd_rawmidi_file out_rfile; /* rawmidi for output */
struct seq_ump_input_buffer input; /* input parser context */
void *ump_info[SNDRV_UMP_MAX_BLOCKS + 1]; /* shadow of seq client ump_info */
@@ -88,6 +89,7 @@ static int seq_ump_process_event(struct snd_seq_event *ev, int direct,
unsigned char type;
int len;
+ guard(read_lock_irqsave)(&client->output_lock);
substream = client->out_rfile.output;
if (!substream)
return -ENODEV;
@@ -106,6 +108,7 @@ static int seq_ump_process_event(struct snd_seq_event *ev, int direct,
static int seq_ump_client_open(struct seq_ump_client *client, int dir)
{
struct snd_ump_endpoint *ump = client->ump;
+ struct snd_rawmidi_file rfile = {};
int err;
guard(mutex)(&ump->open_mutex);
@@ -113,9 +116,11 @@ static int seq_ump_client_open(struct seq_ump_client *client, int dir)
err = snd_rawmidi_kernel_open(&ump->core, 0,
SNDRV_RAWMIDI_LFLG_OUTPUT |
SNDRV_RAWMIDI_LFLG_APPEND,
- &client->out_rfile);
+ &rfile);
if (err < 0)
return err;
+ scoped_guard(write_lock_irqsave, &client->output_lock)
+ client->out_rfile = rfile;
}
client->opened[dir]++;
return 0;
@@ -125,11 +130,19 @@ static int seq_ump_client_open(struct seq_ump_client *client, int dir)
static int seq_ump_client_close(struct seq_ump_client *client, int dir)
{
struct snd_ump_endpoint *ump = client->ump;
+ struct snd_rawmidi_file rfile = {};
guard(mutex)(&ump->open_mutex);
- if (!--client->opened[dir])
- if (dir == STR_OUT)
- snd_rawmidi_kernel_release(&client->out_rfile);
+ if (!--client->opened[dir]) {
+ if (dir == STR_OUT) {
+ scoped_guard(write_lock_irqsave, &client->output_lock) {
+ rfile = client->out_rfile;
+ client->out_rfile = (struct snd_rawmidi_file){};
+ }
+ if (rfile.rmidi)
+ snd_rawmidi_kernel_release(&rfile);
+ }
+ }
return 0;
}
@@ -369,7 +382,7 @@ static void setup_client_group_filter(struct seq_ump_client *client)
cptr = snd_seq_kernel_client_get(client->seq_client);
if (!cptr)
return;
- filter = ~(1U << 0); /* always allow groupless messages */
+ filter = SND_SEQ_GROUP_FILTER_GROUPS; /* always allow groupless messages */
for (p = 0; p < SNDRV_UMP_MAX_GROUPS; p++) {
if (client->ump->groups[p].active)
filter &= ~(1U << (p + 1));
@@ -467,6 +480,7 @@ static int snd_seq_ump_probe(struct snd_seq_device *dev)
INIT_WORK(&client->group_notify_work, handle_group_notify);
client->ump = ump;
+ rwlock_init(&client->output_lock);
client->seq_client =
snd_seq_create_kernel_client(card, ump->core.device,
diff --git a/sound/core/timer.c b/sound/core/timer.c
index 820901d..57583de 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -1007,6 +1007,7 @@ static int snd_timer_dev_register(struct snd_device *dev)
{
struct snd_timer *timer = dev->device_data;
struct snd_timer *timer1;
+ struct list_head *insert_before = &snd_timer_list;
if (snd_BUG_ON(!timer || !timer->hw.start || !timer->hw.stop))
return -ENXIO;
@@ -1016,28 +1017,36 @@ static int snd_timer_dev_register(struct snd_device *dev)
guard(mutex)(®ister_mutex);
list_for_each_entry(timer1, &snd_timer_list, device_list) {
- if (timer1->tmr_class > timer->tmr_class)
+ if (timer1->tmr_class > timer->tmr_class) {
+ insert_before = &timer1->device_list;
break;
+ }
if (timer1->tmr_class < timer->tmr_class)
continue;
if (timer1->card && timer->card) {
- if (timer1->card->number > timer->card->number)
+ if (timer1->card->number > timer->card->number) {
+ insert_before = &timer1->device_list;
break;
+ }
if (timer1->card->number < timer->card->number)
continue;
}
- if (timer1->tmr_device > timer->tmr_device)
+ if (timer1->tmr_device > timer->tmr_device) {
+ insert_before = &timer1->device_list;
break;
+ }
if (timer1->tmr_device < timer->tmr_device)
continue;
- if (timer1->tmr_subdevice > timer->tmr_subdevice)
+ if (timer1->tmr_subdevice > timer->tmr_subdevice) {
+ insert_before = &timer1->device_list;
break;
+ }
if (timer1->tmr_subdevice < timer->tmr_subdevice)
continue;
/* conflicts.. */
return -EBUSY;
}
- list_add_tail(&timer->device_list, &timer1->device_list);
+ list_add_tail(&timer->device_list, insert_before);
return 0;
}
diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c
index aa0d2fc..a37a169 100644
--- a/sound/drivers/aloop.c
+++ b/sound/drivers/aloop.c
@@ -99,6 +99,9 @@ struct loopback_ops {
struct loopback_cable {
spinlock_t lock;
struct loopback_pcm *streams[2];
+ /* in-flight peer stops running outside cable->lock */
+ atomic_t stop_count;
+ wait_queue_head_t stop_wait;
struct snd_pcm_hardware hw;
/* flags */
unsigned int valid;
@@ -366,8 +369,11 @@ static int loopback_check_format(struct loopback_cable *cable, int stream)
return 0;
if (stream == SNDRV_PCM_STREAM_CAPTURE)
return -EIO;
- else if (cruntime->state == SNDRV_PCM_STATE_RUNNING)
+ else if (cruntime->state == SNDRV_PCM_STATE_RUNNING) {
+ /* close must not free the peer runtime below */
+ atomic_inc(&cable->stop_count);
stop_capture = true;
+ }
}
setup = get_setup(dpcm_play);
@@ -396,8 +402,11 @@ static int loopback_check_format(struct loopback_cable *cable, int stream)
}
}
- if (stop_capture)
+ if (stop_capture) {
snd_pcm_stop(dpcm_capt->substream, SNDRV_PCM_STATE_DRAINING);
+ if (atomic_dec_and_test(&cable->stop_count))
+ wake_up(&cable->stop_wait);
+ }
return 0;
}
@@ -1049,23 +1058,29 @@ static void free_cable(struct snd_pcm_substream *substream)
struct loopback *loopback = substream->private_data;
int dev = get_cable_index(substream);
struct loopback_cable *cable;
+ struct loopback_pcm *dpcm;
+ bool other_alive;
cable = loopback->cables[substream->number][dev];
if (!cable)
return;
- if (cable->streams[!substream->stream]) {
- /* other stream is still alive */
- guard(spinlock_irq)(&cable->lock);
- cable->streams[substream->stream] = NULL;
- } else {
- struct loopback_pcm *dpcm = substream->runtime->private_data;
- if (cable->ops && cable->ops->close_cable && dpcm)
- cable->ops->close_cable(dpcm);
- /* free the cable */
- loopback->cables[substream->number][dev] = NULL;
- kfree(cable);
+ scoped_guard(spinlock_irq, &cable->lock) {
+ cable->streams[substream->stream] = NULL;
+ other_alive = cable->streams[!substream->stream];
}
+
+ /* Pair with the stop_count increment in loopback_check_format(). */
+ wait_event(cable->stop_wait, !atomic_read(&cable->stop_count));
+ if (other_alive)
+ return;
+
+ dpcm = substream->runtime->private_data;
+ if (cable->ops && cable->ops->close_cable && dpcm)
+ cable->ops->close_cable(dpcm);
+ /* free the cable */
+ loopback->cables[substream->number][dev] = NULL;
+ kfree(cable);
}
static int loopback_jiffies_timer_open(struct loopback_pcm *dpcm)
@@ -1260,6 +1275,8 @@ static int loopback_open(struct snd_pcm_substream *substream)
goto unlock;
}
spin_lock_init(&cable->lock);
+ atomic_set(&cable->stop_count, 0);
+ init_waitqueue_head(&cable->stop_wait);
cable->hw = loopback_pcm_hardware;
if (loopback->timer_source)
cable->ops = &loopback_snd_timer_ops;
diff --git a/sound/drivers/pcmtest.c b/sound/drivers/pcmtest.c
index 5bfec4c..7f93557 100644
--- a/sound/drivers/pcmtest.c
+++ b/sound/drivers/pcmtest.c
@@ -679,9 +679,9 @@ static ssize_t pattern_read(struct file *file, char __user *u_buff, size_t len,
return 0;
if (copy_to_user(u_buff, patt_buf->buf + *off, to_read))
- to_read = 0;
- else
- *off += to_read;
+ return -EFAULT;
+
+ *off += to_read;
return to_read;
}
diff --git a/sound/firewire/motu/motu-register-dsp-message-parser.c b/sound/firewire/motu/motu-register-dsp-message-parser.c
index a8053e3..4ec23e68 100644
--- a/sound/firewire/motu/motu-register-dsp-message-parser.c
+++ b/sound/firewire/motu/motu-register-dsp-message-parser.c
@@ -386,6 +386,8 @@ unsigned int snd_motu_register_dsp_message_parser_count_event(struct snd_motu *m
{
struct msg_parser *parser = motu->message_parser;
+ guard(spinlock_irqsave)(&parser->lock);
+
if (parser->pull_pos > parser->push_pos)
return EVENT_QUEUE_SIZE - parser->pull_pos + parser->push_pos;
else
@@ -395,13 +397,14 @@ unsigned int snd_motu_register_dsp_message_parser_count_event(struct snd_motu *m
bool snd_motu_register_dsp_message_parser_copy_event(struct snd_motu *motu, u32 *event)
{
struct msg_parser *parser = motu->message_parser;
- unsigned int pos = parser->pull_pos;
-
- if (pos == parser->push_pos)
- return false;
+ unsigned int pos;
guard(spinlock_irqsave)(&parser->lock);
+ if (parser->pull_pos == parser->push_pos)
+ return false;
+
+ pos = parser->pull_pos;
*event = parser->event_queue[pos];
++pos;
diff --git a/sound/firewire/tascam/tascam-hwdep.c b/sound/firewire/tascam/tascam-hwdep.c
index 867b4ea..6270263 100644
--- a/sound/firewire/tascam/tascam-hwdep.c
+++ b/sound/firewire/tascam/tascam-hwdep.c
@@ -73,6 +73,7 @@ static long tscm_hwdep_read_queue(struct snd_tscm *tscm, char __user *buf,
length = rounddown(remained, sizeof(*entries));
if (length == 0)
break;
+ tail_pos = head_pos + length / sizeof(*entries);
spin_unlock_irq(&tscm->lock);
if (copy_to_user(pos, &entries[head_pos], length))
diff --git a/sound/hda/codecs/Makefile b/sound/hda/codecs/Makefile
index e7f03e2..88d2f8a 100644
--- a/sound/hda/codecs/Makefile
+++ b/sound/hda/codecs/Makefile
@@ -7,7 +7,6 @@
snd-hda-codec-analog-y := analog.o
snd-hda-codec-ca0110-y := ca0110.o
snd-hda-codec-ca0132-y := ca0132.o
-snd-hda-codec-cmedia-y := cmedia.o
snd-hda-codec-conexant-y := conexant.o
snd-hda-codec-idt-y := sigmatel.o
snd-hda-codec-senarytech-y := senarytech.o
@@ -26,7 +25,6 @@
obj-$(CONFIG_SND_HDA_CODEC_ANALOG) += snd-hda-codec-analog.o
obj-$(CONFIG_SND_HDA_CODEC_CA0110) += snd-hda-codec-ca0110.o
obj-$(CONFIG_SND_HDA_CODEC_CA0132) += snd-hda-codec-ca0132.o
-obj-$(CONFIG_SND_HDA_CODEC_CMEDIA) += snd-hda-codec-cmedia.o
obj-$(CONFIG_SND_HDA_CODEC_CONEXANT) += snd-hda-codec-conexant.o
obj-$(CONFIG_SND_HDA_CODEC_SIGMATEL) += snd-hda-codec-idt.o
obj-$(CONFIG_SND_HDA_CODEC_SENARYTECH) += snd-hda-codec-senarytech.o
diff --git a/sound/hda/codecs/ca0132.c b/sound/hda/codecs/ca0132.c
index ad533b04..be565ff 100644
--- a/sound/hda/codecs/ca0132.c
+++ b/sound/hda/codecs/ca0132.c
@@ -5498,6 +5498,30 @@ static int zxr_headphone_gain_set(struct hda_codec *codec, long val)
return 0;
}
+/*
+ * Manual output selection (HP/Speaker Playback Switch or alt Output Select)
+ * is meaningful only when HP/Speaker auto-detect is disabled, since the
+ * select_out path always prefers jack presence when auto-detect is on. When
+ * the user explicitly chooses an output, turn auto-detect off so the manual
+ * choice actually takes effect, and notify userspace so the auto-detect
+ * control reflects the new state.
+ */
+static void ca0132_disable_hp_auto_detect(struct hda_codec *codec)
+{
+ struct ca0132_spec *spec = codec->spec;
+ struct snd_kcontrol *kctl;
+
+ if (!spec->vnode_lswitch[VNID_HP_ASEL - VNODE_START_NID])
+ return;
+
+ spec->vnode_lswitch[VNID_HP_ASEL - VNODE_START_NID] = 0;
+ kctl = snd_hda_find_mixer_ctl(codec,
+ "HP/Speaker Auto Detect Playback Switch");
+ if (kctl)
+ snd_ctl_notify(codec->card, SNDRV_CTL_EVENT_MASK_VALUE,
+ &kctl->id);
+}
+
static int ca0132_vnode_switch_set(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
@@ -5510,14 +5534,11 @@ static int ca0132_vnode_switch_set(struct snd_kcontrol *kcontrol,
int auto_jack;
if (nid == VNID_HP_SEL) {
- auto_jack =
- spec->vnode_lswitch[VNID_HP_ASEL - VNODE_START_NID];
- if (!auto_jack) {
- if (ca0132_use_alt_functions(spec))
- ca0132_alt_select_out(codec);
- else
- ca0132_select_out(codec);
- }
+ ca0132_disable_hp_auto_detect(codec);
+ if (ca0132_use_alt_functions(spec))
+ ca0132_alt_select_out(codec);
+ else
+ ca0132_select_out(codec);
return 1;
}
@@ -5978,7 +5999,6 @@ static int ca0132_alt_output_select_put(struct snd_kcontrol *kcontrol,
struct ca0132_spec *spec = codec->spec;
int sel = ucontrol->value.enumerated.item[0];
unsigned int items = NUM_OF_OUTPUTS;
- unsigned int auto_jack;
if (sel >= items)
return 0;
@@ -5988,10 +6008,8 @@ static int ca0132_alt_output_select_put(struct snd_kcontrol *kcontrol,
spec->out_enum_val = sel;
- auto_jack = spec->vnode_lswitch[VNID_HP_ASEL - VNODE_START_NID];
-
- if (!auto_jack)
- ca0132_alt_select_out(codec);
+ ca0132_disable_hp_auto_detect(codec);
+ ca0132_alt_select_out(codec);
return 1;
}
diff --git a/sound/hda/codecs/cirrus/cs420x.c b/sound/hda/codecs/cirrus/cs420x.c
index 42559edb..85c2ecf 100644
--- a/sound/hda/codecs/cirrus/cs420x.c
+++ b/sound/hda/codecs/cirrus/cs420x.c
@@ -582,6 +582,7 @@ static const struct hda_quirk cs4208_mac_fixup_tbl[] = {
SND_PCI_QUIRK(0x106b, 0x7200, "MacBookAir 6,2", CS4208_MBA6),
SND_PCI_QUIRK(0x106b, 0x7800, "MacPro 6,1", CS4208_MACMINI),
SND_PCI_QUIRK(0x106b, 0x7b00, "MacBookPro 12,1", CS4208_MBP11),
+ SND_PCI_QUIRK(0x106b, 0x7f00, "iMac 16,1", CS4208_MBP11),
{} /* terminator */
};
diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c
index 3a9717d..e3b6aaa 100644
--- a/sound/hda/codecs/conexant.c
+++ b/sound/hda/codecs/conexant.c
@@ -1175,6 +1175,7 @@ static void add_cx5051_fake_mutes(struct hda_codec *codec)
static int cx_probe(struct hda_codec *codec, const struct hda_device_id *id)
{
struct conexant_spec *spec;
+ struct hda_jack_callback *callback;
int err;
codec_info(codec, "%s: BIOS auto-probing.\n", codec->core.chip_name);
@@ -1190,7 +1191,12 @@ static int cx_probe(struct hda_codec *codec, const struct hda_device_id *id)
case 0x14f11f86:
case 0x14f11f87:
spec->is_cx11880_sn6140 = true;
- snd_hda_jack_detect_enable_callback(codec, 0x19, cx_update_headset_mic_vref);
+ callback = snd_hda_jack_detect_enable_callback(codec, 0x19,
+ cx_update_headset_mic_vref);
+ if (IS_ERR(callback)) {
+ err = PTR_ERR(callback);
+ goto error;
+ }
break;
}
diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c
index d720565..dcbc669 100644
--- a/sound/hda/codecs/realtek/alc269.c
+++ b/sound/hda/codecs/realtek/alc269.c
@@ -1669,6 +1669,21 @@ static void alc295_fixup_hp_mute_led_coefbit11(struct hda_codec *codec,
}
}
+/* Override wrong pin to NID 0x1b (F.32 BIOS reports 0x18 via DMI OEM string)
+ * on HP pavilion 15-cs1xxx laptops
+ */
+static void alc295_fixup_hp_pavilion_mute_led_1b(struct hda_codec *codec,
+ const struct hda_fixup *fix,
+ int action)
+{
+ struct alc_spec *spec = codec->spec;
+
+ alc269_fixup_hp_mute_led(codec, fix, action);
+
+ if (action == HDA_FIXUP_ACT_PRE_PROBE)
+ spec->mute_led_nid = 0x1b;
+}
+
static void alc233_fixup_lenovo_coef_micmute_led(struct hda_codec *codec,
const struct hda_fixup *fix, int action)
{
@@ -3390,6 +3405,19 @@ static void alc256_fixup_mic_no_presence_and_resume(struct hda_codec *codec,
}
}
+static void alc256_fixup_xiaomi_pro15_resume(struct hda_codec *codec,
+ const struct hda_fixup *fix,
+ int action)
+{
+ /*
+ * On the Xiaomi Mi Laptop Pro 15 (TM1905, SSID 1d72:1905) the ALC256
+ * codec sets coefficient 0x10 bit 9 to 1 after S3 resume, silencing
+ * the internal speaker. Bluetooth and HDMI audio are unaffected.
+ * Clear the bit so the speaker keeps working across suspend cycles.
+ */
+ alc_update_coef_idx(codec, 0x10, 1<<9, 0);
+}
+
static void alc256_decrease_headphone_amp_val(struct hda_codec *codec,
const struct hda_fixup *fix, int action)
{
@@ -3694,6 +3722,17 @@ static void alc287_fixup_lenovo_thinkpad_with_alc1318(struct hda_codec *codec,
spec->power_hook = alc287_s4_power_gpio3_default;
spec->gen.pcm_playback_hook = alc287_alc1318_playback_pcm_hook;
}
+
+static void alc287_fixup_tb_vmaster_led(struct hda_codec *codec,
+ const struct hda_fixup *fix, int action)
+{
+ struct alc_spec *spec = codec->spec;
+
+ if (action == HDA_FIXUP_ACT_PRE_PROBE)
+ spec->gen.vmaster_mute_led = 1;
+
+ alc287_fixup_bind_dacs(codec, fix, action);
+}
/* GPIO2: mute led GPIO3: micmute led */
static void alc245_tas2781_spi_hp_fixup_muteled(struct hda_codec *codec,
const struct hda_fixup *fix, int action)
@@ -3846,6 +3885,7 @@ enum {
ALC290_FIXUP_SUBWOOFER,
ALC290_FIXUP_SUBWOOFER_HSJACK,
ALC295_FIXUP_HP_MUTE_LED_COEFBIT11,
+ ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B,
ALC269_FIXUP_THINKPAD_ACPI,
ALC269_FIXUP_LENOVO_XPAD_ACPI,
ALC269_FIXUP_DMIC_THINKPAD_ACPI,
@@ -4041,6 +4081,7 @@ enum {
ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE,
ALC233_FIXUP_NO_AUDIO_JACK,
ALC256_FIXUP_MIC_NO_PRESENCE_AND_RESUME,
+ ALC256_FIXUP_XIAOMI_PRO15_RESUME,
ALC285_FIXUP_LEGION_Y9000X_SPEAKERS,
ALC285_FIXUP_LEGION_Y9000X_AUTOMUTE,
ALC287_FIXUP_LEGION_16ACHG6,
@@ -4054,6 +4095,7 @@ enum {
ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED,
ALC285_FIXUP_HP_SPEAKERS_MICMUTE_LED,
ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE,
+ ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST,
ALC287_FIXUP_LEGION_16ITHG6,
ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK,
ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN,
@@ -4112,6 +4154,7 @@ enum {
ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED,
ALC236_FIXUP_HP_DMIC,
ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO,
+ ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX,
};
/* A special fixup for Lenovo C940 and Yoga Duet 7;
@@ -5415,7 +5458,7 @@ static const struct hda_fixup alc269_fixups[] = {
[ALC299_FIXUP_PREDATOR_SPK] = {
.type = HDA_FIXUP_PINS,
.v.pins = (const struct hda_pintbl[]) {
- { 0x21, 0x90170150 }, /* use as headset mic, without its own jack detect */
+ { 0x21, 0x90170150 }, /* use as internal speaker */
{ }
}
},
@@ -5689,6 +5732,10 @@ static const struct hda_fixup alc269_fixups[] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc295_fixup_hp_mute_led_coefbit11,
},
+ [ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc295_fixup_hp_pavilion_mute_led_1b,
+ },
[ALC298_FIXUP_SAMSUNG_AMP] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc298_fixup_samsung_amp,
@@ -6229,6 +6276,10 @@ static const struct hda_fixup alc269_fixups[] = {
.chained = true,
.chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC
},
+ [ALC256_FIXUP_XIAOMI_PRO15_RESUME] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc256_fixup_xiaomi_pro15_resume,
+ },
[ALC287_FIXUP_LEGION_16ACHG6] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc287_fixup_legion_16achg6_speakers,
@@ -6296,6 +6347,12 @@ static const struct hda_fixup alc269_fixups[] = {
.chained = true,
.chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC
},
+ [ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc269_fixup_limit_int_mic_boost,
+ .chained = true,
+ .chain_id = ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE,
+ },
[ALC287_FIXUP_LEGION_16ITHG6] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc287_fixup_legion_16ithg6_speakers,
@@ -6448,7 +6505,7 @@ static const struct hda_fixup alc269_fixups[] = {
},
[ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD] = {
.type = HDA_FIXUP_FUNC,
- .v.func = alc287_fixup_bind_dacs,
+ .v.func = alc287_fixup_tb_vmaster_led,
.chained = true,
.chain_id = ALC287_FIXUP_CS35L41_I2C_2_THINKPAD_ACPI,
},
@@ -6664,6 +6721,12 @@ static const struct hda_fixup alc269_fixups[] = {
{ 0x1b, 0x90170110 },
{ }
}
+ },
+ [ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = cs35l41_fixup_i2c_two,
+ .chained = true,
+ .chain_id = ALC245_FIXUP_HP_X360_MUTE_LEDS
}
};
@@ -6717,6 +6780,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC2XX_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x1025, 0x160e, "Acer PT316-51S", ALC2XX_FIXUP_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1025, 0x1640, "Acer Aspire A315-44P", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED),
SND_PCI_QUIRK(0x1025, 0x1679, "Acer Nitro 16 AN16-41", ALC2XX_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED),
SND_PCI_QUIRK(0x1025, 0x171e, "Acer Nitro ANV15-51", ALC245_FIXUP_ACER_MICMUTE_LED),
@@ -6891,9 +6955,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x84da, "HP OMEN dc0019-ur", ALC295_FIXUP_HP_OMEN),
SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3),
SND_PCI_QUIRK(0x103c, 0x8519, "HP Spectre x360 15-df0xxx", ALC285_FIXUP_HP_SPECTRE_X360),
+ SND_PCI_QUIRK(0x103c, 0x8536, "HP ProBook 430 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
SND_PCI_QUIRK(0x103c, 0x8537, "HP ProBook 440 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
SND_PCI_QUIRK(0x103c, 0x8548, "HP EliteBook x360 830 G6", ALC285_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x856a, "HP Pavilion 15-cs1xxx", ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B),
SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11),
SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360),
SND_PCI_QUIRK(0x103c, 0x8603, "HP Omen 17-cb0xxx", ALC285_FIXUP_HP_MUTE_LED),
@@ -7004,6 +7070,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x89d3, "HP EliteBook 645 G9 (MB 89D2)", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
SND_PCI_QUIRK(0x103c, 0x89da, "HP Spectre x360 14t-ea100", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX),
SND_PCI_QUIRK(0x103c, 0x89e7, "HP Elite x2 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x8a06, "HP Dragonfly Folio G3 2-in-1", ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8a0f, "HP Pavilion 14-ec1xxx", ALC287_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8a1f, "HP Laptop 14s-dr5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
SND_PCI_QUIRK(0x103c, 0x8a20, "HP Laptop 15s-fq5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
@@ -7019,6 +7086,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8a30, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8a31, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
+ SND_PCI_QUIRK(0x103c, 0x8a36, "HP Pavilion Plus 14-eh0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT),
SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4),
@@ -7038,6 +7106,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8ad8, "HP 800 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8b0f, "HP Elite mt645 G7 Mobile Thin Client U81", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
SND_PCI_QUIRK(0x103c, 0x8b2f, "HP 255 15.6 inch G10 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
+ SND_PCI_QUIRK(0x103c, 0x8b34, "HP 250 15.6 inch G10 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
SND_PCI_QUIRK(0x103c, 0x8b3a, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8b3f, "HP mt440 Mobile Thin Client U91", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8b42, "HP", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
@@ -7085,7 +7154,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8be6, "HP Envy 16", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8be7, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8be8, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2),
- SND_PCI_QUIRK(0x103c, 0x8be9, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2),
+ SND_PCI_QUIRK(0x103c, 0x8be9, "HP Envy x360 2-in-1 Laptop 15-fh0xxx", ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX),
SND_PCI_QUIRK(0x103c, 0x8bf0, "HP", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8c15, "HP Spectre x360 2-in-1 Laptop 14-eu0xxx", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX),
SND_PCI_QUIRK(0x103c, 0x8c16, "HP Spectre x360 2-in-1 Laptop 16-aa0xxx", ALC245_FIXUP_HP_SPECTRE_X360_16_AA0XXX),
@@ -7135,6 +7204,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8ca4, "HP ZBook Fury", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8ca7, "HP ZBook Fury", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8caf, "HP Elite mt645 G8 Mobile Thin Client", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
+ SND_PCI_QUIRK(0x103c, 0x8cbc, "HP Pavilion Laptop 16-ag0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS),
SND_PCI_QUIRK(0x103c, 0x8cbd, "HP Pavilion Aero Laptop 13-bg0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS),
SND_PCI_QUIRK(0x103c, 0x8cdd, "HP Spectre", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX),
SND_PCI_QUIRK(0x103c, 0x8cde, "HP OmniBook Ultra Flip Laptop 14t", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX),
@@ -7163,7 +7233,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8da0, "HP 16 Clipper OmniBook 7(X360)", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8da1, "HP 16 Clipper OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x103c, 0x8da7, "HP 14 Enstrom OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
- SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
+ SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX),
SND_PCI_QUIRK(0x103c, 0x8dc9, "HP Laptop 15-fc0xxx", ALC236_FIXUP_HP_DMIC),
SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS),
SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
@@ -7175,6 +7245,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x103c, 0x8def, "HP EliteBook 660 G12", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8df0, "HP EliteBook 630 G12", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8df1, "HP EliteBook 630 G12", ALC236_FIXUP_HP_GPIO_LED),
+ SND_PCI_QUIRK(0x103c, 0x8df7, "HP Z66 G6", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8dfb, "HP EliteBook 6 G1a 14", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
SND_PCI_QUIRK(0x103c, 0x8dfc, "HP EliteBook 645 G12", ALC236_FIXUP_HP_GPIO_LED),
SND_PCI_QUIRK(0x103c, 0x8dfd, "HP EliteBook 6 G1a 16", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
@@ -7254,6 +7325,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x1043, 0x11c0, "ASUS X556UR", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE),
HDA_CODEC_QUIRK(0x1043, 0x1204, "ASUS Strix G16 G615JMR", ALC287_FIXUP_TXNW2781_I2C_ASUS),
SND_PCI_QUIRK(0x1043, 0x1204, "ASUS Strix G615JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C),
+ HDA_CODEC_QUIRK(0x1043, 0x1214, "ASUS ROG Strix G615LP", ALC287_FIXUP_TXNW2781_I2C_ASUS),
SND_PCI_QUIRK(0x1043, 0x1214, "ASUS Strix G615LH_LM_LP", ALC287_FIXUP_TAS2781_I2C),
SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0x1043, 0x1271, "ASUS X430UN", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
@@ -7387,12 +7459,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x1043, 0x3e00, "ASUS G814FH/FM/FP", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x1043, 0x3e20, "ASUS G814PH/PM/PP", ALC287_FIXUP_CS35L41_I2C_2),
SND_PCI_QUIRK(0x1043, 0x3e30, "ASUS TP3607SA", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3ee0, "ASUS Strix G815_JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TAS2781_I2C),
- SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TAS2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3ee0, "ASUS Strix G815_JHR_JMR_JPR", ALC287_FIXUP_TXNW2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TXNW2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C),
+ SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TXNW2781_I2C),
SND_PCI_QUIRK(0x1043, 0x3fd0, "ASUS B3605CVA", ALC245_FIXUP_CS35L41_SPI_2),
SND_PCI_QUIRK(0x1043, 0x3ff0, "ASUS B5405CVA", ALC245_FIXUP_CS35L41_SPI_2),
SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC),
@@ -7446,6 +7518,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x144d, 0xc870, "Samsung Galaxy Book2 Pro (NP950XED)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS),
SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS),
SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS),
+ SND_PCI_QUIRK(0x144d, 0xc902, "Samsung Galaxy Book5 360 (NP750QHA)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET),
SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS),
SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS),
SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS),
@@ -7617,6 +7690,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x17aa, 0x3801, "Lenovo Yoga9 14IAP7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN),
HDA_CODEC_QUIRK(0x17aa, 0x3802, "DuetITL 2021", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
SND_PCI_QUIRK(0x17aa, 0x3802, "Lenovo Yoga Pro 9 14IRP8", ALC287_FIXUP_TAS2781_I2C),
+ /* Yoga Pro 9 16IMH9 and Legion 7 16ITHG6 share PCI SSID 17aa:3811
+ * with Legion S7 15IMH05; use codec SSID to distinguish them
+ */
+ HDA_CODEC_QUIRK(0x17aa, 0x38d5, "Lenovo Yoga Pro 9 16IMH9", ALC287_FIXUP_TAS2781_I2C),
+ HDA_CODEC_QUIRK(0x17aa, 0x38d6, "Lenovo Yoga Pro 9 16IMH9", ALC287_FIXUP_TAS2781_I2C),
+ HDA_CODEC_QUIRK(0x17aa, 0x3855, "Legion 7 16ITHG6", ALC287_FIXUP_LEGION_16ITHG6),
SND_PCI_QUIRK(0x17aa, 0x3811, "Legion S7 15IMH05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS),
SND_PCI_QUIRK(0x17aa, 0x3813, "Legion 7i 15IMHG05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS),
SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940 / Yoga Duet 7", ALC298_FIXUP_LENOVO_C940_DUET7),
@@ -7690,6 +7769,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x17aa, 0x38df, "Y990 YG DUAL", ALC287_FIXUP_TAS2781_I2C),
SND_PCI_QUIRK(0x17aa, 0x38f9, "Thinkbook 16P Gen5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
SND_PCI_QUIRK(0x17aa, 0x38fa, "Thinkbook 16P Gen5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
+ SND_PCI_QUIRK(0x17aa, 0x38fc, "Lenovo Yoga Pro 7 15ASH11", ALC245_FIXUP_BASS_HP_DAC),
SND_PCI_QUIRK(0x17aa, 0x38fd, "ThinkBook plus Gen5 Hybrid", ALC287_FIXUP_TAS2781_I2C),
SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN),
@@ -7700,6 +7780,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C),
SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
SND_PCI_QUIRK(0x17aa, 0x392b, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
+ HDA_CODEC_QUIRK(0x17aa, 0x394c, "Lenovo Yoga Slim 7 14AGP11", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN),
SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC),
SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI),
SND_PCI_QUIRK(0x17aa, 0x3bf8, "Quanta FL1", ALC269_FIXUP_PCM_44K),
@@ -7762,9 +7843,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC),
+ SND_PCI_QUIRK(0x1d72, 0x1905, "Xiaomi Mi Laptop Pro 15", ALC256_FIXUP_XIAOMI_PRO15_RESUME),
SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC),
SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
SND_PCI_QUIRK(0x1e39, 0xca14, "MEDION NM14LNL", ALC233_FIXUP_MEDION_MTL_SPK),
+ SND_PCI_QUIRK(0x1e50, 0x7007, "Positivo DN50E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0x1e50, 0x7038, "Positivo DN140", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC),
SND_PCI_QUIRK(0x1ee7, 0x2081, "HONOR MRB-XXX M1020", ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO),
SND_PCI_QUIRK(0x1f4c, 0xe001, "Minisforum V3 (SE)", ALC245_FIXUP_BASS_HP_DAC),
@@ -7790,7 +7874,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0xf111, 0x000b, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
- SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
+ SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop 13 Pro PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0xf111, 0x010f, "Framework Laptop 13 PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST),
#if 0
/* Below is a quirk table taken from the old code.
diff --git a/sound/hda/codecs/side-codecs/Kconfig b/sound/hda/codecs/side-codecs/Kconfig
index fc5651e..e51964c 100644
--- a/sound/hda/codecs/side-codecs/Kconfig
+++ b/sound/hda/codecs/side-codecs/Kconfig
@@ -94,7 +94,6 @@
config SND_HDA_SCODEC_CS35L56_CAL_DEBUGFS
bool "CS35L56 create debugfs for factory calibration"
- default N
depends on DEBUG_FS
select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
help
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c
index b648900..acfccc8 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda.c
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c
@@ -1896,8 +1896,10 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
cs35l41->dacpi = adev;
physdev = get_device(acpi_get_first_physical_node(adev));
- if (!physdev)
+ if (!physdev) {
+ acpi_dev_put(adev);
return -ENODEV;
+ }
sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev));
if (IS_ERR(sub))
diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c
index 1ace4be..a0ea08e 100644
--- a/sound/hda/codecs/side-codecs/cs35l56_hda.c
+++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c
@@ -180,11 +180,15 @@ static int cs35l56_hda_mixer_get(struct snd_kcontrol *kcontrol,
{
struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol);
unsigned int reg_val;
- int i;
+ int i, ret;
cs35l56_hda_wait_dsp_ready(cs35l56);
- regmap_read(cs35l56->base.regmap, kcontrol->private_value, ®_val);
+ ret = regmap_read(cs35l56->base.regmap, kcontrol->private_value,
+ ®_val);
+ if (ret)
+ return ret;
+
reg_val &= CS35L56_ASP_TXn_SRC_MASK;
for (i = 0; i < CS35L56_NUM_INPUT_SRC; ++i) {
@@ -203,15 +207,20 @@ static int cs35l56_hda_mixer_put(struct snd_kcontrol *kcontrol,
struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol);
unsigned int item = ucontrol->value.enumerated.item[0];
bool changed;
+ int ret;
if (item >= CS35L56_NUM_INPUT_SRC)
return -EINVAL;
cs35l56_hda_wait_dsp_ready(cs35l56);
- regmap_update_bits_check(cs35l56->base.regmap, kcontrol->private_value,
- CS35L56_INPUT_MASK, cs35l56_tx_input_values[item],
- &changed);
+ ret = regmap_update_bits_check(cs35l56->base.regmap,
+ kcontrol->private_value,
+ CS35L56_INPUT_MASK,
+ cs35l56_tx_input_values[item],
+ &changed);
+ if (ret)
+ return ret;
return changed;
}
@@ -967,6 +976,7 @@ static int cs35l56_hda_system_resume(struct device *dev)
static int cs35l56_hda_fixup_yoga9(struct cs35l56_hda *cs35l56, int *bus_addr)
{
/* The cirrus,dev-index property has the wrong values */
+ cs35l56->num_amps = 2;
switch (*bus_addr) {
case 0x30:
cs35l56->index = 1;
@@ -1015,8 +1025,7 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id)
u32 values[HDA_MAX_COMPONENTS];
char hid_string[8];
struct acpi_device *adev;
- const char *property, *sub;
- size_t nval;
+ const char *property;
int i, ret;
/*
@@ -1032,12 +1041,14 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id)
return -ENODEV;
}
ACPI_COMPANION_SET(cs35l56->base.dev, adev);
+ acpi_dev_put(adev);
}
/* Initialize things that could be overwritten by a fixup */
cs35l56->index = -1;
- sub = acpi_get_subsystem_id(ACPI_HANDLE(cs35l56->base.dev));
+ const char *sub __free(kfree) = acpi_get_subsystem_id(ACPI_HANDLE(cs35l56->base.dev));
+
ret = cs35l56_hda_apply_platform_fixups(cs35l56, sub, &id);
if (ret)
return ret;
@@ -1052,13 +1063,14 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id)
ret = -EINVAL;
goto err;
}
- nval = ret;
+ cs35l56->num_amps = ret;
- ret = device_property_read_u32_array(cs35l56->base.dev, property, values, nval);
+ ret = device_property_read_u32_array(cs35l56->base.dev, property, values,
+ cs35l56->num_amps);
if (ret)
goto err;
- for (i = 0; i < nval; i++) {
+ for (i = 0; i < cs35l56->num_amps; i++) {
if (values[i] == id) {
cs35l56->index = i;
break;
@@ -1081,17 +1093,19 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id)
"Read ACPI _SUB failed(%ld): fallback to generic firmware\n",
PTR_ERR(sub));
} else {
- ret = cirrus_scodec_get_speaker_id(cs35l56->base.dev, cs35l56->index, nval, -1);
+ ret = cirrus_scodec_get_speaker_id(cs35l56->base.dev, cs35l56->index,
+ cs35l56->num_amps, -1);
if (ret == -ENOENT) {
- cs35l56->system_name = sub;
+ cs35l56->system_name = devm_kstrdup(cs35l56->base.dev, sub, GFP_KERNEL);
} else if (ret >= 0) {
- cs35l56->system_name = kasprintf(GFP_KERNEL, "%s-spkid%d", sub, ret);
- kfree(sub);
- if (!cs35l56->system_name)
- return -ENOMEM;
+ cs35l56->system_name = devm_kasprintf(cs35l56->base.dev, GFP_KERNEL,
+ "%s-spkid%d", sub, ret);
} else {
return ret;
}
+
+ if (!cs35l56->system_name)
+ return -ENOMEM;
}
cs35l56->base.reset_gpio = devm_gpiod_get_index_optional(cs35l56->base.dev,
@@ -1242,7 +1256,6 @@ void cs35l56_hda_remove(struct device *dev)
cs_dsp_remove(&cs35l56->cs_dsp);
- kfree(cs35l56->system_name);
pm_runtime_put_noidle(cs35l56->base.dev);
gpiod_set_value_cansleep(cs35l56->base.reset_gpio, 0);
diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.h b/sound/hda/codecs/side-codecs/cs35l56_hda.h
index cb4b5e7..3705af7 100644
--- a/sound/hda/codecs/side-codecs/cs35l56_hda.h
+++ b/sound/hda/codecs/side-codecs/cs35l56_hda.h
@@ -26,6 +26,7 @@ struct cs35l56_hda {
struct work_struct dsp_work;
int index;
+ int num_amps;
const char *system_name;
const char *amp_name;
diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
index 560f238..0e4f355 100644
--- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
+++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
@@ -132,10 +132,18 @@ static int tasdevice_spi_dev_update_bits(struct tasdevice_priv *tas_priv,
int ret, val;
/*
- * In our TAS2781 SPI mode, read/write was masked in last bit of
- * address, it cause regmap_update_bits() not work as expected.
+ * In TAS2781 SPI mode, when accessing non-book-zero or page numbers
+ * greater than 1 in book 0, an additional byte must be read. The
+ * first byte in such cases is a dummy byte and should be ignored.
*/
- ret = tasdevice_dev_read(tas_priv, chn, reg, &val);
+ if ((TASDEVICE_BOOK_ID(reg) > 0) || (TASDEVICE_PAGE_ID(reg) > 1)) {
+ unsigned char buf[2];
+
+ ret = tasdevice_dev_bulk_read(tas_priv, chn, reg, buf, 2);
+ val = buf[1];
+ } else {
+ ret = tasdevice_dev_read(tas_priv, chn, reg, &val);
+ }
if (ret < 0) {
dev_err(tas_priv->dev, "%s, E=%d\n", __func__, ret);
return ret;
diff --git a/sound/hda/common/auto_parser.c b/sound/hda/common/auto_parser.c
index 8923813..5bc95d3 100644
--- a/sound/hda/common/auto_parser.c
+++ b/sound/hda/common/auto_parser.c
@@ -1013,7 +1013,7 @@ void snd_hda_pick_fixup(struct hda_codec *codec,
const char *name = NULL;
const char *type = NULL;
unsigned int vendor, device;
- u16 pci_vendor, pci_device;
+ u16 pci_vendor = 0, pci_device = 0;
u16 codec_vendor, codec_device;
if (codec->fixup_id != HDA_FIXUP_ID_NOT_SET)
@@ -1066,7 +1066,7 @@ void snd_hda_pick_fixup(struct hda_codec *codec,
/* match primarily with the PCI SSID */
for (q = quirk; q->subvendor || q->subdevice; q++) {
/* if the entry is specific to codec SSID, check with it */
- if (!codec->bus->pci || q->match_codec_ssid) {
+ if (!pci_vendor || !pci_device || q->match_codec_ssid) {
if (hda_quirk_match(codec_vendor, codec_device, q)) {
type = "codec SSID";
goto found_device;
diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c
index c2af251..81f266b 100644
--- a/sound/hda/common/codec.c
+++ b/sound/hda/common/codec.c
@@ -1699,6 +1699,9 @@ int snd_hda_ctl_add(struct hda_codec *codec, hda_nid_t nid,
unsigned short flags = 0;
struct hda_nid_item *item;
+ if (!kctl)
+ return -EINVAL;
+
if (kctl->id.subdevice & HDA_SUBDEV_AMP_FLAG) {
flags |= HDA_NID_ITEM_AMP;
if (nid == 0)
diff --git a/sound/hda/common/controller.c b/sound/hda/common/controller.c
index 5934e5c..afec5c5 100644
--- a/sound/hda/common/controller.c
+++ b/sound/hda/common/controller.c
@@ -97,6 +97,8 @@ static int azx_pcm_close(struct snd_pcm_substream *substream)
trace_azx_pcm_close(chip, azx_dev);
scoped_guard(mutex, &chip->open_mutex) {
+ if (chip->ops->pcm_close)
+ chip->ops->pcm_close(chip, azx_dev);
azx_release_device(azx_dev);
if (hinfo->ops.close)
hinfo->ops.close(hinfo, apcm->codec, substream);
@@ -489,9 +491,9 @@ static int azx_get_time_info(struct snd_pcm_substream *substream,
struct snd_pcm_audio_tstamp_config *audio_tstamp_config,
struct snd_pcm_audio_tstamp_report *audio_tstamp_report)
{
+ struct system_device_crosststamp xtstamp = { .clock_id = CLOCK_REALTIME };
struct azx_dev *azx_dev = get_azx_dev(substream);
struct snd_pcm_runtime *runtime = substream->runtime;
- struct system_device_crosststamp xtstamp;
int ret;
u64 nsec;
@@ -525,7 +527,7 @@ static int azx_get_time_info(struct snd_pcm_substream *substream,
break;
default:
- *system_ts = ktime_to_timespec64(xtstamp.sys_realtime);
+ *system_ts = ktime_to_timespec64(xtstamp.sys_systime);
break;
}
@@ -1264,19 +1266,17 @@ int azx_codec_configure(struct azx *chip)
}
EXPORT_SYMBOL_GPL(azx_codec_configure);
-static int stream_direction(struct azx *chip, unsigned char index)
+void azx_add_stream(struct azx *chip, struct azx_dev *azx_dev, int idx, int tag)
{
- if (index >= chip->capture_index_offset &&
- index < chip->capture_index_offset + chip->capture_streams)
- return SNDRV_PCM_STREAM_CAPTURE;
- return SNDRV_PCM_STREAM_PLAYBACK;
+ snd_hdac_stream_init(azx_bus(chip), azx_stream(azx_dev), idx,
+ azx_stream_direction(chip, idx), tag);
}
+EXPORT_SYMBOL_GPL(azx_add_stream);
/* initialize SD streams */
int azx_init_streams(struct azx *chip)
{
int i;
- int stream_tags[2] = { 0, 0 };
/* initialize each stream (aka device)
* assign the starting bdl address to each stream (device)
@@ -1284,24 +1284,10 @@ int azx_init_streams(struct azx *chip)
*/
for (i = 0; i < chip->num_streams; i++) {
struct azx_dev *azx_dev = kzalloc_obj(*azx_dev);
- int dir, tag;
if (!azx_dev)
return -ENOMEM;
-
- dir = stream_direction(chip, i);
- /* stream tag must be unique throughout
- * the stream direction group,
- * valid values 1...15
- * use separate stream tag if the flag
- * AZX_DCAPS_SEPARATE_STREAM_TAG is used
- */
- if (chip->driver_caps & AZX_DCAPS_SEPARATE_STREAM_TAG)
- tag = ++stream_tags[dir];
- else
- tag = i + 1;
- snd_hdac_stream_init(azx_bus(chip), azx_stream(azx_dev),
- i, dir, tag);
+ azx_add_stream(chip, azx_dev, i, i + 1);
}
return 0;
diff --git a/sound/hda/common/hda_controller.h b/sound/hda/common/hda_controller.h
index 7434f38..38227f8 100644
--- a/sound/hda/common/hda_controller.h
+++ b/sound/hda/common/hda_controller.h
@@ -57,13 +57,12 @@ enum {
struct azx_dev {
struct hdac_stream core;
- unsigned int irq_pending:1;
/*
* For VIA:
* A flag to ensure DMA position is 0
* when link position is not greater than FIFO size
*/
- unsigned int insufficient:1;
+ bool insufficient;
};
#define azx_stream(dev) (&(dev)->core)
@@ -79,6 +78,8 @@ struct hda_controller_ops {
int (*position_check)(struct azx *chip, struct azx_dev *azx_dev);
/* enable/disable the link power */
int (*link_power)(struct azx *chip, bool enable);
+ /* additional hook for PCM */
+ void (*pcm_close)(struct azx *chip, struct azx_dev *azx_dev);
};
struct azx_pcm {
@@ -206,6 +207,15 @@ int azx_bus_init(struct azx *chip, const char *model);
int azx_probe_codecs(struct azx *chip, unsigned int max_slots);
int azx_codec_configure(struct azx *chip);
int azx_init_streams(struct azx *chip);
+void azx_add_stream(struct azx *chip, struct azx_dev *s, int idx, int tag);
void azx_free_streams(struct azx *chip);
+static inline int azx_stream_direction(struct azx *chip, unsigned char index)
+{
+ if (index >= chip->capture_index_offset &&
+ index < chip->capture_index_offset + chip->capture_streams)
+ return SNDRV_PCM_STREAM_CAPTURE;
+ return SNDRV_PCM_STREAM_PLAYBACK;
+}
+
#endif /* __SOUND_HDA_CONTROLLER_H */
diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c
index c87d75d..4b03c64 100644
--- a/sound/hda/controllers/intel.c
+++ b/sound/hda/controllers/intel.c
@@ -615,17 +615,17 @@ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev);
/* called from IRQ */
static int azx_position_check(struct azx *chip, struct azx_dev *azx_dev)
{
- struct hda_intel *hda = container_of(chip, struct hda_intel, chip);
+ struct hda_intel_stream *istream = azx_dev_to_istream(azx_dev);
int ok;
ok = azx_position_ok(chip, azx_dev);
if (ok == 1) {
- azx_dev->irq_pending = 0;
+ istream->irq_pending = false;
return ok;
} else if (ok == 0) {
/* bogus IRQ, process it later */
- azx_dev->irq_pending = 1;
- schedule_work(&hda->irq_pending_work);
+ istream->irq_pending = true;
+ schedule_work(&istream->irq_pending_work);
}
return 0;
}
@@ -721,11 +721,13 @@ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev)
*/
static void azx_irq_pending_work(struct work_struct *work)
{
- struct hda_intel *hda = container_of(work, struct hda_intel, irq_pending_work);
+ struct hda_intel_stream *istream =
+ container_of(work, struct hda_intel_stream, irq_pending_work);
+ struct azx_dev *azx_dev = &istream->azx_dev;
+ struct hda_intel *hda = istream->hda;
struct azx *chip = &hda->chip;
struct hdac_bus *bus = azx_bus(chip);
- struct hdac_stream *s;
- int pending, ok;
+ int ok;
if (!hda->irq_pending_warned) {
dev_info(chip->card->dev,
@@ -735,42 +737,51 @@ static void azx_irq_pending_work(struct work_struct *work)
}
for (;;) {
- pending = 0;
- spin_lock_irq(&bus->reg_lock);
- list_for_each_entry(s, &bus->stream_list, list) {
- struct azx_dev *azx_dev = stream_to_azx_dev(s);
- if (!azx_dev->irq_pending ||
- !s->substream ||
- !s->running)
- continue;
+ scoped_guard(spinlock_irq, &bus->reg_lock) {
+ if (!istream->irq_pending ||
+ !azx_dev->core.substream ||
+ !azx_dev->core.running) {
+ return;
+ }
+
ok = azx_position_ok(chip, azx_dev);
- if (ok > 0) {
- azx_dev->irq_pending = 0;
- spin_unlock(&bus->reg_lock);
- snd_pcm_period_elapsed(s->substream);
- spin_lock(&bus->reg_lock);
- } else if (ok < 0) {
- pending = 0; /* too early */
- } else
- pending++;
+ if (ok < 0)
+ return; /* too early */
+ if (ok > 0)
+ istream->irq_pending = false;
}
- spin_unlock_irq(&bus->reg_lock);
- if (!pending)
+
+ if (ok) {
+ snd_pcm_period_elapsed(azx_dev->core.substream);
return;
+ }
+
msleep(1);
}
}
/* clear irq_pending flags and assure no on-going workq */
+static void hda_intel_stream_clear_irq_pending(struct azx_dev *azx_dev)
+{
+ struct hda_intel_stream *istream = azx_dev_to_istream(azx_dev);
+
+ istream->irq_pending = false;
+ cancel_work_sync(&istream->irq_pending_work);
+}
+
+/* called at PCM close */
+static void hda_intel_pcm_close(struct azx *chip, struct azx_dev *azx_dev)
+{
+ hda_intel_stream_clear_irq_pending(azx_dev);
+}
+
static void azx_clear_irq_pending(struct azx *chip)
{
struct hdac_bus *bus = azx_bus(chip);
struct hdac_stream *s;
- guard(spinlock_irq)(&bus->reg_lock);
list_for_each_entry(s, &bus->stream_list, list) {
- struct azx_dev *azx_dev = stream_to_azx_dev(s);
- azx_dev->irq_pending = 0;
+ hda_intel_stream_clear_irq_pending(stream_to_azx_dev(s));
}
}
@@ -1797,7 +1808,6 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
if (jackpoll_ms[dev] >= 50 && jackpoll_ms[dev] <= 60000)
chip->jackpoll_interval = msecs_to_jiffies(jackpoll_ms[dev]);
INIT_LIST_HEAD(&chip->pcm_list);
- INIT_WORK(&hda->irq_pending_work, azx_irq_pending_work);
INIT_LIST_HEAD(&hda->list);
init_vga_switcheroo(chip);
init_completion(&hda->probe_wait);
@@ -1846,6 +1856,39 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
return 0;
}
+/* create and assign streams */
+static int hda_init_streams(struct azx *chip)
+{
+ int i;
+ int stream_tags[2] = { 0, 0 };
+
+ for (i = 0; i < chip->num_streams; i++) {
+ struct hda_intel_stream *s = kzalloc_obj(*s);
+ int tag, dir;
+
+ if (!s)
+ return -ENOMEM;
+
+ s->hda = container_of(chip, struct hda_intel, chip);
+ INIT_WORK(&s->irq_pending_work, azx_irq_pending_work);
+
+ /* stream tag must be unique throughout
+ * the stream direction group,
+ * valid values 1...15
+ * use separate stream tag if the flag
+ * AZX_DCAPS_SEPARATE_STREAM_TAG is used
+ */
+ dir = azx_stream_direction(chip, i);
+ if (chip->driver_caps & AZX_DCAPS_SEPARATE_STREAM_TAG)
+ tag = ++stream_tags[dir];
+ else
+ tag = i + 1;
+ azx_add_stream(chip, &s->azx_dev, i, tag);
+ }
+
+ return 0;
+}
+
static int azx_first_init(struct azx *chip)
{
int dev = chip->dev_index;
@@ -2000,7 +2043,7 @@ static int azx_first_init(struct azx *chip)
}
/* initialize streams */
- err = azx_init_streams(chip);
+ err = hda_init_streams(chip);
if (err < 0)
return err;
@@ -2099,6 +2142,7 @@ static const struct dmi_system_id driver_denylist_dmi[] = {
static const struct hda_controller_ops pci_hda_ops = {
.disable_msi_reset_irq = disable_msi_reset_irq,
.position_check = azx_position_check,
+ .pcm_close = hda_intel_pcm_close,
};
static DECLARE_BITMAP(probed_devs, SNDRV_CARDS);
diff --git a/sound/hda/controllers/intel.h b/sound/hda/controllers/intel.h
index 2d1725f..4efb3b0 100644
--- a/sound/hda/controllers/intel.h
+++ b/sound/hda/controllers/intel.h
@@ -9,9 +9,6 @@
struct hda_intel {
struct azx chip;
- /* for pending irqs */
- struct work_struct irq_pending_work;
-
/* sync probing */
struct completion probe_wait;
struct delayed_work probe_work;
@@ -35,4 +32,16 @@ struct hda_intel {
int probe_retry; /* being probe-retry */
};
+struct hda_intel_stream {
+ struct azx_dev azx_dev;
+
+ /* for pending irqs */
+ struct hda_intel *hda;
+ struct work_struct irq_pending_work;
+ bool irq_pending;
+};
+
+#define azx_dev_to_istream(azx_dev) \
+ container_of(azx_dev, struct hda_intel_stream, azx_dev)
+
#endif
diff --git a/sound/hda/core/hdmi_chmap.c b/sound/hda/core/hdmi_chmap.c
index 7b276047..c897fc4 100644
--- a/sound/hda/core/hdmi_chmap.c
+++ b/sound/hda/core/hdmi_chmap.c
@@ -353,13 +353,16 @@ static void hdmi_std_setup_channel_mapping(struct hdac_chmap *chmap,
if (hdmi_channel_mapping[ca][1] == 0) {
int hdmi_slot = 0;
/* fill actual channel mappings in ALSA channel (i) order */
- for (i = 0; i < ch_alloc->channels; i++) {
- while (!WARN_ON(hdmi_slot >= 8) &&
- !ch_alloc->speakers[7 - hdmi_slot])
- hdmi_slot++; /* skip zero slots */
+ for (i = 0; i < ch_alloc->channels && hdmi_slot < 8; i++) {
+ while (!ch_alloc->speakers[7 - hdmi_slot]) {
+ /* skip zero slots */
+ if (++hdmi_slot >= 8)
+ goto out;
+ }
hdmi_channel_mapping[ca][i] = (i << 4) | hdmi_slot++;
}
+ out:
/* fill the rest of the slots with ALSA channel 0xf */
for (hdmi_slot = 0; hdmi_slot < 8; hdmi_slot++)
if (!ch_alloc->speakers[7 - hdmi_slot])
diff --git a/sound/pci/asihpi/hpicmn.c b/sound/pci/asihpi/hpicmn.c
index d846777..19f0da2 100644
--- a/sound/pci/asihpi/hpicmn.c
+++ b/sound/pci/asihpi/hpicmn.c
@@ -276,6 +276,12 @@ static short find_control(u16 control_index,
return 0;
}
+ if (control_index >= p_cache->control_count) {
+ HPI_DEBUG_LOG(VERBOSE, "control_index out of bounce %d\n",
+ control_index);
+ return 0;
+ }
+
*pI = p_cache->p_info[control_index];
if (!*pI) {
HPI_DEBUG_LOG(VERBOSE, "Uncached Control %d\n",
diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c
index 1604ed6..309dc9e 100644
--- a/sound/soc/amd/acp-config.c
+++ b/sound/soc/amd/acp-config.c
@@ -30,6 +30,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = {
DMI_MATCH(DMI_PRODUCT_NAME, "HN7306EA"),
},
},
+ {
+ /* ASUS Zenbook S16 UM5606GA (Strix Point, ACP 7.0) */
+ .matches = {
+ DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook S16 UM5606GA"),
+ },
+ },
{}
};
diff --git a/sound/soc/amd/acp/acp-legacy-mach.c b/sound/soc/amd/acp/acp-legacy-mach.c
index a7a5513..235d6cc 100644
--- a/sound/soc/amd/acp/acp-legacy-mach.c
+++ b/sound/soc/amd/acp/acp-legacy-mach.c
@@ -174,7 +174,7 @@ static int acp_asoc_probe(struct platform_device *pdev)
acp_card_drvdata->acp_rev = mach->mach_params.subsystem_rev;
dmi_id = dmi_first_match(acp_quirk_table);
- if (dmi_id && dmi_id->driver_data)
+ if (dmi_id && dmi_id->driver_data == (void *)QUIRK_TDM_MODE_ENABLE)
acp_card_drvdata->tdm_mode = dmi_id->driver_data;
ret = acp_legacy_dai_links_create(card);
diff --git a/sound/soc/amd/acp/acp-mach-common.c b/sound/soc/amd/acp/acp-mach-common.c
index 09f6c9a..ef784cc 100644
--- a/sound/soc/amd/acp/acp-mach-common.c
+++ b/sound/soc/amd/acp/acp-mach-common.c
@@ -20,6 +20,7 @@
#include <sound/soc.h>
#include <linux/input.h>
#include <linux/module.h>
+#include <linux/dmi.h>
#include "../../codecs/rt5682.h"
#include "../../codecs/rt1019.h"
@@ -37,15 +38,21 @@
#define NAU8821_FREQ_OUT 12288000
#define MAX98388_CODEC_DAI "max98388-aif1"
-#define TDM_MODE_ENABLE 1
-
const struct dmi_system_id acp_quirk_table[] = {
{
/* Google skyrim proto-0 */
.matches = {
DMI_EXACT_MATCH(DMI_PRODUCT_FAMILY, "Google_Skyrim"),
},
- .driver_data = (void *)TDM_MODE_ENABLE,
+ .driver_data = (void *)QUIRK_TDM_MODE_ENABLE,
+ },
+ {
+ /* Valve Steam Deck OLED */
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Valve"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Galileo"),
+ },
+ .driver_data = (void *)QUIRK_REMAP_DMIC_BT,
},
{}
};
@@ -1401,6 +1408,7 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card)
struct snd_soc_dai_link *links;
struct device *dev = card->dev;
struct acp_card_drvdata *drv_data = card->drvdata;
+ const struct dmi_system_id *dmi_id = dmi_first_match(acp_quirk_table);
int i = 0, num_links = 0;
if (drv_data->hs_cpu_id)
@@ -1572,6 +1580,9 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card)
links[i].codecs = &snd_soc_dummy_dlc;
links[i].num_codecs = 1;
}
+
+ if (dmi_id && dmi_id->driver_data == (void *)QUIRK_REMAP_DMIC_BT)
+ links[i].id = DMIC_BE_ID;
i++;
}
@@ -1587,6 +1598,11 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card)
links[i].capture_only = 1;
links[i].nonatomic = true;
links[i].no_pcm = 1;
+
+ if (dmi_id && dmi_id->driver_data == (void *)QUIRK_REMAP_DMIC_BT) {
+ links[i].id = BT_BE_ID;
+ dev_dbg(dev, "quirk REMAP_DMIC_BT enabled\n");
+ }
}
card->dai_link = links;
diff --git a/sound/soc/amd/acp/acp-mach.h b/sound/soc/amd/acp/acp-mach.h
index f94c30c..7177d3f 100644
--- a/sound/soc/amd/acp/acp-mach.h
+++ b/sound/soc/amd/acp/acp-mach.h
@@ -26,6 +26,10 @@
#define acp_get_drvdata(card) ((struct acp_card_drvdata *)(card)->drvdata)
+/* List of DMI quirks - check acp-mach-common.c for usage. */
+#define QUIRK_TDM_MODE_ENABLE 1
+#define QUIRK_REMAP_DMIC_BT 2
+
enum be_id {
HEADSET_BE_ID = 0,
AMP_BE_ID,
diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
index 0f21e5f..09b475c 100644
--- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
@@ -260,9 +260,9 @@ static int create_sdw_dailink(struct snd_soc_card *card,
cpus->dai_name = devm_kasprintf(dev, GFP_KERNEL,
"SDW%d Pin%d",
link_num, cpu_pin_id);
- dev_dbg(dev, "cpu->dai_name:%s\n", cpus->dai_name);
if (!cpus->dai_name)
return -ENOMEM;
+ dev_dbg(dev, "cpu->dai_name:%s\n", cpus->dai_name);
codec_maps[j].cpu = 0;
codec_maps[j].codec = j;
diff --git a/sound/soc/amd/acp/acp-sof-mach.c b/sound/soc/amd/acp/acp-sof-mach.c
index 6215e31..36ecef7 100644
--- a/sound/soc/amd/acp/acp-sof-mach.c
+++ b/sound/soc/amd/acp/acp-sof-mach.c
@@ -110,7 +110,7 @@ static int acp_sof_probe(struct platform_device *pdev)
acp_card_drvdata = card->drvdata;
dmi_id = dmi_first_match(acp_quirk_table);
- if (dmi_id && dmi_id->driver_data)
+ if (dmi_id && dmi_id->driver_data == (void *)QUIRK_TDM_MODE_ENABLE)
acp_card_drvdata->tdm_mode = dmi_id->driver_data;
acp_card_drvdata->acp_rev = mach->mach_params.subsystem_rev;
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index c5cf458..7a637d6 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -62,6 +62,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
{
.driver_data = &acp6x_card,
.matches = {
+ DMI_MATCH(DMI_BOARD_VENDOR, "HP"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "OMEN Gaming Laptop 16-ap0xxx"),
+ }
+ },
+ {
+ .driver_data = &acp6x_card,
+ .matches = {
DMI_MATCH(DMI_BOARD_VENDOR, "Dell Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "Dell G15 5525"),
}
@@ -483,6 +490,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
.driver_data = &acp6x_card,
.matches = {
DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 15 C7VE"),
+ }
+ },
+ {
+ .driver_data = &acp6x_card,
+ .matches = {
+ DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."),
DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 15 C7VF"),
}
},
@@ -671,6 +685,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
{
.driver_data = &acp6x_card,
.matches = {
+ DMI_MATCH(DMI_BOARD_VENDOR, "HP"),
+ DMI_MATCH(DMI_BOARD_NAME, "8E35"),
+ }
+ },
+ {
+ .driver_data = &acp6x_card,
+ .matches = {
DMI_MATCH(DMI_BOARD_VENDOR, "MECHREVO"),
DMI_MATCH(DMI_BOARD_NAME, "MRID6"),
}
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index cf94a1c..269c31c 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -898,7 +898,6 @@
config SND_SOC_CS35L56_CAL_DEBUGFS
bool "CS35L56 create debugfs for factory calibration"
- default N
depends on DEBUG_FS
select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
help
@@ -909,7 +908,6 @@
config SND_SOC_CS35L56_CAL_SET_CTRL
bool "CS35L56 ALSA control to restore factory calibration"
- default N
select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
help
Allow restoring factory calibration data through an ALSA
@@ -923,7 +921,6 @@
config SND_SOC_CS35L56_CAL_PERFORM_CTRL
bool "CS35L56 ALSA control to perform factory calibration"
- default N
select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
help
Allow performing factory calibration data through an ALSA
diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c
index fdda1b7..6e8ef9c 100644
--- a/sound/soc/codecs/ab8500-codec.c
+++ b/sound/soc/codecs/ab8500-codec.c
@@ -60,19 +60,6 @@
low before proceeding with the configuration sequence */
#define AB8500_ANC_SM_DELAY 2000
-#define AB8500_FILTER_CONTROL(xname, xcount, xmin, xmax) \
-{ .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \
- .info = filter_control_info, \
- .get = filter_control_get, .put = filter_control_put, \
- .private_value = (unsigned long)&(struct filter_control) \
- {.count = xcount, .min = xmin, .max = xmax} }
-
-struct filter_control {
- long min, max;
- unsigned int count;
- long value[128];
-};
-
/* Sidetone states */
static const char * const enum_sid_state[] = {
"Unconfigured",
@@ -85,45 +72,13 @@ enum sid_state {
SID_FIR_CONFIGURED = 2,
};
-static const char * const enum_anc_state[] = {
- "Unconfigured",
- "Apply FIR and IIR",
- "FIR and IIR are configured",
- "Apply FIR",
- "FIR is configured",
- "Apply IIR",
- "IIR is configured"
-};
-enum anc_state {
- ANC_UNCONFIGURED = 0,
- ANC_APPLY_FIR_IIR = 1,
- ANC_FIR_IIR_CONFIGURED = 2,
- ANC_APPLY_FIR = 3,
- ANC_FIR_CONFIGURED = 4,
- ANC_APPLY_IIR = 5,
- ANC_IIR_CONFIGURED = 6
-};
-
-/* Analog microphones */
-enum amic_idx {
- AMIC_IDX_1A,
- AMIC_IDX_1B,
- AMIC_IDX_2
-};
-
/* Private data for AB8500 device-driver */
struct ab8500_codec_drvdata {
struct regmap *regmap;
struct mutex ctrl_lock;
/* Sidetone */
- long *sid_fir_values;
enum sid_state sid_status;
-
- /* ANC */
- long *anc_fir_values;
- long *anc_iir_values;
- enum anc_state anc_status;
};
static inline const char *amic_micbias_str(enum amic_micbias micbias)
@@ -1024,89 +979,6 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes_mic2_vamicx[] = {
{"MIC2 V-AMICx Enable", NULL, "V-AMIC2"},
};
-/* ANC FIR-coefficients configuration sequence */
-static void anc_fir(struct snd_soc_component *component,
- unsigned int bnk, unsigned int par, unsigned int val)
-{
- if (par == 0 && bnk == 0)
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCFIRUPDATE),
- BIT(AB8500_ANCCONF1_ANCFIRUPDATE));
-
- snd_soc_component_write(component, AB8500_ANCCONF5, val >> 8 & 0xff);
- snd_soc_component_write(component, AB8500_ANCCONF6, val & 0xff);
-
- if (par == AB8500_ANC_FIR_COEFFS - 1 && bnk == 1)
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCFIRUPDATE), 0);
-}
-
-/* ANC IIR-coefficients configuration sequence */
-static void anc_iir(struct snd_soc_component *component, unsigned int bnk,
- unsigned int par, unsigned int val)
-{
- if (par == 0) {
- if (bnk == 0) {
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCIIRINIT),
- BIT(AB8500_ANCCONF1_ANCIIRINIT));
- usleep_range(AB8500_ANC_SM_DELAY, AB8500_ANC_SM_DELAY*2);
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCIIRINIT), 0);
- usleep_range(AB8500_ANC_SM_DELAY, AB8500_ANC_SM_DELAY*2);
- } else {
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCIIRUPDATE),
- BIT(AB8500_ANCCONF1_ANCIIRUPDATE));
- }
- } else if (par > 3) {
- snd_soc_component_write(component, AB8500_ANCCONF7, 0);
- snd_soc_component_write(component, AB8500_ANCCONF8, val >> 16 & 0xff);
- }
-
- snd_soc_component_write(component, AB8500_ANCCONF7, val >> 8 & 0xff);
- snd_soc_component_write(component, AB8500_ANCCONF8, val & 0xff);
-
- if (par == AB8500_ANC_IIR_COEFFS - 1 && bnk == 1)
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ANCIIRUPDATE), 0);
-}
-
-/* ANC IIR-/FIR-coefficients configuration sequence */
-static void anc_configure(struct snd_soc_component *component,
- bool apply_fir, bool apply_iir)
-{
- struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev);
- unsigned int bnk, par, val;
-
- dev_dbg(component->dev, "%s: Enter.\n", __func__);
-
- if (apply_fir)
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ENANC), 0);
-
- snd_soc_component_update_bits(component, AB8500_ANCCONF1,
- BIT(AB8500_ANCCONF1_ENANC), BIT(AB8500_ANCCONF1_ENANC));
-
- if (apply_fir)
- for (bnk = 0; bnk < AB8500_NR_OF_ANC_COEFF_BANKS; bnk++)
- for (par = 0; par < AB8500_ANC_FIR_COEFFS; par++) {
- val = snd_soc_component_read(component,
- drvdata->anc_fir_values[par]);
- anc_fir(component, bnk, par, val);
- }
-
- if (apply_iir)
- for (bnk = 0; bnk < AB8500_NR_OF_ANC_COEFF_BANKS; bnk++)
- for (par = 0; par < AB8500_ANC_IIR_COEFFS; par++) {
- val = snd_soc_component_read(component,
- drvdata->anc_iir_values[par]);
- anc_iir(component, bnk, par, val);
- }
-
- dev_dbg(component->dev, "%s: Exit.\n", __func__);
-}
-
/*
* Control-events
*/
@@ -1130,7 +1002,7 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
{
struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev);
- unsigned int param, sidconf, val;
+ unsigned int param, sidconf;
int status = 1;
dev_dbg(component->dev, "%s: Enter\n", __func__);
@@ -1159,9 +1031,8 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
snd_soc_component_write(component, AB8500_SIDFIRADR, 0);
for (param = 0; param < AB8500_SID_FIR_COEFFS; param++) {
- val = snd_soc_component_read(component, drvdata->sid_fir_values[param]);
- snd_soc_component_write(component, AB8500_SIDFIRCOEF1, val >> 8 & 0xff);
- snd_soc_component_write(component, AB8500_SIDFIRCOEF2, val & 0xff);
+ snd_soc_component_write(component, AB8500_SIDFIRCOEF1, 0);
+ snd_soc_component_write(component, AB8500_SIDFIRCOEF2, 0);
}
snd_soc_component_update_bits(component, AB8500_SIDFIRADR,
@@ -1180,136 +1051,6 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
return status;
}
-static int anc_status_control_get(struct snd_kcontrol *kcontrol,
- struct snd_ctl_elem_value *ucontrol)
-{
- struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev);
-
- mutex_lock(&drvdata->ctrl_lock);
- ucontrol->value.enumerated.item[0] = drvdata->anc_status;
- mutex_unlock(&drvdata->ctrl_lock);
-
- return 0;
-}
-
-static int anc_status_control_put(struct snd_kcontrol *kcontrol,
- struct snd_ctl_elem_value *ucontrol)
-{
- struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
- struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev);
- struct device *dev = component->dev;
- bool apply_fir, apply_iir;
- unsigned int req;
- int status;
-
- dev_dbg(dev, "%s: Enter.\n", __func__);
-
- mutex_lock(&drvdata->ctrl_lock);
-
- req = ucontrol->value.enumerated.item[0];
- if (req >= ARRAY_SIZE(enum_anc_state)) {
- status = -EINVAL;
- goto cleanup;
- }
- if (req != ANC_APPLY_FIR_IIR && req != ANC_APPLY_FIR &&
- req != ANC_APPLY_IIR) {
- dev_err(dev, "%s: ERROR: Unsupported status to set '%s'!\n",
- __func__, enum_anc_state[req]);
- status = -EINVAL;
- goto cleanup;
- }
- apply_fir = req == ANC_APPLY_FIR || req == ANC_APPLY_FIR_IIR;
- apply_iir = req == ANC_APPLY_IIR || req == ANC_APPLY_FIR_IIR;
-
- status = snd_soc_dapm_force_enable_pin(dapm, "ANC Configure Input");
- if (status < 0) {
- dev_err(dev,
- "%s: ERROR: Failed to enable power (status = %d)!\n",
- __func__, status);
- goto cleanup;
- }
- snd_soc_dapm_sync(dapm);
-
- anc_configure(component, apply_fir, apply_iir);
-
- if (apply_fir) {
- if (drvdata->anc_status == ANC_IIR_CONFIGURED)
- drvdata->anc_status = ANC_FIR_IIR_CONFIGURED;
- else if (drvdata->anc_status != ANC_FIR_IIR_CONFIGURED)
- drvdata->anc_status = ANC_FIR_CONFIGURED;
- }
- if (apply_iir) {
- if (drvdata->anc_status == ANC_FIR_CONFIGURED)
- drvdata->anc_status = ANC_FIR_IIR_CONFIGURED;
- else if (drvdata->anc_status != ANC_FIR_IIR_CONFIGURED)
- drvdata->anc_status = ANC_IIR_CONFIGURED;
- }
-
- status = snd_soc_dapm_disable_pin(dapm, "ANC Configure Input");
- snd_soc_dapm_sync(dapm);
-
-cleanup:
- mutex_unlock(&drvdata->ctrl_lock);
-
- if (status < 0)
- dev_err(dev, "%s: Unable to configure ANC! (status = %d)\n",
- __func__, status);
-
- dev_dbg(dev, "%s: Exit.\n", __func__);
-
- return (status < 0) ? status : 1;
-}
-
-static int filter_control_info(struct snd_kcontrol *kcontrol,
- struct snd_ctl_elem_info *uinfo)
-{
- struct filter_control *fc =
- (struct filter_control *)kcontrol->private_value;
-
- uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
- uinfo->count = fc->count;
- uinfo->value.integer.min = fc->min;
- uinfo->value.integer.max = fc->max;
-
- return 0;
-}
-
-static int filter_control_get(struct snd_kcontrol *kcontrol,
- struct snd_ctl_elem_value *ucontrol)
-{
- struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct ab8500_codec_drvdata *drvdata = snd_soc_component_get_drvdata(component);
- struct filter_control *fc =
- (struct filter_control *)kcontrol->private_value;
- unsigned int i;
-
- mutex_lock(&drvdata->ctrl_lock);
- for (i = 0; i < fc->count; i++)
- ucontrol->value.integer.value[i] = fc->value[i];
- mutex_unlock(&drvdata->ctrl_lock);
-
- return 0;
-}
-
-static int filter_control_put(struct snd_kcontrol *kcontrol,
- struct snd_ctl_elem_value *ucontrol)
-{
- struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct ab8500_codec_drvdata *drvdata = snd_soc_component_get_drvdata(component);
- struct filter_control *fc =
- (struct filter_control *)kcontrol->private_value;
- unsigned int i;
-
- mutex_lock(&drvdata->ctrl_lock);
- for (i = 0; i < fc->count; i++)
- fc->value[i] = ucontrol->value.integer.value[i];
- mutex_unlock(&drvdata->ctrl_lock);
-
- return 0;
-}
-
/*
* Controls - Non-DAPM ASoC
*/
@@ -1597,7 +1338,6 @@ static SOC_ENUM_SINGLE_DECL(soc_enum_bfifomast,
static SOC_ENUM_SINGLE_EXT_DECL(soc_enum_sidstate, enum_sid_state);
/* ANC */
-static SOC_ENUM_SINGLE_EXT_DECL(soc_enum_ancstate, enum_anc_state);
static struct snd_kcontrol_new ab8500_ctrls[] = {
/* Charge pump */
@@ -1873,8 +1613,6 @@ static struct snd_kcontrol_new ab8500_ctrls[] = {
AB8500_FIFOCONF6_BFIFOSAMPLE_MAX, 0),
/* ANC */
- SOC_ENUM_EXT("ANC Status", soc_enum_ancstate,
- anc_status_control_get, anc_status_control_put),
SOC_SINGLE_XR_SX("ANC Warp Delay Shift",
AB8500_ANCCONF2, 1, AB8500_ANCCONF2_SHIFT,
AB8500_ANCCONF2_MIN, AB8500_ANCCONF2_MAX, 0),
@@ -1895,21 +1633,6 @@ static struct snd_kcontrol_new ab8500_ctrls[] = {
AB8500_SIDFIRADR, AB8500_SIDFIRADR_FIRSIDSET, 0),
};
-static struct snd_kcontrol_new ab8500_filter_controls[] = {
- AB8500_FILTER_CONTROL("ANC FIR Coefficients", AB8500_ANC_FIR_COEFFS,
- AB8500_ANC_FIR_COEFF_MIN, AB8500_ANC_FIR_COEFF_MAX),
- AB8500_FILTER_CONTROL("ANC IIR Coefficients", AB8500_ANC_IIR_COEFFS,
- AB8500_ANC_IIR_COEFF_MIN, AB8500_ANC_IIR_COEFF_MAX),
- AB8500_FILTER_CONTROL("Sidetone FIR Coefficients",
- AB8500_SID_FIR_COEFFS, AB8500_SID_FIR_COEFF_MIN,
- AB8500_SID_FIR_COEFF_MAX)
-};
-enum ab8500_filter {
- AB8500_FILTER_ANC_FIR = 0,
- AB8500_FILTER_ANC_IIR = 1,
- AB8500_FILTER_SID_FIR = 2,
-};
-
/*
* Extended interface for codec-driver
*/
@@ -2454,7 +2177,6 @@ static int ab8500_codec_probe(struct snd_soc_component *component)
struct device_node *np = dev->of_node;
struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(dev);
struct ab8500_codec_platform_data codec_pdata;
- struct filter_control *fc;
int status;
dev_dbg(dev, "%s: Enter.\n", __func__);
@@ -2486,25 +2208,6 @@ static int ab8500_codec_probe(struct snd_soc_component *component)
snd_soc_component_write(component, AB8500_SHORTCIRCONF,
BIT(AB8500_SHORTCIRCONF_HSZCDDIS));
- /* Add filter controls */
- status = snd_soc_add_component_controls(component, ab8500_filter_controls,
- ARRAY_SIZE(ab8500_filter_controls));
- if (status < 0) {
- dev_err(dev,
- "%s: failed to add ab8500 filter controls (%d).\n",
- __func__, status);
- return status;
- }
- fc = (struct filter_control *)
- &ab8500_filter_controls[AB8500_FILTER_ANC_FIR].private_value;
- drvdata->anc_fir_values = (long *)fc->value;
- fc = (struct filter_control *)
- &ab8500_filter_controls[AB8500_FILTER_ANC_IIR].private_value;
- drvdata->anc_iir_values = (long *)fc->value;
- fc = (struct filter_control *)
- &ab8500_filter_controls[AB8500_FILTER_SID_FIR].private_value;
- drvdata->sid_fir_values = (long *)fc->value;
-
snd_soc_dapm_disable_pin(dapm, "ANC Configure Input");
mutex_init(&drvdata->ctrl_lock);
@@ -2538,7 +2241,6 @@ static int ab8500_codec_driver_probe(struct platform_device *pdev)
if (!drvdata)
return -ENOMEM;
drvdata->sid_status = SID_UNCONFIGURED;
- drvdata->anc_status = ANC_UNCONFIGURED;
dev_set_drvdata(&pdev->dev, drvdata);
drvdata->regmap = devm_regmap_init(&pdev->dev, NULL, &pdev->dev,
diff --git a/sound/soc/codecs/aw88395/aw88395.c b/sound/soc/codecs/aw88395/aw88395.c
index 3602b5b..dd09bac 100644
--- a/sound/soc/codecs/aw88395/aw88395.c
+++ b/sound/soc/codecs/aw88395/aw88395.c
@@ -456,8 +456,6 @@ static void aw88395_hw_reset(struct aw88395 *aw88395)
usleep_range(AW88395_1000_US, AW88395_1000_US + 10);
gpiod_set_value_cansleep(aw88395->reset_gpio, 1);
usleep_range(AW88395_1000_US, AW88395_1000_US + 10);
- } else {
- dev_err(aw88395->aw_pa->dev, "%s failed", __func__);
}
}
@@ -522,9 +520,10 @@ static int aw88395_i2c_probe(struct i2c_client *i2c)
i2c_set_clientdata(i2c, aw88395);
aw88395->reset_gpio = devm_gpiod_get_optional(&i2c->dev, "reset", GPIOD_OUT_LOW);
- if (IS_ERR(aw88395->reset_gpio))
- dev_info(&i2c->dev, "reset gpio not defined\n");
-
+ if (IS_ERR(aw88395->reset_gpio)) {
+ return dev_err_probe(&i2c->dev, PTR_ERR(aw88395->reset_gpio),
+ "failed to get reset gpio\n");
+ }
/* hardware reset */
aw88395_hw_reset(aw88395);
diff --git a/sound/soc/codecs/cs-amp-lib.c b/sound/soc/codecs/cs-amp-lib.c
index b34b1f5..fb5b950 100644
--- a/sound/soc/codecs/cs-amp-lib.c
+++ b/sound/soc/codecs/cs-amp-lib.c
@@ -118,7 +118,7 @@ static int cs_amp_read_cal_coeff(struct cs_dsp *dsp,
}
if (ret < 0) {
- dev_err(dsp->dev, "Failed to write to '%s': %d\n", ctl_name, ret);
+ dev_err(dsp->dev, "Failed to read '%s': %d\n", ctl_name, ret);
return ret;
}
@@ -500,7 +500,7 @@ static int _cs_amp_set_efi_calibration_data(struct device *dev, int amp_index, i
* must be set.
*/
if (data->count == 0)
- data->count = (data->size - sizeof(data)) / sizeof(data->data[0]);
+ data->count = (data->size - struct_offset(data, data)) / sizeof(data->data[0]);
if (amp_index < 0) {
/* Is there already a slot for this target? */
@@ -833,11 +833,18 @@ EXPORT_SYMBOL_NS_GPL(cs_amp_devm_get_vendor_specific_variant_id, "SND_SOC_CS_AMP
*/
struct dentry *cs_amp_create_debugfs(struct device *dev)
{
- struct dentry *dir;
+ struct dentry *dir, *created;
+ /* debugfs_lookup() can return NULL or ERR_PTR on error */
dir = debugfs_lookup("cirrus_logic", NULL);
- if (!dir)
- dir = debugfs_create_dir("cirrus_logic", NULL);
+ if (!IS_ERR_OR_NULL(dir)) {
+ created = debugfs_create_dir(dev_name(dev), dir);
+ dput(dir);
+
+ return created;
+ }
+
+ dir = debugfs_create_dir("cirrus_logic", NULL);
return debugfs_create_dir(dev_name(dev), dir);
}
diff --git a/sound/soc/codecs/cs35l56-sdw.c b/sound/soc/codecs/cs35l56-sdw.c
index 9dc47fe..88e0aac 100644
--- a/sound/soc/codecs/cs35l56-sdw.c
+++ b/sound/soc/codecs/cs35l56-sdw.c
@@ -385,18 +385,19 @@ static int cs35l56_sdw_update_status(struct sdw_slave *peripheral,
switch (status) {
case SDW_SLAVE_ATTACHED:
- dev_dbg(cs35l56->base.dev, "%s: ATTACHED\n", __func__);
cs35l56->sdw_in_clock_stop_1 = false;
if (cs35l56->sdw_attached)
break;
+ dev_dbg(cs35l56->base.dev, "%s: ATTACHED\n", __func__);
if (!cs35l56->base.init_done || cs35l56->soft_resetting)
cs35l56_sdw_init(peripheral);
cs35l56->sdw_attached = true;
break;
case SDW_SLAVE_UNATTACHED:
- dev_dbg(cs35l56->base.dev, "%s: UNATTACHED\n", __func__);
+ if (cs35l56->sdw_attached)
+ dev_dbg(cs35l56->base.dev, "%s: UNATTACHED\n", __func__);
cs35l56->sdw_attached = false;
break;
default:
@@ -584,10 +585,11 @@ static void cs35l56_sdw_remove(struct sdw_slave *peripheral)
/* Disable SoundWire interrupts */
cs35l56->sdw_irq_no_unmask = true;
- cancel_work_sync(&cs35l56->sdw_irq_work);
+ flush_work(&cs35l56->sdw_irq_work);
sdw_write_no_pm(peripheral, CS35L56_SDW_GEN_INT_MASK_1, 0);
sdw_read_no_pm(peripheral, CS35L56_SDW_GEN_INT_STAT_1);
sdw_write_no_pm(peripheral, CS35L56_SDW_GEN_INT_STAT_1, 0xFF);
+ flush_work(&cs35l56->sdw_irq_work);
cs35l56_remove(cs35l56);
}
diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c
index e05d975b..795e276 100644
--- a/sound/soc/codecs/cs35l56-shared.c
+++ b/sound/soc/codecs/cs35l56-shared.c
@@ -108,8 +108,6 @@ int cs35l56_set_patch(struct cs35l56_base *cs35l56_base)
EXPORT_SYMBOL_NS_GPL(cs35l56_set_patch, "SND_SOC_CS35L56_SHARED");
static const struct reg_default cs35l56_reg_defaults[] = {
- /* no defaults for OTP_MEM - first read populates cache */
-
{ CS35L56_ASP1_ENABLES1, 0x00000000 },
{ CS35L56_ASP1_CONTROL1, 0x00000028 },
{ CS35L56_ASP1_CONTROL2, 0x18180200 },
@@ -138,8 +136,6 @@ static const struct reg_default cs35l56_reg_defaults[] = {
};
static const struct reg_default cs35l63_reg_defaults[] = {
- /* no defaults for OTP_MEM - first read populates cache */
-
{ CS35L56_ASP1_ENABLES1, 0x00000000 },
{ CS35L56_ASP1_CONTROL1, 0x00000028 },
{ CS35L56_ASP1_CONTROL2, 0x18180200 },
@@ -282,6 +278,9 @@ static bool cs35l56_common_volatile_reg(unsigned int reg)
case CS35L56_GLOBAL_ENABLES: /* owned by firmware */
case CS35L56_BLOCK_ENABLES: /* owned by firmware */
case CS35L56_BLOCK_ENABLES2: /* owned by firmware */
+ case CS35L56_OTP_MEM_53:
+ case CS35L56_OTP_MEM_54:
+ case CS35L56_OTP_MEM_55:
case CS35L56_SYNC_GPIO1_CFG ... CS35L56_ASP2_DIO_GPIO13_CFG:
case CS35L56_UPDATE_REGS:
case CS35L56_REFCLK_INPUT: /* owned by firmware */
@@ -852,9 +851,11 @@ int cs35l56_runtime_resume_common(struct cs35l56_base *cs35l56_base, bool is_sou
err:
regcache_cache_only(cs35l56_base->regmap, true);
- regmap_multi_reg_write_bypassed(cs35l56_base->regmap,
- cs35l56_hibernate_seq,
- ARRAY_SIZE(cs35l56_hibernate_seq));
+ if (cs35l56_base->can_hibernate) {
+ regmap_multi_reg_write_bypassed(cs35l56_base->regmap,
+ cs35l56_hibernate_seq,
+ ARRAY_SIZE(cs35l56_hibernate_seq));
+ }
return ret;
}
@@ -1729,8 +1730,7 @@ int cs35l56_read_onchip_spkid(struct cs35l56_base *cs35l56_base)
ret = regmap_read(regmap, CS35L56_GPIO_STATUS1, &val);
if (ret) {
- dev_err(cs35l56_base->dev, "GPIO%d status read failed: %d\n",
- cs35l56_base->onchip_spkid_gpios[i] + 1, ret);
+ dev_err(cs35l56_base->dev, "GPIO status read failed: %d\n", ret);
return ret;
}
diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c
index 378017f..4fbbdcc 100644
--- a/sound/soc/codecs/cs35l56.c
+++ b/sound/soc/codecs/cs35l56.c
@@ -867,11 +867,16 @@ static void cs35l56_dsp_work(struct work_struct *work)
if (!cs35l56->base.init_done)
return;
- pm_runtime_get_sync(cs35l56->base.dev);
+ PM_RUNTIME_ACQUIRE(cs35l56->base.dev, pm);
+ ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+ if (ret) {
+ dev_err(cs35l56->base.dev, "dsp_work failed to runtime-resume: %d\n", ret);
+ return;
+ }
ret = cs35l56_read_prot_status(&cs35l56->base, &firmware_missing, &firmware_version);
if (ret)
- goto err;
+ return;
/* Populate fw file qualifier with the revision and security state */
kfree(cs35l56->dsp.fwf_name);
@@ -887,7 +892,7 @@ static void cs35l56_dsp_work(struct work_struct *work)
}
if (!cs35l56->dsp.fwf_name)
- goto err;
+ return;
dev_dbg(cs35l56->base.dev, "DSP fwf name: '%s' system name: '%s'\n",
cs35l56->dsp.fwf_name, cs35l56->dsp.system_name);
@@ -905,8 +910,6 @@ static void cs35l56_dsp_work(struct work_struct *work)
cs35l56_patch(cs35l56, firmware_missing);
cs35l56_log_tuning(&cs35l56->base, &cs35l56->dsp.cs_dsp);
-err:
- pm_runtime_put_autosuspend(cs35l56->base.dev);
}
static struct snd_soc_dapm_context *cs35l56_power_up_for_cal(struct cs35l56_private *cs35l56)
@@ -1956,9 +1959,9 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56)
goto err;
}
- ret = devm_snd_soc_register_component(cs35l56->base.dev,
- &soc_component_dev_cs35l56,
- cs35l56_dai, ARRAY_SIZE(cs35l56_dai));
+ ret = snd_soc_register_component(cs35l56->base.dev,
+ &soc_component_dev_cs35l56,
+ cs35l56_dai, ARRAY_SIZE(cs35l56_dai));
if (ret < 0) {
dev_err_probe(cs35l56->base.dev, ret, "Register codec failed\n");
goto err;
@@ -1970,6 +1973,9 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56)
gpiod_set_value_cansleep(cs35l56->base.reset_gpio, 0);
regulator_bulk_disable(ARRAY_SIZE(cs35l56->supplies), cs35l56->supplies);
+ if (cs35l56->dsp_wq)
+ destroy_workqueue(cs35l56->dsp_wq);
+
return ret;
}
EXPORT_SYMBOL_NS_GPL(cs35l56_common_probe, "SND_SOC_CS35L56_CORE");
@@ -2057,6 +2063,8 @@ EXPORT_SYMBOL_NS_GPL(cs35l56_init, "SND_SOC_CS35L56_CORE");
void cs35l56_remove(struct cs35l56_private *cs35l56)
{
+ snd_soc_unregister_component(cs35l56->base.dev);
+
cs35l56->base.init_done = false;
/*
diff --git a/sound/soc/codecs/es8389.c b/sound/soc/codecs/es8389.c
index 8d418ca..449d957 100644
--- a/sound/soc/codecs/es8389.c
+++ b/sound/soc/codecs/es8389.c
@@ -892,7 +892,7 @@ static int es8389_probe(struct snd_soc_component *component)
return ret;
}
- es8389->mclk = devm_clk_get(component->dev, "mclk");
+ es8389->mclk = devm_clk_get_optional(component->dev, "mclk");
if (IS_ERR(es8389->mclk))
return dev_err_probe(component->dev, PTR_ERR(es8389->mclk),
"ES8389 is unable to get mclk\n");
diff --git a/sound/soc/codecs/fs210x.c b/sound/soc/codecs/fs210x.c
index e6195b7..eda716f 100644
--- a/sound/soc/codecs/fs210x.c
+++ b/sound/soc/codecs/fs210x.c
@@ -968,7 +968,7 @@ static int fs210x_effect_scene_info(struct snd_kcontrol *kcontrol,
if (scene->name)
name = scene->name;
- strscpy(uinfo->value.enumerated.name, name, strlen(name) + 1);
+ strscpy(uinfo->value.enumerated.name, name);
return 0;
}
diff --git a/sound/soc/codecs/pcm512x.c b/sound/soc/codecs/pcm512x.c
index a70e8ea..fdef98c 100644
--- a/sound/soc/codecs/pcm512x.c
+++ b/sound/soc/codecs/pcm512x.c
@@ -235,7 +235,7 @@ static int pcm512x_overclock_pll_put(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_to_dapm(kcontrol);
+ struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
struct pcm512x_priv *pcm512x = snd_soc_component_get_drvdata(component);
switch (snd_soc_dapm_get_bias_level(dapm)) {
@@ -264,7 +264,7 @@ static int pcm512x_overclock_dsp_put(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_to_dapm(kcontrol);
+ struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
struct pcm512x_priv *pcm512x = snd_soc_component_get_drvdata(component);
switch (snd_soc_dapm_get_bias_level(dapm)) {
@@ -293,7 +293,7 @@ static int pcm512x_overclock_dac_put(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
- struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_to_dapm(kcontrol);
+ struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
struct pcm512x_priv *pcm512x = snd_soc_component_get_drvdata(component);
switch (snd_soc_dapm_get_bias_level(dapm)) {
diff --git a/sound/soc/codecs/simple-mux.c b/sound/soc/codecs/simple-mux.c
index 069555f..c2f906a 100644
--- a/sound/soc/codecs/simple-mux.c
+++ b/sound/soc/codecs/simple-mux.c
@@ -51,7 +51,7 @@ static int simple_mux_control_put(struct snd_kcontrol *kcontrol,
struct snd_soc_component *c = snd_soc_dapm_to_component(dapm);
struct simple_mux *priv = snd_soc_component_get_drvdata(c);
- if (ucontrol->value.enumerated.item[0] > e->items)
+ if (ucontrol->value.enumerated.item[0] >= e->items)
return -EINVAL;
if (priv->mux == ucontrol->value.enumerated.item[0])
diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c
index 423b707..5521126 100644
--- a/sound/soc/codecs/tas2764.c
+++ b/sound/soc/codecs/tas2764.c
@@ -684,18 +684,33 @@ static int tas2764_read_die_temp(struct tas2764_priv *tas2764, long *result)
* As per datasheet, subtract 93 from raw value to get degrees
* Celsius. hwmon wants millidegrees.
*
- * NOTE: The chip will initialise the TAS2764_TEMP register to
- * 2.6 *C to avoid triggering temperature protection. Since the
- * ADC is powered down during software shutdown, this value will
- * persist until the chip is fully powered up (e.g. the PCM it's
- * attached to is opened). The ADC will power down again when
- * the chip is put back into software shutdown, with the last
- * value sampled persisting in the ADC's register.
+ * NOTE: The TAS2764 datasheet mentions initialising TAS2764_TEMP
+ * such that the temperature is 2.6 *C, however the register
+ * is actually initialised to 0. The ADC is also powered down during
+ * software shutdown. The last sampled temperature will persist
+ * in the register while the amp is in this power state.
*/
+ if (reg == 0)
+ return -ENODATA;
+
*result = (reg - 93) * 1000;
return 0;
}
+static int tas2764_hwmon_is_fault(struct tas2764_priv *tas2764, long *result)
+{
+ int ret;
+ long temp;
+
+ ret = tas2764_read_die_temp(tas2764, &temp);
+ if (ret == -ENODATA) {
+ *result = true;
+ return 0;
+ }
+
+ return ret;
+}
+
static umode_t tas2764_hwmon_is_visible(const void *data,
enum hwmon_sensor_types type, u32 attr,
int channel)
@@ -705,6 +720,7 @@ static umode_t tas2764_hwmon_is_visible(const void *data,
switch (attr) {
case hwmon_temp_input:
+ case hwmon_temp_fault:
return 0444;
default:
break;
@@ -724,6 +740,9 @@ static int tas2764_hwmon_read(struct device *dev,
case hwmon_temp_input:
ret = tas2764_read_die_temp(tas2764, val);
break;
+ case hwmon_temp_fault:
+ ret = tas2764_hwmon_is_fault(tas2764, val);
+ break;
default:
ret = -EOPNOTSUPP;
break;
@@ -733,7 +752,7 @@ static int tas2764_hwmon_read(struct device *dev,
}
static const struct hwmon_channel_info *const tas2764_hwmon_info[] = {
- HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
+ HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_FAULT),
NULL
};
@@ -904,6 +923,7 @@ static bool tas2764_volatile_register(struct device *dev, unsigned int reg)
{
switch (reg) {
case TAS2764_SW_RST:
+ case TAS2764_TEMP:
case TAS2764_INT_LTCH0 ... TAS2764_INT_LTCH4:
case TAS2764_INT_CLK_CFG:
return true;
diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c
index d4d7d05..dbda9f3 100644
--- a/sound/soc/codecs/tas2770.c
+++ b/sound/soc/codecs/tas2770.c
@@ -624,7 +624,7 @@ static int tas2770_read_die_temp(struct tas2770_priv *tas2770, long *result)
/*
* As per datasheet: divide register by 16 and subtract 93 to get
* degrees Celsius. hwmon requires millidegrees. Let's avoid rounding
- * errors by subtracting 93 * 16 then multiplying by 1000 / 16.
+ * errors by subtracting 93 * 16 and scaling before dividing.
*
* NOTE: The ADC registers are initialised to 0 on reset. This means
* that the temperature will read -93 *C until the chip is brought out
@@ -633,10 +633,27 @@ static int tas2770_read_die_temp(struct tas2770_priv *tas2770, long *result)
* value read back from its registers will be the last value sampled
* before entering software shutdown.
*/
- *result = (reading - (93 * 16)) * (1000 / 16);
+ if (reading == 0)
+ return -ENODATA;
+
+ *result = (reading - (93 * 16)) * 1000 / 16;
return 0;
}
+static int tas2770_hwmon_is_fault(struct tas2770_priv *tas2770, long *result)
+{
+ int ret;
+ long temp;
+
+ ret = tas2770_read_die_temp(tas2770, &temp);
+ if (ret == -ENODATA) {
+ *result = true;
+ return 0;
+ }
+
+ return ret;
+}
+
static umode_t tas2770_hwmon_is_visible(const void *data,
enum hwmon_sensor_types type, u32 attr,
int channel)
@@ -646,6 +663,7 @@ static umode_t tas2770_hwmon_is_visible(const void *data,
switch (attr) {
case hwmon_temp_input:
+ case hwmon_temp_fault:
return 0444;
default:
break;
@@ -665,6 +683,9 @@ static int tas2770_hwmon_read(struct device *dev,
case hwmon_temp_input:
ret = tas2770_read_die_temp(tas2770, val);
break;
+ case hwmon_temp_fault:
+ ret = tas2770_hwmon_is_fault(tas2770, val);
+ break;
default:
ret = -EOPNOTSUPP;
break;
@@ -674,7 +695,7 @@ static int tas2770_hwmon_read(struct device *dev,
}
static const struct hwmon_channel_info *const tas2770_hwmon_info[] = {
- HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
+ HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_FAULT),
NULL
};
diff --git a/sound/soc/codecs/wcd937x.c b/sound/soc/codecs/wcd937x.c
index 10a2d59..72a53f9 100644
--- a/sound/soc/codecs/wcd937x.c
+++ b/sound/soc/codecs/wcd937x.c
@@ -547,6 +547,9 @@ static int wcd937x_codec_aux_dac_event(struct snd_soc_dapm_widget *w,
WCD937X_DIGITAL_CDC_ANA_CLK_CTL,
BIT(2), BIT(2));
snd_soc_component_update_bits(component,
+ WCD937X_AUX_AUXPA,
+ BIT(4), BIT(4));
+ snd_soc_component_update_bits(component,
WCD937X_DIGITAL_CDC_DIG_CLK_CTL,
BIT(2), BIT(2));
snd_soc_component_update_bits(component,
@@ -562,6 +565,9 @@ static int wcd937x_codec_aux_dac_event(struct snd_soc_dapm_widget *w,
snd_soc_component_update_bits(component,
WCD937X_DIGITAL_CDC_ANA_CLK_CTL,
BIT(2), 0x00);
+ snd_soc_component_update_bits(component,
+ WCD937X_AUX_AUXPA,
+ BIT(4), 0x00);
break;
}
@@ -730,10 +736,23 @@ static int wcd937x_codec_enable_aux_pa(struct snd_soc_dapm_widget *w,
snd_soc_component_update_bits(component,
WCD937X_ANA_RX_SUPPLIES,
BIT(1), BIT(1));
+ /* Enable AUX PA related RX supplies */
+ snd_soc_component_update_bits(component,
+ WCD937X_ANA_RX_SUPPLIES,
+ BIT(6), BIT(6));
+ snd_soc_component_update_bits(component,
+ WCD937X_ANA_RX_SUPPLIES,
+ BIT(7), BIT(7));
enable_irq(wcd937x->aux_pdm_wd_int);
break;
case SND_SOC_DAPM_PRE_PMD:
disable_irq_nosync(wcd937x->aux_pdm_wd_int);
+ snd_soc_component_update_bits(component,
+ WCD937X_ANA_RX_SUPPLIES,
+ BIT(6), 0x00);
+ snd_soc_component_update_bits(component,
+ WCD937X_ANA_RX_SUPPLIES,
+ BIT(7), 0x00);
break;
case SND_SOC_DAPM_POST_PMD:
usleep_range(2000, 2010);
@@ -2051,7 +2070,12 @@ static const struct snd_kcontrol_new wcd937x_snd_controls[] = {
wcd937x_get_swr_port, wcd937x_set_swr_port),
SOC_SINGLE_EXT("LO Switch", WCD937X_LO, 0, 1, 0,
wcd937x_get_swr_port, wcd937x_set_swr_port),
-
+ SOC_SINGLE_EXT("CLSH PA Switch", WCD937X_CLSH, 0, 1, 0,
+ wcd937x_get_swr_port, wcd937x_set_swr_port),
+ SOC_SINGLE_EXT("DSD_L Switch", WCD937X_DSD_L, 0, 1, 0,
+ wcd937x_get_swr_port, wcd937x_set_swr_port),
+ SOC_SINGLE_EXT("DSD_R Switch", WCD937X_DSD_R, 0, 1, 0,
+ wcd937x_get_swr_port, wcd937x_set_swr_port),
SOC_SINGLE_EXT("ADC1 Switch", WCD937X_ADC1, 1, 1, 0,
wcd937x_get_swr_port, wcd937x_set_swr_port),
SOC_SINGLE_EXT("ADC2 Switch", WCD937X_ADC2, 1, 1, 0,
diff --git a/sound/soc/codecs/wm_adsp_fw_find_test.c b/sound/soc/codecs/wm_adsp_fw_find_test.c
index d0c7fb3..ae686dc 100644
--- a/sound/soc/codecs/wm_adsp_fw_find_test.c
+++ b/sound/soc/codecs/wm_adsp_fw_find_test.c
@@ -45,6 +45,34 @@ struct wm_adsp_fw_find_test_params {
/* Dummy struct firmware to return from wm_adsp_request_firmware_files */
static const struct firmware wm_adsp_find_test_dummy_firmware;
+static void wm_adsp_fw_find_test_release_firmware_files_stub(struct wm_adsp_fw_files *fw)
+{
+ /*
+ * fw->wmfw.firmware and fw->coeff.firmware allocated by this KUnit
+ * test are dummies not allocated by the real request_firmware() call
+ * so they must not be passed to release_firmware().
+ * This function replaces wm_adsp_release_firmware_files().
+ */
+
+ if (!fw)
+ return;
+
+ kfree(fw->wmfw.filename);
+ kfree(fw->coeff.filename);
+
+ fw->wmfw.firmware = NULL;
+ fw->coeff.firmware = NULL;
+ fw->wmfw.filename = NULL;
+ fw->coeff.filename = NULL;
+}
+
+static void wm_adsp_free_found_fw(struct kunit *test)
+{
+ struct wm_adsp_fw_find_test *priv = test->priv;
+
+ wm_adsp_fw_find_test_release_firmware_files_stub(&priv->found_fw);
+}
+
/* Simple lookup of a filename in a list of names */
static int wm_adsp_fw_find_test_firmware_request_simple_stub(const struct firmware **firmware,
const char *filename,
@@ -97,9 +125,14 @@ static void wm_adsp_fw_find_test_pick_file(struct kunit *test)
kunit_activate_static_stub(test,
wm_adsp_firmware_request,
wm_adsp_fw_find_test_firmware_request_simple_stub);
+ kunit_activate_static_stub(test,
+ wm_adsp_release_firmware_files,
+ wm_adsp_fw_find_test_release_firmware_files_stub);
ret = wm_adsp_request_firmware_files(dsp, &priv->found_fw);
kunit_deactivate_static_stub(test, wm_adsp_firmware_request);
+ kunit_deactivate_static_stub(test, wm_adsp_release_firmware_files);
+
KUNIT_EXPECT_EQ_MSG(test, ret,
(params->expect_wmfw || params->expect_bin) ? 0 : -ENOENT,
"%s\n", priv->searched_fw_files);
@@ -173,10 +206,13 @@ static void wm_adsp_fw_find_test_search_order(struct kunit *test)
kunit_activate_static_stub(test,
wm_adsp_firmware_request,
wm_adsp_fw_find_test_firmware_request_stub);
+ kunit_activate_static_stub(test,
+ wm_adsp_release_firmware_files,
+ wm_adsp_fw_find_test_release_firmware_files_stub);
wm_adsp_request_firmware_files(dsp, &priv->found_fw);
-
kunit_deactivate_static_stub(test, wm_adsp_firmware_request);
+ kunit_deactivate_static_stub(test, wm_adsp_release_firmware_files);
KUNIT_EXPECT_STREQ(test, priv->searched_fw_files, params->expected_searches);
@@ -201,6 +237,7 @@ static void wm_adsp_fw_find_test_find_firmware_byindex(struct kunit *test)
dsp->cs_dsp.name = "cs1234";
dsp->part = "dsp1";
+
for (dsp->fw = 0;; dsp->fw++) {
fw_name = wm_adsp_get_fwf_name_by_index(dsp->fw);
if (!fw_name)
@@ -209,14 +246,22 @@ static void wm_adsp_fw_find_test_find_firmware_byindex(struct kunit *test)
kunit_activate_static_stub(test,
wm_adsp_firmware_request,
wm_adsp_fw_find_test_firmware_request_stub);
+ kunit_activate_static_stub(test,
+ wm_adsp_release_firmware_files,
+ wm_adsp_fw_find_test_release_firmware_files_stub);
wm_adsp_request_firmware_files(dsp, &priv->found_fw);
+
kunit_deactivate_static_stub(test, wm_adsp_firmware_request);
+ kunit_deactivate_static_stub(test, wm_adsp_release_firmware_files);
KUNIT_EXPECT_NOT_NULL_MSG(test,
strstr(priv->searched_fw_files, fw_name),
"fw#%d Did not find '%s' in '%s'\n",
dsp->fw, fw_name, priv->searched_fw_files);
+
+ wm_adsp_free_found_fw(test);
+ memset(priv->searched_fw_files, 0, sizeof(priv->searched_fw_files));
}
}
@@ -255,15 +300,7 @@ static int wm_adsp_fw_find_test_case_init(struct kunit *test)
static void wm_adsp_fw_find_test_case_exit(struct kunit *test)
{
- struct wm_adsp_fw_find_test *priv = test->priv;
-
- /*
- * priv->found_wmfw_firmware and priv->found_bin_firmware are
- * dummies not allocated by the real request_firmware() call they
- * must not be passed to release_firmware().
- */
- kfree(priv->found_fw.wmfw.filename);
- kfree(priv->found_fw.coeff.filename);
+ wm_adsp_free_found_fw(test);
}
static void wm_adsp_fw_find_test_param_desc(const struct wm_adsp_fw_find_test_params *param,
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index bd336d2..e364552 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -1370,6 +1370,31 @@ static int fsl_sai_check_version(struct device *dev)
return 0;
}
+static int fsl_sai_reset_hw(struct device *dev)
+{
+ struct fsl_sai *sai = dev_get_drvdata(dev);
+ unsigned char ofs = sai->soc_data->reg_offset;
+ int ret;
+
+ /*
+ * Clear TCSR/RCSR to reset SAI and disable all interrupts.
+ * Bootloader may leave SAI running causing interrupt storm.
+ */
+ ret = regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), 0);
+ if (ret) {
+ dev_err(dev, "Failed to clear TCSR: %d\n", ret);
+ return ret;
+ }
+
+ ret = regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), 0);
+ if (ret) {
+ dev_err(dev, "Failed to clear RCSR: %d\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
/*
* Calculate the offset between first two datalines, don't
* different offset in one case.
@@ -1575,13 +1600,6 @@ static int fsl_sai_probe(struct platform_device *pdev)
if (irq < 0)
return irq;
- ret = devm_request_irq(dev, irq, fsl_sai_isr, IRQF_SHARED,
- np->name, sai);
- if (ret) {
- dev_err(dev, "failed to claim irq %u\n", irq);
- return ret;
- }
-
memcpy(&sai->cpu_dai_drv, fsl_sai_dai_template,
sizeof(*fsl_sai_dai_template) * ARRAY_SIZE(fsl_sai_dai_template));
@@ -1656,6 +1674,10 @@ static int fsl_sai_probe(struct platform_device *pdev)
if (ret < 0)
dev_warn(dev, "Error reading SAI version: %d\n", ret);
+ ret = fsl_sai_reset_hw(dev);
+ if (ret < 0)
+ dev_warn(dev, "Failed to reset hardware: %d\n", ret);
+
/* Select MCLK direction */
if (sai->mclk_direction_output &&
sai->soc_data->max_register >= FSL_SAI_MCTL) {
@@ -1667,6 +1689,13 @@ static int fsl_sai_probe(struct platform_device *pdev)
if (ret < 0 && ret != -ENOSYS)
goto err_pm_get_sync;
+ ret = devm_request_irq(dev, irq, fsl_sai_isr, IRQF_SHARED,
+ np->name, sai);
+ if (ret) {
+ dev_err(dev, "failed to claim irq %u\n", irq);
+ goto err_pm_get_sync;
+ }
+
if (of_device_is_compatible(np, "fsl,imx952-sai") &&
!of_property_read_string(np, "fsl,sai-amix-mode", &str)) {
if (!strcmp(str, "bypass"))
diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c
index ee16cf6..6677d3b 100644
--- a/sound/soc/fsl/fsl_xcvr.c
+++ b/sound/soc/fsl/fsl_xcvr.c
@@ -228,10 +228,14 @@ static int fsl_xcvr_capds_put(struct snd_kcontrol *kcontrol,
{
struct snd_soc_dai *dai = snd_kcontrol_chip(kcontrol);
struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai);
+ int changed;
- memcpy(xcvr->cap_ds, ucontrol->value.bytes.data, FSL_XCVR_CAPDS_SIZE);
+ changed = memcmp(xcvr->cap_ds, ucontrol->value.bytes.data,
+ sizeof(xcvr->cap_ds)) != 0;
+ memcpy(xcvr->cap_ds, ucontrol->value.bytes.data,
+ sizeof(xcvr->cap_ds));
- return 0;
+ return changed;
}
static struct snd_kcontrol_new fsl_xcvr_earc_capds_kctl = {
@@ -1040,10 +1044,15 @@ static int fsl_xcvr_tx_cs_put(struct snd_kcontrol *kcontrol,
{
struct snd_soc_dai *dai = snd_kcontrol_chip(kcontrol);
struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai);
+ int changed;
- memcpy(xcvr->tx_iec958.status, ucontrol->value.iec958.status, 24);
+ changed = memcmp(xcvr->tx_iec958.status,
+ ucontrol->value.iec958.status,
+ sizeof(xcvr->tx_iec958.status)) != 0;
+ memcpy(xcvr->tx_iec958.status, ucontrol->value.iec958.status,
+ sizeof(xcvr->tx_iec958.status));
- return 0;
+ return changed;
}
static struct snd_kcontrol_new fsl_xcvr_rx_ctls[] = {
diff --git a/sound/soc/intel/boards/bytcht_es8316.c b/sound/soc/intel/boards/bytcht_es8316.c
index 192e2a3..ea387dc 100644
--- a/sound/soc/intel/boards/bytcht_es8316.c
+++ b/sound/soc/intel/boards/bytcht_es8316.c
@@ -40,6 +40,7 @@ struct byt_cht_es8316_private {
struct gpio_desc *speaker_en_gpio;
struct device *codec_dev;
bool speaker_en;
+ bool mclk_enabled;
};
enum {
@@ -170,6 +171,15 @@ static struct snd_soc_jack_pin byt_cht_es8316_jack_pins[] = {
},
};
+static void byt_cht_es8316_disable_mclk(struct byt_cht_es8316_private *priv)
+{
+ if (!priv->mclk_enabled)
+ return;
+
+ clk_disable_unprepare(priv->mclk);
+ priv->mclk_enabled = false;
+}
+
static int byt_cht_es8316_init(struct snd_soc_pcm_runtime *runtime)
{
struct snd_soc_component *codec = snd_soc_rtd_to_codec(runtime, 0)->component;
@@ -227,12 +237,14 @@ static int byt_cht_es8316_init(struct snd_soc_pcm_runtime *runtime)
ret = clk_prepare_enable(priv->mclk);
if (ret)
dev_err(card->dev, "unable to enable MCLK\n");
+ else
+ priv->mclk_enabled = true;
ret = snd_soc_dai_set_sysclk(snd_soc_rtd_to_codec(runtime, 0), 0, 19200000,
SND_SOC_CLOCK_IN);
if (ret < 0) {
dev_err(card->dev, "can't set codec clock %d\n", ret);
- return ret;
+ goto err_disable_mclk;
}
ret = snd_soc_card_jack_new_pins(card, "Headset",
@@ -241,13 +253,25 @@ static int byt_cht_es8316_init(struct snd_soc_pcm_runtime *runtime)
ARRAY_SIZE(byt_cht_es8316_jack_pins));
if (ret) {
dev_err(card->dev, "jack creation failed %d\n", ret);
- return ret;
+ goto err_disable_mclk;
}
snd_jack_set_key(priv->jack.jack, SND_JACK_BTN_0, KEY_PLAYPAUSE);
snd_soc_component_set_jack(codec, &priv->jack, NULL);
return 0;
+
+err_disable_mclk:
+ byt_cht_es8316_disable_mclk(priv);
+ return ret;
+}
+
+static void byt_cht_es8316_exit(struct snd_soc_pcm_runtime *runtime)
+{
+ struct snd_soc_card *card = runtime->card;
+ struct byt_cht_es8316_private *priv = snd_soc_card_get_drvdata(card);
+
+ byt_cht_es8316_disable_mclk(priv);
}
static int byt_cht_es8316_codec_fixup(struct snd_soc_pcm_runtime *rtd,
@@ -353,6 +377,7 @@ static struct snd_soc_dai_link byt_cht_es8316_dais[] = {
| SND_SOC_DAIFMT_CBC_CFC,
.be_hw_params_fixup = byt_cht_es8316_codec_fixup,
.init = byt_cht_es8316_init,
+ .exit = byt_cht_es8316_exit,
SND_SOC_DAILINK_REG(ssp2_port, ssp2_codec, platform),
},
};
diff --git a/sound/soc/intel/boards/bytcr_wm5102.c b/sound/soc/intel/boards/bytcr_wm5102.c
index 4879f79..4aa0cf4 100644
--- a/sound/soc/intel/boards/bytcr_wm5102.c
+++ b/sound/soc/intel/boards/bytcr_wm5102.c
@@ -170,6 +170,7 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w,
ret = byt_wm5102_prepare_and_enable_pll1(codec_dai, 48000);
if (ret) {
dev_err(card->dev, "Error setting codec sysclk: %d\n", ret);
+ clk_disable_unprepare(priv->mclk);
return ret;
}
} else {
diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c
index c18ec60..ce77183 100644
--- a/sound/soc/intel/boards/sof_sdw.c
+++ b/sound/soc/intel/boards/sof_sdw.c
@@ -837,6 +837,14 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = {
SOF_BT_OFFLOAD_SSP(2) |
SOF_SSP_BT_OFFLOAD_PRESENT),
},
+ /* Novalake devices*/
+ {
+ .callback = sof_sdw_quirk_cb,
+ .matches = {
+ DMI_MATCH(DMI_PRODUCT_FAMILY, "Intel_nvlrvp"),
+ },
+ .driver_data = (void *)(SOC_SDW_PCH_DMIC),
+ },
{}
};
diff --git a/sound/soc/intel/common/soc-acpi-intel-arl-match.c b/sound/soc/intel/common/soc-acpi-intel-arl-match.c
index c952f7d..52c5b57 100644
--- a/sound/soc/intel/common/soc-acpi-intel-arl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-arl-match.c
@@ -8,6 +8,7 @@
#include <sound/soc-acpi.h>
#include <sound/soc-acpi-intel-match.h>
#include <sound/soc-acpi-intel-ssp-common.h>
+#include "soc-acpi-intel-sdca-quirks.h"
#include "sof-function-topology-lib.h"
static const struct snd_soc_acpi_endpoint single_endpoint = {
@@ -237,6 +238,15 @@ static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = {
}
};
+static const struct snd_soc_acpi_adr_device rt712_0_agg_adr[] = {
+ {
+ .adr = 0x000030025D071201ull,
+ .num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints),
+ .endpoints = jack_amp_g1_dmic_endpoints,
+ .name_prefix = "rt712"
+ }
+};
+
static const struct snd_soc_acpi_adr_device rt1316_3_single_adr[] = {
{
.adr = 0x000330025D131601ull,
@@ -255,6 +265,15 @@ static const struct snd_soc_acpi_adr_device rt1320_2_single_adr[] = {
}
};
+static const struct snd_soc_acpi_adr_device rt1320_3_group1_adr[] = {
+ {
+ .adr = 0x000330025D132001ull,
+ .num_endpoints = 1,
+ .endpoints = &spk_r_endpoint,
+ .name_prefix = "rt1320-1"
+ }
+};
+
static const struct snd_soc_acpi_link_adr arl_cs42l43_l0[] = {
{
.mask = BIT(0),
@@ -404,6 +423,20 @@ static const struct snd_soc_acpi_link_adr arl_rt722_l0_rt1320_l2[] = {
{}
};
+static const struct snd_soc_acpi_link_adr arl_rt712_l0_rt1320_l3[] = {
+ {
+ .mask = BIT(0),
+ .num_adr = ARRAY_SIZE(rt712_0_agg_adr),
+ .adr_d = rt712_0_agg_adr,
+ },
+ {
+ .mask = BIT(3),
+ .num_adr = ARRAY_SIZE(rt1320_3_group1_adr),
+ .adr_d = rt1320_3_group1_adr,
+ },
+ {}
+};
+
static const struct snd_soc_acpi_codecs arl_essx_83x6 = {
.num_codecs = 3,
.codecs = { "ESSX8316", "ESSX8326", "ESSX8336"},
@@ -483,10 +516,24 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_arl_sdw_machines[] = {
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
- .link_mask = BIT(0),
- .links = arl_cs42l43_l0,
+ .link_mask = BIT(0) | BIT(2),
+ .links = arl_rt722_l0_rt1320_l2,
.drv_name = "sof_sdw",
- .sof_tplg_filename = "sof-arl-cs42l43-l0.tplg",
+ .sof_tplg_filename = "sof-arl-rt722-l0_rt1320-l2.tplg",
+ .get_function_tplg_files = sof_sdw_get_tplg_files,
+ },
+ {
+ .link_mask = BIT(0) | BIT(3),
+ .links = arl_rt711_l0_rt1316_l3,
+ .drv_name = "sof_sdw",
+ .sof_tplg_filename = "sof-arl-rt711-l0-rt1316-l3.tplg",
+ },
+ {
+ .link_mask = BIT(0) | BIT(3),
+ .links = arl_rt712_l0_rt1320_l3,
+ .drv_name = "sof_sdw",
+ .machine_check = snd_soc_acpi_intel_sdca_is_device_rt712_vb,
+ .sof_tplg_filename = "sof-arl-rt712-l0-rt1320-l3.tplg",
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
@@ -497,19 +544,13 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_arl_sdw_machines[] = {
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
- .link_mask = BIT(2),
- .links = arl_cs42l43_l2,
+ .link_mask = BIT(0),
+ .links = arl_cs42l43_l0,
.drv_name = "sof_sdw",
- .sof_tplg_filename = "sof-arl-cs42l43-l2.tplg",
+ .sof_tplg_filename = "sof-arl-cs42l43-l0.tplg",
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
- .link_mask = BIT(0) | BIT(3),
- .links = arl_rt711_l0_rt1316_l3,
- .drv_name = "sof_sdw",
- .sof_tplg_filename = "sof-arl-rt711-l0-rt1316-l3.tplg",
- },
- {
.link_mask = 0x1, /* link0 required */
.links = arl_rvp,
.drv_name = "sof_sdw",
@@ -522,10 +563,10 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_arl_sdw_machines[] = {
.sof_tplg_filename = "sof-arl-rt711-l0.tplg",
},
{
- .link_mask = BIT(0) | BIT(2),
- .links = arl_rt722_l0_rt1320_l2,
+ .link_mask = BIT(2),
+ .links = arl_cs42l43_l2,
.drv_name = "sof_sdw",
- .sof_tplg_filename = "sof-arl-rt722-l0_rt1320-l2.tplg",
+ .sof_tplg_filename = "sof-arl-cs42l43-l2.tplg",
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{},
diff --git a/sound/soc/intel/common/soc-acpi-intel-nvl-match.c b/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
index b8695d47..2172722 100644
--- a/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
@@ -10,7 +10,20 @@
#include <sound/soc-acpi-intel-match.h>
#include "soc-acpi-intel-sdw-mockup-match.h"
+static const struct snd_soc_acpi_codecs nvl_essx_83x6 = {
+ .num_codecs = 3,
+ .codecs = { "ESSX8316", "ESSX8326", "ESSX8336"},
+};
+
struct snd_soc_acpi_mach snd_soc_acpi_intel_nvl_machines[] = {
+ {
+ .comp_ids = &nvl_essx_83x6,
+ .drv_name = "sof-essx8336",
+ .sof_tplg_filename = "sof-nvl-es8336", /* the tplg suffix is added at run time */
+ .tplg_quirk_mask = SND_SOC_ACPI_TPLG_INTEL_SSP_NUMBER |
+ SND_SOC_ACPI_TPLG_INTEL_SSP_MSB |
+ SND_SOC_ACPI_TPLG_INTEL_DMIC_NUMBER,
+ },
{},
};
EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_nvl_machines);
diff --git a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
index 3b78183..f7694b2 100644
--- a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
@@ -92,48 +92,6 @@ static const struct snd_soc_acpi_endpoint spk_r_endpoint = {
.group_id = 1,
};
-static const struct snd_soc_acpi_endpoint spk_1_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 1,
- .group_id = 1,
-};
-
-static const struct snd_soc_acpi_endpoint spk_2_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 2,
- .group_id = 1,
-};
-
-static const struct snd_soc_acpi_endpoint spk_3_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 3,
- .group_id = 1,
-};
-
-static const struct snd_soc_acpi_endpoint spk_4_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 4,
- .group_id = 1,
-};
-
-static const struct snd_soc_acpi_endpoint spk_5_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 5,
- .group_id = 1,
-};
-
-static const struct snd_soc_acpi_endpoint spk_6_endpoint = {
- .num = 0,
- .aggregated = 1,
- .group_position = 6,
- .group_id = 1,
-};
-
static const struct snd_soc_acpi_endpoint jack_dmic_endpoints[] = {
/* Jack Endpoint */
{
@@ -202,15 +160,6 @@ static const struct snd_soc_acpi_endpoint cs42l43_amp_spkagg_endpoints[] = {
},
};
-static const struct snd_soc_acpi_adr_device cs42l43_2_adr[] = {
- {
- .adr = 0x00023001fa424301ull,
- .num_endpoints = ARRAY_SIZE(cs42l43_amp_spkagg_endpoints),
- .endpoints = cs42l43_amp_spkagg_endpoints,
- .name_prefix = "cs42l43"
- }
-};
-
static const struct snd_soc_acpi_adr_device cs42l43_3_agg_adr[] = {
{
.adr = 0x00033001FA424301ull,
@@ -235,48 +184,6 @@ static const struct snd_soc_acpi_adr_device cs35l56_2_lr_adr[] = {
}
};
-static const struct snd_soc_acpi_adr_device cs35l56_1_3amp_adr[] = {
- {
- .adr = 0x00013001fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_1_endpoint,
- .name_prefix = "AMP1"
- },
- {
- .adr = 0x00013101fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_2_endpoint,
- .name_prefix = "AMP2"
- },
- {
- .adr = 0x00013201fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_3_endpoint,
- .name_prefix = "AMP3"
- }
-};
-
-static const struct snd_soc_acpi_adr_device cs35l56_3_3amp_adr[] = {
- {
- .adr = 0x00033301fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_4_endpoint,
- .name_prefix = "AMP4"
- },
- {
- .adr = 0x00033401fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_5_endpoint,
- .name_prefix = "AMP5"
- },
- {
- .adr = 0x00033501fa355601ull,
- .num_endpoints = 1,
- .endpoints = &spk_6_endpoint,
- .name_prefix = "AMP6"
- }
-};
-
static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = {
{
.adr = 0x000030025D071101ull,
@@ -408,25 +315,6 @@ static const struct snd_soc_acpi_link_adr ptl_cs42l43_agg_l3_cs35l56_l2[] = {
{}
};
-static const struct snd_soc_acpi_link_adr ptl_cs42l43_l2_cs35l56x6_l13[] = {
- {
- .mask = BIT(2),
- .num_adr = ARRAY_SIZE(cs42l43_2_adr),
- .adr_d = cs42l43_2_adr,
- },
- {
- .mask = BIT(1),
- .num_adr = ARRAY_SIZE(cs35l56_1_3amp_adr),
- .adr_d = cs35l56_1_3amp_adr,
- },
- {
- .mask = BIT(3),
- .num_adr = ARRAY_SIZE(cs35l56_3_3amp_adr),
- .adr_d = cs35l56_3_3amp_adr,
- },
- {}
-};
-
static const struct snd_soc_acpi_link_adr ptl_rt722_l0_rt1320_l23[] = {
{
.mask = BIT(0),
@@ -493,6 +381,20 @@ static const struct snd_soc_acpi_link_adr ptl_sdw_rt713_vb_l3_rt1320_l12[] = {
{}
};
+static const struct snd_soc_acpi_link_adr ptl_sdw_rt713_vb_l3_rt1320_l1[] = {
+ {
+ .mask = BIT(3),
+ .num_adr = ARRAY_SIZE(rt713_vb_3_adr),
+ .adr_d = rt713_vb_3_adr,
+ },
+ {
+ .mask = BIT(1),
+ .num_adr = ARRAY_SIZE(rt1320_1_group2_adr),
+ .adr_d = rt1320_1_group2_adr,
+ },
+ {}
+};
+
static const struct snd_soc_acpi_link_adr ptl_sdw_rt712_vb_l2_rt1320_l1[] = {
{
.mask = BIT(2),
@@ -579,10 +481,11 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_ptl_sdw_machines[] = {
.get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
- .link_mask = BIT(1) | BIT(2) | BIT(3),
- .links = ptl_cs42l43_l2_cs35l56x6_l13,
+ .link_mask = BIT(1) | BIT(3),
+ .links = ptl_sdw_rt713_vb_l3_rt1320_l1,
.drv_name = "sof_sdw",
- .sof_tplg_filename = "sof-ptl-cs42l43-l2-cs35l56x6-l13.tplg",
+ .sof_tplg_filename = "sof-ptl-rt713-l3-rt1320-l1.tplg",
+ .get_function_tplg_files = sof_sdw_get_tplg_files,
},
{
.link_mask = BIT(0) | BIT(2) | BIT(3),
@@ -611,6 +514,7 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_ptl_sdw_machines[] = {
.link_mask = BIT(2) | BIT(3),
.links = ptl_cs42l43_agg_l3_cs35l56_l2,
.drv_name = "sof_sdw",
+ .machine_check = snd_soc_acpi_intel_no_function_topology,
.sof_tplg_filename = "sof-ptl-cs42l43-agg-l3-cs35l56-l2.tplg",
},
{
diff --git a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c
index 3eaa058..7caabc5 100644
--- a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c
+++ b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c
@@ -6,6 +6,7 @@
*
*/
+#include <linux/dmi.h>
#include <linux/soundwire/sdw_intel.h>
#include <sound/sdca.h>
#include <sound/soc-acpi.h>
@@ -37,6 +38,21 @@ bool snd_soc_acpi_intel_sdca_is_device_rt712_vb(void *arg)
}
EXPORT_SYMBOL_NS(snd_soc_acpi_intel_sdca_is_device_rt712_vb, "SND_SOC_ACPI_INTEL_SDCA_QUIRKS");
+static const struct dmi_system_id function_topology_quirk_table[] = {
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Google"),
+ },
+ },
+ {}
+};
+
+bool snd_soc_acpi_intel_no_function_topology(void *arg)
+{
+ return !!dmi_check_system(function_topology_quirk_table);
+}
+EXPORT_SYMBOL_NS(snd_soc_acpi_intel_no_function_topology, "SND_SOC_ACPI_INTEL_SDCA_QUIRKS");
+
MODULE_DESCRIPTION("ASoC ACPI Intel SDCA quirks");
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS("SND_SOC_SDCA");
diff --git a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h
index bead5ec..2ea0a18 100644
--- a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h
+++ b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h
@@ -10,5 +10,6 @@
#define _SND_SOC_ACPI_INTEL_SDCA_QUIRKS
bool snd_soc_acpi_intel_sdca_is_device_rt712_vb(void *arg);
+bool snd_soc_acpi_intel_no_function_topology(void *arg);
#endif
diff --git a/sound/soc/qcom/qdsp6/q6apm-dai.c b/sound/soc/qcom/qdsp6/q6apm-dai.c
index ede19fd..3a1be41 100644
--- a/sound/soc/qcom/qdsp6/q6apm-dai.c
+++ b/sound/soc/qcom/qdsp6/q6apm-dai.c
@@ -497,7 +497,12 @@ static int q6apm_dai_pcm_new(struct snd_soc_component *component, struct snd_soc
{
struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
struct snd_pcm *pcm = rtd->pcm;
- int size = BUFFER_BYTES_MAX;
+ /*
+ * Allocate one extra page as a workaround for a DSP bug where 32-bit
+ * address arithmetic can overflow when the buffer is placed near the
+ * end of the addressable range.
+ */
+ int size = BUFFER_BYTES_MAX + PAGE_SIZE;
int graph_id, ret;
struct snd_pcm_substream *substream;
diff --git a/sound/soc/qcom/qdsp6/q6asm-dai.c b/sound/soc/qcom/qdsp6/q6asm-dai.c
index 4f8f7db..4f09fdd 100644
--- a/sound/soc/qcom/qdsp6/q6asm-dai.c
+++ b/sound/soc/qcom/qdsp6/q6asm-dai.c
@@ -186,12 +186,10 @@ static void event_handler(uint32_t opcode, uint32_t token,
case ASM_CLIENT_EVENT_CMD_RUN_DONE:
break;
case ASM_CLIENT_EVENT_CMD_EOS_DONE:
- prtd->state = Q6ASM_STREAM_STOPPED;
break;
- case ASM_CLIENT_EVENT_DATA_WRITE_DONE: {
+ case ASM_CLIENT_EVENT_DATA_WRITE_DONE:
snd_pcm_period_elapsed(substream);
break;
- }
case ASM_CLIENT_EVENT_DATA_READ_DONE:
snd_pcm_period_elapsed(substream);
if (prtd->state == Q6ASM_STREAM_RUNNING)
@@ -227,9 +225,19 @@ static int q6asm_dai_prepare(struct snd_soc_component *component,
/* rate and channels are sent to audio driver */
if (prtd->state == Q6ASM_STREAM_RUNNING) {
/* clear the previous setup if any */
- q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE);
- q6asm_unmap_memory_regions(substream->stream,
- prtd->audio_client);
+ ret = q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE);
+ if (ret < 0) {
+ dev_err(dev, "Failed to close q6asm stream %d\n", prtd->stream_id);
+ return ret;
+ }
+
+ ret = q6asm_unmap_memory_regions(substream->stream, prtd->audio_client);
+ if (ret < 0) {
+ dev_err(dev, "Failed to unmap memory regions for q6asm stream %d\n",
+ prtd->stream_id);
+ return ret;
+ }
+
q6routing_stream_close(soc_prtd->dai_link->id,
substream->stream);
prtd->state = Q6ASM_STREAM_STOPPED;
@@ -297,8 +305,6 @@ static int q6asm_dai_prepare(struct snd_soc_component *component,
q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE);
open_err:
q6asm_unmap_memory_regions(substream->stream, prtd->audio_client);
- q6asm_audio_client_free(prtd->audio_client);
- prtd->audio_client = NULL;
return ret;
}
@@ -341,7 +347,6 @@ static int q6asm_dai_trigger(struct snd_soc_component *component,
0, 0, 0);
break;
case SNDRV_PCM_TRIGGER_STOP:
- prtd->state = Q6ASM_STREAM_STOPPED;
ret = q6asm_cmd_nowait(prtd->audio_client, prtd->stream_id,
CMD_EOS);
break;
@@ -378,7 +383,7 @@ static int q6asm_dai_open(struct snd_soc_component *component,
return -EINVAL;
}
- prtd = kzalloc_obj(struct q6asm_dai_rtd);
+ prtd = kzalloc_obj(*prtd);
if (prtd == NULL)
return -ENOMEM;
@@ -457,12 +462,12 @@ static int q6asm_dai_close(struct snd_soc_component *component,
struct q6asm_dai_rtd *prtd = runtime->private_data;
if (prtd->audio_client) {
- if (prtd->state)
+ if (prtd->state == Q6ASM_STREAM_RUNNING) {
q6asm_cmd(prtd->audio_client, prtd->stream_id,
CMD_CLOSE);
-
- q6asm_unmap_memory_regions(substream->stream,
+ q6asm_unmap_memory_regions(substream->stream,
prtd->audio_client);
+ }
q6asm_audio_client_free(prtd->audio_client);
prtd->audio_client = NULL;
}
@@ -555,8 +560,6 @@ static void compress_event_handler(uint32_t opcode, uint32_t token,
snd_compr_drain_notify(prtd->cstream);
prtd->notify_on_drain = false;
- } else {
- prtd->state = Q6ASM_STREAM_STOPPED;
}
break;
@@ -674,7 +677,7 @@ static int q6asm_dai_compr_free(struct snd_soc_component *component,
struct snd_soc_pcm_runtime *rtd = stream->private_data;
if (prtd->audio_client) {
- if (prtd->state) {
+ if (prtd->state == Q6ASM_STREAM_RUNNING) {
q6asm_cmd(prtd->audio_client, prtd->stream_id,
CMD_CLOSE);
if (prtd->next_track_stream_id) {
@@ -682,11 +685,11 @@ static int q6asm_dai_compr_free(struct snd_soc_component *component,
prtd->next_track_stream_id,
CMD_CLOSE);
}
- }
- snd_dma_free_pages(&prtd->dma_buffer);
- q6asm_unmap_memory_regions(stream->direction,
+ q6asm_unmap_memory_regions(stream->direction,
prtd->audio_client);
+ }
+ snd_dma_free_pages(&prtd->dma_buffer);
q6asm_audio_client_free(prtd->audio_client);
prtd->audio_client = NULL;
}
@@ -916,7 +919,7 @@ static int q6asm_dai_compr_set_params(struct snd_soc_component *component,
prtd->session_id, dir);
if (ret) {
dev_err(dev, "Stream reg failed ret:%d\n", ret);
- goto q6_err;
+ goto routing_err;
}
ret = __q6asm_dai_compr_set_codec_params(component, stream,
@@ -942,11 +945,11 @@ static int q6asm_dai_compr_set_params(struct snd_soc_component *component,
return 0;
q6_err:
+ q6routing_stream_close(rtd->dai_link->id, dir);
+routing_err:
q6asm_cmd(prtd->audio_client, prtd->stream_id, CMD_CLOSE);
open_err:
- q6asm_audio_client_free(prtd->audio_client);
- prtd->audio_client = NULL;
return ret;
}
@@ -1014,7 +1017,6 @@ static int q6asm_dai_compr_trigger(struct snd_soc_component *component,
0, 0, 0);
break;
case SNDRV_PCM_TRIGGER_STOP:
- prtd->state = Q6ASM_STREAM_STOPPED;
ret = q6asm_cmd_nowait(prtd->audio_client, prtd->stream_id,
CMD_EOS);
break;
diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c
index 1637cc3f..e440c23 100644
--- a/sound/soc/sdw_utils/soc_sdw_utils.c
+++ b/sound/soc/sdw_utils/soc_sdw_utils.c
@@ -194,6 +194,8 @@ struct asoc_sdw_codec_info codec_info_list[] = {
.dai_type = SOC_SDW_DAI_TYPE_MIC,
.dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_DMIC_DAI_ID},
.rtd_init = asoc_sdw_rt_dmic_rtd_init,
+ .quirk = SOC_SDW_CODEC_MIC,
+ .quirk_exclude = true,
},
},
.dai_num = 3,
@@ -501,6 +503,8 @@ struct asoc_sdw_codec_info codec_info_list[] = {
.dai_type = SOC_SDW_DAI_TYPE_MIC,
.dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_DMIC_DAI_ID},
.rtd_init = asoc_sdw_rt_dmic_rtd_init,
+ .quirk = SOC_SDW_CODEC_MIC,
+ .quirk_exclude = true,
},
},
.dai_num = 3,
@@ -1110,7 +1114,7 @@ int asoc_sdw_rtd_init(struct snd_soc_pcm_runtime *rtd)
struct asoc_sdw_codec_info *codec_info;
struct snd_soc_dai *dai;
struct sdw_slave *sdw_peripheral;
- const char *spk_components="";
+ const char *spk_components = NULL;
int dai_index;
int ret;
int i;
@@ -1193,7 +1197,7 @@ int asoc_sdw_rtd_init(struct snd_soc_pcm_runtime *rtd)
else
component = codec_info->dais[dai_index].component_name;
- if (strlen (spk_components) == 0)
+ if (!spk_components)
spk_components =
devm_kasprintf(card->dev, GFP_KERNEL, "%s", component);
else
@@ -1201,13 +1205,15 @@ int asoc_sdw_rtd_init(struct snd_soc_pcm_runtime *rtd)
spk_components =
devm_kasprintf(card->dev, GFP_KERNEL,
"%s+%s", spk_components, component);
+
+ if (!spk_components)
+ return -ENOMEM;
}
codec_info->dais[dai_index].rtd_init_done = true;
-
}
- if (strlen (spk_components) > 0) {
+ if (spk_components) {
/* Update card components for speaker components */
card->components = devm_kasprintf(card->dev, GFP_KERNEL, "%s spk:%s",
card->components, spk_components);
@@ -1608,6 +1614,7 @@ int asoc_sdw_get_dai_type(u32 type)
switch (type) {
case SDCA_FUNCTION_TYPE_SMART_AMP:
case SDCA_FUNCTION_TYPE_SIMPLE_AMP:
+ case SDCA_FUNCTION_TYPE_COMPANION_AMP:
return SOC_SDW_DAI_TYPE_AMP;
case SDCA_FUNCTION_TYPE_SMART_MIC:
case SDCA_FUNCTION_TYPE_SIMPLE_MIC:
diff --git a/sound/soc/soc-utils.c b/sound/soc/soc-utils.c
index c8adfff..9cb7567 100644
--- a/sound/soc/soc-utils.c
+++ b/sound/soc/soc-utils.c
@@ -36,6 +36,7 @@ int snd_soc_ret(const struct device *dev, int ret, const char *fmt, ...)
vaf.va = &args;
dev_err(dev, "ASoC error (%d): %pV", ret, &vaf);
+ va_end(args);
}
return ret;
diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c
index 71a18f1..f615b8d 100644
--- a/sound/soc/sof/amd/acp.c
+++ b/sound/soc/sof/amd/acp.c
@@ -223,7 +223,7 @@ static int psp_send_cmd(struct acp_dev_data *adata, int cmd)
{
struct snd_sof_dev *sdev = adata->dev;
int ret;
- u32 data;
+ int data;
if (!cmd)
return -EINVAL;
diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c
index b3d61d9..8662b42 100644
--- a/sound/soc/sof/intel/hda.c
+++ b/sound/soc/sof/intel/hda.c
@@ -1412,7 +1412,8 @@ static struct snd_soc_acpi_mach *hda_sdw_machine_select(struct snd_sof_dev *sdev
link_mask |= BIT(peripherals->array[i]->bus->link_id);
link_num = hweight32(link_mask);
- links = devm_kcalloc(sdev->dev, link_num, sizeof(*links), GFP_KERNEL);
+ /* An empty adr_link is needed to terminate the adr_link loop */
+ links = devm_kcalloc(sdev->dev, link_num + 1, sizeof(*links), GFP_KERNEL);
if (!links)
return NULL;
diff --git a/sound/soc/spacemit/k1_i2s.c b/sound/soc/spacemit/k1_i2s.c
index 1cb99f1..5420ca2 100644
--- a/sound/soc/spacemit/k1_i2s.c
+++ b/sound/soc/spacemit/k1_i2s.c
@@ -93,8 +93,8 @@ static void spacemit_i2s_init(struct spacemit_i2s_dev *i2s)
u32 sscr_val, sspsp_val, ssfcr_val, ssrwt_val;
sscr_val = SSCR_TRAIL | SSCR_FRF_PSP;
- ssfcr_val = FIELD_PREP(SSFCR_FIELD_TFT, 5) |
- FIELD_PREP(SSFCR_FIELD_RFT, 5) |
+ ssfcr_val = FIELD_PREP(SSFCR_FIELD_TFT, 0xF) |
+ FIELD_PREP(SSFCR_FIELD_RFT, 0xF) |
SSFCR_RSRE | SSFCR_TSRE;
ssrwt_val = SSRWT_RWOT;
sspsp_val = SSPSP_SFRMP;
@@ -106,6 +106,37 @@ static void spacemit_i2s_init(struct spacemit_i2s_dev *i2s)
writel(0, i2s->base + SSINTEN);
}
+static int spacemit_i2s_startup(struct snd_pcm_substream *substream,
+ struct snd_soc_dai *dai)
+{
+ struct spacemit_i2s_dev *i2s = snd_soc_dai_get_drvdata(dai);
+
+ switch (i2s->dai_fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+ case SND_SOC_DAIFMT_I2S:
+ snd_pcm_hw_constraint_minmax(substream->runtime,
+ SNDRV_PCM_HW_PARAM_CHANNELS,
+ 2, 2);
+ snd_pcm_hw_constraint_mask64(substream->runtime,
+ SNDRV_PCM_HW_PARAM_FORMAT,
+ SNDRV_PCM_FMTBIT_S16_LE);
+ break;
+ case SND_SOC_DAIFMT_DSP_A:
+ case SND_SOC_DAIFMT_DSP_B:
+ snd_pcm_hw_constraint_minmax(substream->runtime,
+ SNDRV_PCM_HW_PARAM_CHANNELS,
+ 1, 1);
+ snd_pcm_hw_constraint_mask64(substream->runtime,
+ SNDRV_PCM_HW_PARAM_FORMAT,
+ SNDRV_PCM_FMTBIT_S32_LE);
+ break;
+ default:
+ dev_dbg(i2s->dev, "unexpected format type");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream,
struct snd_pcm_hw_params *params,
struct snd_soc_dai *dai)
@@ -117,10 +148,6 @@ static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream,
u32 val;
int ret;
- val = readl(i2s->base + SSCR);
- if (val & SSCR_SSE)
- return 0;
-
dma_data = &i2s->playback_dma_data;
if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
@@ -157,22 +184,9 @@ static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream,
dma_data->maxburst = 32;
dma_data->addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
}
-
- snd_pcm_hw_constraint_minmax(substream->runtime,
- SNDRV_PCM_HW_PARAM_CHANNELS,
- 1, 2);
- snd_pcm_hw_constraint_mask64(substream->runtime,
- SNDRV_PCM_HW_PARAM_FORMAT,
- SNDRV_PCM_FMTBIT_S16_LE);
break;
case SND_SOC_DAIFMT_DSP_A:
case SND_SOC_DAIFMT_DSP_B:
- snd_pcm_hw_constraint_minmax(substream->runtime,
- SNDRV_PCM_HW_PARAM_CHANNELS,
- 1, 1);
- snd_pcm_hw_constraint_mask64(substream->runtime,
- SNDRV_PCM_HW_PARAM_FORMAT,
- SNDRV_PCM_FMTBIT_S32_LE);
break;
default:
dev_dbg(i2s->dev, "unexpected format type");
@@ -181,6 +195,9 @@ static int spacemit_i2s_hw_params(struct snd_pcm_substream *substream,
}
val = readl(i2s->base + SSCR);
+ if (val & SSCR_SSE)
+ return 0;
+
val &= ~SSCR_DW_32BYTE;
val |= data_width;
writel(val, i2s->base + SSCR);
@@ -303,6 +320,7 @@ static int spacemit_i2s_dai_remove(struct snd_soc_dai *dai)
static const struct snd_soc_dai_ops spacemit_i2s_dai_ops = {
.probe = spacemit_i2s_dai_probe,
.remove = spacemit_i2s_dai_remove,
+ .startup = spacemit_i2s_startup,
.hw_params = spacemit_i2s_hw_params,
.set_sysclk = spacemit_i2s_set_sysclk,
.set_fmt = spacemit_i2s_set_fmt,
diff --git a/sound/soc/tegra/Makefile b/sound/soc/tegra/Makefile
index 3f396c8..1c18ef6 100644
--- a/sound/soc/tegra/Makefile
+++ b/sound/soc/tegra/Makefile
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
# Tegra platform Support
snd-soc-tegra-pcm-y := tegra_pcm.o
-snd-soc-tegra-utils-y += tegra_asoc_utils.o
snd-soc-tegra20-ac97-y := tegra20_ac97.o
snd-soc-tegra20-das-y := tegra20_das.o
snd-soc-tegra20-i2s-y := tegra20_i2s.o
diff --git a/sound/soc/ti/omap-dmic.c b/sound/soc/ti/omap-dmic.c
index fb92bb8..f6c393c 100644
--- a/sound/soc/ti/omap-dmic.c
+++ b/sound/soc/ti/omap-dmic.c
@@ -328,7 +328,7 @@ static int omap_dmic_select_fclk(struct omap_dmic *dmic, int clk_id,
}
mux = clk_get_parent(dmic->fclk);
- if (IS_ERR(mux)) {
+ if (!mux) {
dev_err(dmic->dev, "can't get fck mux parent\n");
clk_put(parent_clk);
return -ENODEV;
diff --git a/sound/sparc/dbri.c b/sound/sparc/dbri.c
index 75f82a9..2f5f620 100644
--- a/sound/sparc/dbri.c
+++ b/sound/sparc/dbri.c
@@ -592,6 +592,7 @@ static __u32 reverse_bytes(__u32 b, int len)
fallthrough;
case 2:
b = ((b & 0xaaaaaaaa) >> 1) | ((b & 0x55555555) << 1);
+ fallthrough;
case 1:
case 0:
break;
diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c
index 8af0c04..b20aae0 100644
--- a/sound/usb/caiaq/device.c
+++ b/sound/usb/caiaq/device.c
@@ -366,7 +366,7 @@ static int setup_card(struct snd_usb_caiaqdev *cdev)
#ifdef CONFIG_SND_USB_CAIAQ_INPUT
ret = snd_usb_caiaq_input_init(cdev);
- if (ret < 0) {
+ if (ret < 0 && ret != -ENODEV) {
dev_err(dev, "Unable to set up input system (ret=%d)\n", ret);
return ret;
}
@@ -423,6 +423,7 @@ static int create_card(struct usb_device *usb_dev,
cdev = caiaqdev(card);
cdev->chip.dev = usb_get_dev(usb_dev);
+ card->private_free = card_free;
cdev->chip.card = card;
cdev->chip.usb_id = USB_ID(le16_to_cpu(usb_dev->descriptor.idVendor),
le16_to_cpu(usb_dev->descriptor.idProduct));
@@ -511,10 +512,9 @@ static int init_card(struct snd_usb_caiaqdev *cdev)
scnprintf(card->longname, sizeof(card->longname), "%s %s (%s)",
cdev->vendor_name, cdev->product_name, usbpath);
- card->private_free = card_free;
err = setup_card(cdev);
if (err < 0)
- return err;
+ goto err_kill_urb;
return 0;
diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c
index a913089..5c70fdf 100644
--- a/sound/usb/caiaq/input.c
+++ b/sound/usb/caiaq/input.c
@@ -804,7 +804,7 @@ int snd_usb_caiaq_input_init(struct snd_usb_caiaqdev *cdev)
default:
/* no input methods supported on this device */
- ret = -EINVAL;
+ ret = -ENODEV;
goto exit_free_idev;
}
diff --git a/sound/usb/clock.c b/sound/usb/clock.c
index 842ba5b..2e0c18e 100644
--- a/sound/usb/clock.c
+++ b/sound/usb/clock.c
@@ -208,11 +208,18 @@ static bool uac_clock_source_is_valid_quirk(struct snd_usb_audio *chip,
}
/*
- * MOTU MicroBook IIc
- * Sample rate changes takes more than 2 seconds for this device. Clock
- * validity request returns false during that period.
+ * Quirk for older MOTU AVB / hybrid interfaces
+ *
+ * These devices take more than 2 seconds to switch sample rate or
+ * clock source. During this period the clock validity request
+ * returns false, causing ALSA to fail prematurely.
+ *
+ * Affected models (all use vendor 0x07fd):
+ * - MicroBook IIc → 0x0004
+ * - 1248, 624, 8A, UltraLite AVB, 8M, 16A, ... → 0x0005
*/
- if (chip->usb_id == USB_ID(0x07fd, 0x0004)) {
+ if (chip->usb_id == USB_ID(0x07fd, 0x0004) || /* MicroBook IIc */
+ chip->usb_id == USB_ID(0x07fd, 0x0005)) { /* 1248 / 624 / 8A / UltraLite AVB / ... */
count = 0;
while ((!ret) && (count < 50)) {
diff --git a/sound/usb/midi.c b/sound/usb/midi.c
index 0a5b894..d87e3f3 100644
--- a/sound/usb/midi.c
+++ b/sound/usb/midi.c
@@ -1951,15 +1951,17 @@ static struct usb_ms_endpoint_descriptor *find_usb_ms_endpoint_descriptor(
while (extralen > 3) {
struct usb_ms_endpoint_descriptor *ms_ep =
(struct usb_ms_endpoint_descriptor *)extra;
+ int length = ms_ep->bLength;
- if (ms_ep->bLength > 3 &&
+ if (!length || length > extralen)
+ break;
+
+ if (length > 3 &&
ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT &&
ms_ep->bDescriptorSubtype == UAC_MS_GENERAL)
return ms_ep;
- if (!extra[0])
- break;
- extralen -= extra[0];
- extra += extra[0];
+ extralen -= length;
+ extra += length;
}
return NULL;
}
diff --git a/sound/usb/midi2.c b/sound/usb/midi2.c
index 3546ba9..04aeb90 100644
--- a/sound/usb/midi2.c
+++ b/sound/usb/midi2.c
@@ -227,7 +227,7 @@ static void kill_midi_urbs(struct snd_usb_midi2_endpoint *ep, bool suspending)
if (!ep)
return;
if (suspending)
- ep->suspended = ep->running;
+ atomic_set(&ep->suspended, atomic_read(&ep->running));
atomic_set(&ep->running, 0);
for (i = 0; i < ep->num_urbs; i++) {
if (!ep->urbs[i].urb)
@@ -496,15 +496,17 @@ static void *find_usb_ms_endpoint_descriptor(struct usb_host_endpoint *hostep,
while (extralen > 3) {
struct usb_ms_endpoint_descriptor *ms_ep =
(struct usb_ms_endpoint_descriptor *)extra;
+ int length = ms_ep->bLength;
- if (ms_ep->bLength > 3 &&
+ if (!length || length > extralen)
+ break;
+
+ if (length > 3 &&
ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT &&
ms_ep->bDescriptorSubtype == subtype)
return ms_ep;
- if (!extra[0])
- break;
- extralen -= extra[0];
- extra += extra[0];
+ extralen -= length;
+ extra += length;
}
return NULL;
}
@@ -1188,10 +1190,11 @@ void snd_usb_midi_v2_suspend_all(struct snd_usb_audio *chip)
static void resume_midi2_endpoint(struct snd_usb_midi2_endpoint *ep)
{
- ep->running = ep->suspended;
- if (ep->direction == STR_IN)
+ atomic_set(&ep->running, atomic_read(&ep->suspended));
+ atomic_set(&ep->suspended, 0);
+
+ if (ep->direction == STR_IN || atomic_read(&ep->running))
submit_io_urbs(ep);
- /* FIXME: does it all? */
}
void snd_usb_midi_v2_resume_all(struct snd_usb_audio *chip)
diff --git a/sound/usb/misc/ua101.c b/sound/usb/misc/ua101.c
index 49b3dd8..b9a62e9 100644
--- a/sound/usb/misc/ua101.c
+++ b/sound/usb/misc/ua101.c
@@ -894,8 +894,9 @@ find_format_descriptor(struct usb_interface *interface)
struct uac_format_type_i_discrete_descriptor *desc;
desc = (struct uac_format_type_i_discrete_descriptor *)extra;
- if (desc->bLength > extralen) {
- dev_err(&interface->dev, "descriptor overflow\n");
+ if (desc->bLength < sizeof(struct usb_descriptor_header) ||
+ desc->bLength > extralen) {
+ dev_err(&interface->dev, "invalid descriptor length\n");
return NULL;
}
if (desc->bLength == UAC_FORMAT_TYPE_I_DISCRETE_DESC_SIZE(1) &&
@@ -974,6 +975,13 @@ static int detect_usb_format(struct ua101 *ua)
ua->capture.channels = fmt_capture->bNrChannels;
ua->playback.channels = fmt_playback->bNrChannels;
+ if (!ua->capture.channels || !ua->playback.channels) {
+ dev_err(&ua->dev->dev,
+ "invalid channel count: capture %u, playback %u\n",
+ ua->capture.channels, ua->playback.channels);
+ return -EINVAL;
+ }
+
ua->capture.frame_bytes =
fmt_capture->bSubframeSize * ua->capture.channels;
ua->playback.frame_bytes =
diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c
index 8565311..5fba456 100644
--- a/sound/usb/mixer.c
+++ b/sound/usb/mixer.c
@@ -1190,6 +1190,16 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval,
cval->res = 1;
}
break;
+
+ case USB_ID(0x0e6f, 0x024a): /* PDP Riffmaster for PS4 */
+ case USB_ID(0x0e6f, 0x0249): /* PDP Riffmaster for PS5 */
+ if (!strcmp(kctl->id.name, "PCM Playback Volume")) {
+ usb_audio_info(chip,
+ "set volume quirk for PDP Riffmaster for PS4/PS5\n");
+ cval->min = -2560; /* Mute under it */
+ }
+ break;
+
case USB_ID(0x3302, 0x12db): /* MOONDROP Quark2 */
if (!strcmp(kctl->id.name, "PCM Playback Volume")) {
usb_audio_info(chip,
diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c
index 8eaa962..a4fac46 100644
--- a/sound/usb/mixer_scarlett2.c
+++ b/sound/usb/mixer_scarlett2.c
@@ -2504,6 +2504,27 @@ static int scarlett2_has_config_item(
return !!private->config_set->items[config_item_num].offset;
}
+/* Return the configuration item's offset, applying any per-firmware
+ * overrides.
+ *
+ * Firmware 2417 for the 2i2 Gen 4 moved DIRECT_MONITOR_GAIN by 4
+ * bytes. Apply that shift here so that the rest of the driver can
+ * keep using the single config set. This override can be removed
+ * once the multi-config-set framework lands.
+ */
+static int scarlett2_config_item_offset(
+ struct scarlett2_data *private, int config_item_num)
+{
+ int offset = private->config_set->items[config_item_num].offset;
+
+ if (config_item_num == SCARLETT2_CONFIG_DIRECT_MONITOR_GAIN &&
+ private->info == &s2i2_gen4_info &&
+ private->firmware_version >= 2417)
+ offset = 0x2a4;
+
+ return offset;
+}
+
/* Send a USB message to get configuration parameters; result placed in *buf */
static int scarlett2_usb_get_config(
struct usb_mixer_interface *mixer,
@@ -2513,6 +2534,7 @@ static int scarlett2_usb_get_config(
const struct scarlett2_config *config_item =
&private->config_set->items[config_item_num];
int size, err, i;
+ int item_offset;
u8 *buf_8;
u8 value;
@@ -2522,13 +2544,15 @@ static int scarlett2_usb_get_config(
if (!config_item->offset)
return -EFAULT;
+ item_offset = scarlett2_config_item_offset(private, config_item_num);
+
/* Writes to the parameter buffer are always 1 byte */
size = config_item->size ? config_item->size : 8;
/* For byte-sized parameters, retrieve directly into buf */
if (size >= 8) {
size = size / 8 * count;
- err = scarlett2_usb_get(mixer, config_item->offset, buf, size);
+ err = scarlett2_usb_get(mixer, item_offset, buf, size);
if (err < 0)
return err;
if (config_item->size == 16) {
@@ -2546,7 +2570,7 @@ static int scarlett2_usb_get_config(
}
/* For bit-sized parameters, retrieve into value */
- err = scarlett2_usb_get(mixer, config_item->offset, &value, 1);
+ err = scarlett2_usb_get(mixer, item_offset, &value, 1);
if (err < 0)
return err;
@@ -2696,7 +2720,8 @@ static int scarlett2_usb_set_config(
*/
if (config_item->size >= 8) {
size = config_item->size / 8;
- offset = config_item->offset + index * size;
+ offset = scarlett2_config_item_offset(private, config_item_num) +
+ index * size;
/* If updating a bit, retrieve the old value, set/clear the
* bit as needed, and update value
@@ -2705,7 +2730,7 @@ static int scarlett2_usb_set_config(
u8 tmp;
size = 1;
- offset = config_item->offset;
+ offset = scarlett2_config_item_offset(private, config_item_num);
err = scarlett2_usb_get(mixer, offset, &tmp, 1);
if (err < 0)
@@ -6707,6 +6732,8 @@ static int scarlett2_add_line_in_ctls(struct usb_mixer_interface *mixer)
err = scarlett2_add_new_ctl(
mixer, &scarlett2_autogain_status_ctl,
i, 1, s, &private->autogain_status_ctls[i]);
+ if (err < 0)
+ return err;
}
/* Add autogain target controls */
@@ -9185,12 +9212,15 @@ static long scarlett2_hwdep_write(struct snd_hwdep *hw,
flash_size = private->flash_segment_blocks[segment_id] *
SCARLETT2_FLASH_BLOCK_SIZE;
- if (count < 0 || *offset < 0 || *offset + count >= flash_size)
- return -ENOSPC;
+ if (count < 0 || *offset < 0)
+ return -EINVAL;
if (!count)
return 0;
+ if (*offset >= flash_size || count > flash_size - *offset)
+ return -ENOSPC;
+
/* Limit the *req size to SCARLETT2_FLASH_RW_MAX */
if (count > max_data_size)
count = max_data_size;
diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c
index 5f993b8..a000950 100644
--- a/sound/usb/qcom/qc_audio_offload.c
+++ b/sound/usb/qcom/qc_audio_offload.c
@@ -565,6 +565,7 @@ static unsigned long uaudio_iommu_map_pa(enum mem_type mtype, bool dma_coherent,
unsigned long iova = 0;
bool map = true;
int prot = uaudio_iommu_map_prot(dma_coherent);
+ int ret;
switch (mtype) {
case MEM_EVENT_RING:
@@ -582,10 +583,24 @@ static unsigned long uaudio_iommu_map_pa(enum mem_type mtype, bool dma_coherent,
dev_err(uaudio_qdev->data->dev, "unknown mem type %d\n", mtype);
}
- if (!iova || !map)
+ if (!iova)
return 0;
- iommu_map(uaudio_qdev->data->domain, iova, pa, size, prot, GFP_KERNEL);
+ if (!map)
+ return iova;
+
+ ret = iommu_map(uaudio_qdev->data->domain, iova, pa, size, prot,
+ GFP_KERNEL);
+ if (ret) {
+ dev_err(uaudio_qdev->data->dev,
+ "failed to map %zu bytes at iova 0x%08lx: %d\n",
+ size, iova, ret);
+ if (mtype == MEM_XFER_RING)
+ uaudio_put_iova(iova, size,
+ &uaudio_qdev->xfer_ring_list,
+ &uaudio_qdev->xfer_ring_iova_size);
+ return 0;
+ }
return iova;
}
@@ -1054,15 +1069,17 @@ static int uaudio_transfer_buffer_setup(struct snd_usb_substream *subs,
if (!xfer_buf)
return -ENOMEM;
- dma_get_sgtable(subs->dev->bus->sysdev, &xfer_buf_sgt, xfer_buf,
- xfer_buf_dma, len);
+ ret = dma_get_sgtable(subs->dev->bus->sysdev, &xfer_buf_sgt, xfer_buf,
+ xfer_buf_dma, len);
+ if (ret)
+ goto free_xfer_buf;
/* map the physical buffer into sysdev as well */
xfer_buf_dma_sysdev = uaudio_iommu_map_xfer_buf(dma_coherent,
len, &xfer_buf_sgt);
if (!xfer_buf_dma_sysdev) {
ret = -ENOMEM;
- goto unmap_sync;
+ goto free_sgt;
}
mem_info->dma = xfer_buf_dma;
@@ -1073,7 +1090,9 @@ static int uaudio_transfer_buffer_setup(struct snd_usb_substream *subs,
return 0;
-unmap_sync:
+free_sgt:
+ sg_free_table(&xfer_buf_sgt);
+free_xfer_buf:
usb_free_coherent(subs->dev, len, xfer_buf, xfer_buf_dma);
return ret;
diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h
index 803e03d..4e9cfff 100644
--- a/sound/usb/quirks-table.h
+++ b/sound/usb/quirks-table.h
@@ -2652,6 +2652,9 @@ YAMAHA_DEVICE(0x7010, "UB99"),
}
}
},
+
+#endif /* disabled */
+
{
/*
* The AudioBox USB advertises S24_3LE as the only supported format
@@ -2700,7 +2703,6 @@ YAMAHA_DEVICE(0x7010, "UB99"),
}
}
},
-#endif /* disabled */
{
/*
diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
index 7b803ad..3d1b352 100644
--- a/sound/usb/quirks.c
+++ b/sound/usb/quirks.c
@@ -125,7 +125,7 @@ static int add_audio_stream_from_fixed_fmt(struct snd_usb_audio *chip,
snd_usb_audioformat_set_sync_ep(chip, fp);
- err = snd_usb_add_audio_stream(chip, stream, fp);
+ err = snd_usb_add_audio_stream(chip, stream, fp, NULL);
if (err < 0)
return err;
@@ -2277,6 +2277,9 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
QUIRK_FLAG_ALIGN_TRANSFER),
DEVICE_FLG(0x05e1, 0x0480, /* Hauppauge Woodbury */
QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER),
+ DEVICE_FLG(0x05fc, 0x0231, /* JBL Pebbles */
+ QUIRK_FLAG_MIXER_PLAYBACK_LINEAR_VOL | QUIRK_FLAG_MIXER_CAPTURE_LINEAR_VOL |
+ QUIRK_FLAG_GET_SAMPLE_RATE),
DEVICE_FLG(0x0624, 0x3d3f, /* AB13X USB Audio */
QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY),
DEVICE_FLG(0x0644, 0x8043, /* TEAC UD-501/UD-501V2/UD-503/NT-503 */
@@ -2366,6 +2369,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
QUIRK_FLAG_IGNORE_CTL_ERROR),
DEVICE_FLG(0x152a, 0x880a, /* NeuralDSP Quad Cortex */
0), /* Doesn't have the vendor quirk which would otherwise apply */
+ DEVICE_FLG(0x1532, 0x055e, /* Razer Nommo V2 X */
+ QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE),
DEVICE_FLG(0x154e, 0x1002, /* Denon DCD-1500RE */
QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY),
DEVICE_FLG(0x154e, 0x1003, /* Denon DA-300USB */
@@ -2444,6 +2449,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
QUIRK_FLAG_DSD_RAW),
DEVICE_FLG(0x2522, 0x0007, /* LH Labs Geek Out HD Audio 1V5 */
QUIRK_FLAG_SET_IFACE_FIRST),
+ DEVICE_FLG(0x25aa, 0x600b, /* TAE1159 */
+ QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY),
DEVICE_FLG(0x262a, 0x9302, /* ddHiFi TC44C */
QUIRK_FLAG_DSD_RAW),
DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */
@@ -2458,6 +2465,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
QUIRK_FLAG_GENERIC_IMPLICIT_FB),
DEVICE_FLG(0x2b53, 0x0031, /* Fiero SC-01 (firmware v1.1.0) */
QUIRK_FLAG_GENERIC_IMPLICIT_FB),
+ DEVICE_FLG(0x2b73, 0x0047, /* AlphaTheta EUPHONIA */
+ QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB),
DEVICE_FLG(0x2d95, 0x8011, /* VIVO USB-C HEADSET */
QUIRK_FLAG_CTL_MSG_DELAY_1M),
DEVICE_FLG(0x2d95, 0x8021, /* VIVO USB-C-XE710 HEADSET */
@@ -2472,6 +2481,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
QUIRK_FLAG_IGNORE_CTL_ERROR),
DEVICE_FLG(0x3255, 0x0000, /* Luxman D-10X */
QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY),
+ DEVICE_FLG(0x3302, 0x17c2, /* TTGK Technology USB-C Audio */
+ QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY),
DEVICE_FLG(0x339b, 0x3a07, /* Synaptics HONOR USB-C HEADSET */
QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE),
DEVICE_FLG(0x3443, 0x930d, /* NexiGo N930W 60fps Webcam */
diff --git a/sound/usb/stream.c b/sound/usb/stream.c
index 2532bf9..b2c5c81 100644
--- a/sound/usb/stream.c
+++ b/sound/usb/stream.c
@@ -79,7 +79,7 @@ static void snd_usb_audio_pcm_free(struct snd_pcm *pcm)
static void snd_usb_init_substream(struct snd_usb_stream *as,
int stream,
struct audioformat *fp,
- struct snd_usb_power_domain *pd)
+ struct snd_usb_power_domain **pdptr)
{
struct snd_usb_substream *subs = &as->substream[stream];
@@ -105,10 +105,11 @@ static void snd_usb_init_substream(struct snd_usb_stream *as,
if (fp->channels > subs->channels_max)
subs->channels_max = fp->channels;
- if (pd) {
- subs->str_pd = pd;
+ if (pdptr && *pdptr) {
+ subs->str_pd = *pdptr;
+ *pdptr = NULL; /* assigned */
/* Initialize Power Domain to idle status D1 */
- snd_usb_power_domain_set(subs->stream->chip, pd,
+ snd_usb_power_domain_set(subs->stream->chip, subs->str_pd,
UAC3_PD_STATE_D1);
}
@@ -352,6 +353,8 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor
if (len < sizeof(*cs_desc))
break;
cs_len = le16_to_cpu(cs_desc->wLength);
+ if (cs_len < sizeof(*cs_desc))
+ break;
if (len < cs_len)
break;
cs_type = cs_desc->bSegmentType;
@@ -492,11 +495,14 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor
* if not, create a new pcm stream. note, fp is added to the substream
* fmt_list and will be freed on the chip instance release. do not free
* fp or do remove it from the substream fmt_list to avoid double-free.
+ *
+ * pdptr is optional and can be NULL. When it's non-NULL and the PD gets
+ * assigned to the stream, *pdptr is cleared to NULL upon return.
*/
-static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip,
- int stream,
- struct audioformat *fp,
- struct snd_usb_power_domain *pd)
+int snd_usb_add_audio_stream(struct snd_usb_audio *chip,
+ int stream,
+ struct audioformat *fp,
+ struct snd_usb_power_domain **pdptr)
{
struct snd_usb_stream *as;
@@ -529,7 +535,7 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip,
err = snd_pcm_new_stream(as->pcm, stream, 1);
if (err < 0)
return err;
- snd_usb_init_substream(as, stream, fp, pd);
+ snd_usb_init_substream(as, stream, fp, pdptr);
return add_chmap(as->pcm, stream, subs);
}
@@ -558,7 +564,7 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip,
else
strscpy(pcm->name, "USB Audio");
- snd_usb_init_substream(as, stream, fp, pd);
+ snd_usb_init_substream(as, stream, fp, pdptr);
/*
* Keep using head insertion for M-Audio Audiophile USB (tm) which has a
@@ -576,21 +582,6 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip,
return add_chmap(pcm, stream, &as->substream[stream]);
}
-int snd_usb_add_audio_stream(struct snd_usb_audio *chip,
- int stream,
- struct audioformat *fp)
-{
- return __snd_usb_add_audio_stream(chip, stream, fp, NULL);
-}
-
-static int snd_usb_add_audio_stream_v3(struct snd_usb_audio *chip,
- int stream,
- struct audioformat *fp,
- struct snd_usb_power_domain *pd)
-{
- return __snd_usb_add_audio_stream(chip, stream, fp, pd);
-}
-
static int parse_uac_endpoint_attributes(struct snd_usb_audio *chip,
struct usb_host_interface *alts,
int protocol, int iface_no)
@@ -1003,7 +994,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip,
* and request Cluster Descriptor
*/
wLength = le16_to_cpu(hc_header.wLength);
- if (wLength < sizeof(cluster))
+ if (wLength < sizeof(*cluster))
return NULL;
cluster = kzalloc(wLength, GFP_KERNEL);
if (!cluster)
@@ -1113,8 +1104,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip,
}
}
- if (pd)
- *pd_out = pd;
+ *pd_out = pd;
return fp;
}
@@ -1129,7 +1119,6 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip,
struct usb_interface_descriptor *altsd;
int i, altno, err, stream;
struct audioformat *fp = NULL;
- struct snd_usb_power_domain *pd = NULL;
bool set_iface_first;
int num, protocol;
@@ -1171,6 +1160,12 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip,
if (snd_usb_apply_interface_quirk(chip, iface_no, altno))
continue;
+ /* pd may be allocated at snd_usb_get_audioformat_uac3() and
+ * assigned at snd_usb_add_audio_stream(); otherwise it'll be
+ * freed automatically by cleanup at each loop.
+ */
+ struct snd_usb_power_domain *pd __free(kfree) = NULL;
+
/*
* Roland audio streaming interfaces are marked with protocols
* 0/1/2, but are UAC 1 compatible.
@@ -1226,23 +1221,16 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip,
*has_non_pcm = true;
if ((fp->fmt_type == UAC_FORMAT_TYPE_I) == non_pcm) {
audioformat_free(fp);
- kfree(pd);
fp = NULL;
- pd = NULL;
continue;
}
snd_usb_audioformat_set_sync_ep(chip, fp);
dev_dbg(&dev->dev, "%u:%d: add audio endpoint %#x\n", iface_no, altno, fp->endpoint);
- if (protocol == UAC_VERSION_3)
- err = snd_usb_add_audio_stream_v3(chip, stream, fp, pd);
- else
- err = snd_usb_add_audio_stream(chip, stream, fp);
-
+ err = snd_usb_add_audio_stream(chip, stream, fp, &pd);
if (err < 0) {
audioformat_free(fp);
- kfree(pd);
return err;
}
diff --git a/sound/usb/stream.h b/sound/usb/stream.h
index d92e18d..61b9a13 100644
--- a/sound/usb/stream.h
+++ b/sound/usb/stream.h
@@ -7,7 +7,8 @@ int snd_usb_parse_audio_interface(struct snd_usb_audio *chip,
int snd_usb_add_audio_stream(struct snd_usb_audio *chip,
int stream,
- struct audioformat *fp);
+ struct audioformat *fp,
+ struct snd_usb_power_domain **pdptr);
#endif /* __USBAUDIO_STREAM_H */
diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h
index 6673601..eff2964 100644
--- a/tools/arch/x86/include/asm/msr-index.h
+++ b/tools/arch/x86/include/asm/msr-index.h
@@ -793,9 +793,10 @@
#define MSR_AMD64_LBR_SELECT 0xc000010e
/* Zen4 */
-#define MSR_ZEN4_BP_CFG 0xc001102e
+#define MSR_ZEN4_BP_CFG 0xc001102e
#define MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT 4
#define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5
+#define MSR_ZEN2_BP_CFG_BUG_FIX_BIT 33
/* Fam 19h MSRs */
#define MSR_F19H_UMC_PERF_CTL 0xc0010800
diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv
index 9f5155c..45a876d 100644
--- a/tools/arch/x86/kcpuid/cpuid.csv
+++ b/tools/arch/x86/kcpuid/cpuid.csv
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: CC0-1.0
-# Generator: x86-cpuid-db v3.0
+# Generator: x86-cpuid-db v3.1
#
# Auto-generated file.
@@ -177,7 +177,7 @@
0x6, 0, ebx, 3:0, n_therm_thresholds , Digital thermometer thresholds
0x6, 0, ecx, 0, aperfmperf , MPERF/APERF MSRs (effective frequency interface)
0x6, 0, ecx, 3, epb , IA32_ENERGY_PERF_BIAS MSR
- 0x6, 0, ecx, 15:8, thrd_director_nclasses , Number of classes, Intel thread director
+ 0x6, 0, ecx, 15:8, hw_feedback_nclasses , Number of Intel Thread Director classes
0x6, 0, edx, 0, perfcap_reporting , Performance capability reporting
0x6, 0, edx, 1, encap_reporting , Energy efficiency capability reporting
0x6, 0, edx, 11:8, feedback_sz , Feedback interface structure size, in 4K pages
@@ -247,10 +247,10 @@
0x7, 0, edx, 1, sgx_keys , Intel SGX attestation services
0x7, 0, edx, 2, avx512_4vnniw , AVX-512 neural network instructions
0x7, 0, edx, 3, avx512_4fmaps , AVX-512 multiply accumulation single precision
- 0x7, 0, edx, 4, fsrm , Fast short REP MOV
+ 0x7, 0, edx, 4, fsrm , Fast short REP MOVSB
0x7, 0, edx, 5, uintr , User interrupts
0x7, 0, edx, 8, avx512_vp2intersect , VP2INTERSECT{D,Q} instructions
- 0x7, 0, edx, 9, srdbs_ctrl , SRBDS mitigation MSR
+ 0x7, 0, edx, 9, srbds_ctrl , SRBDS mitigation MSR
0x7, 0, edx, 10, md_clear , VERW MD_CLEAR microcode
0x7, 0, edx, 11, rtm_always_abort , XBEGIN (RTM transaction) always aborts
0x7, 0, edx, 13, tsx_force_abort , MSR TSX_FORCE_ABORT, RTM_ABORT bit
@@ -296,8 +296,8 @@
0x7, 2, edx, 0, intel_psfd , Intel predictive store forward disable
0x7, 2, edx, 1, ipred_ctrl , MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S}
0x7, 2, edx, 2, rrsba_ctrl , MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S}
- 0x7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U
- 0x7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S
+ 0x7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U
+ 0x7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S
0x7, 2, edx, 5, mcdt_no , MCDT mitigation not needed
0x7, 2, edx, 6, uclock_disable , UC-lock disable
@@ -368,7 +368,7 @@
0xd, 1, ecx, 8, xss_pt , PT state
0xd, 1, ecx, 10, xss_pasid , PASID state
0xd, 1, ecx, 11, xss_cet_u , CET user state
- 0xd, 1, ecx, 12, xss_cet_p , CET supervisor state
+ 0xd, 1, ecx, 12, xss_cet_s , CET supervisor state
0xd, 1, ecx, 13, xss_hdc , HDC state
0xd, 1, ecx, 14, xss_uintr , UINTR state
0xd, 1, ecx, 15, xss_lbr , LBR state
@@ -433,7 +433,7 @@
0x12, 1, eax, 7, secs_attr_kss , Key Separation and Sharing
0x12, 1, eax, 10, secs_attr_aexnotify , Enclave threads: AEX notifications
0x12, 1, ecx, 0, xfrm_x87 , Enclave XFRM.X87
- 0x12, 1, ecx, 1, xfrm_sse , Enclave XFRM.SEE
+ 0x12, 1, ecx, 1, xfrm_sse , Enclave XFRM.SSE
0x12, 1, ecx, 2, xfrm_avx , Enclave XFRM.AVX
0x12, 1, ecx, 3, xfrm_mpx_bndregs , Enclave XFRM.BNDREGS (MPX BND0-BND3 registers)
0x12, 1, ecx, 4, xfrm_mpx_bndcsr , Enclave XFRM.BNDCSR (MPX BNDCFGU/BNDSTATUS registers)
@@ -466,9 +466,9 @@
0x14, 0, ecx, 0, topa_output , ToPA output scheme
0x14, 0, ecx, 1, topa_multiple_entries , ToPA tables can hold multiple entries
0x14, 0, ecx, 2, single_range_output , Single-range output
- 0x14, 0, ecx, 3, trance_transport_output, Trace Transport subsystem output
+ 0x14, 0, ecx, 3, trace_transport_output , Trace Transport subsystem output
0x14, 0, ecx, 31, ip_payloads_lip , IP payloads have LIP values (CS base included)
- 0x14, 1, eax, 2:0, num_address_ranges , Number of configurable Address Ranges
+ 0x14, 1, eax, 2:0, num_address_ranges , Number of configurable address ranges
0x14, 1, eax, 31:16, mtc_periods_bmp , MTC period encodings bitmap
0x14, 1, ebx, 15:0, cycle_thresholds_bmp , Cycle Threshold encodings bitmap
0x14, 1, ebx, 31:16, psb_periods_bmp , Configurable PSB frequency encodings bitmap
@@ -494,7 +494,7 @@
0x17, 0, ebx, 15:0, soc_vendor_id , SoC vendor ID
0x17, 0, ebx, 16, is_vendor_scheme , Assigned by industry enumeration scheme (not Intel)
0x17, 0, ecx, 31:0, soc_proj_id , SoC project ID, assigned by vendor
- 0x17, 0, edx, 31:0, soc_stepping_id , Soc project stepping ID, assigned by vendor
+ 0x17, 0, edx, 31:0, soc_stepping_id , SoC project stepping ID, assigned by vendor
0x17, 3:1, eax, 31:0, vendor_brand_a , Vendor Brand ID string, bytes subleaf_nr * (0 -> 3)
0x17, 3:1, ebx, 31:0, vendor_brand_b , Vendor Brand ID string, bytes subleaf_nr * (4 -> 7)
0x17, 3:1, ecx, 31:0, vendor_brand_c , Vendor Brand ID string, bytes subleaf_nr * (8 -> 11)
@@ -514,12 +514,12 @@
0x18, 31:0, edx, 4:0, tlb_type , Translation cache type (TLB type)
0x18, 31:0, edx, 7:5, tlb_cache_level , Translation cache level (1-based)
0x18, 31:0, edx, 8, is_fully_associative , Fully-associative
- 0x18, 31:0, edx, 25:14, tlb_max_addressible_ids, Max number of addressable IDs - 1
+ 0x18, 31:0, edx, 25:14, tlb_max_addressable_ids, Max number of addressable IDs - 1
# Leaf 19H
# Intel key locker
- 0x19, 0, eax, 0, kl_cpl0_only , CPL0-only key Locker restriction
+ 0x19, 0, eax, 0, kl_cpl0_only , CPL0-only key locker restriction
0x19, 0, eax, 1, kl_no_encrypt , No-encrypt key locker restriction
0x19, 0, eax, 2, kl_no_decrypt , No-decrypt key locker restriction
0x19, 0, ebx, 0, aes_keylocker , AES key locker instructions
@@ -546,7 +546,7 @@
# Intel LBR (Last Branch Record)
0x1c, 0, eax, 7:0, lbr_depth_mask , Max LBR stack depth bitmask
- 0x1c, 0, eax, 30, lbr_deep_c_reset , LBRs maybe cleared on MWAIT C-state > C1
+ 0x1c, 0, eax, 30, lbr_deep_c_reset , LBRs may be cleared on MWAIT C-state > C1
0x1c, 0, eax, 31, lbr_ip_is_lip , LBR IP contain Last IP (otherwise effective IP)
0x1c, 0, ebx, 0, lbr_cpl , CPL filtering
0x1c, 0, ebx, 1, lbr_branch_filter , Branch filtering
@@ -591,8 +591,8 @@
# Intel TD (Trust Domain)
0x21, 0, ebx, 31:0, tdx_vendorid_0 , TDX vendor ID string bytes 0 - 3
- 0x21, 0, ecx, 31:0, tdx_vendorid_2 , CPU vendor ID string bytes 8 - 11
- 0x21, 0, edx, 31:0, tdx_vendorid_1 , CPU vendor ID string bytes 4 - 7
+ 0x21, 0, ecx, 31:0, tdx_vendorid_2 , TDX vendor ID string bytes 8 - 11
+ 0x21, 0, edx, 31:0, tdx_vendorid_1 , TDX vendor ID string bytes 4 - 7
# Leaf 23H
# Intel Architectural Performance Monitoring Extended (ArchPerfmonExt)
@@ -857,7 +857,7 @@
0x8000000a, 0, edx, 1, lbrv , LBR virtualization
0x8000000a, 0, edx, 2, svm_lock , SVM lock
0x8000000a, 0, edx, 3, nrip_save , NRIP save support on #VMEXIT
-0x8000000a, 0, edx, 4, tsc_scale , MSR based TSC rate control
+0x8000000a, 0, edx, 4, tsc_scale , MSR-based TSC rate control
0x8000000a, 0, edx, 5, vmcb_clean , VMCB clean bits support
0x8000000a, 0, edx, 6, flushbyasid , Flush by ASID + Extended VMCB TLB_Control
0x8000000a, 0, edx, 7, decodeassists , Decode Assists support
@@ -895,7 +895,7 @@
0x8000001a, 0, eax, 0, fp_128 , Internal FP/SIMD exec data path is 128-bits wide
0x8000001a, 0, eax, 1, movu_preferred , SSE: MOVU* better than MOVL*/MOVH*
-0x8000001a, 0, eax, 2, fp_256 , internal FP/SSE exec data path is 256-bits wide
+0x8000001a, 0, eax, 2, fp_256 , Internal FP/SSE exec data path is 256-bits wide
# Leaf 8000001BH
# AMD IBS (Instruction-Based Sampling)
@@ -917,7 +917,7 @@
# AMD LWP (Lightweight Profiling)
0x8000001c, 0, eax, 0, os_lwp_avail , OS: LWP is available to application programs
-0x8000001c, 0, eax, 1, os_lpwval , OS: LWPVAL instruction
+0x8000001c, 0, eax, 1, os_lwpval , OS: LWPVAL instruction
0x8000001c, 0, eax, 2, os_lwp_ire , OS: Instructions Retired Event
0x8000001c, 0, eax, 3, os_lwp_bre , OS: Branch Retired Event
0x8000001c, 0, eax, 4, os_lwp_dme , OS: Dcache Miss Event
@@ -934,13 +934,13 @@
0x8000001c, 0, ecx, 5, lwp_data_addr , Cache miss events report data cache address
0x8000001c, 0, ecx, 8:6, lwp_latency_rnd , Cache latency rounding amount
0x8000001c, 0, ecx, 15:9, lwp_version , LWP version
-0x8000001c, 0, ecx, 23:16, lwp_buf_min_sz , LWP event ring buffer min size, 32 event records units
+0x8000001c, 0, ecx, 23:16, lwp_buf_min_sz , LWP event ring buffer min size, 32 event record units
0x8000001c, 0, ecx, 28, lwp_branch_predict , Branches Retired events can be filtered
0x8000001c, 0, ecx, 29, lwp_ip_filtering , IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP)
0x8000001c, 0, ecx, 30, lwp_cache_levels , Cache-related events: filter by cache level
0x8000001c, 0, ecx, 31, lwp_cache_latency , Cache-related events: filter by latency
0x8000001c, 0, edx, 0, hw_lwp_avail , HW: LWP available
-0x8000001c, 0, edx, 1, hw_lpwval , HW: LWPVAL available
+0x8000001c, 0, edx, 1, hw_lwpval , HW: LWPVAL available
0x8000001c, 0, edx, 2, hw_lwp_ire , HW: Instructions Retired Event
0x8000001c, 0, edx, 3, hw_lwp_bre , HW: Branch Retired Event
0x8000001c, 0, edx, 4, hw_lwp_dme , HW: Dcache Miss Event
@@ -1040,8 +1040,8 @@
0x80000021, 0, eax, 7, upper_addr_ignore , EFER MSR Upper Address Ignore
0x80000021, 0, eax, 8, autoibrs , EFER MSR Automatic IBRS
0x80000021, 0, eax, 9, no_smm_ctl_msr , SMM_CTL MSR not available
-0x80000021, 0, eax, 10, fsrs , Fast Short Rep STOSB
-0x80000021, 0, eax, 11, fsrc , Fast Short Rep CMPSB
+0x80000021, 0, eax, 10, fsrs , Fast Short REP STOSB
+0x80000021, 0, eax, 11, fsrc , Fast Short REP CMPSB
0x80000021, 0, eax, 13, prefetch_ctl_msr , Prefetch control MSR
0x80000021, 0, eax, 16, opcode_reclaim , Reserves opcode space
0x80000021, 0, eax, 17, user_cpuid_disable , #GP when executing CPUID at CPL > 0
@@ -1093,9 +1093,9 @@
# Maximum Transmeta leaf + CPU vendor string
0x80860000, 0, eax, 31:0, max_tra_leaf , Maximum Transmeta leaf
-0x80860000, 0, ebx, 31:0, cpu_vendorid_0 , Transmeta Vendor ID string bytes 0 - 3
-0x80860000, 0, ecx, 31:0, cpu_vendorid_2 , Transmeta Vendor ID string bytes 8 - 11
-0x80860000, 0, edx, 31:0, cpu_vendorid_1 , Transmeta Vendor ID string bytes 4 - 7
+0x80860000, 0, ebx, 31:0, cpu_vendorid_0 , Transmeta vendor ID string bytes 0 - 3
+0x80860000, 0, ecx, 31:0, cpu_vendorid_2 , Transmeta vendor ID string bytes 8 - 11
+0x80860000, 0, edx, 31:0, cpu_vendorid_1 , Transmeta vendor ID string bytes 4 - 7
# Leaf 80860001H
# Transmeta extended CPU features
diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c
index 643f707..ddabde2 100644
--- a/tools/bootconfig/main.c
+++ b/tools/bootconfig/main.c
@@ -390,8 +390,10 @@ static int apply_xbc(const char *path, const char *xbc_path)
/* Backup the bootconfig data */
data = calloc(size + BOOTCONFIG_ALIGN + BOOTCONFIG_FOOTER_SIZE, 1);
- if (!data)
+ if (!data) {
+ free(buf);
return -ENOMEM;
+ }
memcpy(data, buf, size);
/* Check the data format */
diff --git a/tools/include/uapi/linux/stddef.h b/tools/include/uapi/linux/stddef.h
index c53cde4..4574982 100644
--- a/tools/include/uapi/linux/stddef.h
+++ b/tools/include/uapi/linux/stddef.h
@@ -3,7 +3,6 @@
#define _LINUX_STDDEF_H
-
#ifndef __always_inline
#define __always_inline __inline__
#endif
@@ -36,6 +35,11 @@
struct __struct_group_tag(TAG) { MEMBERS } ATTRS NAME; \
} ATTRS
+#ifdef __cplusplus
+/* sizeof(struct{}) is 1 in C++, not 0, can't use C version of the macro. */
+#define __DECLARE_FLEX_ARRAY(T, member) \
+ T member[0]
+#else
/**
* __DECLARE_FLEX_ARRAY() - Declare a flexible array usable in a union
*
@@ -52,3 +56,23 @@
TYPE NAME[]; \
}
#endif
+
+#ifndef __counted_by
+#define __counted_by(m)
+#endif
+
+#ifndef __counted_by_le
+#define __counted_by_le(m)
+#endif
+
+#ifndef __counted_by_be
+#define __counted_by_be(m)
+#endif
+
+#ifndef __counted_by_ptr
+#define __counted_by_ptr(m)
+#endif
+
+#define __kernel_nonstring
+
+#endif /* _LINUX_STDDEF_H */
diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c
index cd5c254..9478b8f 100644
--- a/tools/lib/bpf/gen_loader.c
+++ b/tools/lib/bpf/gen_loader.c
@@ -592,13 +592,12 @@ static void emit_signature_match(struct bpf_gen *gen)
gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start;
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0));
- off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
+ off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;
if (is_simm16(off)) {
emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));
emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off));
} else {
gen->error = -ERANGE;
- emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
}
}
}
diff --git a/tools/net/ynl/Makefile.deps b/tools/net/ynl/Makefile.deps
index 08205f9..cc53b2f 100644
--- a/tools/net/ynl/Makefile.deps
+++ b/tools/net/ynl/Makefile.deps
@@ -15,9 +15,11 @@
get_hdr_inc=-D$(1) -include $(UAPI_PATH)/linux/$(2)
get_hdr_inc2=-D$(1) -D$(2) -include $(UAPI_PATH)/linux/$(3)
+CFLAGS_dev-energymodel:=$(call get_hdr_inc,_LINUX_DEV_ENERGYMODEL_H,dev_energymodel.h)
CFLAGS_devlink:=$(call get_hdr_inc,_LINUX_DEVLINK_H_,devlink.h)
CFLAGS_dpll:=$(call get_hdr_inc,_LINUX_DPLL_H,dpll.h)
-CFLAGS_ethtool:=$(call get_hdr_inc,_LINUX_ETHTOOL_H,ethtool.h) \
+CFLAGS_ethtool:=$(call get_hdr_inc,_LINUX_TYPELIMITS_H,typelimits.h) \
+ $(call get_hdr_inc,_LINUX_ETHTOOL_H,ethtool.h) \
$(call get_hdr_inc,_LINUX_ETHTOOL_NETLINK_H_,ethtool_netlink.h) \
$(call get_hdr_inc,_LINUX_ETHTOOL_NETLINK_GENERATED_H,ethtool_netlink_generated.h)
CFLAGS_handshake:=$(call get_hdr_inc,_LINUX_HANDSHAKE_H,handshake.h)
diff --git a/tools/net/ynl/pyynl/lib/ynl.py b/tools/net/ynl/pyynl/lib/ynl.py
index f63c6f8..010aac0 100644
--- a/tools/net/ynl/pyynl/lib/ynl.py
+++ b/tools/net/ynl/pyynl/lib/ynl.py
@@ -42,6 +42,7 @@
SOL_NETLINK = 270
NETLINK_ADD_MEMBERSHIP = 1
+ NETLINK_LISTEN_ALL_NSID = 8
NETLINK_CAP_ACK = 10
NETLINK_EXT_ACK = 11
NETLINK_GET_STRICT_CHK = 12
@@ -680,6 +681,7 @@
Notification API:
ynl.ntf_subscribe(mcast_name) -- join a multicast group
+ ynl.ntf_listen_all_nsid() -- listen on all netns
ynl.check_ntf() -- drain pending notifications
ynl.poll_ntf(duration=None) -- yield notifications
@@ -748,6 +750,23 @@
self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_ADD_MEMBERSHIP,
mcast_id)
+ def ntf_listen_all_nsid(self):
+ """Enable NETLINK_LISTEN_ALL_NSID to receive notifications from all
+ namespaces that have an nsid mapped in the current one."""
+ self.sock.setsockopt(Netlink.SOL_NETLINK,
+ Netlink.NETLINK_LISTEN_ALL_NSID, 1)
+
+ @staticmethod
+ def _decode_nsid(ancdata):
+ for cmsg_level, cmsg_type, cmsg_data in ancdata:
+ if (cmsg_level == Netlink.SOL_NETLINK and
+ cmsg_type == Netlink.NETLINK_LISTEN_ALL_NSID):
+ nsid = struct.unpack('i', cmsg_data)[0]
+ if nsid >= 0:
+ return nsid
+ return None
+ return None
+
def set_recv_dbg(self, enabled):
self._recv_dbg = enabled
@@ -1235,7 +1254,7 @@
f" when parsing '{attr_spec['name']}'")
return raw
- def handle_ntf(self, decoded):
+ def handle_ntf(self, decoded, nsid=None):
msg = {}
if self.include_raw:
msg['raw'] = decoded
@@ -1246,15 +1265,22 @@
msg['name'] = op['name']
msg['msg'] = attrs
+ if nsid is not None:
+ msg['nsid'] = nsid
self.async_msg_queue.put(msg)
+ def _recvmsg(self, flags=0):
+ reply, ancdata, _, _ = self.sock.recvmsg(self._recv_size, 4096, flags)
+ return reply, ancdata
+
def check_ntf(self):
while True:
try:
- reply = self.sock.recv(self._recv_size, socket.MSG_DONTWAIT)
+ reply, ancdata = self._recvmsg(socket.MSG_DONTWAIT)
except BlockingIOError:
return
+ nsid = self._decode_nsid(ancdata)
nms = NlMsgs(reply)
self._recv_dbg_print(reply, nms)
for nl_msg in nms:
@@ -1271,7 +1297,7 @@
print("Unexpected msg id while checking for ntf", decoded)
continue
- self.handle_ntf(decoded)
+ self.handle_ntf(decoded, nsid)
def poll_ntf(self, duration=None):
start_time = time.time()
@@ -1335,7 +1361,8 @@
rsp = []
op_rsp = []
while not done:
- reply = self.sock.recv(self._recv_size)
+ reply, ancdata = self._recvmsg()
+ nsid = self._decode_nsid(ancdata)
nms = NlMsgs(reply)
self._recv_dbg_print(reply, nms)
for nl_msg in nms:
@@ -1374,7 +1401,7 @@
# Check if this is a reply to our request
if nl_msg.nl_seq not in reqs_by_seq or decoded.cmd() != op.rsp_value:
if decoded.cmd() in self.async_msg_ids:
- self.handle_ntf(decoded)
+ self.handle_ntf(decoded, nsid)
continue
print('Unexpected message: ' + repr(decoded))
continue
diff --git a/tools/net/ynl/pyynl/ynl_gen_c.py b/tools/net/ynl/pyynl/ynl_gen_c.py
index 0e1e486..cdc3646 100755
--- a/tools/net/ynl/pyynl/ynl_gen_c.py
+++ b/tools/net/ynl/pyynl/ynl_gen_c.py
@@ -3212,6 +3212,8 @@
for const in family['definitions']:
if const.get('header'):
continue
+ if const.get('scope', 'uapi') != 'uapi':
+ continue
if const['type'] != 'const':
cw.writes_defines(defines)
@@ -3339,6 +3341,25 @@
cw.p(f'#endif /* {hdr_prot} */')
+def render_scoped_consts(family, cw, scope):
+ defines = []
+ for const in family['definitions']:
+ if const['type'] != 'const':
+ continue
+ if const.get('header'):
+ continue
+ if const.get('scope') != scope:
+ continue
+ name_pfx = const.get('name-prefix', f"{family.ident_name}-")
+ defines.append([
+ c_upper(family.get('c-define-name',
+ f"{name_pfx}{const['name']}")),
+ const['value']])
+ if defines:
+ cw.writes_defines(defines)
+ cw.nl()
+
+
def _render_user_ntf_entry(ri, op):
if not ri.family.is_classic():
ri.cw.block_start(line=f"[{op.enum_name}] = ")
@@ -3504,8 +3525,12 @@
cw.p('#include "ynl.h"')
headers = []
for definition in parsed['definitions'] + parsed['attribute-sets']:
- if 'header' in definition:
- headers.append(definition['header'])
+ if 'header' not in definition:
+ continue
+ scope = definition.get('scope', 'uapi')
+ if scope != 'uapi' and scope != args.mode:
+ continue
+ headers.append(definition['header'])
if args.mode == 'user':
headers.append(parsed.uapi_header)
seen_header = []
@@ -3522,6 +3547,7 @@
for one in args.user_header:
cw.p(f'#include "{one}"')
else:
+ render_scoped_consts(parsed, cw, 'user')
cw.p('struct ynl_sock;')
cw.nl()
render_user_family(parsed, cw, True)
@@ -3529,6 +3555,7 @@
if args.mode == "kernel":
if args.header:
+ render_scoped_consts(parsed, cw, 'kernel')
for _, struct in sorted(parsed.pure_nested_structs.items()):
if struct.request:
cw.p('/* Common nested types */')
diff --git a/tools/objtool/Build b/tools/objtool/Build
index 600da05..93a37b0 100644
--- a/tools/objtool/Build
+++ b/tools/objtool/Build
@@ -12,7 +12,7 @@
objtool-$(BUILD_DISAS) += trace.o
objtool-$(BUILD_ORC) += orc_gen.o orc_dump.o
-objtool-$(BUILD_KLP) += builtin-klp.o klp-diff.o klp-post-link.o
+objtool-$(BUILD_KLP) += builtin-klp.o klp-checksum.o klp-diff.o klp-post-link.o
objtool-y += libstring.o
objtool-y += libctype.o
diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile
index b71d188..a4484fd 100644
--- a/tools/objtool/Makefile
+++ b/tools/objtool/Makefile
@@ -58,7 +58,7 @@
-I$(srctree)/tools/arch/$(SRCARCH)/include \
-I$(srctree)/tools/objtool/include \
-I$(srctree)/tools/objtool/arch/$(SRCARCH)/include \
- -I$(LIBSUBCMD_OUTPUT)/include
+ -I$(srctree)/tools/lib
OBJTOOL_CFLAGS := -std=gnu11 -fomit-frame-pointer -O2 -g $(WARNINGS) \
$(INCLUDES) $(LIBELF_FLAGS) $(LIBXXHASH_CFLAGS) $(HOSTCFLAGS)
@@ -135,7 +135,7 @@
$(Q)$(MAKE) -C $(LIBSUBCMD_DIR) O=$(LIBSUBCMD_OUTPUT) \
DESTDIR=$(LIBSUBCMD_OUTPUT) prefix= subdir= \
$(HOST_OVERRIDES) EXTRA_CFLAGS="$(OBJTOOL_CFLAGS)" \
- $@ install_headers
+ $@
$(LIBSUBCMD)-clean:
$(call QUIET_CLEAN, libsubcmd)
diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
index 350b8ee..1b387d5 100644
--- a/tools/objtool/arch/x86/decode.c
+++ b/tools/objtool/arch/x86/decode.c
@@ -805,14 +805,27 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec
break;
}
- if (ins.immediate.nbytes)
+ if (ins.immediate.nbytes) {
insn->immediate = ins.immediate.value;
- else if (ins.displacement.nbytes)
+ insn->immediate_len = ins.immediate.nbytes;
+ } else if (ins.displacement.nbytes) {
insn->immediate = ins.displacement.value;
+ insn->immediate_len = ins.displacement.nbytes;
+ }
return 0;
}
+size_t arch_jump_opcode_bytes(struct objtool_file *file, struct instruction *insn,
+ unsigned char *buf)
+{
+ size_t len;
+
+ len = insn->len - insn->immediate_len;
+ memcpy(buf, insn->sec->data->d_buf + insn->offset, len);
+ return len;
+}
+
void arch_initial_func_cfi_state(struct cfi_init_state *state)
{
int i;
diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c
index b780df51..118c3de 100644
--- a/tools/objtool/builtin-check.c
+++ b/tools/objtool/builtin-check.c
@@ -73,8 +73,6 @@ static int parse_hacks(const struct option *opt, const char *str, int unset)
static const struct option check_options[] = {
OPT_GROUP("Actions:"),
- OPT_BOOLEAN(0, "checksum", &opts.checksum, "generate per-function checksums"),
- OPT_BOOLEAN(0, "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"),
OPT_STRING_OPTARG('d', "disas", &opts.disas, "function-pattern", "disassemble functions", "*"),
OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks),
OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"),
@@ -85,7 +83,7 @@ static const struct option check_options[] = {
OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"),
OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"),
OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"),
- OPT_INTEGER(0, "prefix", &opts.prefix, "generate prefix symbols"),
+ OPT_INTEGER(0, "prefix", &opts.prefix, "generate or grow prefix symbols for N-byte function padding"),
OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"),
OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"),
OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"),
@@ -93,9 +91,10 @@ static const struct option check_options[] = {
OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump),
OPT_GROUP("Options:"),
+ OPT_BOOLEAN(0, "cfi", &opts.cfi, "grow kCFI preamble symbols (use with --prefix)"),
+ OPT_BOOLEAN(0, "fineibt", &opts.fineibt, "create .cfi_sites section for FineIBT"),
OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"),
OPT_BOOLEAN(0, "backup", &opts.backup, "create backup (.orig) file on warning/error"),
- OPT_STRING(0, "debug-checksum", &opts.debug_checksum, "funcs", "enable checksum debug output"),
OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"),
OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"),
OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"),
@@ -165,20 +164,17 @@ static bool opts_valid(void)
return false;
}
-#ifndef BUILD_KLP
- if (opts.checksum) {
- ERROR("--checksum not supported; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile");
- return false;
- }
-#endif
-
- if (opts.debug_checksum && !opts.checksum) {
- ERROR("--debug-checksum requires --checksum");
+ if (opts.cfi && !opts.prefix) {
+ ERROR("--cfi requires --prefix");
return false;
}
- if (opts.checksum ||
- opts.disas ||
+ if (opts.fineibt && !opts.cfi) {
+ ERROR("--fineibt requires --cfi");
+ return false;
+ }
+
+ if (opts.disas ||
opts.hack_jump_label ||
opts.hack_noinstr ||
opts.ibt ||
diff --git a/tools/objtool/builtin-klp.c b/tools/objtool/builtin-klp.c
index 56d5a5b..58c3b9b 100644
--- a/tools/objtool/builtin-klp.c
+++ b/tools/objtool/builtin-klp.c
@@ -13,6 +13,7 @@ struct subcmd {
};
static struct subcmd subcmds[] = {
+ { "checksum", "Generate per-function checksums", cmd_klp_checksum, },
{ "diff", "Generate binary diff of two object files", cmd_klp_diff, },
{ "post-link", "Finalize klp symbols/relocs after module linking", cmd_klp_post_link, },
};
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 9b11cf3..10b18cf 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -18,7 +18,6 @@
#include <objtool/special.h>
#include <objtool/trace.h>
#include <objtool/warn.h>
-#include <objtool/checksum.h>
#include <objtool/util.h>
#include <linux/objtool_types.h>
@@ -64,8 +63,8 @@ struct instruction *next_insn_same_sec(struct objtool_file *file,
return insn;
}
-static struct instruction *next_insn_same_func(struct objtool_file *file,
- struct instruction *insn)
+struct instruction *next_insn_same_func(struct objtool_file *file,
+ struct instruction *insn)
{
struct instruction *next = next_insn_same_sec(file, insn);
struct symbol *func = insn_func(insn);
@@ -113,10 +112,6 @@ static struct instruction *prev_insn_same_sym(struct objtool_file *file,
for_each_sec(file->elf, __sec) \
sec_for_each_insn(file, __sec, insn)
-#define func_for_each_insn(file, func, insn) \
- for (insn = find_insn(file, func->sec, func->offset); \
- insn; \
- insn = next_insn_same_func(file, insn))
#define sym_for_each_insn(file, sym, insn) \
for (insn = find_insn(file, sym->sec, sym->offset); \
@@ -491,7 +486,7 @@ static int decode_instructions(struct objtool_file *file)
return -1;
}
- if (func->embedded_insn || func->alias != func)
+ if (func->embedded_insn || is_alias_sym(func))
continue;
if (!find_insn(file, sec, func->offset)) {
@@ -500,7 +495,7 @@ static int decode_instructions(struct objtool_file *file)
}
sym_for_each_insn(file, func, insn) {
- insn->sym = func;
+ insn->_sym = func;
if (is_func_sym(func) &&
insn->type == INSN_ENDBR &&
list_empty(&insn->call_node)) {
@@ -864,15 +859,14 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file)
list_for_each_entry(insn, &file->endbr_list, call_node) {
int *site = (int *)sec->data->d_buf + idx;
- struct symbol *sym = insn->sym;
+ struct symbol *func = insn_func(insn);
*site = 0;
- if (opts.module && sym && is_func_sym(sym) &&
- insn->offset == sym->offset &&
- (!strcmp(sym->name, "init_module") ||
- !strcmp(sym->name, "cleanup_module"))) {
+ if (opts.module && func && insn->offset == func->offset &&
+ (!strcmp(func->name, "init_module") ||
+ !strcmp(func->name, "cleanup_module"))) {
ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",
- sym->name);
+ func->name);
return -1;
}
@@ -887,6 +881,31 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file)
return 0;
}
+/*
+* Grow __cfi_ symbols to fill the NOP gap between the 'mov <hash>, %rax' and
+* the start of the function.
+*/
+static int grow_cfi_symbols(struct objtool_file *file)
+{
+ struct symbol *sym;
+
+ for_each_sym(file->elf, sym) {
+ if (!is_func_sym(sym) || !strstarts(sym->name, "__cfi_") ||
+ sym->len != 5)
+ continue;
+
+ if (!find_func_by_offset(sym->sec, sym->offset + sym->len + opts.prefix))
+ continue;
+
+ sym->len += opts.prefix;
+ sym->sym.st_size = sym->len;
+ if (elf_write_symbol(file->elf, sym))
+ return -1;
+ }
+
+ return 0;
+}
+
static int create_cfi_sections(struct objtool_file *file)
{
struct section *sec;
@@ -1023,59 +1042,6 @@ static int create_direct_call_sections(struct objtool_file *file)
return 0;
}
-#ifdef BUILD_KLP
-static int create_sym_checksum_section(struct objtool_file *file)
-{
- struct section *sec;
- struct symbol *sym;
- unsigned int idx = 0;
- struct sym_checksum *checksum;
- size_t entsize = sizeof(struct sym_checksum);
-
- sec = find_section_by_name(file->elf, ".discard.sym_checksum");
- if (sec) {
- if (!opts.dryrun)
- WARN("file already has .discard.sym_checksum section, skipping");
-
- return 0;
- }
-
- for_each_sym(file->elf, sym)
- if (sym->csum.checksum)
- idx++;
-
- if (!idx)
- return 0;
-
- sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize,
- idx, idx);
- if (!sec)
- return -1;
-
- idx = 0;
- for_each_sym(file->elf, sym) {
- if (!sym->csum.checksum)
- continue;
-
- if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize,
- sym, 0, R_TEXT64))
- return -1;
-
- checksum = (struct sym_checksum *)sec->data->d_buf + idx;
- checksum->addr = 0; /* reloc */
- checksum->checksum = sym->csum.checksum;
-
- mark_sec_changed(file->elf, sec, true);
-
- idx++;
- }
-
- return 0;
-}
-#else
-static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; }
-#endif
-
/*
* Warnings shouldn't be reported for ignored functions.
*/
@@ -1349,14 +1315,11 @@ __weak bool arch_is_embedded_insn(struct symbol *sym)
return false;
}
-static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
+struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
{
struct reloc *reloc;
- if (insn->no_reloc)
- return NULL;
-
- if (!file)
+ if (!file || insn->no_reloc || insn->fake)
return NULL;
reloc = find_reloc_by_dest_range(file->elf, insn->sec,
@@ -1642,7 +1605,7 @@ static int add_jump_destinations(struct objtool_file *file)
}
if (!dest_sym || is_sec_sym(dest_sym)) {
- dest_sym = dest_insn->sym;
+ dest_sym = insn_sym(dest_insn);
if (!dest_sym)
goto set_jump_dest;
}
@@ -1658,7 +1621,7 @@ static int add_jump_destinations(struct objtool_file *file)
continue;
}
- if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc)
+ if (!insn_sym(insn) || insn_sym(insn)->pfunc == dest_sym->pfunc)
goto set_jump_dest;
/*
@@ -1831,7 +1794,6 @@ static int handle_group_alt(struct objtool_file *file,
nop->offset = special_alt->new_off + special_alt->new_len;
nop->len = special_alt->orig_len - special_alt->new_len;
nop->type = INSN_NOP;
- nop->sym = orig_insn->sym;
nop->alt_group = new_alt_group;
nop->fake = 1;
}
@@ -1850,7 +1812,6 @@ static int handle_group_alt(struct objtool_file *file,
last_new_insn = insn;
- insn->sym = orig_insn->sym;
insn->alt_group = new_alt_group;
/*
@@ -2232,7 +2193,7 @@ static int add_jump_table_alts(struct objtool_file *file)
return 0;
for_each_sym(file->elf, func) {
- if (!is_func_sym(func) || func->alias != func)
+ if (!is_func_sym(func) || is_alias_sym(func))
continue;
mark_func_jump_tables(file, func);
@@ -2493,12 +2454,12 @@ static int __annotate_late(struct objtool_file *file, int type, struct instructi
break;
case ANNOTYPE_NOCFI:
- sym = insn->sym;
+ sym = insn_sym(insn);
if (!sym) {
ERROR_INSN(insn, "dodgy NOCFI annotation");
return -1;
}
- insn->sym->nocfi = 1;
+ sym->nocfi = 1;
break;
default:
@@ -2566,7 +2527,6 @@ static int classify_symbols(struct objtool_file *file)
static void mark_rodata(struct objtool_file *file)
{
struct section *sec;
- bool found = false;
/*
* Search for the following rodata sections, each of which can
@@ -2579,15 +2539,11 @@ static void mark_rodata(struct objtool_file *file)
* .rodata.str1.* sections are ignored; they don't contain jump tables.
*/
for_each_sec(file->elf, sec) {
- if ((!strncmp(sec->name, ".rodata", 7) &&
- !strstr(sec->name, ".str1.")) ||
- !strncmp(sec->name, ".data.rel.ro", 12)) {
- sec->rodata = true;
- found = true;
+ if (is_rodata_sec(sec)) {
+ file->rodata = true;
+ return;
}
}
-
- file->rodata = found;
}
static void mark_holes(struct objtool_file *file)
@@ -2604,7 +2560,7 @@ static void mark_holes(struct objtool_file *file)
* favour of a regular symbol, but leaves the code in place.
*/
for_each_insn(file, insn) {
- if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) {
+ if (insn_sym(insn) || !find_symbol_hole_containing(insn->sec, insn->offset)) {
in_hole = false;
continue;
}
@@ -2622,7 +2578,7 @@ static void mark_holes(struct objtool_file *file)
if (insn->jump_dest) {
struct symbol *dest_func = insn_func(insn->jump_dest);
- if (dest_func && dest_func->cold)
+ if (dest_func && is_cold_func(dest_func))
dest_func->ignore = true;
}
}
@@ -2630,14 +2586,35 @@ static void mark_holes(struct objtool_file *file)
static bool validate_branch_enabled(void)
{
- return opts.stackval ||
- opts.orc ||
- opts.uaccess ||
+ return opts.stackval ||
+ opts.orc ||
+ opts.uaccess;
+}
+
+static bool alts_needed(void)
+{
+ return validate_branch_enabled() ||
+ opts.noinstr ||
+ opts.hack_jump_label ||
+ opts.disas ||
opts.checksum;
}
-static int decode_sections(struct objtool_file *file)
+int decode_file(struct objtool_file *file)
{
+ arch_initial_func_cfi_state(&initial_func_cfi);
+ init_cfi_state(&init_cfi);
+ init_cfi_state(&func_cfi);
+ set_func_state(&func_cfi);
+ init_cfi_state(&force_undefined_cfi);
+ force_undefined_cfi.force_undefined = true;
+
+ if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
+ return -1;
+
+ cfi_hash_add(&init_cfi);
+ cfi_hash_add(&func_cfi);
+
file->klp = is_livepatch_module(file);
mark_rodata(file);
@@ -2666,7 +2643,7 @@ static int decode_sections(struct objtool_file *file)
* Must be before add_jump_destinations(), which depends on 'func'
* being set for alternatives, to enable proper sibling call detection.
*/
- if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label || opts.disas) {
+ if (alts_needed()) {
if (add_special_section_alts(file))
return -1;
}
@@ -3027,7 +3004,7 @@ static int update_cfi_state(struct instruction *insn,
}
if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
- insn->sym->frame_pointer) {
+ insn_sym(insn)->frame_pointer) {
/* addi.d fp,sp,imm on LoongArch */
if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
cfa->base = CFI_BP;
@@ -3039,7 +3016,7 @@ static int update_cfi_state(struct instruction *insn,
if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
/* addi.d sp,fp,imm on LoongArch */
if (cfa->base == CFI_BP && cfa->offset == 0) {
- if (insn->sym->frame_pointer) {
+ if (insn_sym(insn)->frame_pointer) {
cfa->base = CFI_SP;
cfa->offset = -op->src.offset;
}
@@ -3662,88 +3639,6 @@ static bool skip_alt_group(struct instruction *insn)
return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC;
}
-static int checksum_debug_init(struct objtool_file *file)
-{
- char *dup, *s;
-
- if (!opts.debug_checksum)
- return 0;
-
- dup = strdup(opts.debug_checksum);
- if (!dup) {
- ERROR_GLIBC("strdup");
- return -1;
- }
-
- s = dup;
- while (*s) {
- struct symbol *func;
- char *comma;
-
- comma = strchr(s, ',');
- if (comma)
- *comma = '\0';
-
- func = find_symbol_by_name(file->elf, s);
- if (!func || !is_func_sym(func))
- WARN("--debug-checksum: can't find '%s'", s);
- else
- func->debug_checksum = 1;
-
- if (!comma)
- break;
-
- s = comma + 1;
- }
-
- free(dup);
- return 0;
-}
-
-static void checksum_update_insn(struct objtool_file *file, struct symbol *func,
- struct instruction *insn)
-{
- struct reloc *reloc = insn_reloc(file, insn);
- unsigned long offset;
- struct symbol *sym;
-
- if (insn->fake)
- return;
-
- checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len);
-
- if (!reloc) {
- struct symbol *call_dest = insn_call_dest(insn);
-
- if (call_dest)
- checksum_update(func, insn, call_dest->demangled_name,
- strlen(call_dest->demangled_name));
- return;
- }
-
- sym = reloc->sym;
- offset = arch_insn_adjusted_addend(insn, reloc);
-
- if (is_string_sec(sym->sec)) {
- char *str;
-
- str = sym->sec->data->d_buf + sym->offset + offset;
- checksum_update(func, insn, str, strlen(str));
- return;
- }
-
- if (is_sec_sym(sym)) {
- sym = find_symbol_containing(reloc->sym->sec, offset);
- if (!sym)
- return;
-
- offset -= sym->offset;
- }
-
- checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name));
- checksum_update(func, insn, &offset, sizeof(offset));
-}
-
static int validate_branch(struct objtool_file *file, struct symbol *func,
struct instruction *insn, struct insn_state state);
static int do_validate_branch(struct objtool_file *file, struct symbol *func,
@@ -4025,9 +3920,6 @@ static int do_validate_branch(struct objtool_file *file, struct symbol *func,
insn->trace = 0;
next_insn = next_insn_to_validate(file, insn);
- if (opts.checksum && func && insn->sec)
- checksum_update_insn(file, func, insn);
-
if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
/* Ignore KCFI type preambles, which always fall through */
if (is_prefix_func(func))
@@ -4093,9 +3985,6 @@ static int validate_unwind_hint(struct objtool_file *file,
struct symbol *func = insn_func(insn);
int ret;
- if (opts.checksum)
- checksum_init(func);
-
ret = validate_branch(file, func, insn, *state);
if (ret)
BT_INSN(insn, "<=== (hint)");
@@ -4304,7 +4193,7 @@ static int validate_retpoline(struct objtool_file *file)
* broken.
*/
list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
- struct symbol *sym = insn->sym;
+ struct symbol *sym = insn_sym(insn);
if (sym && (is_notype_sym(sym) ||
is_func_sym(sym)) && !sym->nocfi) {
@@ -4407,17 +4296,6 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio
* For FineIBT or kCFI, a certain number of bytes preceding the function may be
* NOPs. Those NOPs may be rewritten at runtime and executed, so give them a
* proper function name: __pfx_<func>.
- *
- * The NOPs may not exist for the following cases:
- *
- * - compiler cloned functions (*.cold, *.part0, etc)
- * - asm functions created with inline asm or without SYM_FUNC_START()
- *
- * Also, the function may already have a prefix from a previous objtool run
- * (livepatch extracted functions, or manually running objtool multiple times).
- *
- * So return 0 if the NOPs are missing or the function already has a prefix
- * symbol.
*/
static int create_prefix_symbol(struct objtool_file *file, struct symbol *func)
{
@@ -4425,10 +4303,6 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func)
char name[SYM_NAME_LEN];
struct cfi_state *cfi;
- if (!is_func_sym(func) || is_prefix_func(func) ||
- func->cold || func->static_call_tramp)
- return 0;
-
if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) {
WARN("%s: symbol name too long, can't create __pfx_ symbol",
func->name);
@@ -4438,59 +4312,21 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func)
if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name))
return -1;
- if (file->klp) {
- struct symbol *pfx;
-
- pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix);
- if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name))
- return 0;
- }
-
- insn = find_insn(file, func->sec, func->offset);
- if (!insn) {
- WARN("%s: can't find starting instruction", func->name);
+ if (!elf_create_symbol(file->elf, name, func->sec,
+ GELF_ST_BIND(func->sym.st_info),
+ GELF_ST_TYPE(func->sym.st_info),
+ func->offset - opts.prefix, opts.prefix))
return -1;
- }
-
- for (prev = prev_insn_same_sec(file, insn);
- prev;
- prev = prev_insn_same_sec(file, prev)) {
- u64 offset;
-
- if (prev->type != INSN_NOP)
- return 0;
-
- offset = func->offset - prev->offset;
-
- if (offset > opts.prefix)
- return 0;
-
- if (offset < opts.prefix)
- continue;
-
- if (!elf_create_symbol(file->elf, name, func->sec,
- GELF_ST_BIND(func->sym.st_info),
- GELF_ST_TYPE(func->sym.st_info),
- prev->offset, opts.prefix))
- return -1;
-
- break;
- }
-
- if (!prev)
- return 0;
-
- if (!insn->cfi) {
- /*
- * This can happen if stack validation isn't enabled or the
- * function is annotated with STACK_FRAME_NON_STANDARD.
- */
- return 0;
- }
/* Propagate insn->cfi to the prefix code */
+ insn = find_insn(file, func->sec, func->offset);
+ if (!insn || !insn->cfi)
+ return 0;
+
cfi = cfi_hash_find_or_add(insn->cfi);
- for (; prev != insn; prev = next_insn_same_sec(file, prev))
+ for (prev = find_insn(file, func->sec, func->offset - opts.prefix);
+ prev && prev != insn;
+ prev = next_insn_same_sec(file, prev))
prev->cfi = cfi;
return 0;
@@ -4498,15 +4334,20 @@ static int create_prefix_symbol(struct objtool_file *file, struct symbol *func)
static int create_prefix_symbols(struct objtool_file *file)
{
- struct section *sec;
+ struct section *pfe_sec;
struct symbol *func;
+ struct reloc *reloc;
- for_each_sec(file->elf, sec) {
- if (!is_text_sec(sec))
+ for_each_sec(file->elf, pfe_sec) {
+ if (strcmp(pfe_sec->name, "__patchable_function_entries"))
+ continue;
+ if (!pfe_sec->rsec)
continue;
- sec_for_each_sym(sec, func) {
- if (create_prefix_symbol(file, func))
+ for_each_reloc(pfe_sec->rsec, reloc) {
+ func = find_func_by_offset(reloc->sym->sec,
+ reloc->sym->offset + reloc_addend(reloc) + opts.prefix);
+ if (func && create_prefix_symbol(file, func))
return -1;
}
}
@@ -4526,7 +4367,7 @@ static int validate_symbol(struct objtool_file *file, struct section *sec,
return 1;
}
- if (sym->pfunc != sym || sym->alias != sym)
+ if (sym->pfunc != sym || is_alias_sym(sym))
return 0;
insn = find_insn(file, sec, sym->offset);
@@ -4538,9 +4379,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec,
func = insn_func(insn);
- if (opts.checksum)
- checksum_init(func);
-
if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) {
trace_enable();
TRACE("%s: validation begin\n", sym->name);
@@ -4553,9 +4391,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec,
TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end");
trace_disable();
- if (opts.checksum)
- checksum_finish(func);
-
return ret;
}
@@ -4948,7 +4783,7 @@ struct insn_chunk {
* which can trigger more allocations for .debug_* sections whose data hasn't
* been read yet.
*/
-static void free_insns(struct objtool_file *file)
+void free_insns(struct objtool_file *file)
{
struct instruction *insn;
struct insn_chunk *chunks = NULL, *chunk;
@@ -4995,26 +4830,7 @@ int check(struct objtool_file *file)
objtool_disas_ctx = disas_ctx;
}
- arch_initial_func_cfi_state(&initial_func_cfi);
- init_cfi_state(&init_cfi);
- init_cfi_state(&func_cfi);
- set_func_state(&func_cfi);
- init_cfi_state(&force_undefined_cfi);
- force_undefined_cfi.force_undefined = true;
-
- if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) {
- ret = -1;
- goto out;
- }
-
- cfi_hash_add(&init_cfi);
- cfi_hash_add(&func_cfi);
-
- ret = checksum_debug_init(file);
- if (ret)
- goto out;
-
- ret = decode_sections(file);
+ ret = decode_file(file);
if (ret)
goto out;
@@ -5064,12 +4880,6 @@ int check(struct objtool_file *file)
goto out;
}
- if (opts.cfi) {
- ret = create_cfi_sections(file);
- if (ret)
- goto out;
- }
-
if (opts.rethunk) {
ret = create_return_sites_sections(file);
if (ret)
@@ -5089,9 +4899,21 @@ int check(struct objtool_file *file)
}
if (opts.prefix) {
- ret = create_prefix_symbols(file);
- if (ret)
- goto out;
+ if (!opts.cfi) {
+ ret = create_prefix_symbols(file);
+ if (ret)
+ goto out;
+ } else {
+ ret = grow_cfi_symbols(file);
+ if (ret)
+ goto out;
+
+ if (opts.fineibt) {
+ ret = create_cfi_sections(file);
+ if (ret)
+ goto out;
+ }
+ }
}
if (opts.ibt) {
@@ -5103,12 +4925,6 @@ int check(struct objtool_file *file)
if (opts.noabs)
warnings += check_abs_references(file);
- if (opts.checksum) {
- ret = create_sym_checksum_section(file);
- if (ret)
- goto out;
- }
-
if (opts.orc && nr_insns) {
ret = orc_create(file);
if (ret)
diff --git a/tools/objtool/disas.c b/tools/objtool/disas.c
index 5909023..e6a54a8 100644
--- a/tools/objtool/disas.c
+++ b/tools/objtool/disas.c
@@ -210,7 +210,7 @@ static bool disas_print_addr_alt(bfd_vma addr, struct disassemble_info *dinfo)
offset = addr - alt_group->first_insn->offset;
addr = orig_first_insn->offset + offset;
- sym = orig_first_insn->sym;
+ sym = insn_sym(orig_first_insn);
disas_print_addr_sym(orig_first_insn->sec, sym, addr, dinfo);
@@ -222,15 +222,13 @@ static void disas_print_addr_noreloc(bfd_vma addr,
{
struct disas_context *dctx = dinfo->application_data;
struct instruction *insn = dctx->insn;
- struct symbol *sym = NULL;
+ struct symbol *sym = insn_sym(insn);
if (disas_print_addr_alt(addr, dinfo))
return;
- if (insn->sym && addr >= insn->sym->offset &&
- addr < insn->sym->offset + insn->sym->len) {
- sym = insn->sym;
- }
+ if (sym && (addr < sym->offset || addr >= sym->offset + sym->len))
+ sym = NULL;
disas_print_addr_sym(insn->sec, sym, addr, dinfo);
}
@@ -291,9 +289,9 @@ static void disas_print_address(bfd_vma addr, struct disassemble_info *dinfo)
* up. So check it first.
*/
jump_dest = insn->jump_dest;
- if (jump_dest && jump_dest->sym && jump_dest->offset == addr) {
+ if (jump_dest && insn_sym(jump_dest) && jump_dest->offset == addr) {
if (!disas_print_addr_alt(addr, dinfo))
- disas_print_addr_sym(jump_dest->sec, jump_dest->sym,
+ disas_print_addr_sym(jump_dest->sec, insn_sym(jump_dest),
addr, dinfo);
return;
}
@@ -768,8 +766,8 @@ static int disas_alt_jump(struct disas_alt *dalt)
if (orig_insn->len == 5)
suffix[0] = 'q';
str = strfmt("jmp%-3s %lx <%s+0x%lx>", suffix,
- dest_insn->offset, dest_insn->sym->name,
- dest_insn->offset - dest_insn->sym->offset);
+ dest_insn->offset, insn_sym(dest_insn)->name,
+ dest_insn->offset - insn_sym(dest_insn)->offset);
nops = 0;
} else {
str = strfmt("nop%d", orig_insn->len);
@@ -794,8 +792,8 @@ static int disas_alt_extable(struct disas_alt *dalt)
alt_insn = dalt->alt->insn;
str = strfmt("resume at 0x%lx <%s+0x%lx>",
- alt_insn->offset, alt_insn->sym->name,
- alt_insn->offset - alt_insn->sym->offset);
+ alt_insn->offset, insn_sym(alt_insn)->name,
+ alt_insn->offset - insn_sym(alt_insn)->offset);
if (!str)
return -1;
diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
index f3df2bd..33c95a7 100644
--- a/tools/objtool/elf.c
+++ b/tools/objtool/elf.c
@@ -27,27 +27,16 @@
static ssize_t demangled_name_len(const char *name);
-static inline u32 str_hash(const char *str)
-{
- return jhash(str, strlen(str), 0);
-}
-
-static inline u32 str_hash_demangled(const char *str)
+u32 str_hash_demangled(const char *str)
{
return jhash(str, demangled_name_len(str), 0);
}
-#define __elf_table(name) (elf->name##_hash)
-#define __elf_bits(name) (elf->name##_bits)
-
-#define __elf_table_entry(name, key) \
- __elf_table(name)[hash_min(key, __elf_bits(name))]
-
#define elf_hash_add(name, node, key) \
({ \
struct elf_hash_node *__node = node; \
- __node->next = __elf_table_entry(name, key); \
- __elf_table_entry(name, key) = __node; \
+ __node->next = __elf_table_entry(elf, name, key); \
+ __elf_table_entry(elf, name, key) = __node; \
})
static inline void __elf_hash_del(struct elf_hash_node *node,
@@ -69,30 +58,20 @@ static inline void __elf_hash_del(struct elf_hash_node *node,
}
#define elf_hash_del(name, node, key) \
- __elf_hash_del(node, &__elf_table_entry(name, key))
-
-#define elf_list_entry(ptr, type, member) \
-({ \
- typeof(ptr) __ptr = (ptr); \
- __ptr ? container_of(__ptr, type, member) : NULL; \
-})
-
-#define elf_hash_for_each_possible(name, obj, member, key) \
- for (obj = elf_list_entry(__elf_table_entry(name, key), typeof(*obj), member); \
- obj; \
- obj = elf_list_entry(obj->member.next, typeof(*(obj)), member))
+ __elf_hash_del(node, &__elf_table_entry(elf, name, key))
#define elf_alloc_hash(name, size) \
({ \
- __elf_bits(name) = max(10, ilog2(size)); \
- __elf_table(name) = mmap(NULL, sizeof(struct elf_hash_node *) << __elf_bits(name), \
+ __elf_bits(elf, name) = max(10, ilog2(size)); \
+ __elf_table(elf, name) = mmap(NULL, \
+ sizeof(struct elf_hash_node *) << __elf_bits(elf, name), \
PROT_READ|PROT_WRITE, \
MAP_PRIVATE|MAP_ANON, -1, 0); \
- if (__elf_table(name) == (void *)-1L) { \
+ if (__elf_table(elf, name) == (void *)-1L) { \
ERROR_GLIBC("mmap fail " #name); \
- __elf_table(name) = NULL; \
+ __elf_table(elf, name) = NULL; \
} \
- __elf_table(name); \
+ __elf_table(elf, name); \
})
static inline unsigned long __sym_start(struct symbol *s)
@@ -141,7 +120,7 @@ struct section *find_section_by_name(const struct elf *elf, const char *name)
{
struct section *sec;
- elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) {
+ elf_hash_for_each_possible(elf, section_name, sec, name_hash, str_hash(name)) {
if (!strcmp(sec->name, name))
return sec;
}
@@ -154,7 +133,7 @@ static struct section *find_section_by_index(struct elf *elf,
{
struct section *sec;
- elf_hash_for_each_possible(section, sec, hash, idx) {
+ elf_hash_for_each_possible(elf, section, sec, hash, idx) {
if (sec->idx == idx)
return sec;
}
@@ -166,7 +145,7 @@ static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx)
{
struct symbol *sym;
- elf_hash_for_each_possible(symbol, sym, hash, idx) {
+ elf_hash_for_each_possible(elf, symbol, sym, hash, idx) {
if (sym->idx == idx)
return sym;
}
@@ -229,6 +208,20 @@ struct symbol *find_symbol_containing(const struct section *sec, unsigned long o
}
/*
+ * Also match the symbol end address which can be used for a bounds comparison.
+ */
+struct symbol *find_symbol_containing_inclusive(const struct section *sec,
+ unsigned long offset)
+{
+ struct symbol *sym = find_symbol_containing(sec, offset);
+
+ if (!sym && offset)
+ sym = find_symbol_containing(sec, offset - 1);
+
+ return sym;
+}
+
+/*
* Returns size of hole starting at @offset.
*/
int find_symbol_hole_containing(const struct section *sec, unsigned long offset)
@@ -285,7 +278,7 @@ struct symbol *find_symbol_by_name(const struct elf *elf, const char *name)
{
struct symbol *sym;
- elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) {
+ elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash(name)) {
if (!strcmp(sym->name, name))
return sym;
}
@@ -300,7 +293,7 @@ static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf,
{
struct symbol *sym;
- elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) {
+ elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) {
if (sym->bind == STB_LOCAL && sym->file == file &&
!strcmp(sym->name, name)) {
return sym;
@@ -314,7 +307,7 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam
{
struct symbol *sym;
- elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) {
+ elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, str_hash_demangled(name)) {
if (!strcmp(sym->name, name) && !is_local_sym(sym))
return sym;
}
@@ -322,21 +315,9 @@ struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *nam
return NULL;
}
-void iterate_global_symbol_by_demangled_name(const struct elf *elf,
- const char *demangled_name,
- void (*process)(struct symbol *sym, void *data),
- void *data)
-{
- struct symbol *sym;
-
- elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(demangled_name)) {
- if (!strcmp(sym->demangled_name, demangled_name) && !is_local_sym(sym))
- process(sym, data);
- }
-}
-
+/* If there are multiple matches, return the first one in the range */
struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec,
- unsigned long offset, unsigned int len)
+ unsigned long offset, unsigned int len)
{
struct reloc *reloc, *r = NULL;
struct section *rsec;
@@ -347,7 +328,7 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se
return NULL;
for_offset_range(o, offset, offset + len) {
- elf_hash_for_each_possible(reloc, reloc, hash,
+ elf_hash_for_each_possible(elf, reloc, reloc, hash,
sec_offset_hash(rsec, o)) {
if (reloc->sec != rsec)
continue;
@@ -358,11 +339,11 @@ struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *se
r = reloc;
}
}
- if (r)
+ if (r && (reloc_offset(r) & OFFSET_STRIDE_MASK) == o)
return r;
}
- return NULL;
+ return r;
}
struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset)
@@ -668,7 +649,7 @@ static int read_symbols(struct elf *elf)
if (is_file_sym(sym))
file = sym;
- else if (sym->bind == STB_LOCAL)
+ else if (sym->bind == STB_LOCAL && !is_sec_sym(sym))
sym->file = file;
}
@@ -1016,6 +997,26 @@ struct symbol *elf_create_symbol(struct elf *elf, const char *name,
return sym;
}
+int elf_write_symbol(struct elf *elf, struct symbol *sym)
+{
+ struct section *symtab, *symtab_shndx;
+
+ symtab = find_section_by_name(elf, ".symtab");
+ if (!symtab) {
+ ERROR("no .symtab");
+ return -1;
+ }
+
+ symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
+
+ if (elf_update_symbol(elf, symtab, symtab_shndx, sym))
+ return -1;
+
+ mark_sec_changed(elf, symtab, true);
+
+ return 0;
+}
+
struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec)
{
struct symbol *sym = calloc(1, sizeof(*sym));
@@ -1172,6 +1173,17 @@ static int read_relocs(struct elf *elf)
return 0;
}
+static void mark_rodata(struct elf *elf)
+{
+ struct section *sec;
+
+ for_each_sec(elf, sec) {
+ if ((strstarts(sec->name, ".rodata") && !strstr(sec->name, ".str1.")) ||
+ strstarts(sec->name, ".data.rel.ro"))
+ sec->rodata = true;
+ }
+}
+
struct elf *elf_open_read(const char *name, int flags)
{
struct elf *elf;
@@ -1222,6 +1234,8 @@ struct elf *elf_open_read(const char *name, int flags)
if (read_sections(elf))
goto err;
+ mark_rodata(elf);
+
if (read_symbols(elf))
goto err;
diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h
index 8866158..96d828a 100644
--- a/tools/objtool/include/objtool/arch.h
+++ b/tools/objtool/include/objtool/arch.h
@@ -79,6 +79,9 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec
unsigned long offset, unsigned int maxlen,
struct instruction *insn);
+size_t arch_jump_opcode_bytes(struct objtool_file *file, struct instruction *insn,
+ unsigned char *buf);
+
bool arch_callee_saved_reg(unsigned char reg);
unsigned long arch_jump_destination(struct instruction *insn);
diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h
index b9e229e..e844e9c 100644
--- a/tools/objtool/include/objtool/builtin.h
+++ b/tools/objtool/include/objtool/builtin.h
@@ -9,8 +9,8 @@
struct opts {
/* actions: */
- bool cfi;
bool checksum;
+ const char *disas;
bool dump_orc;
bool hack_jump_label;
bool hack_noinstr;
@@ -20,6 +20,7 @@ struct opts {
bool noabs;
bool noinstr;
bool orc;
+ int prefix;
bool retpoline;
bool rethunk;
bool unret;
@@ -27,14 +28,14 @@ struct opts {
bool stackval;
bool static_call;
bool uaccess;
- int prefix;
- const char *disas;
/* options: */
bool backtrace;
bool backup;
+ bool cfi;
const char *debug_checksum;
bool dryrun;
+ bool fineibt;
bool link;
bool mnop;
bool module;
diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h
index 5f2f77b..063f5985 100644
--- a/tools/objtool/include/objtool/check.h
+++ b/tools/objtool/include/objtool/check.h
@@ -68,6 +68,7 @@ struct instruction {
s8 instr;
u32 idx : INSN_CHUNK_BITS,
+ immediate_len : 4,
dead_end : 1,
ignore_alts : 1,
hint : 1,
@@ -81,7 +82,7 @@ struct instruction {
hole : 1,
fake : 1,
trace : 1;
- /* 9 bit hole */
+ /* 4 bit hole */
struct alt_group *alt_group;
struct instruction *jump_dest;
@@ -94,14 +95,30 @@ struct instruction {
};
};
struct alternative *alts;
- struct symbol *sym;
+ struct symbol *_sym;
struct stack_op *stack_ops;
struct cfi_state *cfi;
};
+/*
+ * Return the symbol associated with an instruction. For alternative
+ * replacements, return the symbol of the original code being replaced rather
+ * than NULL. insn->_sym reflects the actual location in the ELF file.
+ */
+static inline struct symbol *insn_sym(struct instruction *insn)
+{
+ struct symbol *sym = insn->_sym;
+
+ if ((!sym || !is_func_sym(sym)) &&
+ insn->alt_group && insn->alt_group->orig_group)
+ sym = insn->alt_group->orig_group->first_insn->_sym;
+
+ return sym;
+}
+
static inline struct symbol *insn_func(struct instruction *insn)
{
- struct symbol *sym = insn->sym;
+ struct symbol *sym = insn_sym(insn);
if (sym && sym->type != STT_FUNC)
sym = NULL;
@@ -144,6 +161,12 @@ struct instruction *find_insn(struct objtool_file *file,
struct section *sec, unsigned long offset);
struct instruction *next_insn_same_sec(struct objtool_file *file, struct instruction *insn);
+struct instruction *next_insn_same_func(struct objtool_file *file, struct instruction *insn);
+
+#define func_for_each_insn(file, func, insn) \
+ for (insn = find_insn(file, func->sec, func->offset); \
+ insn; \
+ insn = next_insn_same_func(file, insn))
#define sec_for_each_insn(file, _sec, insn) \
for (insn = find_insn(file, _sec, 0); \
@@ -155,6 +178,11 @@ struct instruction *next_insn_same_sec(struct objtool_file *file, struct instruc
insn && insn->offset < sym->offset + sym->len; \
insn = next_insn_same_sec(file, insn))
+struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn);
+
+int decode_file(struct objtool_file *file);
+void free_insns(struct objtool_file *file);
+
const char *objtool_disas_insn(struct instruction *insn);
extern size_t sym_name_max_len;
diff --git a/tools/objtool/include/objtool/checksum.h b/tools/objtool/include/objtool/checksum.h
index 7fe2160..d46293f 100644
--- a/tools/objtool/include/objtool/checksum.h
+++ b/tools/objtool/include/objtool/checksum.h
@@ -6,37 +6,54 @@
#ifdef BUILD_KLP
-static inline void checksum_init(struct symbol *func)
+static inline void checksum_init(struct symbol *sym)
{
- if (func && !func->csum.state) {
- func->csum.state = XXH3_createState();
- XXH3_64bits_reset(func->csum.state);
+ if (sym && !sym->csum.state) {
+ sym->csum.state = XXH3_createState();
+ XXH3_64bits_reset(sym->csum.state);
}
}
-static inline void checksum_update(struct symbol *func,
- struct instruction *insn,
- const void *data, size_t size)
+static inline void __checksum_update(struct symbol *sym, const void *data,
+ size_t size)
{
- XXH3_64bits_update(func->csum.state, data, size);
- dbg_checksum(func, insn, XXH3_64bits_digest(func->csum.state));
+ XXH3_64bits_update(sym->csum.state, data, size);
}
-static inline void checksum_finish(struct symbol *func)
+static inline void __checksum_update_insn(struct symbol *sym,
+ struct instruction *insn,
+ const void *data, size_t size)
{
- if (func && func->csum.state) {
- func->csum.checksum = XXH3_64bits_digest(func->csum.state);
- func->csum.state = NULL;
+ __checksum_update(sym, data, size);
+ dbg_checksum_insn(sym, insn, XXH3_64bits_digest(sym->csum.state));
+}
+
+static inline void __checksum_update_object(struct symbol *sym,
+ unsigned long offset,
+ const char *what, const void *data,
+ size_t size)
+{
+ __checksum_update(sym, &offset, sizeof(offset));
+ __checksum_update(sym, data, size);
+ dbg_checksum_object(sym, offset, what, XXH3_64bits_digest(sym->csum.state));
+}
+
+static inline void checksum_finish(struct symbol *sym)
+{
+ if (sym && sym->csum.state) {
+ sym->csum.checksum = XXH3_64bits_digest(sym->csum.state);
+ XXH3_freeState(sym->csum.state);
+ sym->csum.state = NULL;
}
}
+int calculate_checksums(struct objtool_file *file);
+int create_sym_checksum_section(struct objtool_file *file);
+
#else /* !BUILD_KLP */
-static inline void checksum_init(struct symbol *func) {}
-static inline void checksum_update(struct symbol *func,
- struct instruction *insn,
- const void *data, size_t size) {}
-static inline void checksum_finish(struct symbol *func) {}
+static inline int calculate_checksums(struct objtool_file *file) { return -ENOSYS; }
+static inline int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; }
#endif /* !BUILD_KLP */
diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h
index 25573e5..d9c44df 100644
--- a/tools/objtool/include/objtool/elf.h
+++ b/tools/objtool/include/objtool/elf.h
@@ -21,6 +21,13 @@
#define SEC_NAME_LEN 1024
#define SYM_NAME_LEN 512
+static inline u32 str_hash(const char *str)
+{
+ return jhash(str, strlen(str), 0);
+}
+
+u32 str_hash_demangled(const char *str);
+
#define bswap_if_needed(elf, val) __bswap_if_needed(&elf->ehdr, val)
#ifdef LIBELF_USE_DEPRECATED
@@ -89,6 +96,7 @@ struct symbol {
u8 changed : 1;
u8 included : 1;
u8 klp : 1;
+ u8 dont_correlate : 1;
struct list_head pv_target;
struct reloc *relocs;
struct section *group_sec;
@@ -130,6 +138,23 @@ struct elf {
struct symbol *symbol_data;
};
+#define __elf_table(elf, name) ((elf)->name##_hash)
+#define __elf_bits(elf, name) ((elf)->name##_bits)
+
+#define __elf_table_entry(elf, name, key) \
+ __elf_table(elf, name)[hash_min(key, __elf_bits(elf, name))]
+
+#define elf_list_entry(ptr, type, member) \
+({ \
+ typeof(ptr) __ptr = (ptr); \
+ __ptr ? container_of(__ptr, type, member) : NULL; \
+})
+
+#define elf_hash_for_each_possible(elf, name, obj, member, key) \
+ for (obj = elf_list_entry(__elf_table_entry(elf, name, key), typeof(*obj), member); \
+ obj; \
+ obj = elf_list_entry(obj->member.next, typeof(*(obj)), member))
+
struct elf *elf_open_read(const char *name, int flags);
struct elf *elf_create_file(GElf_Ehdr *ehdr, const char *name);
@@ -175,6 +200,7 @@ struct reloc *elf_init_reloc_data_sym(struct elf *elf, struct section *sec,
struct symbol *sym,
s64 addend);
+int elf_write_symbol(struct elf *elf, struct symbol *sym);
int elf_write_insn(struct elf *elf, struct section *sec, unsigned long offset,
unsigned int len, const char *insn);
@@ -186,10 +212,8 @@ struct symbol *find_func_by_offset(struct section *sec, unsigned long offset);
struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset);
struct symbol *find_symbol_by_name(const struct elf *elf, const char *name);
struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name);
-void iterate_global_symbol_by_demangled_name(const struct elf *elf, const char *demangled_name,
- void (*process)(struct symbol *sym, void *data),
- void *data);
struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset);
+struct symbol *find_symbol_containing_inclusive(const struct section *sec, unsigned long offset);
int find_symbol_hole_containing(const struct section *sec, unsigned long offset);
struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset);
struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec,
@@ -276,11 +300,21 @@ static inline bool is_local_sym(struct symbol *sym)
return sym->bind == STB_LOCAL;
}
+static inline bool is_alias_sym(struct symbol *sym)
+{
+ return sym->alias != sym;
+}
+
static inline bool is_prefix_func(struct symbol *sym)
{
return sym->prefix;
}
+static inline bool is_cold_func(struct symbol *sym)
+{
+ return sym->cold;
+}
+
static inline bool is_reloc_sec(struct section *sec)
{
return sec->sh.sh_type == SHT_RELA || sec->sh.sh_type == SHT_REL;
@@ -296,6 +330,11 @@ static inline bool is_text_sec(struct section *sec)
return sec->sh.sh_flags & SHF_EXECINSTR;
}
+static inline bool is_rodata_sec(struct section *sec)
+{
+ return sec->rodata;
+}
+
static inline bool sec_changed(struct section *sec)
{
return sec->_changed;
@@ -468,6 +507,16 @@ static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next)
#define for_each_sym_continue(elf, sym) \
list_for_each_entry_continue(sym, &elf->symbols, global_list)
+#define for_each_sym_by_name(elf, _name, sym) \
+ elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, \
+ str_hash_demangled(_name)) \
+ if (strcmp(sym->name, _name)) {} else
+
+#define for_each_sym_by_demangled_name(elf, name, sym) \
+ elf_hash_for_each_possible(elf, symbol_name, sym, name_hash, \
+ str_hash(name)) \
+ if (strcmp(sym->demangled_name, name)) {} else
+
#define rsec_next_reloc(rsec, reloc) \
reloc_idx(reloc) < sec_num_entries(rsec) - 1 ? reloc + 1 : NULL
@@ -491,10 +540,10 @@ static inline struct symbol *get_func_prefix(struct symbol *func)
{
struct symbol *prev;
- if (!is_func_sym(func))
+ if (!is_func_sym(func) || !func->offset)
return NULL;
- prev = sec_prev_sym(func);
+ prev = find_func_containing(func->sec, func->offset - 1);
if (prev && is_prefix_func(prev))
return prev;
diff --git a/tools/objtool/include/objtool/klp.h b/tools/objtool/include/objtool/klp.h
index e32e5e8..6f60cf0 100644
--- a/tools/objtool/include/objtool/klp.h
+++ b/tools/objtool/include/objtool/klp.h
@@ -29,6 +29,7 @@ struct klp_reloc {
u32 type;
};
+int cmd_klp_checksum(int argc, const char **argv);
int cmd_klp_diff(int argc, const char **argv);
int cmd_klp_post_link(int argc, const char **argv);
diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h
index fa8b7d2..870e147 100644
--- a/tools/objtool/include/objtool/warn.h
+++ b/tools/objtool/include/objtool/warn.h
@@ -77,13 +77,13 @@ static inline char *offstr(struct section *sec, unsigned long offset)
#define WARN_INSN(insn, format, ...) \
({ \
struct instruction *_insn = (insn); \
- if (!_insn->sym || !_insn->sym->warned) { \
+ if (!insn_sym(_insn) || !insn_sym(_insn)->warned) { \
WARN_FUNC(_insn->sec, _insn->offset, format, \
##__VA_ARGS__); \
BT_INSN(_insn, ""); \
} \
- if (_insn->sym) \
- _insn->sym->warned = 1; \
+ if (insn_sym(_insn)) \
+ insn_sym(_insn)->warned = 1; \
})
#define BT_INSN(insn, format, ...) \
@@ -109,7 +109,7 @@ static inline char *offstr(struct section *sec, unsigned long offset)
#define ERROR_FUNC(sec, offset, format, ...) __WARN_FUNC(ERROR_STR, sec, offset, format, ##__VA_ARGS__)
#define ERROR_INSN(insn, format, ...) ERROR_FUNC(insn->sec, insn->offset, format, ##__VA_ARGS__)
-extern bool debug;
+extern bool debug, debug_correlate, debug_clone;
extern int indent;
static inline void unindent(int *unused) { indent--; }
@@ -130,32 +130,39 @@ static inline void unindent(int *unused) { indent--; }
objname ? ": " : "", \
##__VA_ARGS__)
-#define dbg(args...) \
+#define dbg_checksum_insn(func, insn, checksum) \
({ \
- if (unlikely(debug)) \
- __dbg(args); \
-})
-
-#define __dbg_indent(format, ...) \
-({ \
- if (unlikely(debug)) \
- __dbg("%*s" format, indent * 8, "", ##__VA_ARGS__); \
-})
-
-#define dbg_indent(args...) \
- int __cleanup(unindent) __dummy_##__COUNTER__; \
- __dbg_indent(args); \
- indent++
-
-#define dbg_checksum(func, insn, checksum) \
-({ \
- if (unlikely(insn->sym && insn->sym->pfunc && \
- insn->sym->pfunc->debug_checksum)) { \
+ if (unlikely(func->debug_checksum)) { \
char *insn_off = offstr(insn->sec, insn->offset); \
- __dbg("checksum: %s %s %016llx", \
+ __dbg("checksum: %s(): %s %016llx", \
func->name, insn_off, (unsigned long long)checksum);\
free(insn_off); \
} \
})
+#define dbg_checksum_object(sym, offset, what, checksum) \
+({ \
+ if (unlikely(sym->debug_checksum)) \
+ __dbg("checksum: %s+0x%lx: %s %016llx", \
+ sym->name, offset, what, \
+ (unsigned long long)checksum); \
+})
+
+#define dbg_correlate(args...) \
+({ \
+ if (unlikely(debug_correlate)) \
+ __dbg(args); \
+})
+
+#define __dbg_clone(format, ...) \
+({ \
+ if (unlikely(debug_clone)) \
+ __dbg("%*s" format, indent * 8, "", ##__VA_ARGS__); \
+})
+
+#define dbg_clone(args...) \
+ int __cleanup(unindent) __dummy_##__COUNTER__; \
+ __dbg_clone(args); \
+ indent++
+
#endif /* _WARN_H */
diff --git a/tools/objtool/klp-checksum.c b/tools/objtool/klp-checksum.c
new file mode 100644
index 0000000..b8e47f2
--- /dev/null
+++ b/tools/objtool/klp-checksum.c
@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <string.h>
+#include <subcmd/parse-options.h>
+
+#include <objtool/arch.h>
+#include <objtool/builtin.h>
+#include <objtool/check.h>
+#include <objtool/elf.h>
+#include <objtool/klp.h>
+#include <objtool/objtool.h>
+#include <objtool/warn.h>
+#include <objtool/checksum.h>
+
+static int checksum_debug_init(struct objtool_file *file)
+{
+ char *dup, *s;
+
+ if (!opts.debug_checksum)
+ return 0;
+
+ dup = strdup(opts.debug_checksum);
+ if (!dup) {
+ ERROR_GLIBC("strdup");
+ return -1;
+ }
+
+ s = dup;
+ while (*s) {
+ bool found = false;
+ struct symbol *sym;
+ char *comma;
+
+ comma = strchr(s, ',');
+ if (comma)
+ *comma = '\0';
+
+ for_each_sym_by_name(file->elf, s, sym) {
+ if (!is_func_sym(sym) && !is_object_sym(sym))
+ continue;
+ sym->debug_checksum = 1;
+ found = true;
+ }
+
+ if (!found)
+ WARN("--debug-checksum: can't find '%s'", s);
+
+ if (!comma)
+ break;
+
+ s = comma + 1;
+ }
+
+ free(dup);
+ return 0;
+}
+
+static void checksum_update_insn(struct objtool_file *file, struct symbol *func,
+ struct instruction *insn)
+{
+ struct reloc *reloc = insn_reloc(file, insn);
+ struct alternative *alt;
+ unsigned long offset;
+ struct symbol *sym;
+ static bool in_alt;
+
+ if (insn->fake)
+ return;
+
+ if (!reloc) {
+ struct symbol *call_dest = insn_call_dest(insn);
+ struct instruction *jump_dest = insn->jump_dest;
+
+ /*
+ * For a jump/call non-relocated dest offset embedded in the
+ * instruction, the offset may vary due to changes in
+ * surrounding code. Just hash the opcode and a
+ * position-independent representation of the destination.
+ */
+
+ if (call_dest || jump_dest) {
+ unsigned char buf[16];
+ size_t len;
+
+ len = arch_jump_opcode_bytes(file, insn, buf);
+ __checksum_update_insn(func, insn, buf, len);
+
+ if (call_dest) {
+ __checksum_update_insn(func, insn, call_dest->demangled_name,
+ strlen(call_dest->demangled_name));
+
+ } else if (jump_dest) {
+ struct symbol *dest_sym;
+ unsigned long offset;
+
+ /*
+ * use insn->_sym instead of insn_sym() here.
+ * For alternative replacements, the latter
+ * would give the function of the code being
+ * replaced.
+ */
+ dest_sym = jump_dest->_sym;
+ if (!dest_sym)
+ goto alts;
+
+ __checksum_update_insn(func, insn, dest_sym->demangled_name,
+ strlen(dest_sym->demangled_name));
+
+ offset = jump_dest->offset - dest_sym->offset;
+ __checksum_update_insn(func, insn, &offset, sizeof(offset));
+ }
+
+ goto alts;
+ }
+ }
+
+ __checksum_update_insn(func, insn, insn->sec->data->d_buf + insn->offset, insn->len);
+
+ if (!reloc)
+ goto alts;
+
+ sym = reloc->sym;
+ offset = arch_insn_adjusted_addend(insn, reloc);
+
+ if (is_string_sec(sym->sec)) {
+ char *str;
+
+ str = sym->sec->data->d_buf + sym->offset + offset;
+ __checksum_update_insn(func, insn, str, strlen(str));
+ goto alts;
+ }
+
+ if (is_sec_sym(sym)) {
+ sym = find_symbol_containing(reloc->sym->sec, offset);
+ if (!sym)
+ goto alts;
+
+ offset -= sym->offset;
+ }
+
+ __checksum_update_insn(func, insn, sym->demangled_name,
+ strlen(sym->demangled_name));
+ __checksum_update_insn(func, insn, &offset, sizeof(offset));
+
+alts:
+ for (alt = insn->alts; alt; alt = alt->next) {
+ struct alt_group *alt_group = alt->insn->alt_group;
+
+ /* Prevent __ex_table recursion, e.g. LOAD_SEGMENT() */
+ if (in_alt)
+ break;
+ in_alt = true;
+
+ __checksum_update_insn(func, insn, &alt->type,
+ sizeof(alt->type));
+
+ if (alt_group && alt_group->orig_group) {
+ struct instruction *alt_insn;
+
+ __checksum_update_insn(func, insn, &alt_group->feature,sizeof(alt_group->feature));
+
+ for (alt_insn = alt->insn; alt_insn; alt_insn = next_insn_same_sec(file, alt_insn)) {
+ checksum_update_insn(file, func, alt_insn);
+ if (!alt_group->last_insn || alt_insn == alt_group->last_insn)
+ break;
+ }
+ } else {
+ checksum_update_insn(file, func, alt->insn);
+ }
+
+ in_alt = false;
+ }
+}
+
+static void checksum_update_object(struct objtool_file *file, struct symbol *sym)
+{
+ struct reloc *reloc;
+
+ __checksum_update_object(sym, 0, "len", &sym->len, sizeof(sym->len));
+
+ if (sym->sec->data->d_buf)
+ __checksum_update_object(sym, 0, "data",
+ sym->sec->data->d_buf + sym->offset,
+ sym->len);
+
+ sym_for_each_reloc(file->elf, sym, reloc) {
+ unsigned long sym_offset = reloc_offset(reloc) - sym->offset;
+ struct symbol *target = reloc->sym;
+ s64 offset;
+
+ offset = reloc_addend(reloc);
+
+ if (is_string_sec(target->sec)) {
+ char *str;
+
+ str = target->sec->data->d_buf + target->offset + offset;
+ __checksum_update_object(sym, sym_offset,
+ "reloc string", str, strlen(str));
+ continue;
+ }
+
+ if (is_sec_sym(target)) {
+ target = find_symbol_containing(reloc->sym->sec, offset);
+ if (!target)
+ continue;
+
+ offset -= target->offset;
+ }
+
+ __checksum_update_object(sym, sym_offset, "reloc name",
+ target->demangled_name,
+ strlen(target->demangled_name));
+ __checksum_update_object(sym, sym_offset, "reloc addend",
+ &offset, sizeof(offset));
+ }
+}
+
+int calculate_checksums(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct symbol *sym;
+
+ if (checksum_debug_init(file))
+ return -1;
+
+ for_each_sym(file->elf, sym) {
+
+ /*
+ * Skip cold subfunctions and aliases: they share the
+ * parent's checksum via func_for_each_insn() which
+ * follows func->cfunc into the cold subfunction.
+ */
+ if (is_cold_func(sym) || is_alias_sym(sym) || !sym->len ||
+ !sym->sec || !sym->sec->data)
+ continue;
+
+ if (is_func_sym(sym)) {
+ checksum_init(sym);
+ func_for_each_insn(file, sym, insn)
+ checksum_update_insn(file, sym, insn);
+ checksum_finish(sym);
+
+ } else if (is_object_sym(sym)) {
+ checksum_init(sym);
+ checksum_update_object(file, sym);
+ checksum_finish(sym);
+ }
+
+ }
+
+ return 0;
+}
+
+int create_sym_checksum_section(struct objtool_file *file)
+{
+ struct section *sec;
+ struct symbol *sym;
+ unsigned int idx = 0;
+ struct sym_checksum *checksum;
+ size_t entsize = sizeof(struct sym_checksum);
+
+ sec = find_section_by_name(file->elf, ".discard.sym_checksum");
+ if (sec) {
+ if (!opts.dryrun)
+ WARN("file already has .discard.sym_checksum section, skipping");
+
+ return 0;
+ }
+
+ for_each_sym(file->elf, sym)
+ if (sym->csum.checksum)
+ idx++;
+
+ sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize,
+ idx, idx);
+ if (!sec)
+ return -1;
+
+ idx = 0;
+ for_each_sym(file->elf, sym) {
+ if (!sym->csum.checksum)
+ continue;
+
+ if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize,
+ sym, 0, R_TEXT64))
+ return -1;
+
+ checksum = (struct sym_checksum *)sec->data->d_buf + idx;
+ checksum->addr = 0; /* reloc */
+ checksum->checksum = sym->csum.checksum;
+
+ mark_sec_changed(file->elf, sec, true);
+
+ idx++;
+ }
+
+ return 0;
+}
+
+static const char * const klp_checksum_usage[] = {
+ "objtool klp checksum [<options>] file.o",
+ NULL,
+};
+
+int cmd_klp_checksum(int argc, const char **argv)
+{
+ struct objtool_file *file;
+ int ret;
+
+ const struct option options[] = {
+ OPT_STRING(0, "debug-checksum", &opts.debug_checksum, "syms", "enable checksum debug output"),
+ OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"),
+ OPT_END(),
+ };
+
+ argc = parse_options(argc, argv, options, klp_checksum_usage, 0);
+ if (argc != 1)
+ usage_with_options(klp_checksum_usage, options);
+
+ opts.checksum = true;
+
+ objname = argv[0];
+
+ file = objtool_open_read(objname);
+ if (!file)
+ return 1;
+
+ ret = decode_file(file);
+ if (ret)
+ goto out;
+
+ ret = calculate_checksums(file);
+ if (ret)
+ goto out;
+
+ ret = create_sym_checksum_section(file);
+
+out:
+ free_insns(file);
+
+ if (ret)
+ return ret;
+
+ if (!opts.dryrun && file->elf->changed && elf_write(file->elf))
+ return 1;
+
+ return elf_close(file->elf);
+}
diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c
index 0b0d150..f8787d7 100644
--- a/tools/objtool/klp-diff.c
+++ b/tools/objtool/klp-diff.c
@@ -33,6 +33,9 @@ struct export {
char *mod, *sym;
};
+bool debug, debug_correlate, debug_clone;
+int indent;
+
static const char * const klp_diff_usage[] = {
"objtool klp diff [<options>] <in1.o> <in2.o> <out.o>",
NULL,
@@ -40,17 +43,14 @@ static const char * const klp_diff_usage[] = {
static const struct option klp_diff_options[] = {
OPT_GROUP("Options:"),
- OPT_BOOLEAN('d', "debug", &debug, "enable debug output"),
+ OPT_BOOLEAN('d', "debug", &debug, "enable all debug output"),
+ OPT_BOOLEAN(0, "debug-correlate", &debug_correlate, "enable correlation debug output"),
+ OPT_BOOLEAN(0, "debug-clone", &debug_clone, "enable cloning debug output"),
OPT_END(),
};
static DEFINE_HASHTABLE(exports, 15);
-static inline u32 str_hash(const char *str)
-{
- return jhash(str, strlen(str), 0);
-}
-
static char *escape_str(const char *orig)
{
size_t len = 0;
@@ -171,7 +171,7 @@ static int read_sym_checksums(struct elf *elf)
sec = find_section_by_name(elf, ".discard.sym_checksum");
if (!sec) {
- ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool --checksum'?",
+ ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool klp checksum'?",
elf->name);
return -1;
}
@@ -206,7 +206,7 @@ static int read_sym_checksums(struct elf *elf)
return -1;
}
- if (is_func_sym(sym))
+ if (is_func_sym(sym) || is_object_sym(sym))
sym->csum.checksum = sym_checksum->checksum;
}
@@ -242,25 +242,39 @@ static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym)
static bool is_uncorrelated_static_local(struct symbol *sym)
{
static const char * const vars[] = {
- "__already_done.",
- "__func__.",
- "__key.",
- "__warned.",
- "_entry.",
- "_entry_ptr.",
- "_rs.",
- "descriptor.",
- "CSWTCH.",
+ "__already_done",
+ "__func__",
+ "__key",
+ "__warned",
+ "_entry",
+ "_entry_ptr",
+ "_rs",
+ "descriptor",
+ "CSWTCH",
};
+ const char *dot;
if (!is_object_sym(sym) || !is_local_sym(sym))
return false;
- if (!strcmp(sym->sec->name, ".data.once"))
+ /* WARN_ONCE, etc */
+ if (!strcmp(sym->sec->name, ".data..once"))
return true;
+ dot = strchr(sym->name, '.');
+ if (!dot)
+ return false;
+
for (int i = 0; i < ARRAY_SIZE(vars); i++) {
- if (strstarts(sym->name, vars[i]))
+ size_t len = strlen(vars[i]);
+
+ /* GCC: <var>.<id> */
+ if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.'))
+ return true;
+
+ /* Clang: <func>.<var>[.<id>] */
+ if (strstarts(dot + 1, vars[i]) &&
+ (dot[1 + len] == '.' || dot[1 + len] == '\0'))
return true;
}
@@ -268,20 +282,21 @@ static bool is_uncorrelated_static_local(struct symbol *sym)
}
/*
- * Clang emits several useless .Ltmp_* code labels.
+ * .L symbols are assembler-local labels not present in kallsyms. They must
+ * never become KLP relocations; instead their data is cloned into the patch
+ * module. This covers .Ltmp* (Clang temp labels), .L__const.* (Clang local
+ * constants), and any other assembler-local pattern.
*/
-static bool is_clang_tmp_label(struct symbol *sym)
+static bool is_local_label(struct symbol *sym)
{
- return is_notype_sym(sym) &&
- is_text_sec(sym->sec) &&
- strstarts(sym->name, ".Ltmp") &&
- isdigit(sym->name[5]);
+ return strstarts(sym->name, ".L");
}
static bool is_special_section(struct section *sec)
{
static const char * const specials[] = {
".altinstructions",
+ ".kcfi_traps",
".smp_locks",
"__bug_table",
"__ex_table",
@@ -339,6 +354,45 @@ static bool is_special_section_aux(struct section *sec)
}
/*
+ * Symbols created by ___ADDRESSABLE() are only used to convince the toolchain
+ * not to optimize out the referenced symbol.
+ */
+static bool is_addressable_sym(struct symbol *sym)
+{
+ return !strcmp(sym->sec->name, ".discard.addressable");
+}
+
+/*
+ * ABS symbols are typically assembly .set/.equ constants which are never
+ * referenced by relocations. (Exclude FILE symbols which are also SHN_ABS.)
+ */
+static bool is_abs_sym(struct symbol *sym)
+{
+ return sym->sym.st_shndx == SHN_ABS && !is_file_sym(sym);
+}
+
+static bool is_initcall_sym(struct symbol *sym)
+{
+ return strstarts(sym->name, "__initcall__") ||
+ strstarts(sym->name, "__initstub__");
+}
+
+/*
+ * Some .rodata is anonymous and can't be correlated due to there being no
+ * symbol names.
+ *
+ * The .rodata.cst* sections aren't technically anonymous, they're SHF_MERGE
+ * constant pool sections containing small fixed-size data (lookup tables,
+ * bitmasks) which are only read by value, so pointer equivalence isn't needed.
+ * They are typically referenced by UBSAN data sections.
+ */
+static bool is_anonymous_rodata(struct symbol *sym)
+{
+ return is_rodata_sec(sym->sec) &&
+ (!is_object_sym(sym) || strstarts(sym->sec->name, ".rodata.cst"));
+}
+
+/*
* These symbols should never be correlated, so their local patched versions
* are used instead of linking to the originals.
*/
@@ -347,96 +401,391 @@ static bool dont_correlate(struct symbol *sym)
return is_file_sym(sym) ||
is_null_sym(sym) ||
is_sec_sym(sym) ||
+ is_abs_sym(sym) ||
is_prefix_func(sym) ||
is_uncorrelated_static_local(sym) ||
- is_clang_tmp_label(sym) ||
+ is_local_label(sym) ||
is_string_sec(sym->sec) ||
+ is_anonymous_rodata(sym) ||
+ is_initcall_sym(sym) ||
+ is_addressable_sym(sym) ||
is_special_section(sym->sec) ||
- is_special_section_aux(sym->sec) ||
- strstarts(sym->name, "__initcall__");
+ is_special_section_aux(sym->sec);
}
-struct process_demangled_name_data {
- struct symbol *ret;
- int count;
-};
-
-static void process_demangled_name(struct symbol *sym, void *d)
+static const char *llvm_suffix(const char *name)
{
- struct process_demangled_name_data *data = d;
+ return strstr(name, ".llvm.");
+}
- if (sym->twin)
- return;
-
- data->count++;
- data->ret = sym;
+static bool is_llvm_sym(struct symbol *sym)
+{
+ return llvm_suffix(sym->name);
}
/*
- * When there is no full name match, try match demangled_name. This would
- * match original foo.llvm.123 to patched foo.llvm.456.
+ * Determine if two symbols have compatible source file origins:
*
- * Note that, in very rare cases, it is possible to have multiple
- * foo.llvm.<hash> in the same kernel. When this happens, report error and
- * fail the diff.
+ * - If both symbols are local, only return true if they belong to the same
+ * ELF file symbol.
+ *
+ * - If both symbols are global, always return true, as globals don't have
+ * file associations.
+ *
+ * - If they have different scopes, also return true, as the patch might have
+ * changed the symbol's scope.
+ *
+ * Works for both same-ELF (direct pointer compare) and cross-ELF
+ * (compare via file->twin) cases.
*/
-static int find_global_symbol_by_demangled_name(struct elf *elf, struct symbol *sym,
- struct symbol **out_sym)
+static bool maybe_same_file(struct symbol *sym1, struct symbol *sym2)
{
- struct process_demangled_name_data data = {};
+ if (!sym1->file || !sym2->file)
+ return true;
+ if (sym1->file == sym2->file)
+ return true;
+ return sym1->file->twin == sym2->file;
+}
- iterate_global_symbol_by_demangled_name(elf, sym->demangled_name,
- process_demangled_name,
- &data);
- if (data.count > 1) {
- ERROR("Multiple (%d) correlation candidates for %s", data.count, sym->name);
- return -1;
+/*
+ * Similar to maybe_same_file(), but strict: no scope changes allowed.
+ *
+ * Works for both same-ELF (direct pointer compare) and cross-ELF
+ * (compare via file->twin) cases.
+ */
+static bool same_file(struct symbol *sym1, struct symbol *sym2)
+{
+ if (llvm_suffix(sym1->name) && llvm_suffix(sym2->name))
+ return true;
+ if (!sym1->file && !sym2->file)
+ return true;
+ if (!sym1->file || !sym2->file)
+ return false;
+ if (sym1->file == sym2->file)
+ return true;
+ return sym1->file->twin == sym2->file;
+}
+
+/*
+ * Is it a local symbol, or at least was it local in the translation unit
+ * before LLVM promoted it?
+ */
+static bool is_tu_local_sym(struct symbol *sym)
+{
+ return is_local_sym(sym) || is_llvm_sym(sym);
+}
+
+/*
+ * Try to find sym1's twin in patched using deterministic matching.
+ *
+ * Multiple symbols can share a demangled name (e.g., static functions in
+ * different TUs). This function counts same-named candidates through a
+ * funnel of progressively tighter filters. Each level is a strict subset
+ * of the previous one.
+ *
+ * The widest level that yields a 1:1 match wins. Narrower levels are only
+ * needed when the wider level is ambiguous (count > 1).
+ *
+ * Candidates are pre-filtered by maybe_same_file(), which narrows most
+ * local symbols to their own TU. For example, 19 different static
+ * type_show() functions across vmlinux.o each see only one candidate after
+ * pre-filtering, so they match immediately at Level 1.
+ *
+ * Level 1 (name): Works when the demangled name is unique after
+ * pre-filtering. Handles most symbols: unique globals like copy_signal(),
+ * or per-TU locals like pcspkr_probe().
+ *
+ * Level 2 (scope): Filters by local-vs-global (TU-local-vs-not). Example:
+ * parse_header() exists as both a static and a global function. Level 1
+ * sees both (same demangled name), but Level 2 separates them by scope.
+ *
+ * Level 3 (file): Strict file matching via same_file(), which rejects scope
+ * changes. Example: LLVM-promoted foo.llvm.12345 (global, no FILE symbol)
+ * vs genuine local foo (has FILE symbol). Both are TU-local so Level 2
+ * can't distinguish them, but same_file() rejects the pair because one has
+ * a file association and the other doesn't.
+ *
+ * Level 4 (checksum): Distinguishes by function checksum. Example:
+ * usb_devnode.llvm.AAA and usb_devnode.llvm.BBB are two LLVM-promoted
+ * functions from different TUs with the same demangled name. After a TU
+ * change, the .llvm. hashes change but the functions themselves may be
+ * unchanged. Level 4 matches each to the patched candidate with the
+ * same checksum.
+ */
+static struct symbol *find_twin(struct elfs *e, struct symbol *sym1)
+{
+ struct symbol *name_last = NULL, *scope_last = NULL,
+ *file_last = NULL, *csum_last = NULL;
+ unsigned int name_orig = 0, name_patched = 0;
+ unsigned int scope_orig = 0, scope_patched = 0;
+ unsigned int file_orig = 0, file_patched = 0;
+ unsigned int csum_orig = 0, csum_patched = 0;
+ struct symbol *sym2, *match = NULL;
+
+ /* Count orig candidates */
+ for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
+ if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
+ (!maybe_same_file(sym1, sym2)))
+ continue;
+
+ /* Level 1: name match (widest filter) */
+ name_orig++;
+
+ /* Level 2: scope (scope changes allowed) */
+ if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
+ continue;
+ scope_orig++;
+
+ /* Level 3: file (scope changes disallowed) */
+ if (!same_file(sym1, sym2))
+ continue;
+ file_orig++;
+
+ /* Level 4: checksum (unchanged symbols) */
+ if (sym1->len != sym2->len || !sym1->csum.checksum ||
+ sym1->csum.checksum != sym2->csum.checksum)
+ continue;
+ csum_orig++;
}
- *out_sym = data.ret;
+
+ /* Count patched candidates */
+ for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
+ if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
+ !maybe_same_file(sym1, sym2))
+ continue;
+
+ /* Level 1 */
+ name_patched++;
+ name_last = sym2;
+
+ /* Level 2 */
+ if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2))
+ continue;
+ scope_patched++;
+ scope_last = sym2;
+
+ /* Level 3 */
+ if (!same_file(sym1, sym2))
+ continue;
+ file_patched++;
+ file_last = sym2;
+
+ /* Level 4 */
+ if (sym1->len != sym2->len || !sym1->csum.checksum ||
+ sym1->csum.checksum != sym2->csum.checksum)
+ continue;
+ csum_patched++;
+ csum_last = sym2;
+ }
+
+ /* Return the widest level that yields a unique (1:1) match */
+ if (name_orig == 1 && name_patched == 1)
+ match = name_last;
+ else if (scope_orig == 1 && scope_patched == 1)
+ match = scope_last;
+ else if (file_orig == 1 && file_patched == 1)
+ match = file_last;
+ else if (csum_orig == 1 && csum_patched == 1)
+ match = csum_last;
+
+ if (!match)
+ return NULL;
+
+ if (name_orig != 1 || name_patched != 1)
+ dbg_correlate("find_twin(): %s%s -> %s%s",
+ sym1->name, is_func_sym(sym1) ? "()" : "",
+ match->name, is_func_sym(match) ? "()" : "");
+
+ return match;
+}
+
+struct llvm_suffix_pair {
+ struct hlist_node hash;
+ const char *orig;
+ const char *patched;
+};
+
+static DECLARE_HASHTABLE(suffix_map, 7);
+
+/*
+ * Build a mapping of known orig-to-patched LLVM suffixes based on
+ * already-correlated symbol pairs. All promoted symbols from the same TU
+ * share the same .llvm.<hash> suffix, so one correlated pair seeds the map
+ * for the entire TU.
+ */
+static int update_suffix_map(struct elf *elf)
+{
+ struct llvm_suffix_pair *entry;
+ struct symbol *sym;
+
+ for_each_sym(elf, sym) {
+ const char *s1, *s2;
+ bool found;
+
+ if (!sym->twin)
+ continue;
+
+ s1 = llvm_suffix(sym->name);
+ s2 = llvm_suffix(sym->twin->name);
+
+ if (!s1 || !s2)
+ continue;
+
+ found = false;
+ hash_for_each_possible(suffix_map, entry, hash, str_hash(s1)) {
+ if (!strcmp(entry->orig, s1)) {
+ found = true;
+ break;
+ }
+ }
+ if (found)
+ continue;
+
+ entry = calloc(1, sizeof(*entry));
+ if (!entry) {
+ ERROR_GLIBC("calloc");
+ return -1;
+ }
+
+ entry->orig = s1;
+ entry->patched = s2;
+ hash_add(suffix_map, &entry->hash, str_hash(s1));
+ }
+
return 0;
}
/*
- * For each symbol in the original kernel, find its corresponding "twin" in the
- * patched kernel.
+ * Match by translating the symbol's .llvm.<hash> suffix through the suffix
+ * map to find the corresponding hash suffix for the patched object.
+ *
+ * Example: In the original kernel, TU drivers/base/core.c contains
+ * foo.llvm.12345 and bar.llvm.12345 (same TU, same hash). After patching,
+ * they become foo.llvm.67890 and bar.llvm.67890. If foo was already
+ * correlated by find_twin() (e.g., unique by name), the suffix map records
+ * .llvm.12345 -> .llvm.67890. When processing bar.llvm.12345, this
+ * function looks up .llvm.12345, gets .llvm.67890, constructs the name
+ * bar.llvm.67890, and finds the match.
+ */
+static struct symbol *find_twin_suffixed(struct elf *elf, struct symbol *sym1)
+{
+ const char *suffix, *patched_suffix = NULL;
+ struct symbol *sym2, *match = NULL;
+ char name[SYM_NAME_LEN];
+ struct llvm_suffix_pair *entry;
+ int count = 0;
+
+ suffix = llvm_suffix(sym1->name);
+ if (!suffix)
+ return NULL;
+
+ hash_for_each_possible(suffix_map, entry, hash, str_hash(suffix)) {
+ if (!strcmp(entry->orig, suffix)) {
+ patched_suffix = entry->patched;
+ break;
+ }
+ }
+ if (!patched_suffix)
+ return NULL;
+
+ if (snprintf_check(name, SYM_NAME_LEN, "%s%s",
+ sym1->demangled_name, patched_suffix))
+ return NULL;
+
+ for_each_sym_by_name(elf, name, sym2) {
+ if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate)
+ continue;
+ count++;
+ match = sym2;
+ }
+
+ if (count != 1)
+ return NULL;
+
+ dbg_correlate("find_suffixed_twin(): %s%s -> %s%s",
+ sym1->name, is_func_sym(sym1) ? "()" : "",
+ match->name, is_func_sym(match) ? "()" : "");
+
+ return match;
+}
+
+/*
+ * Last-resort positional matching.
+ *
+ * Finds a symbol with the same position in the symbol table among
+ * same-demangled-name candidates, similar to livepatch sympos. Note that
+ * LLVM-promoted symbols are globals, which come after locals in the symbol
+ * table, so we have to be careful not to compare different scopes.
+ *
+ * Example: arch/x86/events/intel/core.c defines many __quirk variables via
+ * X86_MATCH_*() macros. In the symbol table they appear as __quirk.90,
+ * __quirk.97, __quirk.101, etc., all with demangled name __quirk, same
+ * scope, and same FILE symbol. No deterministic filter can distinguish
+ * them, so they're matched by position: the 1st __quirk in orig matches the
+ * 1st in patched, the 2nd matches the 2nd, etc.
+ *
+ * This is less deterministic than the other strategies, so it's done last.
+ */
+static struct symbol *find_twin_positional(struct elfs *e, struct symbol *sym1)
+{
+ unsigned int idx_orig = 0, idx_patched = 0;
+ unsigned int sym1_pos = 0;
+ struct symbol *sym2, *match = NULL;
+
+ for_each_sym_by_demangled_name(e->orig, sym1->demangled_name, sym2) {
+ if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
+ !maybe_same_file(sym1, sym2))
+ continue;
+ if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
+ is_llvm_sym(sym1) != is_llvm_sym(sym2))
+ continue;
+ if (sym1 == sym2)
+ sym1_pos = idx_orig;
+ idx_orig++;
+ }
+
+ for_each_sym_by_demangled_name(e->patched, sym1->demangled_name, sym2) {
+ if (sym2->twin || sym1->type != sym2->type || sym2->dont_correlate ||
+ !maybe_same_file(sym1, sym2))
+ continue;
+ if (is_tu_local_sym(sym1) != is_tu_local_sym(sym2) ||
+ is_llvm_sym(sym1) != is_llvm_sym(sym2))
+ continue;
+ if (idx_patched == sym1_pos)
+ match = sym2;
+ idx_patched++;
+ }
+
+ if (idx_orig != idx_patched)
+ return NULL;
+
+ dbg_correlate("find_twin_positional(): %s%s -> %s%s",
+ sym1->name, is_func_sym(sym1) ? "()" : "",
+ match->name, is_func_sym(match) ? "()" : "");
+
+ return match;
+}
+
+/*
+ * Correlate symbols between the orig and patched objects. This is a
+ * prerequisite for detecting changed functions, as well as for properly
+ * translating relocations so they point to the correct symbol.
*/
static int correlate_symbols(struct elfs *e)
{
struct symbol *file1_sym, *file2_sym;
struct symbol *sym1, *sym2;
+ bool progress;
+ for_each_sym(e->orig, sym1)
+ sym1->dont_correlate = dont_correlate(sym1);
+ for_each_sym(e->patched, sym2)
+ sym2->dont_correlate = dont_correlate(sym2);
+
+ /* Correlate FILE symbols */
file1_sym = first_file_symbol(e->orig);
file2_sym = first_file_symbol(e->patched);
- /*
- * Correlate any locals before the first FILE symbol. This has been
- * seen when LTO inexplicably strips the initramfs_data.o FILE symbol
- * due to the file only containing data and no code.
- */
- for_each_sym(e->orig, sym1) {
- if (sym1 == file1_sym || !is_local_sym(sym1))
- break;
-
- if (dont_correlate(sym1))
- continue;
-
- for_each_sym(e->patched, sym2) {
- if (sym2 == file2_sym || !is_local_sym(sym2))
- break;
-
- if (sym2->twin || dont_correlate(sym2))
- continue;
-
- if (strcmp(sym1->demangled_name, sym2->demangled_name))
- continue;
-
- sym1->twin = sym2;
- sym2->twin = sym1;
- break;
- }
- }
-
- /* Correlate locals after the first FILE symbol */
for (; ; file1_sym = next_file_symbol(e->orig, file1_sym),
file2_sym = next_file_symbol(e->patched, file2_sym)) {
@@ -460,96 +809,56 @@ static int correlate_symbols(struct elfs *e)
file1_sym->twin = file2_sym;
file2_sym->twin = file1_sym;
-
- sym1 = file1_sym;
-
- for_each_sym_continue(e->orig, sym1) {
- if (is_file_sym(sym1) || !is_local_sym(sym1))
- break;
-
- if (dont_correlate(sym1))
- continue;
-
- sym2 = file2_sym;
- for_each_sym_continue(e->patched, sym2) {
- if (is_file_sym(sym2) || !is_local_sym(sym2))
- break;
-
- if (sym2->twin || dont_correlate(sym2))
- continue;
-
- if (strcmp(sym1->demangled_name, sym2->demangled_name))
- continue;
-
- sym1->twin = sym2;
- sym2->twin = sym1;
- break;
- }
- }
}
- /* Correlate globals */
- for_each_sym(e->orig, sym1) {
- if (sym1->bind == STB_LOCAL)
- continue;
-
- sym2 = find_global_symbol_by_name(e->patched, sym1->name);
- if (sym2 && !sym2->twin) {
- sym1->twin = sym2;
- sym2->twin = sym1;
- }
- }
/*
- * Correlate globals with demangled_name.
- * A separate loop is needed because we want to finish all the
- * full name correlations first.
+ * Correlate in two phases: loop deterministic levels until no more
+ * progress, then use positional fallback for the rest. This prevents
+ * the nondeterministic positional matching from stealing symbols that
+ * have deterministic matches.
*/
+ hash_init(suffix_map);
+ do {
+ progress = false;
+ for_each_sym(e->orig, sym1) {
+ if (sym1->twin || sym1->dont_correlate)
+ continue;
+ sym2 = find_twin(e, sym1);
+ if (!sym2)
+ continue;
+ sym1->twin = sym2;
+ sym2->twin = sym1;
+ progress = true;
+ }
+
+ if (update_suffix_map(e->orig))
+ return -1;
+
+ for_each_sym(e->orig, sym1) {
+ if (sym1->twin || sym1->dont_correlate)
+ continue;
+ sym2 = find_twin_suffixed(e->patched, sym1);
+ if (!sym2)
+ continue;
+ sym1->twin = sym2;
+ sym2->twin = sym1;
+ progress = true;
+ }
+ } while (progress);
+
for_each_sym(e->orig, sym1) {
- if (sym1->bind == STB_LOCAL || sym1->twin)
+ if (sym1->twin || sym1->dont_correlate)
continue;
-
- if (find_global_symbol_by_demangled_name(e->patched, sym1, &sym2))
- return -1;
-
- if (sym2 && !sym2->twin) {
- sym1->twin = sym2;
- sym2->twin = sym1;
- }
- }
-
- /* Correlate original locals with patched globals */
- for_each_sym(e->orig, sym1) {
- if (sym1->twin || dont_correlate(sym1) || !is_local_sym(sym1))
+ sym2 = find_twin_positional(e, sym1);
+ if (!sym2)
continue;
-
- sym2 = find_global_symbol_by_name(e->patched, sym1->name);
- if (!sym2 && find_global_symbol_by_demangled_name(e->patched, sym1, &sym2))
- return -1;
-
- if (sym2 && !sym2->twin) {
- sym1->twin = sym2;
- sym2->twin = sym1;
- }
- }
-
- /* Correlate original globals with patched locals */
- for_each_sym(e->patched, sym2) {
- if (sym2->twin || dont_correlate(sym2) || !is_local_sym(sym2))
- continue;
-
- sym1 = find_global_symbol_by_name(e->orig, sym2->name);
- if (!sym1 && find_global_symbol_by_demangled_name(e->orig, sym2, &sym1))
- return -1;
-
- if (sym1 && !sym1->twin) {
- sym2->twin = sym1;
- sym1->twin = sym2;
- }
+ sym1->twin = sym2;
+ sym2->twin = sym1;
}
for_each_sym(e->orig, sym1) {
- if (sym1->twin || dont_correlate(sym1))
+ if (sym1->twin || sym1->dont_correlate)
continue;
WARN("no correlation: %s", sym1->name);
}
@@ -655,7 +964,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym
size_t size;
/* bss doesn't have data */
- if (patched_sym->sec->data->d_buf)
+ if (patched_sym->sec->data && patched_sym->sec->data->d_buf)
data = patched_sym->sec->data->d_buf + patched_sym->offset;
if (is_sec_sym(patched_sym))
@@ -715,7 +1024,7 @@ static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym,
if (patched_sym->clone)
return patched_sym->clone;
- dbg_indent("%s%s", patched_sym->name, data_too ? " [+DATA]" : "");
+ dbg_clone("%s%s", patched_sym->name, data_too ? " [+DATA]" : "");
/* Make sure the prefix gets cloned first */
if (is_func_sym(patched_sym) && data_too) {
@@ -757,19 +1066,24 @@ static void mark_included_function(struct symbol *func)
*/
static int mark_changed_functions(struct elfs *e)
{
- struct symbol *sym_orig, *patched_sym;
+ struct symbol *orig_sym, *patched_sym;
bool changed = false;
/* Find changed functions */
- for_each_sym(e->orig, sym_orig) {
- if (!is_func_sym(sym_orig) || is_prefix_func(sym_orig))
+ for_each_sym(e->orig, orig_sym) {
+ if (orig_sym->dont_correlate)
continue;
- patched_sym = sym_orig->twin;
+ patched_sym = orig_sym->twin;
if (!patched_sym)
continue;
- if (sym_orig->csum.checksum != patched_sym->csum.checksum) {
+ if (orig_sym->csum.checksum != patched_sym->csum.checksum) {
+ if (!is_func_sym(orig_sym)) {
+ ERROR("changed data: %s", orig_sym->name);
+ return -1;
+ }
+
patched_sym->changed = 1;
mark_included_function(patched_sym);
changed = true;
@@ -778,7 +1092,7 @@ static int mark_changed_functions(struct elfs *e)
/* Find added functions and print them */
for_each_sym(e->patched, patched_sym) {
- if (!is_func_sym(patched_sym) || is_prefix_func(patched_sym))
+ if (!is_func_sym(patched_sym) || patched_sym->dont_correlate)
continue;
if (!patched_sym->twin) {
@@ -794,7 +1108,7 @@ static int mark_changed_functions(struct elfs *e)
printf("%s: changed function: %s\n", objname, patched_sym->name);
}
- return !changed ? -1 : 0;
+ return !changed ? 1 : 0;
}
static int clone_included_functions(struct elfs *e)
@@ -811,39 +1125,6 @@ static int clone_included_functions(struct elfs *e)
return 0;
}
-/*
- * Determine whether a relocation should reference the section rather than the
- * underlying symbol.
- */
-static bool section_reference_needed(struct section *sec)
-{
- /*
- * String symbols are zero-length and uncorrelated. It's easier to
- * deal with them as section symbols.
- */
- if (is_string_sec(sec))
- return true;
-
- /*
- * .rodata has mostly anonymous data so there's no way to determine the
- * length of a needed reference. just copy the whole section if needed.
- */
- if (strstarts(sec->name, ".rodata"))
- return true;
-
- /* UBSAN anonymous data */
- if (strstarts(sec->name, ".data..Lubsan") || /* GCC */
- strstarts(sec->name, ".data..L__unnamed_")) /* Clang */
- return true;
-
- return false;
-}
-
-static bool is_reloc_allowed(struct reloc *reloc)
-{
- return section_reference_needed(reloc->sym->sec) == is_sec_sym(reloc->sym);
-}
-
static struct export *find_export(struct symbol *sym)
{
struct export *export;
@@ -917,7 +1198,7 @@ static bool klp_reloc_needed(struct reloc *patched_reloc)
struct export *export;
/* no external symbol to reference */
- if (dont_correlate(patched_sym))
+ if (patched_sym->dont_correlate)
return false;
/* For included functions, a regular reloc will do. */
@@ -952,34 +1233,60 @@ static bool klp_reloc_needed(struct reloc *patched_reloc)
return true;
}
+/* Return -1 error, 0 success, 1 skip */
static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc)
{
struct symbol *sym = reloc->sym;
struct section *sec = sym->sec;
+ if (is_sec_sym(sym))
+ return 0;
+
if (!sec->sym && !elf_create_section_symbol(elf, sec))
return -1;
reloc->sym = sec->sym;
- set_reloc_sym(elf, reloc, sym->idx);
+ set_reloc_sym(elf, reloc, sec->sym->idx);
set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc));
return 0;
}
+/* Return -1 error, 0 success, 1 skip */
static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc)
{
struct symbol *sym = reloc->sym;
struct section *sec = sym->sec;
+ if (!is_sec_sym(sym))
+ return 0;
+
/* If the symbol has a dedicated section, it's easy to find */
sym = find_symbol_by_offset(sec, 0);
if (sym && sym->len == sec_size(sec))
goto found_sym;
/* No dedicated section; find the symbol manually */
- sym = find_symbol_containing(sec, arch_adjusted_addend(reloc));
+ sym = find_symbol_containing_inclusive(sec, arch_adjusted_addend(reloc));
if (!sym) {
/*
+ * This is presumably an .altinstr_replacement section which is
+ * empty due to it only having zero-length replacement(s).
+ */
+ if (!sec_size(sec))
+ return 1;
+
+ /*
+ * .rodata is a mixed bag of named objects and anonymous data.
+ *
+ * Convert section symbol references to named object symbols
+ * when possible, to preserve pointer identity for const
+ * structs like file_operations. Otherwise a section symbol is
+ * fine.
+ */
+ if (is_rodata_sec(sec))
+ return 0;
+
+ /*
* This can happen for special section references to weak code
* whose symbol has been stripped by the linker.
*/
@@ -994,18 +1301,33 @@ static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc)
}
/*
+ * Sections with anonymous or uncorrelated data (strings, UBSAN data, Clang
+ * anonymous constants) need section symbol references.
+ */
+static bool is_uncorrelated_section(struct section *sec)
+{
+ return is_string_sec(sec) ||
+ strstarts(sec->name, ".data..Lubsan") || /* GCC */
+ strstarts(sec->name, ".data..L__unnamed_") || /* Clang */
+ strstarts(sec->name, ".data..Lanon."); /* Clang */
+}
+
+/*
* Convert a relocation symbol reference to the needed format: either a section
- * symbol or the underlying symbol itself.
+ * symbol or the underlying symbol itself. Return -1 error, 0 success, 1 skip.
*/
static int convert_reloc_sym(struct elf *elf, struct reloc *reloc)
{
- if (is_reloc_allowed(reloc))
- return 0;
+ struct section *sec = reloc->sym->sec;
- if (section_reference_needed(reloc->sym->sec))
+ if (reloc_type(reloc) == R_NONE)
+ return 1;
+
+ if (is_uncorrelated_section(sec))
return convert_reloc_sym_to_secsym(elf, reloc);
- else
- return convert_reloc_secsym_to_sym(elf, reloc);
+
+ /* Everything else: references should use named symbols. */
+ return convert_reloc_secsym_to_sym(elf, reloc);
}
/*
@@ -1079,7 +1401,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc,
klp_sym = find_symbol_by_name(e->out, sym_name);
if (!klp_sym) {
- __dbg_indent("%s", sym_name);
+ __dbg_clone("%s", sym_name);
/* STB_WEAK: avoid modpost undefined symbol warnings */
klp_sym = elf_create_symbol(e->out, sym_name, NULL,
@@ -1130,7 +1452,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc,
}
#define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \
- dbg_indent("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \
+ dbg_clone("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \
sec->name, offset, patched_sym->name, \
addend >= 0 ? "+" : "-", labs(addend), \
sym_type(patched_sym), \
@@ -1150,13 +1472,6 @@ static int clone_reloc(struct elfs *e, struct reloc *patched_reloc,
struct symbol *out_sym;
bool klp;
- if (!is_reloc_allowed(patched_reloc)) {
- ERROR_FUNC(patched_reloc->sec->base, reloc_offset(patched_reloc),
- "missing symbol for reference to %s+%ld",
- patched_sym->name, addend);
- return -1;
- }
-
klp = klp_reloc_needed(patched_reloc);
dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp);
@@ -1186,13 +1501,13 @@ static int clone_reloc(struct elfs *e, struct reloc *patched_reloc,
/*
* For strings, all references use section symbols, thanks to
- * section_reference_needed(). clone_symbol() has cloned an empty
+ * convert_reloc_sym(). clone_symbol() has cloned an empty
* version of the string section. Now copy the string itself.
*/
if (is_string_sec(patched_sym->sec)) {
const char *str = patched_sym->sec->data->d_buf + addend;
- __dbg_indent("\"%s\"", escape_str(str));
+ __dbg_clone("\"%s\"", escape_str(str));
addend = elf_add_string(e->out, out_sym->sec, str);
if (addend == -1)
@@ -1239,6 +1554,7 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym)
for_each_reloc(patched_rsec, patched_reloc) {
unsigned long offset;
+ int ret;
if (reloc_offset(patched_reloc) < start ||
reloc_offset(patched_reloc) >= end)
@@ -1252,12 +1568,15 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym)
!strcmp(patched_reloc->sym->sec->name, ".altinstr_aux"))
continue;
- if (convert_reloc_sym(e->patched, patched_reloc)) {
+ ret = convert_reloc_sym(e->patched, patched_reloc);
+ if (ret < 0) {
ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc),
"failed to convert reloc sym '%s' to its proper format",
patched_reloc->sym->name);
return -1;
}
+ if (ret > 0)
+ continue;
offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset);
@@ -1334,7 +1653,7 @@ static int create_fake_symbols(struct elf *elf)
sec = find_section_by_name(elf, ".discard.annotate_data");
if (!sec || !sec->rsec)
- return 0;
+ goto entsize;
for_each_reloc(sec->rsec, reloc) {
unsigned long offset, size;
@@ -1366,7 +1685,7 @@ static int create_fake_symbols(struct elf *elf)
/*
* 2) Make symbols for sh_entsize, and simple arrays of pointers:
*/
-
+entsize:
for_each_sec(elf, sec) {
unsigned int entry_size;
unsigned long offset;
@@ -1400,6 +1719,7 @@ static int create_fake_symbols(struct elf *elf)
/* Keep a special section entry if it references an included function */
static bool should_keep_special_sym(struct elf *elf, struct symbol *sym)
{
+ bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn");
struct reloc *reloc;
if (is_sec_sym(sym) || !sym->sec->rsec)
@@ -1409,7 +1729,16 @@ static bool should_keep_special_sym(struct elf *elf, struct symbol *sym)
if (convert_reloc_sym(elf, reloc))
continue;
- if (is_func_sym(reloc->sym) && reloc->sym->included)
+ if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone))
+ continue;
+
+ /*
+ * Keep special section references to cloned functions.
+ * In some cases annotate_insn can also reference cloned alt
+ * replacement fake symbols; keep those references as well.
+ */
+ if (is_func_sym(reloc->sym) ||
+ (annotate_insn && is_notype_sym(reloc->sym)))
return true;
}
@@ -1553,15 +1882,28 @@ static int clone_special_section(struct elfs *e, struct section *patched_sec)
/* Extract only the needed bits from special sections */
static int clone_special_sections(struct elfs *e)
{
- struct section *patched_sec;
+ struct section *sec, *annotate_insn = NULL;
- for_each_sec(e->patched, patched_sec) {
- if (is_special_section(patched_sec)) {
- if (clone_special_section(e, patched_sec))
+ for_each_sec(e->patched, sec) {
+ if (is_special_section(sec)) {
+ if (!strcmp(sec->name, ".discard.annotate_insn")) {
+ annotate_insn = sec;
+ continue;
+ }
+ if (clone_special_section(e, sec))
return -1;
}
}
+ /*
+ * Do .discard.annotate_insn last, it can reference other special
+ * sections (alt replacements) so they need to be cloned first.
+ */
+ if (annotate_insn) {
+ if (clone_special_section(e, annotate_insn))
+ return -1;
+ }
+
return 0;
}
@@ -1638,7 +1980,8 @@ static int create_klp_sections(struct elfs *e)
unsigned long sympos;
void *func_data;
- if (!is_func_sym(sym) || sym->cold || !sym->clone || !sym->clone->changed)
+ if (!is_func_sym(sym) || is_cold_func(sym) ||
+ !sym->clone || !sym->clone->changed)
continue;
/* allocate klp_func_ext */
@@ -1798,11 +2141,17 @@ static int copy_import_ns(struct elfs *e)
int cmd_klp_diff(int argc, const char **argv)
{
struct elfs e = {0};
+ int ret;
argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0);
if (argc != 3)
usage_with_options(klp_diff_usage, klp_diff_options);
+ if (debug) {
+ debug_correlate = true;
+ debug_clone = true;
+ }
+
objname = argv[0];
e.orig = elf_open_read(argv[0], O_RDONLY);
@@ -1824,7 +2173,10 @@ int cmd_klp_diff(int argc, const char **argv)
if (correlate_symbols(&e))
return -1;
- if (mark_changed_functions(&e))
+ ret = mark_changed_functions(&e);
+ if (ret < 0)
+ return -1;
+ if (ret > 0)
return 0;
e.out = elf_create_file(&e.orig->ehdr, argv[2]);
diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c
index 1c36221..a4e139d 100644
--- a/tools/objtool/objtool.c
+++ b/tools/objtool/objtool.c
@@ -16,9 +16,6 @@
#include <objtool/objtool.h>
#include <objtool/warn.h>
-bool debug;
-int indent;
-
static struct objtool_file file;
struct objtool_file *objtool_open_read(const char *filename)
diff --git a/tools/objtool/trace.c b/tools/objtool/trace.c
index 5dec44d..61c6aa3 100644
--- a/tools/objtool/trace.c
+++ b/tools/objtool/trace.c
@@ -169,8 +169,8 @@ void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt,
*/
TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s for instruction at 0x%lx <%s+0x%lx>",
alt_name,
- orig_insn->offset, orig_insn->sym->name,
- orig_insn->offset - orig_insn->sym->offset);
+ orig_insn->offset, insn_sym(orig_insn)->name,
+ orig_insn->offset - insn_sym(orig_insn)->offset);
} else {
TRACE_ALT_INFO_NOADDR(orig_insn, "/ ", "%s", alt_name);
}
@@ -185,8 +185,8 @@ void trace_alt_begin(struct instruction *orig_insn, struct alternative *alt,
if (orig_insn->type == INSN_NOP) {
suffix[0] = (orig_insn->len == 5) ? 'q' : '\0';
TRACE_ADDR(orig_insn, "jmp%-3s %lx <%s+0x%lx>", suffix,
- alt_insn->offset, alt_insn->sym->name,
- alt_insn->offset - alt_insn->sym->offset);
+ alt_insn->offset, insn_sym(alt_insn)->name,
+ alt_insn->offset - insn_sym(alt_insn)->offset);
} else {
TRACE_ADDR(orig_insn, "nop%d", orig_insn->len);
trace_depth--;
diff --git a/tools/perf/Makefile b/tools/perf/Makefile
index 816d5d8..5b71383 100644
--- a/tools/perf/Makefile
+++ b/tools/perf/Makefile
@@ -111,6 +111,9 @@
build-test-tarball:
@$(MAKE) -f tests/make REUSE_FEATURES_DUMP=1 MK=Makefile SET_PARALLEL=1 --no-print-directory out
+check-headers:
+ @./check-headers.sh
+
#
# All other targets get passed through:
#
@@ -118,4 +121,4 @@
$(print_msg)
$(make)
-.PHONY: tags TAGS FORCE Makefile
+.PHONY: tags TAGS FORCE Makefile build-test build-test-tarball check-headers
diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
index cee19c9..76b35ac 100644
--- a/tools/perf/Makefile.perf
+++ b/tools/perf/Makefile.perf
@@ -285,7 +285,6 @@
$(goals) all: sub-make
sub-make: fixdep
- @./check-headers.sh
$(Q)$(MAKE) FIXDEP_BUILT=1 -f Makefile.perf $(goals)
else # force_fixdep
@@ -565,6 +564,12 @@
$(fsmount_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_tbls)
$(Q)$(SHELL) '$(fsmount_tbls)' $(beauty_uapi_linux_dir) > $@
+fsmount_attr_arrays := $(beauty_outdir)/fsmount_attr_arrays.c
+fsmount_attr_tbls := $(srctree)/tools/perf/trace/beauty/fsmount_attr.sh
+
+$(fsmount_attr_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_attr_tbls)
+ $(Q)$(SHELL) '$(fsmount_attr_tbls)' $(beauty_uapi_linux_dir) > $@
+
fspick_arrays := $(beauty_outdir)/fspick_arrays.c
fspick_tbls := $(srctree)/tools/perf/trace/beauty/fspick.sh
@@ -855,6 +860,7 @@
$(fadvise_advice_array) \
$(fsconfig_arrays) \
$(fsmount_arrays) \
+ $(fsmount_attr_arrays) \
$(fspick_arrays) \
$(pkey_alloc_access_rights_array) \
$(sndrv_pcm_ioctl_array) \
@@ -1302,6 +1308,7 @@
$(OUTPUT)$(fadvise_advice_array) \
$(OUTPUT)$(fsconfig_arrays) \
$(OUTPUT)$(fsmount_arrays) \
+ $(OUTPUT)$(fsmount_attr_arrays) \
$(OUTPUT)$(fspick_arrays) \
$(OUTPUT)$(madvise_behavior_array) \
$(OUTPUT)$(mmap_flags_array) \
diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c
index e58c49d..48615dd 100644
--- a/tools/perf/builtin-trace.c
+++ b/tools/perf/builtin-trace.c
@@ -771,11 +771,6 @@ static const char *bpf_cmd[] = {
};
static DEFINE_STRARRAY(bpf_cmd, "BPF_");
-static const char *fsmount_flags[] = {
- [1] = "CLOEXEC",
-};
-static DEFINE_STRARRAY(fsmount_flags, "FSMOUNT_");
-
#include "trace/beauty/generated/fsconfig_arrays.c"
static DEFINE_STRARRAY(fsconfig_cmds, "FSCONFIG_");
@@ -1202,7 +1197,9 @@ static const struct syscall_fmt syscall_fmts[] = {
{ .name = "fsconfig",
.arg = { [1] = STRARRAY(cmd, fsconfig_cmds), }, },
{ .name = "fsmount",
- .arg = { [1] = STRARRAY_FLAGS(flags, fsmount_flags),
+ .arg = { [1] = { .scnprintf = SCA_FSMOUNT_FLAGS, /* fsmount_flags */
+ .strtoul = STUL_STRARRAYS,
+ .show_zero = true, },
[2] = { .scnprintf = SCA_FSMOUNT_ATTR_FLAGS, /* attr_flags */ }, }, },
{ .name = "fspick",
.arg = { [0] = { .scnprintf = SCA_FDAT, /* dfd */ },
diff --git a/tools/perf/trace/beauty/beauty.h b/tools/perf/trace/beauty/beauty.h
index 0a07ad1..a90c35f 100644
--- a/tools/perf/trace/beauty/beauty.h
+++ b/tools/perf/trace/beauty/beauty.h
@@ -179,6 +179,9 @@ size_t syscall_arg__scnprintf_fcntl_arg(char *bf, size_t size, struct syscall_ar
size_t syscall_arg__scnprintf_flock(char *bf, size_t size, struct syscall_arg *arg);
#define SCA_FLOCK syscall_arg__scnprintf_flock
+size_t syscall_arg__scnprintf_fsmount_flags(char *bf, size_t size, struct syscall_arg *arg);
+#define SCA_FSMOUNT_FLAGS syscall_arg__scnprintf_fsmount_flags
+
size_t syscall_arg__scnprintf_fsmount_attr_flags(char *bf, size_t size, struct syscall_arg *arg);
#define SCA_FSMOUNT_ATTR_FLAGS syscall_arg__scnprintf_fsmount_attr_flags
diff --git a/tools/perf/trace/beauty/clone.sh b/tools/perf/trace/beauty/clone.sh
index 18b6c0d..98cb1f8 100755
--- a/tools/perf/trace/beauty/clone.sh
+++ b/tools/perf/trace/beauty/clone.sh
@@ -14,4 +14,8 @@
grep -E $regex ${linux_sched} | \
sed -r "s/$regex/\2 \1/g" | \
xargs printf "\t[ilog2(%s) + 1] = \"%s\",\n"
+regex='^[[:space:]]*#[[:space:]]*define[[:space:]]+CLONE_([^_]+[[:alnum:]_]+)[[:space:]]+\(1ULL[[:space:]]*<<[[:space:]]*([[:digit:]]+)\)[[:space:]]*.*'
+grep -E $regex ${linux_sched} | \
+ sed -r "s/$regex/\2 \1/g" | \
+ xargs printf "\t[%s + 1] = \"%s\",\n"
printf "};\n"
diff --git a/tools/perf/trace/beauty/fsmount.c b/tools/perf/trace/beauty/fsmount.c
index 28c2c16..179e649 100644
--- a/tools/perf/trace/beauty/fsmount.c
+++ b/tools/perf/trace/beauty/fsmount.c
@@ -16,9 +16,25 @@
#define MOUNT_ATTR_RELATIME 0x00000000 /* - Update atime relative to mtime/ctime. */
#endif
-static size_t fsmount__scnprintf_attr_flags(unsigned long flags, char *bf, size_t size, bool show_prefix)
+
+static size_t fsmount__scnprintf_flags(unsigned long flags, char *bf, size_t size, bool show_prefix)
{
#include "trace/beauty/generated/fsmount_arrays.c"
+ static DEFINE_STRARRAY(fsmount_flags, "FSMOUNT_");
+
+ return strarray__scnprintf_flags(&strarray__fsmount_flags, bf, size, show_prefix, flags);
+}
+
+size_t syscall_arg__scnprintf_fsmount_flags(char *bf, size_t size, struct syscall_arg *arg)
+{
+ unsigned long flags = arg->val;
+
+ return fsmount__scnprintf_flags(flags, bf, size, arg->show_string_prefix);
+}
+
+static size_t fsmount__scnprintf_attr_flags(unsigned long flags, char *bf, size_t size, bool show_prefix)
+{
+#include "trace/beauty/generated/fsmount_attr_arrays.c"
static DEFINE_STRARRAY(fsmount_attr_flags, "MOUNT_ATTR_");
size_t printed = 0;
diff --git a/tools/perf/trace/beauty/fsmount.sh b/tools/perf/trace/beauty/fsmount.sh
index 6b67a54..6d1e80b 100755
--- a/tools/perf/trace/beauty/fsmount.sh
+++ b/tools/perf/trace/beauty/fsmount.sh
@@ -9,14 +9,9 @@
linux_mount=${beauty_uapi_linux_dir}/mount.h
-# Remove MOUNT_ATTR_RELATIME as it is zeros, handle it a special way in the beautifier
-# Only handle MOUNT_ATTR_ followed by a capital letter/num as __ is special case
-# for things like MOUNT_ATTR__ATIME that is a mask for the possible ATIME handling
-# bits. Special case it as well in the beautifier
-
-printf "static const char *fsmount_attr_flags[] = {\n"
-regex='^[[:space:]]*#[[:space:]]*define[[:space:]]+MOUNT_ATTR_([[:alnum:]][[:alnum:]_]+)[[:space:]]+(0x[[:xdigit:]]+)[[:space:]]*.*'
-grep -E $regex ${linux_mount} | grep -v MOUNT_ATTR_RELATIME | \
+printf "static const char *fsmount_flags[] = {\n"
+regex='^[[:space:]]*#[[:space:]]*define[[:space:]]+FSMOUNT_([[:alnum:]][[:alnum:]_]+)[[:space:]]+(0x[[:xdigit:]]+)[[:space:]]*.*'
+grep -E $regex ${linux_mount} | \
sed -r "s/$regex/\2 \1/g" | \
xargs printf "\t[ilog2(%s) + 1] = \"%s\",\n"
printf "};\n"
diff --git a/tools/perf/trace/beauty/fsmount_attr.sh b/tools/perf/trace/beauty/fsmount_attr.sh
new file mode 100644
index 0000000..6b67a54
--- /dev/null
+++ b/tools/perf/trace/beauty/fsmount_attr.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+# SPDX-License-Identifier: LGPL-2.1
+
+if [ $# -ne 1 ] ; then
+ beauty_uapi_linux_dir=tools/perf/trace/beauty/include/uapi/linux/
+else
+ beauty_uapi_linux_dir=$1
+fi
+
+linux_mount=${beauty_uapi_linux_dir}/mount.h
+
+# Remove MOUNT_ATTR_RELATIME as it is zeros, handle it a special way in the beautifier
+# Only handle MOUNT_ATTR_ followed by a capital letter/num as __ is special case
+# for things like MOUNT_ATTR__ATIME that is a mask for the possible ATIME handling
+# bits. Special case it as well in the beautifier
+
+printf "static const char *fsmount_attr_flags[] = {\n"
+regex='^[[:space:]]*#[[:space:]]*define[[:space:]]+MOUNT_ATTR_([[:alnum:]][[:alnum:]_]+)[[:space:]]+(0x[[:xdigit:]]+)[[:space:]]*.*'
+grep -E $regex ${linux_mount} | grep -v MOUNT_ATTR_RELATIME | \
+ sed -r "s/$regex/\2 \1/g" | \
+ xargs printf "\t[ilog2(%s) + 1] = \"%s\",\n"
+printf "};\n"
diff --git a/tools/perf/trace/beauty/include/linux/socket.h b/tools/perf/trace/beauty/include/linux/socket.h
index ec715ad..ec4a0a0 100644
--- a/tools/perf/trace/beauty/include/linux/socket.h
+++ b/tools/perf/trace/beauty/include/linux/socket.h
@@ -415,7 +415,7 @@ struct __kernel_timespec;
struct old_timespec32;
struct scm_timestamping_internal {
- struct timespec64 ts[3];
+ ktime_t ts[3];
};
extern void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_internal *tss);
diff --git a/tools/perf/trace/beauty/include/uapi/linux/fs.h b/tools/perf/trace/beauty/include/uapi/linux/fs.h
index 70b2b66..13f7120 100644
--- a/tools/perf/trace/beauty/include/uapi/linux/fs.h
+++ b/tools/perf/trace/beauty/include/uapi/linux/fs.h
@@ -657,4 +657,16 @@ struct procmap_query {
__u64 build_id_addr; /* in */
};
+/*
+ * Shutdown the filesystem.
+ */
+#define FS_IOC_SHUTDOWN _IOR('X', 125, __u32)
+
+/*
+ * Flags for FS_IOC_SHUTDOWN
+ */
+#define FS_SHUTDOWN_FLAGS_DEFAULT 0x0
+#define FS_SHUTDOWN_FLAGS_LOGFLUSH 0x1 /* flush log but not data*/
+#define FS_SHUTDOWN_FLAGS_NOLOGFLUSH 0x2 /* don't flush log nor data */
+
#endif /* _UAPI_LINUX_FS_H */
diff --git a/tools/perf/trace/beauty/include/uapi/linux/mount.h b/tools/perf/trace/beauty/include/uapi/linux/mount.h
index d9d8659..2204708 100644
--- a/tools/perf/trace/beauty/include/uapi/linux/mount.h
+++ b/tools/perf/trace/beauty/include/uapi/linux/mount.h
@@ -110,6 +110,7 @@ enum fsconfig_command {
* fsmount() flags.
*/
#define FSMOUNT_CLOEXEC 0x00000001
+#define FSMOUNT_NAMESPACE 0x00000002 /* Create the mount in a new mount namespace */
/*
* Mount attributes.
diff --git a/tools/perf/trace/beauty/include/uapi/linux/sched.h b/tools/perf/trace/beauty/include/uapi/linux/sched.h
index 359a14c..33a4624 100644
--- a/tools/perf/trace/beauty/include/uapi/linux/sched.h
+++ b/tools/perf/trace/beauty/include/uapi/linux/sched.h
@@ -34,8 +34,12 @@
#define CLONE_IO 0x80000000 /* Clone io context */
/* Flags for the clone3() syscall. */
-#define CLONE_CLEAR_SIGHAND 0x100000000ULL /* Clear any signal handler and reset to SIG_DFL. */
-#define CLONE_INTO_CGROUP 0x200000000ULL /* Clone into a specific cgroup given the right permissions. */
+#define CLONE_CLEAR_SIGHAND (1ULL << 32) /* Clear any signal handler and reset to SIG_DFL. */
+#define CLONE_INTO_CGROUP (1ULL << 33) /* Clone into a specific cgroup given the right permissions. */
+#define CLONE_AUTOREAP (1ULL << 34) /* Auto-reap child on exit. */
+#define CLONE_NNP (1ULL << 35) /* Set no_new_privs on child. */
+#define CLONE_PIDFD_AUTOKILL (1ULL << 36) /* Kill child when clone pidfd closes. */
+#define CLONE_EMPTY_MNTNS (1ULL << 37) /* Create an empty mount namespace. */
/*
* cloning flags intersect with CSIGNAL so can be used with unshare and clone3
@@ -43,6 +47,12 @@
*/
#define CLONE_NEWTIME 0x00000080 /* New time namespace */
+/*
+ * unshare flags share the bit space with clone flags but only apply to the
+ * unshare syscall:
+ */
+#define UNSHARE_EMPTY_MNTNS 0x00100000 /* Unshare an empty mount namespace. */
+
#ifndef __ASSEMBLY__
/**
* struct clone_args - arguments for the clone3 syscall
@@ -146,4 +156,7 @@ struct clone_args {
SCHED_FLAG_KEEP_ALL | \
SCHED_FLAG_UTIL_CLAMP)
+/* Only for sched_getattr() own flag param, if task is SCHED_DEADLINE */
+#define SCHED_GETATTR_FLAG_DL_DYNAMIC 0x01
+
#endif /* _UAPI_LINUX_SCHED_H */
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index b68abb9..aad698f 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -159,13 +159,7 @@ static s32 pick_direct_dispatch_cpu(struct task_struct *p, s32 prev_cpu)
static struct task_ctx *lookup_task_ctx(struct task_struct *p)
{
- struct task_ctx *tctx;
-
- if (!(tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0))) {
- scx_bpf_error("task_ctx lookup failed");
- return NULL;
- }
- return tctx;
+ return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
}
s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p,
@@ -175,7 +169,7 @@ s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p,
s32 cpu;
if (!(tctx = lookup_task_ctx(p)))
- return -ESRCH;
+ return prev_cpu;
if (p->scx.weight < 2 && !(p->flags & PF_KTHREAD))
return prev_cpu;
@@ -540,13 +534,9 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev)
*/
if (prev) {
tctx = bpf_task_storage_get(&task_ctx_stor, prev, 0, 0);
- if (!tctx) {
- scx_bpf_error("task_ctx lookup failed");
- return;
- }
-
- tctx->core_sched_seq =
- core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
+ if (tctx)
+ tctx->core_sched_seq =
+ core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
}
}
@@ -584,10 +574,8 @@ static s64 task_qdist(struct task_struct *p)
s64 qdist;
tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
- if (!tctx) {
- scx_bpf_error("task_ctx lookup failed");
+ if (!tctx)
return 0;
- }
qdist = tctx->core_sched_seq - core_sched_head_seqs[idx];
diff --git a/tools/sched_ext/scx_show_state.py b/tools/sched_ext/scx_show_state.py
index 02e43c1..446d828 100644
--- a/tools/sched_ext/scx_show_state.py
+++ b/tools/sched_ext/scx_show_state.py
@@ -27,18 +27,25 @@
def state_str(state):
return prog['scx_enable_state_str'][state].string_().decode()
+def read_root_ops_name():
+ if root:
+ return root.ops.name.string_().decode()
+ return ''
+
+def read_root_field(name, default):
+ if root:
+ return getattr(root, name).value_()
+ return default
+
root = prog['scx_root']
enable_state = read_atomic("scx_enable_state_var")
-if root:
- print(f'ops : {root.ops.name.string_().decode()}')
-else:
- print('ops : ')
+print(f'ops : {read_root_ops_name()}')
print(f'enabled : {read_static_key("__scx_enabled")}')
print(f'switching_all : {read_int("scx_switching_all")}')
print(f'switched_all : {read_static_key("__scx_switched_all")}')
print(f'enable_state : {state_str(enable_state)} ({enable_state})')
-print(f'aborting : {prog["scx_aborting"].value_()}')
-print(f'bypass_depth : {prog["scx_bypass_depth"].value_()}')
+print(f'aborting : {read_root_field("aborting", False)}')
+print(f'bypass_depth : {read_root_field("bypass_depth", 0)}')
print(f'nr_rejected : {read_atomic("scx_nr_rejected")}')
print(f'enable_seq : {read_atomic("scx_enable_seq")}')
diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
index 4186699..296516e 100644
--- a/tools/testing/cxl/test/cxl.c
+++ b/tools/testing/cxl/test/cxl.c
@@ -1523,6 +1523,23 @@ static void mock_companion(struct acpi_device *adev, struct device *dev)
#define SZ_64G (SZ_32G * 2)
#endif
+static int cxl_mock_platform_device_add(struct platform_device *pdev,
+ struct platform_device **ppdev)
+{
+ int rc;
+
+ if (ppdev)
+ *ppdev = pdev;
+ rc = platform_device_add(pdev);
+ if (rc) {
+ platform_device_put(pdev);
+ if (ppdev)
+ *ppdev = NULL;
+ }
+
+ return rc;
+}
+
static __init int cxl_rch_topo_init(void)
{
int rc, i;
@@ -1537,13 +1554,10 @@ static __init int cxl_rch_topo_init(void)
goto err_bridge;
mock_companion(adev, &pdev->dev);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_rch[i]);
+ if (rc)
goto err_bridge;
- }
- cxl_rch[i] = pdev;
mock_pci_bus[idx].bridge = &pdev->dev;
rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
"firmware_node");
@@ -1595,13 +1609,10 @@ static __init int cxl_single_topo_init(void)
goto err_bridge;
mock_companion(adev, &pdev->dev);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_hb_single[i]);
+ if (rc)
goto err_bridge;
- }
- cxl_hb_single[i] = pdev;
mock_pci_bus[i + NR_CXL_HOST_BRIDGES].bridge = &pdev->dev;
rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
"physical_node");
@@ -1620,12 +1631,9 @@ static __init int cxl_single_topo_init(void)
goto err_port;
pdev->dev.parent = &bridge->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_root_single[i]);
+ if (rc)
goto err_port;
- }
- cxl_root_single[i] = pdev;
}
for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++) {
@@ -1638,12 +1646,9 @@ static __init int cxl_single_topo_init(void)
goto err_uport;
pdev->dev.parent = &root_port->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_swu_single[i]);
+ if (rc)
goto err_uport;
- }
- cxl_swu_single[i] = pdev;
}
for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++) {
@@ -1657,12 +1662,9 @@ static __init int cxl_single_topo_init(void)
goto err_dport;
pdev->dev.parent = &uport->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_swd_single[i]);
+ if (rc)
goto err_dport;
- }
- cxl_swd_single[i] = pdev;
}
return 0;
@@ -1735,12 +1737,9 @@ static int cxl_mem_init(void)
pdev->dev.parent = &dport->dev;
set_dev_node(&pdev->dev, i % 2);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]);
+ if (rc)
goto err_mem;
- }
- cxl_mem[i] = pdev;
}
for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++) {
@@ -1753,12 +1752,9 @@ static int cxl_mem_init(void)
pdev->dev.parent = &dport->dev;
set_dev_node(&pdev->dev, i % 2);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_mem_single[i]);
+ if (rc)
goto err_single;
- }
- cxl_mem_single[i] = pdev;
}
for (i = 0; i < ARRAY_SIZE(cxl_rcd); i++) {
@@ -1772,12 +1768,9 @@ static int cxl_mem_init(void)
pdev->dev.parent = &rch->dev;
set_dev_node(&pdev->dev, i % 2);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_rcd[i]);
+ if (rc)
goto err_rcd;
- }
- cxl_rcd[i] = pdev;
}
return 0;
@@ -1869,13 +1862,10 @@ static __init int cxl_test_init(void)
goto err_bridge;
mock_companion(adev, &pdev->dev);
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_host_bridge[i]);
+ if (rc)
goto err_bridge;
- }
- cxl_host_bridge[i] = pdev;
mock_pci_bus[i].bridge = &pdev->dev;
rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
"physical_node");
@@ -1893,12 +1883,9 @@ static __init int cxl_test_init(void)
goto err_port;
pdev->dev.parent = &bridge->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_root_port[i]);
+ if (rc)
goto err_port;
- }
- cxl_root_port[i] = pdev;
}
BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port));
@@ -1911,12 +1898,9 @@ static __init int cxl_test_init(void)
goto err_uport;
pdev->dev.parent = &root_port->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_switch_uport[i]);
+ if (rc)
goto err_uport;
- }
- cxl_switch_uport[i] = pdev;
}
for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) {
@@ -1929,12 +1913,9 @@ static __init int cxl_test_init(void)
goto err_dport;
pdev->dev.parent = &uport->dev;
- rc = platform_device_add(pdev);
- if (rc) {
- platform_device_put(pdev);
+ rc = cxl_mock_platform_device_add(pdev, &cxl_switch_dport[i]);
+ if (rc)
goto err_dport;
- }
- cxl_switch_dport[i] = pdev;
}
rc = cxl_single_topo_init();
@@ -1953,9 +1934,9 @@ static __init int cxl_test_init(void)
acpi0017_mock.dev.bus = &platform_bus_type;
cxl_acpi->dev.groups = cxl_acpi_groups;
- rc = platform_device_add(cxl_acpi);
+ rc = cxl_mock_platform_device_add(cxl_acpi, NULL);
if (rc)
- goto err_root;
+ goto err_rch;
rc = cxl_mem_init();
if (rc)
diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c
index feedd5a..0607913 100644
--- a/tools/testing/radix-tree/maple.c
+++ b/tools/testing/radix-tree/maple.c
@@ -2,7 +2,7 @@
/*
* maple_tree.c: Userspace testing for maple tree test-suite
* Copyright (c) 2018-2022 Oracle Corporation
- * Author: Liam R. Howlett <Liam.Howlett@Oracle.com>
+ * Author: Liam R. Howlett <liam@infradead.org>
*
* Any tests that require internal knowledge of the tree or threads and other
* difficult to handle in kernel tests.
diff --git a/tools/testing/selftests/arm64/gcs/gcs-util.h b/tools/testing/selftests/arm64/gcs/gcs-util.h
index c99a6b3..7a81bb0 100644
--- a/tools/testing/selftests/arm64/gcs/gcs-util.h
+++ b/tools/testing/selftests/arm64/gcs/gcs-util.h
@@ -18,12 +18,6 @@
#ifndef NT_ARM_GCS
#define NT_ARM_GCS 0x410
-
-struct user_gcs {
- __u64 features_enabled;
- __u64 features_locked;
- __u64 gcspr_el0;
-};
#endif
/* Shadow Stack/Guarded Control Stack interface */
diff --git a/tools/testing/selftests/arm64/gcs/libc-gcs.c b/tools/testing/selftests/arm64/gcs/libc-gcs.c
index 17b2fab..72e82bf 100644
--- a/tools/testing/selftests/arm64/gcs/libc-gcs.c
+++ b/tools/testing/selftests/arm64/gcs/libc-gcs.c
@@ -16,6 +16,7 @@
#include <asm/hwcap.h>
#include <asm/mman.h>
+#include <asm/ptrace.h>
#include <linux/compiler.h>
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
index f829b6f..fe30181 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
@@ -112,6 +112,10 @@ static void test_cubic(void)
ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called");
+ ASSERT_TRUE(cubic_skel->bss->nodelay_init_reject, "init reject nodelay option");
+ ASSERT_TRUE(cubic_skel->bss->nodelay_cwnd_event_tx_start_reject,
+ "cwnd_event_tx_start reject nodelay option");
+
bpf_link__destroy(link);
bpf_cubic__destroy(cubic_skel);
}
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c
index 8cd298b..04aaf4c 100644
--- a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c
+++ b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c
@@ -14,7 +14,7 @@ static struct {
const char *prog_name;
int expected_runtime_err;
} kfunc_dynptr_tests[] = {
- {"dynptr_data_null", -EBADMSG},
+ {"dynptr_data_null", -EINVAL},
};
static bool kfunc_not_supported;
diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c
new file mode 100644
index 0000000..2a8b238
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+
+/*
+ * Test that replacing an inner percpu array map with one that has different
+ * max_entries is rejected. percpu_array_map_gen_lookup() inlines the
+ * template's index_mask, so allowing a smaller replacement would cause OOB.
+ */
+void test_percpu_array_inner_map(void)
+{
+ LIBBPF_OPTS(bpf_map_create_opts, opts);
+ int outer_fd, tmpl_fd, good_fd, bad_fd, err;
+ int zero = 0;
+
+ /* Create template: percpu array with 8 entries */
+ tmpl_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "tmpl",
+ sizeof(int), sizeof(long), 8, NULL);
+ if (!ASSERT_OK_FD(tmpl_fd, "create_tmpl"))
+ return;
+
+ /* Create outer array-of-maps using template */
+ opts.inner_map_fd = tmpl_fd;
+ outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer",
+ sizeof(int), sizeof(int), 1, &opts);
+ if (!ASSERT_OK_FD(outer_fd, "create_outer"))
+ goto close_tmpl;
+
+ /* Insert template as initial inner map */
+ err = bpf_map_update_elem(outer_fd, &zero, &tmpl_fd, 0);
+ if (!ASSERT_OK(err, "insert_tmpl"))
+ goto close_outer;
+
+ /* Replacement with same max_entries should succeed */
+ good_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "good",
+ sizeof(int), sizeof(long), 8, NULL);
+ if (!ASSERT_OK_FD(good_fd, "create_good"))
+ goto close_outer;
+
+ err = bpf_map_update_elem(outer_fd, &zero, &good_fd, 0);
+ ASSERT_OK(err, "replace_same_max_entries");
+ close(good_fd);
+
+ /* Replacement with fewer max_entries must fail */
+ bad_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "bad",
+ sizeof(int), sizeof(long), 2, NULL);
+ if (!ASSERT_OK_FD(bad_fd, "create_bad"))
+ goto close_outer;
+
+ err = bpf_map_update_elem(outer_fd, &zero, &bad_fd, 0);
+ ASSERT_ERR(err, "replace_smaller_max_entries");
+ close(bad_fd);
+
+close_outer:
+ close(outer_fd);
+close_tmpl:
+ close(tmpl_fd);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
index b87e7f3..6ed8e14 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
@@ -417,6 +417,107 @@ static void run_tests(int family, enum bpf_map_type map_type)
close(map);
}
+/*
+ * Regression test for the KTLS + sockmap (verdict) reverse-order UAF.
+ *
+ * Vulnerable sequence:
+ * 1. Insert receiver socket into sockmap with BPF_SK_SKB_VERDICT program.
+ * sk->sk_data_ready becomes sk_psock_verdict_data_ready.
+ * 2. Configure TLS RX: tls_sw_strparser_arm() saves
+ * sk_psock_verdict_data_ready as rx_ctx->saved_data_ready.
+ *
+ * When data arrives, tls_rx_msg_ready() calls saved_data_ready() =
+ * sk_psock_verdict_data_ready(), which calls tcp_read_skb() and drains
+ * sk_receive_queue via __skb_unlink() without advancing copied_seq.
+ * tls_strp_msg_load() then finds the queue empty while tcp_inq() is still
+ * non-zero, hits WARN_ON_ONCE(!first), and leaves a dangling frag_list
+ * pointer that tls_decrypt_sg() walks — a use-after-free.
+ *
+ * The fix adds a tls_sw_has_ctx_rx() check to sk_psock_verdict_data_ready(),
+ * mirroring what sk_psock_strp_data_ready() already does: when a TLS RX
+ * context is present, defer to psock->saved_data_ready (sock_def_readable)
+ * instead of calling tcp_read_skb(), so TLS retains sole ownership of the
+ * receive queue. Data is then decrypted and returned correctly by
+ * tls_sw_recvmsg().
+ */
+static void test_sockmap_ktls_verdict_with_tls_rx(int family, int sotype)
+{
+ struct tls12_crypto_info_aes_gcm_128 crypto_info = {};
+ char send_buf[] = "hello ktls sockmap reverse order";
+ char recv_buf[sizeof(send_buf)] = {};
+ struct test_sockmap_ktls *skel;
+ int c = -1, p = -1, zero = 0;
+ int prog_fd, map_fd;
+ ssize_t n;
+ int err;
+
+ skel = test_sockmap_ktls__open_and_load();
+ if (!ASSERT_TRUE(skel, "open_and_load"))
+ return;
+
+ err = create_pair(family, sotype, &c, &p);
+ if (!ASSERT_OK(err, "create_pair"))
+ goto out;
+
+ prog_fd = bpf_program__fd(skel->progs.prog_skb_verdict_pass);
+ map_fd = bpf_map__fd(skel->maps.sock_map_verdict);
+
+ err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_SKB_VERDICT, 0);
+ if (!ASSERT_OK(err, "bpf_prog_attach sk_skb verdict"))
+ goto out;
+
+ /* Step 1: configure TLS TX on sender (no sockmap involvement) */
+ err = setsockopt(c, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
+ if (!ASSERT_OK(err, "setsockopt(TCP_ULP) client"))
+ goto out;
+
+ crypto_info.info.version = TLS_1_2_VERSION;
+ crypto_info.info.cipher_type = TLS_CIPHER_AES_GCM_128;
+ memset(crypto_info.key, 0x01, sizeof(crypto_info.key));
+ memset(crypto_info.salt, 0x02, sizeof(crypto_info.salt));
+
+ err = setsockopt(c, SOL_TLS, TLS_TX, &crypto_info, sizeof(crypto_info));
+ if (!ASSERT_OK(err, "setsockopt(TLS_TX)"))
+ goto out;
+
+ /* Step 2: insert receiver into sockmap BEFORE TLS RX */
+ err = bpf_map_update_elem(map_fd, &zero, &p, BPF_NOEXIST);
+ if (!ASSERT_OK(err, "bpf_map_update_elem"))
+ goto out;
+
+ /* Step 3: configure TLS RX AFTER sockmap insertion */
+ err = setsockopt(p, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
+ if (!ASSERT_OK(err, "setsockopt(TCP_ULP) server"))
+ goto out;
+
+ err = setsockopt(p, SOL_TLS, TLS_RX, &crypto_info, sizeof(crypto_info));
+ if (!ASSERT_OK(err, "setsockopt(TLS_RX)"))
+ goto out;
+
+ /*
+ * A buggy kernel hits WARN_ON_ONCE in tls_strp_load_anchor_with_queue
+ * and may UAF in tls_decrypt_sg here. With the fix,
+ * sk_psock_verdict_data_ready defers to sock_def_readable and TLS
+ * decrypts the record normally.
+ */
+ n = send(c, send_buf, sizeof(send_buf), 0);
+ if (!ASSERT_EQ(n, (ssize_t)sizeof(send_buf), "send"))
+ goto out;
+
+ n = recv_timeout(p, recv_buf, sizeof(recv_buf), 0, 5);
+ if (!ASSERT_EQ(n, (ssize_t)sizeof(send_buf), "recv"))
+ goto out;
+
+ ASSERT_OK(memcmp(send_buf, recv_buf, sizeof(send_buf)), "data integrity");
+
+out:
+ if (c != -1)
+ close(c);
+ if (p != -1)
+ close(p);
+ test_sockmap_ktls__destroy(skel);
+}
+
static void run_ktls_test(int family, int sotype)
{
if (test__start_subtest("tls simple offload"))
@@ -429,6 +530,8 @@ static void run_ktls_test(int family, int sotype)
test_sockmap_ktls_tx_no_buf(family, sotype, true);
if (test__start_subtest("tls tx with pop"))
test_sockmap_ktls_tx_pop(family, sotype);
+ if (test__start_subtest("tls verdict with tls rx"))
+ test_sockmap_ktls_verdict_with_tls_rx(family, sotype);
}
void test_sockmap_ktls(void)
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
index 5363743..3a41c51 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
@@ -190,7 +190,7 @@ static int getsetsockopt(void)
fd = socket(AF_NETLINK, SOCK_RAW, 0);
if (fd < 0) {
log_err("Failed to create AF_NETLINK socket");
- return -1;
+ goto err;
}
buf.u32 = 1;
@@ -211,6 +211,21 @@ static int getsetsockopt(void)
}
ASSERT_EQ(optlen, 8, "Unexpected NETLINK_LIST_MEMBERSHIPS value");
+ /* Trick bpf_tcp_sock() with IPPROTO_TCP */
+ close(fd);
+ fd = socket(AF_INET, SOCK_RAW, IPPROTO_TCP);
+ if (!ASSERT_OK_FD(fd, "socket"))
+ goto err;
+
+ /* The BPF prog intercepts this before the kernel sees it, any
+ * optlen works. Go with 4 bytes for simplicity.
+ */
+ buf.u32 = 1;
+ optlen = sizeof(buf.u32);
+ err = setsockopt(fd, SOL_TCP, TCP_SAVED_SYN, &buf, optlen);
+ if (!ASSERT_ERR(err, "setsockopt(TCP_SAVED_SYN)"))
+ goto err;
+
free(big_buf);
close(fd);
return 0;
diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
index 56685fc..80e6315 100644
--- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
+++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
@@ -507,6 +507,10 @@ static void misc(void)
ASSERT_EQ(misc_skel->bss->nr_hwtstamp, 0, "nr_hwtstamp");
+ ASSERT_TRUE(misc_skel->bss->nodelay_est_ok, "nodelay_est_ok");
+ ASSERT_TRUE(misc_skel->bss->nodelay_hdr_len_reject, "nodelay_hdr_len_reject");
+ ASSERT_TRUE(misc_skel->bss->nodelay_write_hdr_reject, "nodelay_write_hdr_reject");
+
check_linum:
ASSERT_FALSE(check_error_linum(&sk_fds), "check_error_linum");
sk_fds_close(&sk_fds);
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index a96b25e..06cd24e 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -22,6 +22,7 @@
#include "verifier_bswap.skel.h"
#include "verifier_btf_ctx_access.skel.h"
#include "verifier_btf_unreliable_prog.skel.h"
+#include "verifier_call_large_imm.skel.h"
#include "verifier_cfg.skel.h"
#include "verifier_cgroup_inv_retcode.skel.h"
#include "verifier_cgroup_skb.skel.h"
@@ -170,6 +171,7 @@ void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); }
void test_verifier_bswap(void) { RUN(verifier_bswap); }
void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); }
void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); }
+void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); }
void test_verifier_cfg(void) { RUN(verifier_cfg); }
void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); }
void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); }
diff --git a/tools/testing/selftests/bpf/progs/bpf_cubic.c b/tools/testing/selftests/bpf/progs/bpf_cubic.c
index ce18a4d..ebd5a1e 100644
--- a/tools/testing/selftests/bpf/progs/bpf_cubic.c
+++ b/tools/testing/selftests/bpf/progs/bpf_cubic.c
@@ -16,6 +16,7 @@
#include "bpf_tracing_net.h"
#include <bpf/bpf_tracing.h>
+#include <errno.h>
char _license[] SEC("license") = "GPL";
@@ -170,10 +171,18 @@ static void bictcp_hystart_reset(struct sock *sk)
ca->sample_cnt = 0;
}
+bool nodelay_init_reject = false;
+bool nodelay_cwnd_event_tx_start_reject = false;
+
SEC("struct_ops")
void BPF_PROG(bpf_cubic_init, struct sock *sk)
{
struct bpf_bictcp *ca = inet_csk_ca(sk);
+ int true_val = 1, ret;
+
+ ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
+ if (ret == -EOPNOTSUPP)
+ nodelay_init_reject = true;
bictcp_reset(ca);
@@ -189,8 +198,13 @@ void BPF_PROG(bpf_cubic_cwnd_event_tx_start, struct sock *sk)
{
struct bpf_bictcp *ca = inet_csk_ca(sk);
__u32 now = tcp_jiffies32;
+ int true_val = 1, ret;
__s32 delta;
+ ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
+ if (ret == -EOPNOTSUPP)
+ nodelay_cwnd_event_tx_start_reject = true;
+
delta = now - tcp_sk(sk)->lsndtime;
/* We were application limited (idle) for a while.
diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c
index 051e2b6..ac44d60 100644
--- a/tools/testing/selftests/bpf/progs/exceptions_fail.c
+++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c
@@ -208,6 +208,28 @@ int reject_with_reference(void *ctx)
return 0;
}
+__noinline int global_subprog_may_throw(struct __sk_buff *ctx)
+{
+ if (ctx->len)
+ bpf_throw(0);
+ return 0;
+}
+
+SEC("?tc")
+__failure __msg("Unreleased reference")
+int reject_global_subprog_throw_with_reference(struct __sk_buff *ctx)
+{
+ struct foo *f;
+
+ f = bpf_obj_new(typeof(*f));
+ if (!f)
+ return 0;
+ if (ctx->protocol)
+ global_subprog_may_throw(ctx);
+ bpf_obj_drop(f);
+ return 0;
+}
+
__noinline static int subprog_ref(struct __sk_buff *ctx)
{
struct foo *f;
diff --git a/tools/testing/selftests/bpf/progs/sockopt_sk.c b/tools/testing/selftests/bpf/progs/sockopt_sk.c
index cb990a7..5e0b27e 100644
--- a/tools/testing/selftests/bpf/progs/sockopt_sk.c
+++ b/tools/testing/selftests/bpf/progs/sockopt_sk.c
@@ -149,6 +149,20 @@ int _setsockopt(struct bpf_sockopt *ctx)
if (sk && sk->family == AF_NETLINK)
goto out;
+ if (sk && sk->family == AF_INET && sk->type == SOCK_RAW) {
+ struct bpf_tcp_sock *tp = bpf_tcp_sock(sk);
+
+ if (tp) {
+ char saved_syn[60];
+
+ bpf_getsockopt(sk, SOL_TCP, TCP_SAVED_SYN,
+ &saved_syn, sizeof(saved_syn));
+ goto consumed;
+ }
+
+ goto out;
+ }
+
/* Make sure bpf_get_netns_cookie is callable.
*/
if (bpf_get_netns_cookie(NULL) == 0)
@@ -224,6 +238,8 @@ int _setsockopt(struct bpf_sockopt *ctx)
return 0; /* couldn't get sk storage */
storage->val = optval[0];
+
+consumed:
ctx->optlen = -1; /* BPF has consumed this option, don't call kernel
* setsockopt handler.
*/
diff --git a/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c b/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c
index d487153..ed5a0011 100644
--- a/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c
+++ b/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c
@@ -29,6 +29,10 @@ unsigned int nr_syn = 0;
unsigned int nr_fin = 0;
unsigned int nr_hwtstamp = 0;
+bool nodelay_est_ok = false;
+bool nodelay_hdr_len_reject = false;
+bool nodelay_write_hdr_reject = false;
+
/* Check the header received from the active side */
static int __check_active_hdr_in(struct bpf_sock_ops *skops, bool check_syn)
{
@@ -300,7 +304,7 @@ static int handle_passive_estab(struct bpf_sock_ops *skops)
SEC("sockops")
int misc_estab(struct bpf_sock_ops *skops)
{
- int true_val = 1;
+ int true_val = 1, false_val = 0, ret;
switch (skops->op) {
case BPF_SOCK_OPS_TCP_LISTEN_CB:
@@ -316,10 +320,19 @@ int misc_estab(struct bpf_sock_ops *skops)
case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:
return handle_parse_hdr(skops);
case BPF_SOCK_OPS_HDR_OPT_LEN_CB:
+ ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
+ if (ret == -EOPNOTSUPP)
+ nodelay_hdr_len_reject = true;
return handle_hdr_opt_len(skops);
case BPF_SOCK_OPS_WRITE_HDR_OPT_CB:
+ ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val));
+ if (ret == -EOPNOTSUPP)
+ nodelay_write_hdr_reject = true;
return handle_write_hdr_opt(skops);
case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
+ ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &false_val, sizeof(false_val));
+ if (!ret)
+ nodelay_est_ok = true;
return handle_passive_estab(skops);
}
diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c b/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c
index 83df491..facafea 100644
--- a/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c
+++ b/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c
@@ -17,6 +17,13 @@ struct {
__type(value, int);
} sock_map SEC(".maps");
+struct {
+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
+ __uint(max_entries, 2);
+ __type(key, int);
+ __type(value, int);
+} sock_map_verdict SEC(".maps");
+
SEC("sk_msg")
int prog_sk_policy(struct sk_msg_md *msg)
{
@@ -38,3 +45,17 @@ int prog_sk_policy_redir(struct sk_msg_md *msg)
bpf_msg_apply_bytes(msg, apply_bytes);
return bpf_msg_redirect_map(msg, &sock_map, two, 0);
}
+
+/*
+ * Verdict program for the reverse-order TLS/sockmap regression test.
+ * Returns SK_PASS so tcp_read_skb() drains the receive queue via
+ * sk_psock_verdict_recv() without calling tcp_eat_skb(), which is
+ * the precondition for the KTLS strparser frag_list UAF.
+ */
+SEC("sk_skb/verdict")
+int prog_skb_verdict_pass(struct __sk_buff *skb)
+{
+ return SK_PASS;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c
new file mode 100644
index 0000000..7998df07
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+int call_happened = 0;
+
+/*
+ * 32765 is the exact minimum number of padding instructions needed to
+ * trigger the verifier failure, because:
+ * 1. Counting the wrapper instructions around the padding block (one
+ * "r0=0" and two "exit" instructions), the actual jump distance
+ * evaluates to N + 3.
+ * 2. To overflow the s16 max bound (32767), we need N + 3 > 32767.
+ * Thus, N = 32765 is the exact minimum padding size required.
+ */
+static __attribute__((noinline)) void padding_subprog(void)
+{
+ asm volatile (
+ "r0 = 0;"
+ ".rept 32765;"
+ "r0 += 0;"
+ ".endr;"
+ ::: __clobber_all);
+}
+
+static __attribute__((noinline)) int target_subprog(void)
+{
+ /* Use volatile variable here to prevent optimization. */
+ volatile int magic_ret = 3;
+ return magic_ret;
+}
+
+SEC("syscall")
+__success __retval(3)
+int call_large_imm_test(void *ctx)
+{
+ /*
+ * Landing pad to handle call error on kernel without the fix,
+ * preventing kernel panic.
+ */
+ asm volatile (
+ "r0 = 0;"
+ ".rept 32768;"
+ "r0 += 0;"
+ ".endr;"
+ ::: __clobber_all);
+
+ /*
+ * The call_happened variable is 1 only when the call insn wrongly
+ * go back to the landing pad above.
+ */
+ if (call_happened == 1) {
+ /* Use volatile variable here to prevent optimization. */
+ volatile int flag = -1;
+ return flag;
+ }
+
+ call_happened = 1;
+
+ padding_subprog();
+
+ return target_subprog();
+}
+
+char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/cgroup/lib/cgroup_util.c b/tools/testing/selftests/cgroup/lib/cgroup_util.c
index 6a72953..42f5493 100644
--- a/tools/testing/selftests/cgroup/lib/cgroup_util.c
+++ b/tools/testing/selftests/cgroup/lib/cgroup_util.c
@@ -106,8 +106,9 @@ int cg_read_strcmp(const char *cgroup, const char *control,
/* Handle the case of comparing against empty string */
if (!expected)
return -1;
- else
- size = strlen(expected) + 1;
+
+ /* needs size > 1, otherwise cg_read() reads 0 bytes */
+ size = (expected[0] == '\0') ? 2 : strlen(expected) + 1;
buf = malloc(size);
if (!buf)
diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
index a56f415..683b050 100755
--- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh
+++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
@@ -492,6 +492,16 @@
" C1-5:P1 . C1-4:P1 C2-3 . . \
. . . P1 . . p1:5|c11:1-4|c12:5 \
p1:P1|c11:P1|c12:P-1"
+ # Narrowing cpuset.cpus to previously sibling-excluded CPUs should
+ # not return CPUs that were never actually owned.
+ " C1-4:P1 . C1-2:P1 C1-3:P2 . . \
+ . . . C3 . . p1:4|c11:1-2|c12:3 \
+ p1:P1|c11:P1|c12:P2 3"
+ # Expanding cpuset.cpus to include a previously sibling-excluded CPU
+ # after the sibling has become a member should correctly request it.
+ " C1-4:P1 . C1-2:P1 C1-3:P2 . . \
+ . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \
+ p1:P1|c11:P0|c12:P2 2-3"
)
#
diff --git a/tools/testing/selftests/cgroup/test_cpuset_v1_base.sh b/tools/testing/selftests/cgroup/test_cpuset_v1_base.sh
index 42a6628..1c04447 100755
--- a/tools/testing/selftests/cgroup/test_cpuset_v1_base.sh
+++ b/tools/testing/selftests/cgroup/test_cpuset_v1_base.sh
@@ -18,7 +18,7 @@
echo "testing $interface $value"
echo $value > $dir/$interface
new=$(cat $dir/$interface)
- [[ $value -ne $(cat $dir/$interface) ]] && {
+ [[ "$value" != "$new" ]] && {
echo "$interface write $value failed: new:$new"
exit 1
}
diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c
index eeabd34..12f5992 100644
--- a/tools/testing/selftests/cgroup/test_kmem.c
+++ b/tools/testing/selftests/cgroup/test_kmem.c
@@ -368,11 +368,15 @@ static int test_percpu_basic(const char *root)
for (i = 0; i < 1000; i++) {
child = cg_name_indexed(parent, "child", i);
- if (!child)
- return -1;
-
- if (cg_create(child))
+ if (!child) {
+ ret = -1;
goto cleanup_children;
+ }
+
+ if (cg_create(child)) {
+ free(child);
+ goto cleanup_children;
+ }
free(child);
}
diff --git a/tools/testing/selftests/drivers/net/README.rst b/tools/testing/selftests/drivers/net/README.rst
index c858843..c6bed9a 100644
--- a/tools/testing/selftests/drivers/net/README.rst
+++ b/tools/testing/selftests/drivers/net/README.rst
@@ -211,8 +211,8 @@
Test files should be relatively self contained. The libraries should
only include very core or non-trivial code.
-It may be tempting to "factor out" the common code, but fight that urge.
-Library code increases the barrier of entry, and complexity in general.
+It may be tempting to "factor out" the common code to lib/py/, but fight that
+urge. Library code increases the barrier of entry, and complexity in general.
Avoid mixing test code and boilerplate
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -290,6 +290,12 @@
def test(cfg, mode, protocol):
pass
+Linters
+~~~~~~~
+
+We expect clean ``ruff check`` and ``pylint --disable=R``.
+The code should be clean, avoid disabling pylint warnings explicitly!
+
Running tests CI-style
======================
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index 85ca4d1..82809d5 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -31,6 +31,7 @@
hw_stats_l3.sh \
hw_stats_l3_gre.sh \
iou-zcrx.py \
+ ipsec_vxlan.py \
irq.py \
loopback.sh \
nic_timestamp.py \
diff --git a/tools/testing/selftests/drivers/net/hw/config b/tools/testing/selftests/drivers/net/hw/config
index dd50cb8..8c132ac 100644
--- a/tools/testing/selftests/drivers/net/hw/config
+++ b/tools/testing/selftests/drivers/net/hw/config
@@ -3,6 +3,10 @@
CONFIG_FAULT_INJECTION=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y
CONFIG_FUNCTION_ERROR_INJECTION=y
+CONFIG_INET6_ESP=y
+CONFIG_INET6_ESP_OFFLOAD=y
+CONFIG_INET_ESP=y
+CONFIG_INET_ESP_OFFLOAD=y
CONFIG_IO_URING=y
CONFIG_IPV6=y
CONFIG_IPV6_GRE=y
@@ -14,3 +18,4 @@
CONFIG_NET_SCH_INGRESS=y
CONFIG_UDMABUF=y
CONFIG_VXLAN=y
+CONFIG_XFRM_USER=y
diff --git a/tools/testing/selftests/drivers/net/hw/ipsec_vxlan.py b/tools/testing/selftests/drivers/net/hw/ipsec_vxlan.py
new file mode 100755
index 0000000..0740a4d
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/ipsec_vxlan.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+"""Traffic test for VXLAN + IPsec crypto-offload."""
+
+import os
+
+from lib.py import ksft_run, ksft_exit, ksft_eq, ksft_ge
+from lib.py import ksft_variants, KsftNamedVariant, KsftSkipEx
+from lib.py import CmdExitFailure, NetDrvEpEnv, cmd, defer, ethtool, ip
+from lib.py import Iperf3Runner
+
+# Inner tunnel addresses - TEST-NET-2 (RFC 5737) / doc prefix (RFC 3849)
+INNER_V4_LOCAL = "198.51.100.1"
+INNER_V4_REMOTE = "198.51.100.2"
+INNER_V6_LOCAL = "2001:db8:100::1"
+INNER_V6_REMOTE = "2001:db8:100::2"
+
+# ESP parameters
+SPI_OUT = "0x1000"
+SPI_IN = "0x1001"
+# 128-bit key + 32-bit salt = 20 bytes hex, 128-bit ICV
+ESP_AEAD = "aead 'rfc4106(gcm(aes))' 0x" + "01" * 20 + " 128"
+
+
+def xfrm(args, host=None):
+ """Runs 'ip xfrm' via shell to preserve parentheses in algo names."""
+ cmd(f"ip xfrm {args}", shell=True, host=host)
+
+
+def check_xfrm_offload_support():
+ """Skips if iproute2 lacks xfrm offload support."""
+ out = cmd("ip xfrm state help", fail=False)
+ if "offload" not in out.stdout + out.stderr:
+ raise KsftSkipEx("iproute2 too old, missing xfrm offload")
+
+
+def check_esp_hw_offload(cfg):
+ """Skips if device lacks esp-hw-offload support."""
+ check_xfrm_offload_support()
+ try:
+ feat = ethtool(f"-k {cfg.ifname}", json=True)[0]
+ except (CmdExitFailure, IndexError) as e:
+ raise KsftSkipEx(f"can't query features: {e}") from e
+ if not feat.get("esp-hw-offload", {}).get("active"):
+ raise KsftSkipEx("Device does not support esp-hw-offload")
+
+
+def get_tx_drops(cfg):
+ """Returns TX dropped counter from the physical device."""
+ stats = ip("-s -s link show dev " + cfg.ifname, json=True)[0]
+ return stats["stats64"]["tx"]["dropped"]
+
+
+def setup_vxlan_ipsec(cfg, outer_ipver, inner_ipver):
+ """Sets up VXLAN tunnel with IPsec transport-mode crypto-offload."""
+ vxlan_name = f"vx{os.getpid()}"
+ local_addr = cfg.addr_v[outer_ipver]
+ remote_addr = cfg.remote_addr_v[outer_ipver]
+
+ if inner_ipver == "4":
+ inner_local = f"{INNER_V4_LOCAL}/24"
+ inner_remote = f"{INNER_V4_REMOTE}/24"
+ addr_extra = ""
+ else:
+ inner_local = f"{INNER_V6_LOCAL}/64"
+ inner_remote = f"{INNER_V6_REMOTE}/64"
+ addr_extra = " nodad"
+
+ if outer_ipver == "6":
+ vxlan_opts = "udp6zerocsumtx udp6zerocsumrx"
+ else:
+ vxlan_opts = "noudpcsum"
+
+ # VXLAN tunnel - local side
+ ip(f"link add {vxlan_name} type vxlan id 100 dstport 4789 {vxlan_opts} "
+ f"local {local_addr} remote {remote_addr} dev {cfg.ifname}")
+ defer(ip, f"link del {vxlan_name}")
+ ip(f"addr add {inner_local} dev {vxlan_name}{addr_extra}")
+ ip(f"link set {vxlan_name} up")
+
+ # VXLAN tunnel - remote side
+ ip(f"link add {vxlan_name} type vxlan id 100 dstport 4789 {vxlan_opts} "
+ f"local {remote_addr} remote {local_addr} dev {cfg.remote_ifname}",
+ host=cfg.remote)
+ defer(ip, f"link del {vxlan_name}", host=cfg.remote)
+ ip(f"addr add {inner_remote} dev {vxlan_name}{addr_extra}",
+ host=cfg.remote)
+ ip(f"link set {vxlan_name} up", host=cfg.remote)
+
+ # xfrm state - local outbound SA
+ xfrm(f"state add src {local_addr} dst {remote_addr} "
+ f"proto esp spi {SPI_OUT} "
+ f"{ESP_AEAD} "
+ f"mode transport offload crypto dev {cfg.ifname} dir out")
+ defer(xfrm, f"state del src {local_addr} dst {remote_addr} "
+ f"proto esp spi {SPI_OUT}")
+
+ # xfrm state - local inbound SA
+ xfrm(f"state add src {remote_addr} dst {local_addr} "
+ f"proto esp spi {SPI_IN} "
+ f"{ESP_AEAD} "
+ f"mode transport offload crypto dev {cfg.ifname} dir in")
+ defer(xfrm, f"state del src {remote_addr} dst {local_addr} "
+ f"proto esp spi {SPI_IN}")
+
+ # xfrm state - remote outbound SA (mirror, software crypto)
+ xfrm(f"state add src {remote_addr} dst {local_addr} "
+ f"proto esp spi {SPI_IN} "
+ f"{ESP_AEAD} "
+ f"mode transport",
+ host=cfg.remote)
+ defer(xfrm, f"state del src {remote_addr} dst {local_addr} "
+ f"proto esp spi {SPI_IN}", host=cfg.remote)
+
+ # xfrm state - remote inbound SA (mirror, software crypto)
+ xfrm(f"state add src {local_addr} dst {remote_addr} "
+ f"proto esp spi {SPI_OUT} "
+ f"{ESP_AEAD} "
+ f"mode transport",
+ host=cfg.remote)
+ defer(xfrm, f"state del src {local_addr} dst {remote_addr} "
+ f"proto esp spi {SPI_OUT}", host=cfg.remote)
+
+ # xfrm policy - local out
+ xfrm(f"policy add src {local_addr} dst {remote_addr} "
+ f"proto udp dport 4789 dir out "
+ f"tmpl src {local_addr} dst {remote_addr} proto esp mode transport")
+ defer(xfrm, f"policy del src {local_addr} dst {remote_addr} "
+ f"proto udp dport 4789 dir out")
+
+ # xfrm policy - local in
+ xfrm(f"policy add src {remote_addr} dst {local_addr} "
+ f"proto udp dport 4789 dir in "
+ f"tmpl src {remote_addr} dst {local_addr} proto esp mode transport")
+ defer(xfrm, f"policy del src {remote_addr} dst {local_addr} "
+ f"proto udp dport 4789 dir in")
+
+ # xfrm policy - remote out
+ xfrm(f"policy add src {remote_addr} dst {local_addr} "
+ f"proto udp dport 4789 dir out "
+ f"tmpl src {remote_addr} dst {local_addr} proto esp mode transport",
+ host=cfg.remote)
+ defer(xfrm, f"policy del src {remote_addr} dst {local_addr} "
+ f"proto udp dport 4789 dir out", host=cfg.remote)
+
+ # xfrm policy - remote in
+ xfrm(f"policy add src {local_addr} dst {remote_addr} "
+ f"proto udp dport 4789 dir in "
+ f"tmpl src {local_addr} dst {remote_addr} proto esp mode transport",
+ host=cfg.remote)
+ defer(xfrm, f"policy del src {local_addr} dst {remote_addr} "
+ f"proto udp dport 4789 dir in", host=cfg.remote)
+
+
+def _vxlan_ipsec_variants():
+ """Generates outer/inner IP version variants."""
+ for outer in ["4", "6"]:
+ for inner in ["4", "6"]:
+ yield KsftNamedVariant(f"outer_v{outer}_inner_v{inner}", outer, inner)
+
+
+@ksft_variants(_vxlan_ipsec_variants())
+def test_vxlan_ipsec_crypto_offload(cfg, outer_ipver, inner_ipver):
+ """Tests VXLAN+IPsec crypto-offload has no TX drops."""
+ cfg.require_ipver(outer_ipver)
+ check_esp_hw_offload(cfg)
+
+ setup_vxlan_ipsec(cfg, outer_ipver, inner_ipver)
+
+ if inner_ipver == "4":
+ inner_local = INNER_V4_LOCAL
+ inner_remote = INNER_V4_REMOTE
+ ping = "ping"
+ else:
+ inner_local = INNER_V6_LOCAL
+ inner_remote = INNER_V6_REMOTE
+ ping = "ping -6"
+
+ cmd(f"{ping} -c 1 -W 2 {inner_remote}")
+
+ drops_before = get_tx_drops(cfg)
+
+ runner = Iperf3Runner(cfg, server_ip=inner_local,
+ client_ip=inner_remote)
+ bw_gbps = runner.measure_bandwidth(reverse=True)
+
+ cfg.wait_hw_stats_settle()
+ drops_after = get_tx_drops(cfg)
+
+ ksft_eq(drops_after - drops_before, 0,
+ comment="TX drops during VXLAN+IPsec")
+ ksft_ge(bw_gbps, 0.1,
+ comment="Minimum 100Mbps over VXLAN+IPsec")
+
+
+def main():
+ """Runs VXLAN+IPsec crypto-offload GSO selftest."""
+ with NetDrvEpEnv(__file__, nsim_test=False) as cfg:
+ ksft_run([test_vxlan_ipsec_crypto_offload], args=(cfg,))
+ ksft_exit()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/testing/selftests/drivers/net/lib/py/load.py b/tools/testing/selftests/drivers/net/lib/py/load.py
index f181fa2..e24660e 100644
--- a/tools/testing/selftests/drivers/net/lib/py/load.py
+++ b/tools/testing/selftests/drivers/net/lib/py/load.py
@@ -48,7 +48,10 @@
Starts the iperf3 client with the configured options.
"""
cmdline = self._build_client(streams, duration, reverse)
- return cmd(cmdline, background=background, host=self.env.remote)
+ kwargs = {"background": background, "host": self.env.remote}
+ if not background:
+ kwargs["timeout"] = duration + 5
+ return cmd(cmdline, **kwargs)
def measure_bandwidth(self, reverse=False):
"""
diff --git a/tools/testing/selftests/drivers/net/shaper.py b/tools/testing/selftests/drivers/net/shaper.py
index 11310f1..e39d270 100755
--- a/tools/testing/selftests/drivers/net/shaper.py
+++ b/tools/testing/selftests/drivers/net/shaper.py
@@ -1,7 +1,10 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
-from lib.py import ksft_run, ksft_exit, ksft_eq, ksft_true, KsftSkipEx
+import errno
+
+from lib.py import ksft_run, ksft_exit
+from lib.py import ksft_eq, ksft_raises, ksft_true, KsftSkipEx
from lib.py import EthtoolFamily, NetshaperFamily
from lib.py import NetDrvEnv
from lib.py import NlError
@@ -438,6 +441,21 @@
nl_shaper.delete({'ifindex': cfg.ifindex,
'handle': {'scope': 'queue', 'id': i}})
+def dup_leaves(cfg, nl_shaper) -> None:
+ """ Ensure that the kernel rejects duplicate leaves. """
+ if not cfg.groups:
+ raise KsftSkipEx("device does not support node scope")
+
+ with ksft_raises(NlError) as cm:
+ nl_shaper.group({
+ 'ifindex': cfg.ifindex,
+ 'leaves':[{'handle': {'scope': 'queue', 'id': 0}},
+ {'handle': {'scope': 'queue', 'id': 0}}],
+ 'handle': {'scope':'node'},
+ 'metric': 'bps',
+ 'bw-max': 10000})
+ ksft_eq(cm.exception.error, errno.EINVAL)
+
def main() -> None:
with NetDrvEnv(__file__, queue_count=4) as cfg:
cfg.queues = False
@@ -453,7 +471,9 @@
basic_groups,
qgroups,
delegation,
- queue_update], args=(cfg, NetshaperFamily()))
+ dup_leaves,
+ queue_update],
+ args=(cfg, NetshaperFamily()))
ksft_exit()
diff --git a/tools/testing/selftests/futex/functional/robust_list.c b/tools/testing/selftests/futex/functional/robust_list.c
index e7d1254..b3fab60 100644
--- a/tools/testing/selftests/futex/functional/robust_list.c
+++ b/tools/testing/selftests/futex/functional/robust_list.c
@@ -27,12 +27,15 @@
#include "futextest.h"
#include "../../kselftest_harness.h"
+#include <dlfcn.h>
#include <errno.h>
#include <pthread.h>
#include <signal.h>
+#include <stdint.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stddef.h>
+#include <sys/auxv.h>
#include <sys/mman.h>
#include <sys/wait.h>
@@ -42,6 +45,10 @@
#define SLEEP_US 100
+#if __SIZEOF_LONG__ == 8
+# define BUILD_64
+#endif
+
static pthread_barrier_t barrier, barrier2;
static int set_robust_list(struct robust_list_head *head, size_t len)
@@ -54,6 +61,12 @@ static int get_robust_list(int pid, struct robust_list_head **head, size_t *len_
return syscall(SYS_get_robust_list, pid, head, len_ptr);
}
+static int sys_futex_robust_unlock(_Atomic(uint32_t) *uaddr, unsigned int op, int val,
+ void *list_op_pending, unsigned int val3)
+{
+ return syscall(SYS_futex, uaddr, op, val, NULL, list_op_pending, val3, 0);
+}
+
/*
* Basic lock struct, contains just the futex word and the robust list element
* Real implementations have also a *prev to easily walk in the list
@@ -549,4 +562,230 @@ TEST(test_circular_list)
ksft_test_result_pass("%s\n", __func__);
}
+/*
+ * Below are tests for the fix of robust release race condition. Please read the following
+ * thread to learn more about the issue in the first place and why the following functions fix it:
+ * https://lore.kernel.org/lkml/20260316162316.356674433@kernel.org/
+ */
+
+/*
+ * Auxiliary code for binding the vDSO functions
+ */
+static void *get_vdso_func_addr(const char *function)
+{
+ const char *vdso_names[] = {
+ "linux-vdso.so.1", "linux-gate.so.1", "linux-vdso32.so.1", "linux-vdso64.so.1",
+ };
+
+ for (int i = 0; i < ARRAY_SIZE(vdso_names); i++) {
+ void *vdso = dlopen(vdso_names[i], RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
+
+ if (vdso)
+ return dlsym(vdso, function);
+ }
+ return NULL;
+}
+
+/*
+ * These are the real vDSO function signatures:
+ *
+ * __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop)
+ * __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop)
+ *
+ * So for the generic entry point we need to use a void pointer as the last argument
+ */
+FIXTURE(vdso_unlock)
+{
+ uint32_t (*vdso)(_Atomic(uint32_t) *lock, uint32_t tid, void *pop);
+};
+
+FIXTURE_VARIANT(vdso_unlock)
+{
+ bool is_32;
+ char func_name[];
+};
+
+FIXTURE_SETUP(vdso_unlock)
+{
+ self->vdso = get_vdso_func_addr(variant->func_name);
+}
+
+FIXTURE_TEARDOWN(vdso_unlock) {}
+
+FIXTURE_VARIANT_ADD(vdso_unlock, 32)
+{
+ .func_name = "__vdso_futex_robust_list32_try_unlock",
+ .is_32 = true,
+};
+
+FIXTURE_VARIANT_ADD(vdso_unlock, 64)
+{
+ .func_name = "__vdso_futex_robust_list64_try_unlock",
+ .is_32 = false,
+};
+
+/*
+ * Test the vDSO robust_listXX_try_unlock() for the uncontended case. The virtual syscall should
+ * return the thread ID of the lock owner, the lock word must be 0 and the list_op_pending should
+ * be NULL.
+ */
+TEST_F(vdso_unlock, test_robust_try_unlock_uncontended)
+{
+ struct lock_struct lock = { .futex = 0 };
+ _Atomic(unsigned int) *futex = &lock.futex;
+ struct robust_list_head head;
+ uintptr_t exp = (uintptr_t) NULL;
+ pid_t tid = gettid();
+ int ret;
+
+ if (!self->vdso) {
+ ksft_test_result_skip("%s not found\n", variant->func_name);
+ return;
+ }
+
+ *futex = tid;
+
+ ret = set_list(&head);
+ if (ret)
+ ksft_test_result_fail("set_robust_list error\n");
+
+ head.list_op_pending = &lock.list;
+
+ ret = self->vdso(futex, tid, &head.list_op_pending);
+
+ ASSERT_EQ(ret, tid);
+ ASSERT_EQ(*futex, 0);
+
+ /* Check only the lower 32 bits for the 32-bit entry point */
+ if (variant->is_32) {
+ exp = (uintptr_t)(unsigned long)&lock.list;
+ exp &= ~0xFFFFFFFFULL;
+ }
+
+ ASSERT_EQ((uintptr_t)(unsigned long)head.list_op_pending, exp);
+}
+
+/*
+ * If the lock is contended, the operation fails. The return value is the value found at the
+ * futex word (tid | FUTEX_WAITERS), the futex word is not modified and the list_op_pending is_32
+ * not cleared.
+ */
+TEST_F(vdso_unlock, test_robust_try_unlock_contended)
+{
+ struct lock_struct lock = { .futex = 0 };
+ _Atomic(unsigned int) *futex = &lock.futex;
+ struct robust_list_head head;
+ pid_t tid = gettid();
+ int ret;
+
+ if (!self->vdso) {
+ ksft_test_result_skip("%s not found\n", variant->func_name);
+ return;
+ }
+
+ *futex = tid | FUTEX_WAITERS;
+
+ ret = set_list(&head);
+ if (ret)
+ ksft_test_result_fail("set_robust_list error\n");
+
+ head.list_op_pending = &lock.list;
+
+ ret = self->vdso(futex, tid, &head.list_op_pending);
+
+ ASSERT_EQ(ret, tid | FUTEX_WAITERS);
+ ASSERT_EQ(*futex, tid | FUTEX_WAITERS);
+ ASSERT_EQ(head.list_op_pending, &lock.list);
+}
+
+FIXTURE(futex_op) {};
+
+FIXTURE_VARIANT(futex_op)
+{
+ unsigned int op;
+ unsigned int val3;
+};
+
+FIXTURE_SETUP(futex_op) {}
+
+FIXTURE_TEARDOWN(futex_op) {}
+
+FIXTURE_VARIANT_ADD(futex_op, wake)
+{
+ .op = FUTEX_WAKE,
+ .val3 = 0,
+};
+
+FIXTURE_VARIANT_ADD(futex_op, wake_bitset)
+{
+ .op = FUTEX_WAKE_BITSET,
+ .val3 = FUTEX_BITSET_MATCH_ANY,
+};
+
+FIXTURE_VARIANT_ADD(futex_op, unlock_pi)
+{
+ .op = FUTEX_UNLOCK_PI,
+ .val3 = 0,
+};
+
+FIXTURE_VARIANT_ADD(futex_op, wake32)
+{
+ .op = FUTEX_WAKE | FUTEX_ROBUST_LIST32,
+ .val3 = 0,
+};
+
+FIXTURE_VARIANT_ADD(futex_op, wake_bitset32)
+{
+ .op = FUTEX_WAKE_BITSET | FUTEX_ROBUST_LIST32,
+ .val3 = FUTEX_BITSET_MATCH_ANY,
+};
+
+FIXTURE_VARIANT_ADD(futex_op, unlock_pi32)
+{
+ .op = FUTEX_UNLOCK_PI | FUTEX_ROBUST_LIST32,
+ .val3 = 0,
+};
+
+/*
+ * The syscall should return the number of tasks waken (for this test, 0), clear the futex word and
+ * clear list_op_pending
+ */
+TEST_F(futex_op, test_futex_robust_unlock)
+{
+ struct lock_struct lock = { .futex = 0 };
+ _Atomic(unsigned int) *futex = &lock.futex;
+ uintptr_t exp = (uintptr_t) NULL;
+ struct robust_list_head head;
+ pid_t tid = gettid();
+ int ret;
+
+#ifndef BUILD_64
+ if (!(variant->op & FUTEX_ROBUST_LIST32)) {
+ ksft_test_result_skip("Not supported for 32 bit build\n");
+ return;
+ }
+#endif
+
+ *futex = tid | FUTEX_WAITERS;
+
+ ret = set_list(&head);
+ if (ret)
+ ksft_test_result_fail("set_robust_list error\n");
+
+ head.list_op_pending = &lock.list;
+
+ ret = sys_futex_robust_unlock(futex, FUTEX_ROBUST_UNLOCK | variant->op, tid,
+ &head.list_op_pending, variant->val3);
+
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(*futex, 0);
+
+ if (variant->op & FUTEX_ROBUST_LIST32) {
+ exp = (uint64_t)(unsigned long)&lock.list;
+ exp &= ~0xFFFFFFFFULL;
+ }
+
+ ASSERT_EQ((uintptr_t)(unsigned long)head.list_op_pending, exp);
+}
+
TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/include/futextest.h b/tools/testing/selftests/futex/include/futextest.h
index 3d48e97..df33f31 100644
--- a/tools/testing/selftests/futex/include/futextest.h
+++ b/tools/testing/selftests/futex/include/futextest.h
@@ -38,6 +38,12 @@ typedef volatile u_int32_t futex_t;
#ifndef FUTEX_CMP_REQUEUE_PI
#define FUTEX_CMP_REQUEUE_PI 12
#endif
+#ifndef FUTEX_ROBUST_UNLOCK
+#define FUTEX_ROBUST_UNLOCK 512
+#endif
+#ifndef FUTEX_ROBUST_LIST32
+#define FUTEX_ROBUST_LIST32 1024
+#endif
#ifndef FUTEX_WAIT_REQUEUE_PI_PRIVATE
#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \
FUTEX_PRIVATE_FLAG)
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h
index 6d809f0..60838b6 100644
--- a/tools/testing/selftests/kselftest.h
+++ b/tools/testing/selftests/kselftest.h
@@ -450,7 +450,7 @@ static inline __noreturn __printf(1, 2) void ksft_exit_skip(const char *msg, ...
*/
if (ksft_plan || ksft_test_num()) {
ksft_cnt.ksft_xskip++;
- printf("ok %u # SKIP ", 1 + ksft_test_num());
+ printf("ok %u # SKIP ", ksft_test_num());
} else {
printf("1..0 # SKIP ");
}
diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h
index 75fb016..cfdce9c 100644
--- a/tools/testing/selftests/kselftest_harness.h
+++ b/tools/testing/selftests/kselftest_harness.h
@@ -76,7 +76,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n)
memset(s, c, n);
}
-#define KSELFTEST_PRIO_TEST_F 20000
+#define KSELFTEST_PRIO_TEST 20000
#define KSELFTEST_PRIO_XFAIL 20001
#define TEST_TIMEOUT_DEFAULT 30
@@ -194,7 +194,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n)
.fixture = &_fixture_global, \
.termsig = _signal, \
.timeout = TEST_TIMEOUT_DEFAULT, }; \
- static void __attribute__((constructor)) _register_##test_name(void) \
+ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) _register_##test_name(void) \
{ \
__register_test(&_##test_name##_object); \
} \
@@ -238,7 +238,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n)
FIXTURE_VARIANT(fixture_name); \
static struct __fixture_metadata _##fixture_name##_fixture_object = \
{ .name = #fixture_name, }; \
- static void __attribute__((constructor)) \
+ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \
_register_##fixture_name##_data(void) \
{ \
__register_fixture(&_##fixture_name##_fixture_object); \
@@ -364,7 +364,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n)
_##fixture_name##_##variant_name##_object = \
{ .name = #variant_name, \
.data = &_##fixture_name##_##variant_name##_variant}; \
- static void __attribute__((constructor)) \
+ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \
_register_##fixture_name##_##variant_name(void) \
{ \
__register_fixture_variant(&_##fixture_name##_fixture_object, \
@@ -468,7 +468,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n)
fixture_name##_teardown(_metadata, self, variant); \
} \
static struct __test_metadata *_##fixture_name##_##test_name##_object; \
- static void __attribute__((constructor(KSELFTEST_PRIO_TEST_F))) \
+ static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \
_register_##fixture_name##_##test_name(void) \
{ \
struct __test_metadata *object = mmap(NULL, sizeof(*object), \
@@ -1323,7 +1323,7 @@ static int test_harness_run(int argc, char **argv)
return KSFT_FAIL;
}
-static void __attribute__((constructor)) __constructor_order_first(void)
+static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) __constructor_order_first(void)
{
__constructor_order_forward = true;
}
diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c
index b058f27..4415c94 100644
--- a/tools/testing/selftests/kvm/access_tracking_perf_test.c
+++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c
@@ -41,10 +41,10 @@
#include <inttypes.h>
#include <limits.h>
#include <pthread.h>
-#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include "kvm_syscalls.h"
#include "kvm_util.h"
#include "test_util.h"
#include "memstress.h"
@@ -101,15 +101,15 @@ struct test_params {
enum vm_mem_backing_src_type backing_src;
/* The amount of memory to allocate for each vCPU. */
- uint64_t vcpu_memory_bytes;
+ u64 vcpu_memory_bytes;
/* The number of vCPUs to create in the VM. */
int nr_vcpus;
};
-static uint64_t pread_uint64(int fd, const char *filename, uint64_t index)
+static u64 pread_u64(int fd, const char *filename, u64 index)
{
- uint64_t value;
+ u64 value;
off_t offset = index * sizeof(value);
TEST_ASSERT(pread(fd, &value, sizeof(value), offset) == sizeof(value),
@@ -123,13 +123,13 @@ static uint64_t pread_uint64(int fd, const char *filename, uint64_t index)
#define PAGEMAP_PRESENT (1ULL << 63)
#define PAGEMAP_PFN_MASK ((1ULL << 55) - 1)
-static uint64_t lookup_pfn(int pagemap_fd, struct kvm_vm *vm, uint64_t gva)
+static u64 lookup_pfn(int pagemap_fd, struct kvm_vm *vm, gva_t gva)
{
- uint64_t hva = (uint64_t) addr_gva2hva(vm, gva);
- uint64_t entry;
- uint64_t pfn;
+ u64 hva = (u64)addr_gva2hva(vm, gva);
+ u64 entry;
+ u64 pfn;
- entry = pread_uint64(pagemap_fd, "pagemap", hva / getpagesize());
+ entry = pread_u64(pagemap_fd, "pagemap", hva / getpagesize());
if (!(entry & PAGEMAP_PRESENT))
return 0;
@@ -139,16 +139,16 @@ static uint64_t lookup_pfn(int pagemap_fd, struct kvm_vm *vm, uint64_t gva)
return pfn;
}
-static bool is_page_idle(int page_idle_fd, uint64_t pfn)
+static bool is_page_idle(int page_idle_fd, u64 pfn)
{
- uint64_t bits = pread_uint64(page_idle_fd, "page_idle", pfn / 64);
+ u64 bits = pread_u64(page_idle_fd, "page_idle", pfn / 64);
return !!((bits >> (pfn % 64)) & 1);
}
-static void mark_page_idle(int page_idle_fd, uint64_t pfn)
+static void mark_page_idle(int page_idle_fd, u64 pfn)
{
- uint64_t bits = 1ULL << (pfn % 64);
+ u64 bits = 1ULL << (pfn % 64);
TEST_ASSERT(pwrite(page_idle_fd, &bits, 8, 8 * (pfn / 64)) == 8,
"Set page_idle bits for PFN 0x%" PRIx64, pfn);
@@ -174,11 +174,11 @@ static void pageidle_mark_vcpu_memory_idle(struct kvm_vm *vm,
struct memstress_vcpu_args *vcpu_args)
{
int vcpu_idx = vcpu_args->vcpu_idx;
- uint64_t base_gva = vcpu_args->gva;
- uint64_t pages = vcpu_args->pages;
- uint64_t page;
- uint64_t still_idle = 0;
- uint64_t no_pfn = 0;
+ gva_t base_gva = vcpu_args->gva;
+ u64 pages = vcpu_args->pages;
+ u64 page;
+ u64 still_idle = 0;
+ u64 no_pfn = 0;
int page_idle_fd;
int pagemap_fd;
@@ -193,8 +193,8 @@ static void pageidle_mark_vcpu_memory_idle(struct kvm_vm *vm,
TEST_ASSERT(pagemap_fd > 0, "Failed to open pagemap.");
for (page = 0; page < pages; page++) {
- uint64_t gva = base_gva + page * memstress_args.guest_page_size;
- uint64_t pfn = lookup_pfn(pagemap_fd, vm, gva);
+ gva_t gva = base_gva + page * memstress_args.guest_page_size;
+ u64 pfn = lookup_pfn(pagemap_fd, vm, gva);
if (!pfn) {
no_pfn++;
@@ -297,10 +297,10 @@ static void lru_gen_mark_memory_idle(struct kvm_vm *vm)
lru_gen_last_gen = new_gen;
}
-static void assert_ucall(struct kvm_vcpu *vcpu, uint64_t expected_ucall)
+static void assert_ucall(struct kvm_vcpu *vcpu, u64 expected_ucall)
{
struct ucall uc;
- uint64_t actual_ucall = get_ucall(vcpu, &uc);
+ u64 actual_ucall = get_ucall(vcpu, &uc);
TEST_ASSERT(expected_ucall == actual_ucall,
"Guest exited unexpectedly (expected ucall %" PRIu64
@@ -417,7 +417,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
*/
test_pages = params->nr_vcpus * params->vcpu_memory_bytes /
max(memstress_args.guest_page_size,
- (uint64_t)getpagesize());
+ (u64)getpagesize());
memstress_start_vcpu_threads(nr_vcpus, vcpu_thread_main);
diff --git a/tools/testing/selftests/kvm/arch_timer.c b/tools/testing/selftests/kvm/arch_timer.c
index cf8fb67..90c475a 100644
--- a/tools/testing/selftests/kvm/arch_timer.c
+++ b/tools/testing/selftests/kvm/arch_timer.c
@@ -78,9 +78,9 @@ static void *test_vcpu_run(void *arg)
return NULL;
}
-static uint32_t test_get_pcpu(void)
+static u32 test_get_pcpu(void)
{
- uint32_t pcpu;
+ u32 pcpu;
unsigned int nproc_conf;
cpu_set_t online_cpuset;
@@ -98,7 +98,7 @@ static uint32_t test_get_pcpu(void)
static int test_migrate_vcpu(unsigned int vcpu_idx)
{
int ret;
- uint32_t new_pcpu = test_get_pcpu();
+ u32 new_pcpu = test_get_pcpu();
pr_debug("Migrating vCPU: %u to pCPU: %u\n", vcpu_idx, new_pcpu);
diff --git a/tools/testing/selftests/kvm/arm64/aarch32_id_regs.c b/tools/testing/selftests/kvm/arm64/aarch32_id_regs.c
index 713005b..8a019cb 100644
--- a/tools/testing/selftests/kvm/arm64/aarch32_id_regs.c
+++ b/tools/testing/selftests/kvm/arm64/aarch32_id_regs.c
@@ -66,7 +66,7 @@ static void test_guest_raz(struct kvm_vcpu *vcpu)
}
}
-static uint64_t raz_wi_reg_ids[] = {
+static u64 raz_wi_reg_ids[] = {
KVM_ARM64_SYS_REG(SYS_ID_PFR0_EL1),
KVM_ARM64_SYS_REG(SYS_ID_PFR1_EL1),
KVM_ARM64_SYS_REG(SYS_ID_DFR0_EL1),
@@ -94,8 +94,8 @@ static void test_user_raz_wi(struct kvm_vcpu *vcpu)
int i;
for (i = 0; i < ARRAY_SIZE(raz_wi_reg_ids); i++) {
- uint64_t reg_id = raz_wi_reg_ids[i];
- uint64_t val;
+ u64 reg_id = raz_wi_reg_ids[i];
+ u64 val;
val = vcpu_get_reg(vcpu, reg_id);
TEST_ASSERT_EQ(val, 0);
@@ -111,7 +111,7 @@ static void test_user_raz_wi(struct kvm_vcpu *vcpu)
}
}
-static uint64_t raz_invariant_reg_ids[] = {
+static u64 raz_invariant_reg_ids[] = {
KVM_ARM64_SYS_REG(SYS_ID_AFR0_EL1),
KVM_ARM64_SYS_REG(sys_reg(3, 0, 0, 3, 3)),
KVM_ARM64_SYS_REG(SYS_ID_DFR1_EL1),
@@ -123,8 +123,8 @@ static void test_user_raz_invariant(struct kvm_vcpu *vcpu)
int i, r;
for (i = 0; i < ARRAY_SIZE(raz_invariant_reg_ids); i++) {
- uint64_t reg_id = raz_invariant_reg_ids[i];
- uint64_t val;
+ u64 reg_id = raz_invariant_reg_ids[i];
+ u64 val;
val = vcpu_get_reg(vcpu, reg_id);
TEST_ASSERT_EQ(val, 0);
@@ -142,7 +142,7 @@ static void test_user_raz_invariant(struct kvm_vcpu *vcpu)
static bool vcpu_aarch64_only(struct kvm_vcpu *vcpu)
{
- uint64_t val, el0;
+ u64 val, el0;
val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1));
diff --git a/tools/testing/selftests/kvm/arm64/arch_timer.c b/tools/testing/selftests/kvm/arm64/arch_timer.c
index d592a45..5fa5c0e 100644
--- a/tools/testing/selftests/kvm/arm64/arch_timer.c
+++ b/tools/testing/selftests/kvm/arm64/arch_timer.c
@@ -56,7 +56,7 @@ static void guest_validate_irq(unsigned int intid,
struct test_vcpu_shared_data *shared_data)
{
enum guest_stage stage = shared_data->guest_stage;
- uint64_t xcnt = 0, xcnt_diff_us, cval = 0;
+ u64 xcnt = 0, xcnt_diff_us, cval = 0;
unsigned long xctl = 0;
unsigned int timer_irq = 0;
unsigned int accessor;
@@ -105,7 +105,7 @@ static void guest_validate_irq(unsigned int intid,
static void guest_irq_handler(struct ex_regs *regs)
{
unsigned int intid = gic_get_and_ack_irq();
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
guest_validate_irq(intid, shared_data);
@@ -116,7 +116,7 @@ static void guest_irq_handler(struct ex_regs *regs)
static void guest_run_stage(struct test_vcpu_shared_data *shared_data,
enum guest_stage stage)
{
- uint32_t irq_iter, config_iter;
+ u32 irq_iter, config_iter;
shared_data->guest_stage = stage;
shared_data->nr_iter = 0;
@@ -140,7 +140,7 @@ static void guest_run_stage(struct test_vcpu_shared_data *shared_data,
static void guest_code(void)
{
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
local_irq_disable();
diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
index 993c9e3..f7625eb 100644
--- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
+++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
@@ -23,25 +23,25 @@
#include "vgic.h"
/* Depends on counter width. */
-static uint64_t CVAL_MAX;
+static u64 CVAL_MAX;
/* tval is a signed 32-bit int. */
-static const int32_t TVAL_MAX = INT32_MAX;
-static const int32_t TVAL_MIN = INT32_MIN;
+static const s32 TVAL_MAX = INT32_MAX;
+static const s32 TVAL_MIN = INT32_MIN;
/* After how much time we say there is no IRQ. */
-static const uint32_t TIMEOUT_NO_IRQ_US = 50000;
+static const u32 TIMEOUT_NO_IRQ_US = 50000;
/* Counter value to use as the starting one for most tests. Set to CVAL_MAX/2 */
-static uint64_t DEF_CNT;
+static u64 DEF_CNT;
/* Number of runs. */
-static const uint32_t NR_TEST_ITERS_DEF = 5;
+static const u32 NR_TEST_ITERS_DEF = 5;
/* Default wait test time in ms. */
-static const uint32_t WAIT_TEST_MS = 10;
+static const u32 WAIT_TEST_MS = 10;
/* Default "long" wait test time in ms. */
-static const uint32_t LONG_WAIT_TEST_MS = 100;
+static const u32 LONG_WAIT_TEST_MS = 100;
/* Shared with IRQ handler. */
struct test_vcpu_shared_data {
@@ -53,9 +53,9 @@ struct test_args {
/* Virtual or physical timer and counter tests. */
enum arch_timer timer;
/* Delay used for most timer tests. */
- uint64_t wait_ms;
+ u64 wait_ms;
/* Delay used in the test_long_timer_delays test. */
- uint64_t long_wait_ms;
+ u64 long_wait_ms;
/* Number of iterations. */
int iterations;
/* Whether to test the physical timer. */
@@ -82,12 +82,12 @@ enum sync_cmd {
NO_USERSPACE_CMD,
};
-typedef void (*sleep_method_t)(enum arch_timer timer, uint64_t usec);
+typedef void (*sleep_method_t)(enum arch_timer timer, u64 usec);
-static void sleep_poll(enum arch_timer timer, uint64_t usec);
-static void sleep_sched_poll(enum arch_timer timer, uint64_t usec);
-static void sleep_in_userspace(enum arch_timer timer, uint64_t usec);
-static void sleep_migrate(enum arch_timer timer, uint64_t usec);
+static void sleep_poll(enum arch_timer timer, u64 usec);
+static void sleep_sched_poll(enum arch_timer timer, u64 usec);
+static void sleep_in_userspace(enum arch_timer timer, u64 usec);
+static void sleep_migrate(enum arch_timer timer, u64 usec);
sleep_method_t sleep_method[] = {
sleep_poll,
@@ -115,14 +115,14 @@ enum timer_view {
TIMER_TVAL,
};
-static void assert_irqs_handled(uint32_t n)
+static void assert_irqs_handled(u32 n)
{
int h = atomic_read(&shared_data.handled);
__GUEST_ASSERT(h == n, "Handled %d IRQS but expected %d", h, n);
}
-static void userspace_cmd(uint64_t cmd)
+static void userspace_cmd(u64 cmd)
{
GUEST_SYNC_ARGS(cmd, 0, 0, 0, 0);
}
@@ -132,12 +132,12 @@ static void userspace_migrate_vcpu(void)
userspace_cmd(USERSPACE_MIGRATE_SELF);
}
-static void userspace_sleep(uint64_t usecs)
+static void userspace_sleep(u64 usecs)
{
GUEST_SYNC_ARGS(USERSPACE_USLEEP, usecs, 0, 0, 0);
}
-static void set_counter(enum arch_timer timer, uint64_t counter)
+static void set_counter(enum arch_timer timer, u64 counter)
{
GUEST_SYNC_ARGS(SET_COUNTER_VALUE, counter, timer, 0, 0);
}
@@ -146,8 +146,8 @@ static void guest_irq_handler(struct ex_regs *regs)
{
unsigned int intid = gic_get_and_ack_irq();
enum arch_timer timer;
- uint64_t cnt, cval;
- uint32_t ctl;
+ u64 cnt, cval;
+ u32 ctl;
bool timer_condition, istatus;
if (intid == IAR_SPURIOUS) {
@@ -178,8 +178,8 @@ static void guest_irq_handler(struct ex_regs *regs)
gic_set_eoi(intid);
}
-static void set_cval_irq(enum arch_timer timer, uint64_t cval_cycles,
- uint32_t ctl)
+static void set_cval_irq(enum arch_timer timer, u64 cval_cycles,
+ u32 ctl)
{
atomic_set(&shared_data.handled, 0);
atomic_set(&shared_data.spurious, 0);
@@ -187,8 +187,8 @@ static void set_cval_irq(enum arch_timer timer, uint64_t cval_cycles,
timer_set_ctl(timer, ctl);
}
-static void set_tval_irq(enum arch_timer timer, uint64_t tval_cycles,
- uint32_t ctl)
+static void set_tval_irq(enum arch_timer timer, u64 tval_cycles,
+ u32 ctl)
{
atomic_set(&shared_data.handled, 0);
atomic_set(&shared_data.spurious, 0);
@@ -196,7 +196,7 @@ static void set_tval_irq(enum arch_timer timer, uint64_t tval_cycles,
timer_set_ctl(timer, ctl);
}
-static void set_xval_irq(enum arch_timer timer, uint64_t xval, uint32_t ctl,
+static void set_xval_irq(enum arch_timer timer, u64 xval, u32 ctl,
enum timer_view tv)
{
switch (tv) {
@@ -275,13 +275,13 @@ static void wait_migrate_poll_for_irq(void)
* Sleep for usec microseconds by polling in the guest or in
* userspace (e.g. userspace_cmd=USERSPACE_SCHEDULE).
*/
-static void guest_poll(enum arch_timer test_timer, uint64_t usec,
+static void guest_poll(enum arch_timer test_timer, u64 usec,
enum sync_cmd usp_cmd)
{
- uint64_t cycles = usec_to_cycles(usec);
+ u64 cycles = usec_to_cycles(usec);
/* Whichever timer we are testing with, sleep with the other. */
enum arch_timer sleep_timer = 1 - test_timer;
- uint64_t start = timer_get_cntct(sleep_timer);
+ u64 start = timer_get_cntct(sleep_timer);
while ((timer_get_cntct(sleep_timer) - start) < cycles) {
if (usp_cmd == NO_USERSPACE_CMD)
@@ -291,22 +291,22 @@ static void guest_poll(enum arch_timer test_timer, uint64_t usec,
}
}
-static void sleep_poll(enum arch_timer timer, uint64_t usec)
+static void sleep_poll(enum arch_timer timer, u64 usec)
{
guest_poll(timer, usec, NO_USERSPACE_CMD);
}
-static void sleep_sched_poll(enum arch_timer timer, uint64_t usec)
+static void sleep_sched_poll(enum arch_timer timer, u64 usec)
{
guest_poll(timer, usec, USERSPACE_SCHED_YIELD);
}
-static void sleep_migrate(enum arch_timer timer, uint64_t usec)
+static void sleep_migrate(enum arch_timer timer, u64 usec)
{
guest_poll(timer, usec, USERSPACE_MIGRATE_SELF);
}
-static void sleep_in_userspace(enum arch_timer timer, uint64_t usec)
+static void sleep_in_userspace(enum arch_timer timer, u64 usec)
{
userspace_sleep(usec);
}
@@ -315,15 +315,15 @@ static void sleep_in_userspace(enum arch_timer timer, uint64_t usec)
* Reset the timer state to some nice values like the counter not being close
* to the edge, and the control register masked and disabled.
*/
-static void reset_timer_state(enum arch_timer timer, uint64_t cnt)
+static void reset_timer_state(enum arch_timer timer, u64 cnt)
{
set_counter(timer, cnt);
timer_set_ctl(timer, CTL_IMASK);
}
-static void test_timer_xval(enum arch_timer timer, uint64_t xval,
+static void test_timer_xval(enum arch_timer timer, u64 xval,
enum timer_view tv, irq_wait_method_t wm, bool reset_state,
- uint64_t reset_cnt)
+ u64 reset_cnt)
{
local_irq_disable();
@@ -348,23 +348,23 @@ static void test_timer_xval(enum arch_timer timer, uint64_t xval,
* the "runner", like: tools/testing/selftests/kselftest/runner.sh.
*/
-static void test_timer_cval(enum arch_timer timer, uint64_t cval,
+static void test_timer_cval(enum arch_timer timer, u64 cval,
irq_wait_method_t wm, bool reset_state,
- uint64_t reset_cnt)
+ u64 reset_cnt)
{
test_timer_xval(timer, cval, TIMER_CVAL, wm, reset_state, reset_cnt);
}
-static void test_timer_tval(enum arch_timer timer, int32_t tval,
+static void test_timer_tval(enum arch_timer timer, s32 tval,
irq_wait_method_t wm, bool reset_state,
- uint64_t reset_cnt)
+ u64 reset_cnt)
{
- test_timer_xval(timer, (uint64_t) tval, TIMER_TVAL, wm, reset_state,
+ test_timer_xval(timer, (u64)tval, TIMER_TVAL, wm, reset_state,
reset_cnt);
}
-static void test_xval_check_no_irq(enum arch_timer timer, uint64_t xval,
- uint64_t usec, enum timer_view timer_view,
+static void test_xval_check_no_irq(enum arch_timer timer, u64 xval,
+ u64 usec, enum timer_view timer_view,
sleep_method_t guest_sleep)
{
local_irq_disable();
@@ -379,17 +379,17 @@ static void test_xval_check_no_irq(enum arch_timer timer, uint64_t xval,
assert_irqs_handled(0);
}
-static void test_cval_no_irq(enum arch_timer timer, uint64_t cval,
- uint64_t usec, sleep_method_t wm)
+static void test_cval_no_irq(enum arch_timer timer, u64 cval,
+ u64 usec, sleep_method_t wm)
{
test_xval_check_no_irq(timer, cval, usec, TIMER_CVAL, wm);
}
-static void test_tval_no_irq(enum arch_timer timer, int32_t tval, uint64_t usec,
+static void test_tval_no_irq(enum arch_timer timer, s32 tval, u64 usec,
sleep_method_t wm)
{
- /* tval will be cast to an int32_t in test_xval_check_no_irq */
- test_xval_check_no_irq(timer, (uint64_t) tval, usec, TIMER_TVAL, wm);
+ /* tval will be cast to an s32 in test_xval_check_no_irq */
+ test_xval_check_no_irq(timer, (u64)tval, usec, TIMER_TVAL, wm);
}
/* Test masking/unmasking a timer using the timer mask (not the IRQ mask). */
@@ -463,7 +463,7 @@ static void test_timers_fired_multiple_times(enum arch_timer timer)
* timeout for the wait: we use the wfi instruction.
*/
static void test_reprogramming_timer(enum arch_timer timer, irq_wait_method_t wm,
- int32_t delta_1_ms, int32_t delta_2_ms)
+ s32 delta_1_ms, s32 delta_2_ms)
{
local_irq_disable();
reset_timer_state(timer, DEF_CNT);
@@ -488,7 +488,7 @@ static void test_reprogramming_timer(enum arch_timer timer, irq_wait_method_t wm
static void test_reprogram_timers(enum arch_timer timer)
{
int i;
- uint64_t base_wait = test_args.wait_ms;
+ u64 base_wait = test_args.wait_ms;
for (i = 0; i < ARRAY_SIZE(irq_wait_method); i++) {
/*
@@ -504,8 +504,8 @@ static void test_reprogram_timers(enum arch_timer timer)
static void test_basic_functionality(enum arch_timer timer)
{
- int32_t tval = (int32_t) msec_to_cycles(test_args.wait_ms);
- uint64_t cval = DEF_CNT + msec_to_cycles(test_args.wait_ms);
+ s32 tval = (s32)msec_to_cycles(test_args.wait_ms);
+ u64 cval = DEF_CNT + msec_to_cycles(test_args.wait_ms);
int i;
for (i = 0; i < ARRAY_SIZE(irq_wait_method); i++) {
@@ -593,7 +593,7 @@ static void test_set_cnt_after_tval_max(enum arch_timer timer, irq_wait_method_t
reset_timer_state(timer, DEF_CNT);
set_cval_irq(timer,
- (uint64_t) TVAL_MAX +
+ (u64)TVAL_MAX +
msec_to_cycles(test_args.wait_ms) / 2, CTL_ENABLE);
set_counter(timer, TVAL_MAX);
@@ -608,7 +608,7 @@ static void test_set_cnt_after_tval_max(enum arch_timer timer, irq_wait_method_t
/* Test timers set for: cval = now + TVAL_MAX + wait_ms / 2 */
static void test_timers_above_tval_max(enum arch_timer timer)
{
- uint64_t cval;
+ u64 cval;
int i;
/*
@@ -638,8 +638,8 @@ static void test_timers_above_tval_max(enum arch_timer timer)
* sets the counter to cnt_1, the [c|t]val, the counter to cnt_2, and
* then waits for an IRQ.
*/
-static void test_set_cnt_after_xval(enum arch_timer timer, uint64_t cnt_1,
- uint64_t xval, uint64_t cnt_2,
+static void test_set_cnt_after_xval(enum arch_timer timer, u64 cnt_1,
+ u64 xval, u64 cnt_2,
irq_wait_method_t wm, enum timer_view tv)
{
local_irq_disable();
@@ -662,8 +662,8 @@ static void test_set_cnt_after_xval(enum arch_timer timer, uint64_t cnt_1,
* then waits for an IRQ.
*/
static void test_set_cnt_after_xval_no_irq(enum arch_timer timer,
- uint64_t cnt_1, uint64_t xval,
- uint64_t cnt_2,
+ u64 cnt_1, u64 xval,
+ u64 cnt_2,
sleep_method_t guest_sleep,
enum timer_view tv)
{
@@ -684,31 +684,31 @@ static void test_set_cnt_after_xval_no_irq(enum arch_timer timer,
timer_set_ctl(timer, CTL_IMASK);
}
-static void test_set_cnt_after_tval(enum arch_timer timer, uint64_t cnt_1,
- int32_t tval, uint64_t cnt_2,
+static void test_set_cnt_after_tval(enum arch_timer timer, u64 cnt_1,
+ s32 tval, u64 cnt_2,
irq_wait_method_t wm)
{
test_set_cnt_after_xval(timer, cnt_1, tval, cnt_2, wm, TIMER_TVAL);
}
-static void test_set_cnt_after_cval(enum arch_timer timer, uint64_t cnt_1,
- uint64_t cval, uint64_t cnt_2,
+static void test_set_cnt_after_cval(enum arch_timer timer, u64 cnt_1,
+ u64 cval, u64 cnt_2,
irq_wait_method_t wm)
{
test_set_cnt_after_xval(timer, cnt_1, cval, cnt_2, wm, TIMER_CVAL);
}
static void test_set_cnt_after_tval_no_irq(enum arch_timer timer,
- uint64_t cnt_1, int32_t tval,
- uint64_t cnt_2, sleep_method_t wm)
+ u64 cnt_1, s32 tval,
+ u64 cnt_2, sleep_method_t wm)
{
test_set_cnt_after_xval_no_irq(timer, cnt_1, tval, cnt_2, wm,
TIMER_TVAL);
}
static void test_set_cnt_after_cval_no_irq(enum arch_timer timer,
- uint64_t cnt_1, uint64_t cval,
- uint64_t cnt_2, sleep_method_t wm)
+ u64 cnt_1, u64 cval,
+ u64 cnt_2, sleep_method_t wm)
{
test_set_cnt_after_xval_no_irq(timer, cnt_1, cval, cnt_2, wm,
TIMER_CVAL);
@@ -718,7 +718,7 @@ static void test_set_cnt_after_cval_no_irq(enum arch_timer timer,
static void test_move_counters_ahead_of_timers(enum arch_timer timer)
{
int i;
- int32_t tval;
+ s32 tval;
for (i = 0; i < ARRAY_SIZE(irq_wait_method); i++) {
irq_wait_method_t wm = irq_wait_method[i];
@@ -730,8 +730,7 @@ static void test_move_counters_ahead_of_timers(enum arch_timer timer)
test_set_cnt_after_tval(timer, 0, -1, DEF_CNT + 1, wm);
test_set_cnt_after_tval(timer, 0, -1, TVAL_MAX, wm);
tval = TVAL_MAX;
- test_set_cnt_after_tval(timer, 0, tval, (uint64_t) tval + 1,
- wm);
+ test_set_cnt_after_tval(timer, 0, tval, (u64)tval + 1, wm);
}
}
@@ -754,8 +753,8 @@ static void test_move_counters_behind_timers(enum arch_timer timer)
static void test_timers_in_the_past(enum arch_timer timer)
{
- int32_t tval = -1 * (int32_t) msec_to_cycles(test_args.wait_ms);
- uint64_t cval;
+ s32 tval = -1 * (s32)msec_to_cycles(test_args.wait_ms);
+ u64 cval;
int i;
for (i = 0; i < ARRAY_SIZE(irq_wait_method); i++) {
@@ -790,8 +789,8 @@ static void test_timers_in_the_past(enum arch_timer timer)
static void test_long_timer_delays(enum arch_timer timer)
{
- int32_t tval = (int32_t) msec_to_cycles(test_args.long_wait_ms);
- uint64_t cval = DEF_CNT + msec_to_cycles(test_args.long_wait_ms);
+ s32 tval = (s32)msec_to_cycles(test_args.long_wait_ms);
+ u64 cval = DEF_CNT + msec_to_cycles(test_args.long_wait_ms);
int i;
for (i = 0; i < ARRAY_SIZE(irq_wait_method); i++) {
@@ -846,11 +845,11 @@ static void guest_code(enum arch_timer timer)
static cpu_set_t default_cpuset;
-static uint32_t next_pcpu(void)
+static u32 next_pcpu(void)
{
- uint32_t max = get_nprocs();
- uint32_t cur = sched_getcpu();
- uint32_t next = cur;
+ u32 max = get_nprocs();
+ u32 cur = sched_getcpu();
+ u32 next = cur;
cpu_set_t cpuset = default_cpuset;
TEST_ASSERT(max > 1, "Need at least two physical cpus");
@@ -862,7 +861,7 @@ static uint32_t next_pcpu(void)
return next;
}
-static void kvm_set_cntxct(struct kvm_vcpu *vcpu, uint64_t cnt,
+static void kvm_set_cntxct(struct kvm_vcpu *vcpu, u64 cnt,
enum arch_timer timer)
{
if (timer == PHYSICAL)
@@ -874,7 +873,7 @@ static void kvm_set_cntxct(struct kvm_vcpu *vcpu, uint64_t cnt,
static void handle_sync(struct kvm_vcpu *vcpu, struct ucall *uc)
{
enum sync_cmd cmd = uc->args[1];
- uint64_t val = uc->args[2];
+ u64 val = uc->args[2];
enum arch_timer timer = uc->args[3];
switch (cmd) {
@@ -1018,8 +1017,8 @@ static bool parse_args(int argc, char *argv[])
static void set_counter_defaults(void)
{
- const uint64_t MIN_ROLLOVER_SECS = 40ULL * 365 * 24 * 3600;
- uint64_t freq = read_sysreg(CNTFRQ_EL0);
+ const u64 MIN_ROLLOVER_SECS = 40ULL * 365 * 24 * 3600;
+ u64 freq = read_sysreg(CNTFRQ_EL0);
int width = ilog2(MIN_ROLLOVER_SECS * freq);
width = clamp(width, 56, 64);
diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c
index 1d431de..3eb4b1b 100644
--- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c
+++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c
@@ -31,14 +31,14 @@
extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start, hw_bp_ctx;
extern unsigned char iter_ss_begin, iter_ss_end;
-static volatile uint64_t sw_bp_addr, hw_bp_addr;
-static volatile uint64_t wp_addr, wp_data_addr;
-static volatile uint64_t svc_addr;
-static volatile uint64_t ss_addr[4], ss_idx;
-#define PC(v) ((uint64_t)&(v))
+static volatile u64 sw_bp_addr, hw_bp_addr;
+static volatile u64 wp_addr, wp_data_addr;
+static volatile u64 svc_addr;
+static volatile u64 ss_addr[4], ss_idx;
+#define PC(v) ((u64)&(v))
#define GEN_DEBUG_WRITE_REG(reg_name) \
-static void write_##reg_name(int num, uint64_t val) \
+static void write_##reg_name(int num, u64 val) \
{ \
switch (num) { \
case 0: \
@@ -102,8 +102,8 @@ GEN_DEBUG_WRITE_REG(dbgwvr)
static void reset_debug_state(void)
{
- uint8_t brps, wrps, i;
- uint64_t dfr0;
+ u8 brps, wrps, i;
+ u64 dfr0;
asm volatile("msr daifset, #8");
@@ -140,7 +140,7 @@ static void enable_os_lock(void)
static void enable_monitor_debug_exceptions(void)
{
- uint64_t mdscr;
+ u64 mdscr;
asm volatile("msr daifclr, #8");
@@ -149,9 +149,9 @@ static void enable_monitor_debug_exceptions(void)
isb();
}
-static void install_wp(uint8_t wpn, uint64_t addr)
+static void install_wp(u8 wpn, u64 addr)
{
- uint32_t wcr;
+ u32 wcr;
wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E;
write_dbgwcr(wpn, wcr);
@@ -162,9 +162,9 @@ static void install_wp(uint8_t wpn, uint64_t addr)
enable_monitor_debug_exceptions();
}
-static void install_hw_bp(uint8_t bpn, uint64_t addr)
+static void install_hw_bp(u8 bpn, u64 addr)
{
- uint32_t bcr;
+ u32 bcr;
bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E;
write_dbgbcr(bpn, bcr);
@@ -174,11 +174,10 @@ static void install_hw_bp(uint8_t bpn, uint64_t addr)
enable_monitor_debug_exceptions();
}
-static void install_wp_ctx(uint8_t addr_wp, uint8_t ctx_bp, uint64_t addr,
- uint64_t ctx)
+static void install_wp_ctx(u8 addr_wp, u8 ctx_bp, u64 addr, u64 ctx)
{
- uint32_t wcr;
- uint64_t ctx_bcr;
+ u32 wcr;
+ u64 ctx_bcr;
/* Setup a context-aware breakpoint for Linked Context ID Match */
ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
@@ -188,7 +187,7 @@ static void install_wp_ctx(uint8_t addr_wp, uint8_t ctx_bp, uint64_t addr,
/* Setup a linked watchpoint (linked to the context-aware breakpoint) */
wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E |
- DBGWCR_WT_LINK | ((uint32_t)ctx_bp << DBGWCR_LBN_SHIFT);
+ DBGWCR_WT_LINK | ((u32)ctx_bp << DBGWCR_LBN_SHIFT);
write_dbgwcr(addr_wp, wcr);
write_dbgwvr(addr_wp, addr);
isb();
@@ -196,10 +195,9 @@ static void install_wp_ctx(uint8_t addr_wp, uint8_t ctx_bp, uint64_t addr,
enable_monitor_debug_exceptions();
}
-void install_hw_bp_ctx(uint8_t addr_bp, uint8_t ctx_bp, uint64_t addr,
- uint64_t ctx)
+void install_hw_bp_ctx(u8 addr_bp, u8 ctx_bp, u64 addr, u64 ctx)
{
- uint32_t addr_bcr, ctx_bcr;
+ u32 addr_bcr, ctx_bcr;
/* Setup a context-aware breakpoint for Linked Context ID Match */
ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
@@ -213,7 +211,7 @@ void install_hw_bp_ctx(uint8_t addr_bp, uint8_t ctx_bp, uint64_t addr,
*/
addr_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
DBGBCR_BT_ADDR_LINK_CTX |
- ((uint32_t)ctx_bp << DBGBCR_LBN_SHIFT);
+ ((u32)ctx_bp << DBGBCR_LBN_SHIFT);
write_dbgbcr(addr_bp, addr_bcr);
write_dbgbvr(addr_bp, addr);
isb();
@@ -223,7 +221,7 @@ void install_hw_bp_ctx(uint8_t addr_bp, uint8_t ctx_bp, uint64_t addr,
static void install_ss(void)
{
- uint64_t mdscr;
+ u64 mdscr;
asm volatile("msr daifclr, #8");
@@ -234,9 +232,9 @@ static void install_ss(void)
static volatile char write_data;
-static void guest_code(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
+static void guest_code(u8 bpn, u8 wpn, u8 ctx_bpn)
{
- uint64_t ctx = 0xabcdef; /* a random context number */
+ u64 ctx = 0xabcdef; /* a random context number */
/* Software-breakpoint */
reset_debug_state();
@@ -377,8 +375,8 @@ static void guest_svc_handler(struct ex_regs *regs)
static void guest_code_ss(int test_cnt)
{
- uint64_t i;
- uint64_t bvr, wvr, w_bvr, w_wvr;
+ u64 i;
+ u64 bvr, wvr, w_bvr, w_wvr;
for (i = 0; i < test_cnt; i++) {
/* Bits [1:0] of dbg{b,w}vr are RES0 */
@@ -416,12 +414,12 @@ static void guest_code_ss(int test_cnt)
GUEST_DONE();
}
-static int debug_version(uint64_t id_aa64dfr0)
+static int debug_version(u64 id_aa64dfr0)
{
return FIELD_GET(ID_AA64DFR0_EL1_DebugVer, id_aa64dfr0);
}
-static void test_guest_debug_exceptions(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
+static void test_guest_debug_exceptions(u8 bpn, u8 wpn, u8 ctx_bpn)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -468,8 +466,8 @@ void test_single_step_from_userspace(int test_cnt)
struct kvm_vm *vm;
struct ucall uc;
struct kvm_run *run;
- uint64_t pc, cmd;
- uint64_t test_pc = 0;
+ u64 pc, cmd;
+ u64 test_pc = 0;
bool ss_enable = false;
struct kvm_guest_debug debug = {};
@@ -506,7 +504,7 @@ void test_single_step_from_userspace(int test_cnt)
"Unexpected pc 0x%lx (expected 0x%lx)",
pc, test_pc);
- if ((pc + 4) == (uint64_t)&iter_ss_end) {
+ if ((pc + 4) == (u64)&iter_ss_end) {
test_pc = 0;
debug.control = KVM_GUESTDBG_ENABLE;
ss_enable = false;
@@ -519,8 +517,8 @@ void test_single_step_from_userspace(int test_cnt)
* iter_ss_end, the pc for the next KVM_EXIT_DEBUG should
* be the current pc + 4.
*/
- if ((pc >= (uint64_t)&iter_ss_begin) &&
- (pc < (uint64_t)&iter_ss_end))
+ if ((pc >= (u64)&iter_ss_begin) &&
+ (pc < (u64)&iter_ss_end))
test_pc = pc + 4;
else
test_pc = 0;
@@ -533,9 +531,9 @@ void test_single_step_from_userspace(int test_cnt)
* Run debug testing using the various breakpoint#, watchpoint# and
* context-aware breakpoint# with the given ID_AA64DFR0_EL1 configuration.
*/
-void test_guest_debug_exceptions_all(uint64_t aa64dfr0)
+void test_guest_debug_exceptions_all(u64 aa64dfr0)
{
- uint8_t brp_num, wrp_num, ctx_brp_num, normal_brp_num, ctx_brp_base;
+ u8 brp_num, wrp_num, ctx_brp_num, normal_brp_num, ctx_brp_base;
int b, w, c;
/* Number of breakpoints */
@@ -580,7 +578,7 @@ int main(int argc, char *argv[])
struct kvm_vm *vm;
int opt;
int ss_iteration = 10000;
- uint64_t aa64dfr0;
+ u64 aa64dfr0;
vm = vm_create_with_one_vcpu(&vcpu, guest_code);
aa64dfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1));
diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c
index bf038a0..5d96cdf 100644
--- a/tools/testing/selftests/kvm/arm64/hypercalls.c
+++ b/tools/testing/selftests/kvm/arm64/hypercalls.c
@@ -29,9 +29,9 @@
#define KVM_REG_ARM_VENDOR_HYP_BMAP_2_RESET_VAL 0
struct kvm_fw_reg_info {
- uint64_t reg; /* Register definition */
- uint64_t max_feat_bit; /* Bit that represents the upper limit of the feature-map */
- uint64_t reset_val; /* Reset value for the register */
+ u64 reg; /* Register definition */
+ u64 max_feat_bit; /* Bit that represents the upper limit of the feature-map */
+ u64 reset_val; /* Reset value for the register */
};
#define FW_REG_INFO(r) \
@@ -59,8 +59,8 @@ enum test_stage {
static int stage = TEST_STAGE_REG_IFACE;
struct test_hvc_info {
- uint32_t func_id;
- uint64_t arg1;
+ u32 func_id;
+ u64 arg1;
};
#define TEST_HVC_INFO(f, a1) \
@@ -152,9 +152,9 @@ static void guest_code(void)
}
struct st_time {
- uint32_t rev;
- uint32_t attr;
- uint64_t st_time;
+ u32 rev;
+ u32 attr;
+ u64 st_time;
};
#define STEAL_TIME_SIZE ((sizeof(struct st_time) + 63) & ~63)
@@ -162,7 +162,7 @@ struct st_time {
static void steal_time_init(struct kvm_vcpu *vcpu)
{
- uint64_t st_ipa = (ulong)ST_GPA_BASE;
+ u64 st_ipa = (ulong)ST_GPA_BASE;
unsigned int gpages;
gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE);
@@ -174,13 +174,13 @@ static void steal_time_init(struct kvm_vcpu *vcpu)
static void test_fw_regs_before_vm_start(struct kvm_vcpu *vcpu)
{
- uint64_t val;
+ u64 val;
unsigned int i;
int ret;
for (i = 0; i < ARRAY_SIZE(fw_reg_info); i++) {
const struct kvm_fw_reg_info *reg_info = &fw_reg_info[i];
- uint64_t set_val;
+ u64 set_val;
/* First 'read' should be the reset value for the reg */
val = vcpu_get_reg(vcpu, reg_info->reg);
@@ -229,7 +229,7 @@ static void test_fw_regs_before_vm_start(struct kvm_vcpu *vcpu)
static void test_fw_regs_after_vm_start(struct kvm_vcpu *vcpu)
{
- uint64_t val;
+ u64 val;
unsigned int i;
int ret;
diff --git a/tools/testing/selftests/kvm/arm64/idreg-idst.c b/tools/testing/selftests/kvm/arm64/idreg-idst.c
index 9ca9f12..a3e8470 100644
--- a/tools/testing/selftests/kvm/arm64/idreg-idst.c
+++ b/tools/testing/selftests/kvm/arm64/idreg-idst.c
@@ -13,7 +13,7 @@ static volatile bool sys64, undef;
#define __check_sr_read(r) \
({ \
- uint64_t val; \
+ u64 val; \
\
sys64 = false; \
undef = false; \
@@ -101,7 +101,7 @@ int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint64_t mmfr2;
+ u64 mmfr2;
test_disable_default_vgic();
diff --git a/tools/testing/selftests/kvm/arm64/no-vgic.c b/tools/testing/selftests/kvm/arm64/no-vgic.c
index b14686e..25b2e32 100644
--- a/tools/testing/selftests/kvm/arm64/no-vgic.c
+++ b/tools/testing/selftests/kvm/arm64/no-vgic.c
@@ -15,7 +15,7 @@ static volatile bool handled;
#define __check_sr_read(r) \
({ \
- uint64_t val; \
+ u64 val; \
\
handled = false; \
dsb(sy); \
@@ -33,7 +33,7 @@ static volatile bool handled;
#define __check_gicv5_gicr_op(r) \
({ \
- uint64_t val; \
+ u64 val; \
\
handled = false; \
dsb(sy); \
@@ -82,7 +82,7 @@ static volatile bool handled;
static void guest_code_gicv3(void)
{
- uint64_t val;
+ u64 val;
/*
* Check that we advertise that ID_AA64PFR0_EL1.GIC == 0, having
@@ -262,7 +262,7 @@ int main(int argc, char *argv[])
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
bool has_v3, has_v5;
- uint64_t pfr;
+ u64 pfr;
test_disable_default_vgic();
diff --git a/tools/testing/selftests/kvm/arm64/page_fault_test.c b/tools/testing/selftests/kvm/arm64/page_fault_test.c
index 4ccbd38..6bb3d82 100644
--- a/tools/testing/selftests/kvm/arm64/page_fault_test.c
+++ b/tools/testing/selftests/kvm/arm64/page_fault_test.c
@@ -23,7 +23,7 @@
#define TEST_PTE_GVA 0xb0000000
#define TEST_DATA 0x0123456789ABCDEF
-static uint64_t *guest_test_memory = (uint64_t *)TEST_GVA;
+static u64 *guest_test_memory = (u64 *)TEST_GVA;
#define CMD_NONE (0)
#define CMD_SKIP_TEST (1ULL << 1)
@@ -48,7 +48,7 @@ static struct event_cnt {
struct test_desc {
const char *name;
- uint64_t mem_mark_cmd;
+ u64 mem_mark_cmd;
/* Skip the test if any prepare function returns false */
bool (*guest_prepare[PREPARE_FN_NR])(void);
void (*guest_test)(void);
@@ -59,8 +59,8 @@ struct test_desc {
void (*iabt_handler)(struct ex_regs *regs);
void (*mmio_handler)(struct kvm_vm *vm, struct kvm_run *run);
void (*fail_vcpu_run_handler)(int ret);
- uint32_t pt_memslot_flags;
- uint32_t data_memslot_flags;
+ u32 pt_memslot_flags;
+ u32 data_memslot_flags;
bool skip;
struct event_cnt expected_events;
};
@@ -70,9 +70,9 @@ struct test_params {
struct test_desc *test_desc;
};
-static inline void flush_tlb_page(uint64_t vaddr)
+static inline void flush_tlb_page(gva_t gva)
{
- uint64_t page = vaddr >> 12;
+ gva_t page = gva >> 12;
dsb(ishst);
asm volatile("tlbi vaae1is, %0" :: "r" (page));
@@ -82,7 +82,7 @@ static inline void flush_tlb_page(uint64_t vaddr)
static void guest_write64(void)
{
- uint64_t val;
+ u64 val;
WRITE_ONCE(*guest_test_memory, TEST_DATA);
val = READ_ONCE(*guest_test_memory);
@@ -92,8 +92,8 @@ static void guest_write64(void)
/* Check the system for atomic instructions. */
static bool guest_check_lse(void)
{
- uint64_t isar0 = read_sysreg(id_aa64isar0_el1);
- uint64_t atomic;
+ u64 isar0 = read_sysreg(id_aa64isar0_el1);
+ u64 atomic;
atomic = FIELD_GET(ID_AA64ISAR0_EL1_ATOMIC, isar0);
return atomic >= 2;
@@ -101,8 +101,8 @@ static bool guest_check_lse(void)
static bool guest_check_dc_zva(void)
{
- uint64_t dczid = read_sysreg(dczid_el0);
- uint64_t dzp = FIELD_GET(DCZID_EL0_DZP, dczid);
+ u64 dczid = read_sysreg(dczid_el0);
+ u64 dzp = FIELD_GET(DCZID_EL0_DZP, dczid);
return dzp == 0;
}
@@ -110,7 +110,7 @@ static bool guest_check_dc_zva(void)
/* Compare and swap instruction. */
static void guest_cas(void)
{
- uint64_t val;
+ u64 val;
GUEST_ASSERT(guest_check_lse());
asm volatile(".arch_extension lse\n"
@@ -122,7 +122,7 @@ static void guest_cas(void)
static void guest_read64(void)
{
- uint64_t val;
+ u64 val;
val = READ_ONCE(*guest_test_memory);
GUEST_ASSERT_EQ(val, 0);
@@ -131,7 +131,7 @@ static void guest_read64(void)
/* Address translation instruction */
static void guest_at(void)
{
- uint64_t par;
+ u64 par;
asm volatile("at s1e1r, %0" :: "r" (guest_test_memory));
isb();
@@ -148,7 +148,7 @@ static void guest_at(void)
*/
static void guest_dc_zva(void)
{
- uint16_t val;
+ u16 val;
asm volatile("dc zva, %0" :: "r" (guest_test_memory));
dsb(ish);
@@ -164,8 +164,8 @@ static void guest_dc_zva(void)
*/
static void guest_ld_preidx(void)
{
- uint64_t val;
- uint64_t addr = TEST_GVA - 8;
+ u64 val;
+ u64 addr = TEST_GVA - 8;
/*
* This ends up accessing "TEST_GVA + 8 - 8", where "TEST_GVA - 8" is
@@ -179,8 +179,8 @@ static void guest_ld_preidx(void)
static void guest_st_preidx(void)
{
- uint64_t val = TEST_DATA;
- uint64_t addr = TEST_GVA - 8;
+ u64 val = TEST_DATA;
+ u64 addr = TEST_GVA - 8;
asm volatile("str %0, [%1, #8]!"
: "+r" (val), "+r" (addr));
@@ -191,8 +191,8 @@ static void guest_st_preidx(void)
static bool guest_set_ha(void)
{
- uint64_t mmfr1 = read_sysreg(id_aa64mmfr1_el1);
- uint64_t hadbs, tcr;
+ u64 mmfr1 = read_sysreg(id_aa64mmfr1_el1);
+ u64 hadbs, tcr;
/* Skip if HA is not supported. */
hadbs = FIELD_GET(ID_AA64MMFR1_EL1_HAFDBS, mmfr1);
@@ -208,7 +208,7 @@ static bool guest_set_ha(void)
static bool guest_clear_pte_af(void)
{
- *((uint64_t *)TEST_PTE_GVA) &= ~PTE_AF;
+ *((u64 *)TEST_PTE_GVA) &= ~PTE_AF;
flush_tlb_page(TEST_GVA);
return true;
@@ -217,7 +217,7 @@ static bool guest_clear_pte_af(void)
static void guest_check_pte_af(void)
{
dsb(ish);
- GUEST_ASSERT_EQ(*((uint64_t *)TEST_PTE_GVA) & PTE_AF, PTE_AF);
+ GUEST_ASSERT_EQ(*((u64 *)TEST_PTE_GVA) & PTE_AF, PTE_AF);
}
static void guest_check_write_in_dirty_log(void)
@@ -302,26 +302,26 @@ static void no_iabt_handler(struct ex_regs *regs)
static struct uffd_args {
char *copy;
void *hva;
- uint64_t paging_size;
+ u64 paging_size;
} pt_args, data_args;
/* Returns true to continue the test, and false if it should be skipped. */
static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg,
struct uffd_args *args)
{
- uint64_t addr = msg->arg.pagefault.address;
- uint64_t flags = msg->arg.pagefault.flags;
+ u64 addr = msg->arg.pagefault.address;
+ u64 flags = msg->arg.pagefault.flags;
struct uffdio_copy copy;
int ret;
TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING,
"The only expected UFFD mode is MISSING");
- TEST_ASSERT_EQ(addr, (uint64_t)args->hva);
+ TEST_ASSERT_EQ(addr, (u64)args->hva);
pr_debug("uffd fault: addr=%p write=%d\n",
(void *)addr, !!(flags & UFFD_PAGEFAULT_FLAG_WRITE));
- copy.src = (uint64_t)args->copy;
+ copy.src = (u64)args->copy;
copy.dst = addr;
copy.len = args->paging_size;
copy.mode = 0;
@@ -407,7 +407,7 @@ static bool punch_hole_in_backing_store(struct kvm_vm *vm,
struct userspace_mem_region *region)
{
void *hva = (void *)region->region.userspace_addr;
- uint64_t paging_size = region->region.memory_size;
+ u64 paging_size = region->region.memory_size;
int ret, fd = region->fd;
if (fd != -1) {
@@ -438,7 +438,7 @@ static void mmio_on_test_gpa_handler(struct kvm_vm *vm, struct kvm_run *run)
static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run)
{
- uint64_t data;
+ u64 data;
memcpy(&data, run->mmio.data, sizeof(data));
pr_debug("addr=%lld len=%d w=%d data=%lx\n",
@@ -449,11 +449,11 @@ static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run)
static bool check_write_in_dirty_log(struct kvm_vm *vm,
struct userspace_mem_region *region,
- uint64_t host_pg_nr)
+ u64 host_pg_nr)
{
unsigned long *bmap;
bool first_page_dirty;
- uint64_t size = region->region.memory_size;
+ u64 size = region->region.memory_size;
/* getpage_size() is not always equal to vm->page_size */
bmap = bitmap_zalloc(size / getpagesize());
@@ -468,7 +468,7 @@ static bool handle_cmd(struct kvm_vm *vm, int cmd)
{
struct userspace_mem_region *data_region, *pt_region;
bool continue_test = true;
- uint64_t pte_gpa, pte_pg;
+ u64 pte_gpa, pte_pg;
data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
pt_region = vm_get_mem_region(vm, MEM_REGION_PT);
@@ -510,7 +510,7 @@ void fail_vcpu_run_mmio_no_syndrome_handler(int ret)
events.fail_vcpu_runs += 1;
}
-typedef uint32_t aarch64_insn_t;
+typedef u32 aarch64_insn_t;
extern aarch64_insn_t __exec_test[2];
noinline void __return_0x77(void)
@@ -525,7 +525,7 @@ noinline void __return_0x77(void)
*/
static void load_exec_code_for_test(struct kvm_vm *vm)
{
- uint64_t *code;
+ u64 *code;
struct userspace_mem_region *region;
void *hva;
@@ -552,7 +552,7 @@ static void setup_abort_handlers(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
static void setup_gva_maps(struct kvm_vm *vm)
{
struct userspace_mem_region *region;
- uint64_t pte_gpa;
+ u64 pte_gpa;
region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
/* Map TEST_GVA first. This will install a new PTE. */
@@ -574,12 +574,12 @@ enum pf_test_memslots {
*/
static void setup_memslots(struct kvm_vm *vm, struct test_params *p)
{
- uint64_t backing_src_pagesz = get_backing_src_pagesz(p->src_type);
- uint64_t guest_page_size = vm->page_size;
- uint64_t max_gfn = vm_compute_max_gfn(vm);
+ u64 backing_src_pagesz = get_backing_src_pagesz(p->src_type);
+ u64 guest_page_size = vm->page_size;
+ u64 max_gfn = vm_compute_max_gfn(vm);
/* Enough for 2M of code when using 4K guest pages. */
- uint64_t code_npages = 512;
- uint64_t pt_size, data_size, data_gpa;
+ u64 code_npages = 512;
+ u64 pt_size, data_size, data_gpa;
/*
* This test requires 1 pgd, 2 pud, 4 pmd, and 6 pte pages when using
diff --git a/tools/testing/selftests/kvm/arm64/psci_test.c b/tools/testing/selftests/kvm/arm64/psci_test.c
index 98e49f7..e775faf 100644
--- a/tools/testing/selftests/kvm/arm64/psci_test.c
+++ b/tools/testing/selftests/kvm/arm64/psci_test.c
@@ -22,8 +22,7 @@
#define CPU_ON_ENTRY_ADDR 0xfeedf00dul
#define CPU_ON_CONTEXT_ID 0xdeadc0deul
-static uint64_t psci_cpu_on(uint64_t target_cpu, uint64_t entry_addr,
- uint64_t context_id)
+static u64 psci_cpu_on(u64 target_cpu, u64 entry_addr, u64 context_id)
{
struct arm_smccc_res res;
@@ -33,8 +32,7 @@ static uint64_t psci_cpu_on(uint64_t target_cpu, uint64_t entry_addr,
return res.a0;
}
-static uint64_t psci_affinity_info(uint64_t target_affinity,
- uint64_t lowest_affinity_level)
+static u64 psci_affinity_info(u64 target_affinity, u64 lowest_affinity_level)
{
struct arm_smccc_res res;
@@ -44,7 +42,7 @@ static uint64_t psci_affinity_info(uint64_t target_affinity,
return res.a0;
}
-static uint64_t psci_system_suspend(uint64_t entry_addr, uint64_t context_id)
+static u64 psci_system_suspend(u64 entry_addr, u64 context_id)
{
struct arm_smccc_res res;
@@ -54,7 +52,7 @@ static uint64_t psci_system_suspend(uint64_t entry_addr, uint64_t context_id)
return res.a0;
}
-static uint64_t psci_system_off2(uint64_t type, uint64_t cookie)
+static u64 psci_system_off2(u64 type, u64 cookie)
{
struct arm_smccc_res res;
@@ -63,7 +61,7 @@ static uint64_t psci_system_off2(uint64_t type, uint64_t cookie)
return res.a0;
}
-static uint64_t psci_features(uint32_t func_id)
+static u64 psci_features(u32 func_id)
{
struct arm_smccc_res res;
@@ -110,7 +108,7 @@ static void enter_guest(struct kvm_vcpu *vcpu)
static void assert_vcpu_reset(struct kvm_vcpu *vcpu)
{
- uint64_t obs_pc, obs_x0;
+ u64 obs_pc, obs_x0;
obs_pc = vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc));
obs_x0 = vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.regs[0]));
@@ -123,9 +121,9 @@ static void assert_vcpu_reset(struct kvm_vcpu *vcpu)
obs_x0, CPU_ON_CONTEXT_ID);
}
-static void guest_test_cpu_on(uint64_t target_cpu)
+static void guest_test_cpu_on(u64 target_cpu)
{
- uint64_t target_state;
+ u64 target_state;
GUEST_ASSERT(!psci_cpu_on(target_cpu, CPU_ON_ENTRY_ADDR, CPU_ON_CONTEXT_ID));
@@ -142,7 +140,7 @@ static void guest_test_cpu_on(uint64_t target_cpu)
static void host_test_cpu_on(void)
{
struct kvm_vcpu *source, *target;
- uint64_t target_mpidr;
+ u64 target_mpidr;
struct kvm_vm *vm;
struct ucall uc;
@@ -166,7 +164,7 @@ static void host_test_cpu_on(void)
static void guest_test_system_suspend(void)
{
- uint64_t ret;
+ u64 ret;
/* assert that SYSTEM_SUSPEND is discoverable */
GUEST_ASSERT(!psci_features(PSCI_1_0_FN_SYSTEM_SUSPEND));
@@ -200,7 +198,7 @@ static void host_test_system_suspend(void)
static void guest_test_system_off2(void)
{
- uint64_t ret;
+ u64 ret;
/* assert that SYSTEM_OFF2 is discoverable */
GUEST_ASSERT(psci_features(PSCI_1_3_FN_SYSTEM_OFF2) &
@@ -238,7 +236,7 @@ static void host_test_system_off2(void)
{
struct kvm_vcpu *source, *target;
struct kvm_mp_state mps;
- uint64_t psci_version = 0;
+ u64 psci_version = 0;
int nr_shutdowns = 0;
struct kvm_run *run;
struct ucall uc;
diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c
index 573dd79..e96d898 100644
--- a/tools/testing/selftests/kvm/arm64/sea_to_user.c
+++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c
@@ -51,18 +51,16 @@
#define EINJ_OFFSET 0x01234badUL
#define EINJ_GVA ((START_GVA) + (EINJ_OFFSET))
-static vm_paddr_t einj_gpa;
+static gpa_t einj_gpa;
static void *einj_hva;
-static uint64_t einj_hpa;
+static u64 einj_hpa;
static bool far_invalid;
-static uint64_t translate_to_host_paddr(unsigned long vaddr)
+static u64 translate_hva_to_hpa(unsigned long hva)
{
- uint64_t pinfo;
- int64_t offset = vaddr / getpagesize() * sizeof(pinfo);
+ u64 pinfo;
+ s64 offset = hva / getpagesize() * sizeof(pinfo);
int fd;
- uint64_t page_addr;
- uint64_t paddr;
fd = open("/proc/self/pagemap", O_RDONLY);
if (fd < 0)
@@ -77,12 +75,11 @@ static uint64_t translate_to_host_paddr(unsigned long vaddr)
if ((pinfo & PAGE_PRESENT) == 0)
ksft_exit_fail_perror("Page not present");
- page_addr = (pinfo & PAGE_PHYSICAL) << MIN_PAGE_SHIFT;
- paddr = page_addr + (vaddr & (getpagesize() - 1));
- return paddr;
+ return ((pinfo & PAGE_PHYSICAL) << MIN_PAGE_SHIFT) +
+ (hva & (getpagesize() - 1));
}
-static void write_einj_entry(const char *einj_path, uint64_t val)
+static void write_einj_entry(const char *einj_path, u64 val)
{
char cmd[256] = {0};
FILE *cmdfile = NULL;
@@ -96,7 +93,7 @@ static void write_einj_entry(const char *einj_path, uint64_t val)
ksft_exit_fail_perror("Failed to write EINJ entry");
}
-static void inject_uer(uint64_t paddr)
+static void inject_uer(u64 hpa)
{
if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1)
ksft_test_result_skip("EINJ table no available in firmware");
@@ -106,7 +103,7 @@ static void inject_uer(uint64_t paddr)
write_einj_entry(EINJ_ETYPE, ERROR_TYPE_MEMORY_UER);
write_einj_entry(EINJ_FLAGS, MASK_MEMORY_UER);
- write_einj_entry(EINJ_ADDR, paddr);
+ write_einj_entry(EINJ_ADDR, hpa);
write_einj_entry(EINJ_MASK, ~0x0UL);
write_einj_entry(EINJ_NOTRIGGER, 1);
write_einj_entry(EINJ_DOIT, 1);
@@ -145,10 +142,10 @@ static void setup_sigbus_handler(void)
static void guest_code(void)
{
- uint64_t guest_data;
+ u64 guest_data;
/* Consumes error will cause a SEA. */
- guest_data = *(uint64_t *)EINJ_GVA;
+ guest_data = *(u64 *)EINJ_GVA;
GUEST_FAIL("Poison not protected by SEA: gva=%#lx, guest_data=%#lx\n",
EINJ_GVA, guest_data);
@@ -253,8 +250,8 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu)
size_t backing_page_size;
size_t guest_page_size;
size_t alignment;
- uint64_t num_guest_pages;
- vm_paddr_t start_gpa;
+ u64 num_guest_pages;
+ gpa_t start_gpa;
enum vm_mem_backing_src_type src_type = VM_MEM_SRC_ANONYMOUS_HUGETLB_1GB;
struct kvm_vm *vm;
@@ -278,7 +275,7 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu)
vm_userspace_mem_region_add(
/*vm=*/vm,
/*src_type=*/src_type,
- /*guest_paddr=*/start_gpa,
+ /*gpa=*/start_gpa,
/*slot=*/1,
/*npages=*/num_guest_pages,
/*flags=*/0);
@@ -292,18 +289,18 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu)
static void vm_inject_memory_uer(struct kvm_vm *vm)
{
- uint64_t guest_data;
+ u64 guest_data;
einj_gpa = addr_gva2gpa(vm, EINJ_GVA);
einj_hva = addr_gva2hva(vm, EINJ_GVA);
/* Populate certain data before injecting UER. */
- *(uint64_t *)einj_hva = 0xBAADCAFE;
- guest_data = *(uint64_t *)einj_hva;
+ *(u64 *)einj_hva = 0xBAADCAFE;
+ guest_data = *(u64 *)einj_hva;
ksft_print_msg("Before EINJect: data=%#lx\n",
guest_data);
- einj_hpa = translate_to_host_paddr((unsigned long)einj_hva);
+ einj_hpa = translate_hva_to_hpa((unsigned long)einj_hva);
ksft_print_msg("EINJ_GVA=%#lx, einj_gpa=%#lx, einj_hva=%p, einj_hpa=%#lx\n",
EINJ_GVA, einj_gpa, einj_hva, einj_hpa);
diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c
index 3a7e5fe..7429a10 100644
--- a/tools/testing/selftests/kvm/arm64/set_id_regs.c
+++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c
@@ -30,20 +30,20 @@ struct reg_ftr_bits {
char *name;
bool sign;
enum ftr_type type;
- uint8_t shift;
- uint64_t mask;
+ u8 shift;
+ u64 mask;
/*
* For FTR_EXACT, safe_val is used as the exact safe value.
* For FTR_LOWER_SAFE, safe_val is used as the minimal safe value.
*/
- int64_t safe_val;
+ s64 safe_val;
/* Allowed to be changed by the host after run */
bool mutable;
};
struct test_feature_reg {
- uint32_t reg;
+ u32 reg;
const struct reg_ftr_bits *ftr_bits;
};
@@ -275,9 +275,9 @@ static void guest_code(void)
}
/* Return a safe value to a given ftr_bits an ftr value */
-uint64_t get_safe_value(const struct reg_ftr_bits *ftr_bits, uint64_t ftr)
+u64 get_safe_value(const struct reg_ftr_bits *ftr_bits, u64 ftr)
{
- uint64_t ftr_max = ftr_bits->mask >> ftr_bits->shift;
+ u64 ftr_max = ftr_bits->mask >> ftr_bits->shift;
TEST_ASSERT(ftr_max > 1, "This test doesn't support single bit features");
@@ -329,16 +329,16 @@ uint64_t get_safe_value(const struct reg_ftr_bits *ftr_bits, uint64_t ftr)
}
/* Return an invalid value to a given ftr_bits an ftr value */
-uint64_t get_invalid_value(const struct reg_ftr_bits *ftr_bits, uint64_t ftr)
+u64 get_invalid_value(const struct reg_ftr_bits *ftr_bits, u64 ftr)
{
- uint64_t ftr_max = ftr_bits->mask >> ftr_bits->shift;
+ u64 ftr_max = ftr_bits->mask >> ftr_bits->shift;
TEST_ASSERT(ftr_max > 1, "This test doesn't support single bit features");
if (ftr_bits->sign == FTR_UNSIGNED) {
switch (ftr_bits->type) {
case FTR_EXACT:
- ftr = max((uint64_t)ftr_bits->safe_val + 1, ftr + 1);
+ ftr = max((u64)ftr_bits->safe_val + 1, ftr + 1);
break;
case FTR_LOWER_SAFE:
ftr++;
@@ -358,7 +358,7 @@ uint64_t get_invalid_value(const struct reg_ftr_bits *ftr_bits, uint64_t ftr)
} else if (ftr != ftr_max) {
switch (ftr_bits->type) {
case FTR_EXACT:
- ftr = max((uint64_t)ftr_bits->safe_val + 1, ftr + 1);
+ ftr = max((u64)ftr_bits->safe_val + 1, ftr + 1);
break;
case FTR_LOWER_SAFE:
ftr++;
@@ -382,12 +382,12 @@ uint64_t get_invalid_value(const struct reg_ftr_bits *ftr_bits, uint64_t ftr)
return ftr;
}
-static uint64_t test_reg_set_success(struct kvm_vcpu *vcpu, uint64_t reg,
- const struct reg_ftr_bits *ftr_bits)
+static u64 test_reg_set_success(struct kvm_vcpu *vcpu, u64 reg,
+ const struct reg_ftr_bits *ftr_bits)
{
- uint8_t shift = ftr_bits->shift;
- uint64_t mask = ftr_bits->mask;
- uint64_t val, new_val, ftr;
+ u8 shift = ftr_bits->shift;
+ u64 mask = ftr_bits->mask;
+ u64 val, new_val, ftr;
val = vcpu_get_reg(vcpu, reg);
ftr = (val & mask) >> shift;
@@ -405,12 +405,12 @@ static uint64_t test_reg_set_success(struct kvm_vcpu *vcpu, uint64_t reg,
return new_val;
}
-static void test_reg_set_fail(struct kvm_vcpu *vcpu, uint64_t reg,
+static void test_reg_set_fail(struct kvm_vcpu *vcpu, u64 reg,
const struct reg_ftr_bits *ftr_bits)
{
- uint8_t shift = ftr_bits->shift;
- uint64_t mask = ftr_bits->mask;
- uint64_t val, old_val, ftr;
+ u8 shift = ftr_bits->shift;
+ u64 mask = ftr_bits->mask;
+ u64 val, old_val, ftr;
int r;
val = vcpu_get_reg(vcpu, reg);
@@ -431,7 +431,7 @@ static void test_reg_set_fail(struct kvm_vcpu *vcpu, uint64_t reg,
TEST_ASSERT_EQ(val, old_val);
}
-static uint64_t test_reg_vals[KVM_ARM_FEATURE_ID_RANGE_SIZE];
+static u64 test_reg_vals[KVM_ARM_FEATURE_ID_RANGE_SIZE];
#define encoding_to_range_idx(encoding) \
KVM_ARM_FEATURE_ID_RANGE_IDX(sys_reg_Op0(encoding), sys_reg_Op1(encoding), \
@@ -441,7 +441,7 @@ static uint64_t test_reg_vals[KVM_ARM_FEATURE_ID_RANGE_SIZE];
static void test_vm_ftr_id_regs(struct kvm_vcpu *vcpu, bool aarch64_only)
{
- uint64_t masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
+ u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
struct reg_mask_range range = {
.addr = (__u64)masks,
};
@@ -458,8 +458,8 @@ static void test_vm_ftr_id_regs(struct kvm_vcpu *vcpu, bool aarch64_only)
for (int i = 0; i < ARRAY_SIZE(test_regs); i++) {
const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits;
- uint32_t reg_id = test_regs[i].reg;
- uint64_t reg = KVM_ARM64_SYS_REG(reg_id);
+ u32 reg_id = test_regs[i].reg;
+ u64 reg = KVM_ARM64_SYS_REG(reg_id);
int idx;
/* Get the index to masks array for the idreg */
@@ -489,11 +489,11 @@ static void test_vm_ftr_id_regs(struct kvm_vcpu *vcpu, bool aarch64_only)
#define MPAM_IDREG_TEST 6
static void test_user_set_mpam_reg(struct kvm_vcpu *vcpu)
{
- uint64_t masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
+ u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
struct reg_mask_range range = {
.addr = (__u64)masks,
};
- uint64_t val;
+ u64 val;
int idx, err;
/*
@@ -584,13 +584,13 @@ static void test_user_set_mpam_reg(struct kvm_vcpu *vcpu)
#define MTE_IDREG_TEST 1
static void test_user_set_mte_reg(struct kvm_vcpu *vcpu)
{
- uint64_t masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
+ u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
struct reg_mask_range range = {
.addr = (__u64)masks,
};
- uint64_t val;
- uint64_t mte;
- uint64_t mte_frac;
+ u64 val;
+ u64 mte;
+ u64 mte_frac;
int idx, err;
val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1));
@@ -644,7 +644,7 @@ static void test_user_set_mte_reg(struct kvm_vcpu *vcpu)
ksft_test_result_pass("ID_AA64PFR1_EL1.MTE_frac no longer 0xF\n");
}
-static uint64_t reset_mutable_bits(uint32_t id, uint64_t val)
+static u64 reset_mutable_bits(u32 id, u64 val)
{
struct test_feature_reg *reg = NULL;
@@ -674,7 +674,7 @@ static void test_guest_reg_read(struct kvm_vcpu *vcpu)
struct ucall uc;
while (!done) {
- uint64_t val;
+ u64 val;
vcpu_run(vcpu);
@@ -707,7 +707,7 @@ static void test_guest_reg_read(struct kvm_vcpu *vcpu)
static void test_clidr(struct kvm_vcpu *vcpu)
{
- uint64_t clidr;
+ u64 clidr;
int level;
clidr = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CLIDR_EL1));
@@ -772,10 +772,10 @@ static void test_vcpu_non_ftr_id_regs(struct kvm_vcpu *vcpu)
ksft_test_result_pass("%s\n", __func__);
}
-static void test_assert_id_reg_unchanged(struct kvm_vcpu *vcpu, uint32_t encoding)
+static void test_assert_id_reg_unchanged(struct kvm_vcpu *vcpu, u32 encoding)
{
size_t idx = encoding_to_range_idx(encoding);
- uint64_t observed;
+ u64 observed;
observed = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(encoding));
TEST_ASSERT_EQ(reset_mutable_bits(encoding, test_reg_vals[idx]),
@@ -808,7 +808,7 @@ int main(void)
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
bool aarch64_only;
- uint64_t val, el0;
+ u64 val, el0;
int test_cnt, i, j;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES));
diff --git a/tools/testing/selftests/kvm/arm64/smccc_filter.c b/tools/testing/selftests/kvm/arm64/smccc_filter.c
index 1763b9d..21e4188 100644
--- a/tools/testing/selftests/kvm/arm64/smccc_filter.c
+++ b/tools/testing/selftests/kvm/arm64/smccc_filter.c
@@ -37,7 +37,7 @@ static bool test_runs_at_el2(void)
for (conduit = test_runs_at_el2() ? SMC_INSN : HVC_INSN; \
conduit <= SMC_INSN; conduit++)
-static void guest_main(uint32_t func_id, enum smccc_conduit conduit)
+static void guest_main(u32 func_id, enum smccc_conduit conduit)
{
struct arm_smccc_res res;
@@ -49,7 +49,7 @@ static void guest_main(uint32_t func_id, enum smccc_conduit conduit)
GUEST_SYNC(res.a0);
}
-static int __set_smccc_filter(struct kvm_vm *vm, uint32_t start, uint32_t nr_functions,
+static int __set_smccc_filter(struct kvm_vm *vm, u32 start, u32 nr_functions,
enum kvm_smccc_filter_action action)
{
struct kvm_smccc_filter filter = {
@@ -62,7 +62,7 @@ static int __set_smccc_filter(struct kvm_vm *vm, uint32_t start, uint32_t nr_fun
KVM_ARM_VM_SMCCC_FILTER, &filter);
}
-static void set_smccc_filter(struct kvm_vm *vm, uint32_t start, uint32_t nr_functions,
+static void set_smccc_filter(struct kvm_vm *vm, u32 start, u32 nr_functions,
enum kvm_smccc_filter_action action)
{
int ret = __set_smccc_filter(vm, start, nr_functions, action);
@@ -112,7 +112,7 @@ static void test_filter_reserved_range(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = setup_vm(&vcpu);
- uint32_t smc64_fn;
+ u32 smc64_fn;
int r;
r = __set_smccc_filter(vm, ARM_SMCCC_ARCH_WORKAROUND_1,
@@ -217,7 +217,7 @@ static void test_filter_denied(void)
}
}
-static void expect_call_fwd_to_user(struct kvm_vcpu *vcpu, uint32_t func_id,
+static void expect_call_fwd_to_user(struct kvm_vcpu *vcpu, u32 func_id,
enum smccc_conduit conduit)
{
struct kvm_run *run = vcpu->run;
diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c
index 8d6d3a4..47e34b4 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_init.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_init.c
@@ -19,7 +19,7 @@
#define NR_VCPUS 4
-#define REG_OFFSET(vcpu, offset) (((uint64_t)vcpu << 32) | offset)
+#define REG_OFFSET(vcpu, offset) (((u64)vcpu << 32) | offset)
#define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2)
#define VGIC_DEV_IS_V3(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V3)
@@ -27,10 +27,10 @@
struct vm_gic {
struct kvm_vm *vm;
int gic_fd;
- uint32_t gic_dev_type;
+ u32 gic_dev_type;
};
-static uint64_t max_phys_size;
+static u64 max_phys_size;
/*
* Helpers to access a redistributor register and verify the ioctl() failed or
@@ -39,17 +39,17 @@ static uint64_t max_phys_size;
static void v3_redist_reg_get_errno(int gicv3_fd, int vcpu, int offset,
int want, const char *msg)
{
- uint32_t ignored_val;
+ u32 ignored_val;
int ret = __kvm_device_attr_get(gicv3_fd, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS,
REG_OFFSET(vcpu, offset), &ignored_val);
TEST_ASSERT(ret && errno == want, "%s; want errno = %d", msg, want);
}
-static void v3_redist_reg_get(int gicv3_fd, int vcpu, int offset, uint32_t want,
+static void v3_redist_reg_get(int gicv3_fd, int vcpu, int offset, u32 want,
const char *msg)
{
- uint32_t val;
+ u32 val;
kvm_device_attr_get(gicv3_fd, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS,
REG_OFFSET(vcpu, offset), &val);
@@ -71,8 +71,8 @@ static int run_vcpu(struct kvm_vcpu *vcpu)
return __vcpu_run(vcpu) ? -errno : 0;
}
-static struct vm_gic vm_gic_create_with_vcpus(uint32_t gic_dev_type,
- uint32_t nr_vcpus,
+static struct vm_gic vm_gic_create_with_vcpus(u32 gic_dev_type,
+ u32 nr_vcpus,
struct kvm_vcpu *vcpus[])
{
struct vm_gic v;
@@ -84,7 +84,7 @@ static struct vm_gic vm_gic_create_with_vcpus(uint32_t gic_dev_type,
return v;
}
-static struct vm_gic vm_gic_create_barebones(uint32_t gic_dev_type)
+static struct vm_gic vm_gic_create_barebones(u32 gic_dev_type)
{
struct vm_gic v;
@@ -103,9 +103,9 @@ static void vm_gic_destroy(struct vm_gic *v)
}
struct vgic_region_attr {
- uint64_t attr;
- uint64_t size;
- uint64_t alignment;
+ u64 attr;
+ u64 size;
+ u64 alignment;
};
struct vgic_region_attr gic_v3_dist_region = {
@@ -143,7 +143,7 @@ struct vgic_region_attr gic_v2_cpu_region = {
static void subtest_dist_rdist(struct vm_gic *v)
{
int ret;
- uint64_t addr;
+ u64 addr;
struct vgic_region_attr rdist; /* CPU interface in GICv2*/
struct vgic_region_attr dist;
@@ -223,7 +223,7 @@ static void subtest_dist_rdist(struct vm_gic *v)
/* Test the new REDIST region API */
static void subtest_v3_redist_regions(struct vm_gic *v)
{
- uint64_t addr, expected_addr;
+ u64 addr, expected_addr;
int ret;
ret = __kvm_has_device_attr(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
@@ -332,7 +332,7 @@ static void subtest_v3_redist_regions(struct vm_gic *v)
* VGIC KVM device is created and initialized before the secondary CPUs
* get created
*/
-static void test_vgic_then_vcpus(uint32_t gic_dev_type)
+static void test_vgic_then_vcpus(u32 gic_dev_type)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
@@ -353,7 +353,7 @@ static void test_vgic_then_vcpus(uint32_t gic_dev_type)
}
/* All the VCPUs are created before the VGIC KVM device gets initialized */
-static void test_vcpus_then_vgic(uint32_t gic_dev_type)
+static void test_vcpus_then_vgic(u32 gic_dev_type)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
@@ -408,7 +408,7 @@ static void test_v3_new_redist_regions(void)
struct kvm_vcpu *vcpus[NR_VCPUS];
void *dummy = NULL;
struct vm_gic v;
- uint64_t addr;
+ u64 addr;
int ret;
v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus);
@@ -460,7 +460,7 @@ static void test_v3_new_redist_regions(void)
static void test_v3_typer_accesses(void)
{
struct vm_gic v;
- uint64_t addr;
+ u64 addr;
int ret, i;
v.vm = vm_create(NR_VCPUS);
@@ -518,7 +518,7 @@ static void test_v3_typer_accesses(void)
}
static struct vm_gic vm_gic_v3_create_with_vcpuids(int nr_vcpus,
- uint32_t vcpuids[])
+ u32 vcpuids[])
{
struct vm_gic v;
int i;
@@ -544,9 +544,9 @@ static struct vm_gic vm_gic_v3_create_with_vcpuids(int nr_vcpus,
*/
static void test_v3_last_bit_redist_regions(void)
{
- uint32_t vcpuids[] = { 0, 3, 5, 4, 1, 2 };
+ u32 vcpuids[] = { 0, 3, 5, 4, 1, 2 };
struct vm_gic v;
- uint64_t addr;
+ u64 addr;
v = vm_gic_v3_create_with_vcpuids(ARRAY_SIZE(vcpuids), vcpuids);
@@ -578,9 +578,9 @@ static void test_v3_last_bit_redist_regions(void)
/* Test last bit with legacy region */
static void test_v3_last_bit_single_rdist(void)
{
- uint32_t vcpuids[] = { 0, 3, 5, 4, 1, 2 };
+ u32 vcpuids[] = { 0, 3, 5, 4, 1, 2 };
struct vm_gic v;
- uint64_t addr;
+ u64 addr;
v = vm_gic_v3_create_with_vcpuids(ARRAY_SIZE(vcpuids), vcpuids);
@@ -606,7 +606,7 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void)
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
int ret, i;
- uint64_t addr;
+ u64 addr;
v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, vcpus);
@@ -638,7 +638,7 @@ static void test_v3_its_region(void)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
- uint64_t addr;
+ u64 addr;
int its_fd, ret;
v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus);
@@ -717,11 +717,11 @@ static void test_v3_nassgicap(void)
/*
* Returns 0 if it's possible to create GIC device of a given type (V2 or V3).
*/
-int test_kvm_device(uint32_t gic_dev_type)
+int test_kvm_device(u32 gic_dev_type)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
- uint32_t other;
+ u32 other;
int ret;
v.vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus);
@@ -968,7 +968,7 @@ static void test_v3_sysregs(void)
kvm_vm_free(vm);
}
-void run_tests(uint32_t gic_dev_type)
+void run_tests(u32 gic_dev_type)
{
test_vcpus_then_vgic(gic_dev_type);
test_vgic_then_vcpus(gic_dev_type);
diff --git a/tools/testing/selftests/kvm/arm64/vgic_irq.c b/tools/testing/selftests/kvm/arm64/vgic_irq.c
index 2fb2c79..5e23199 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_irq.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_irq.c
@@ -24,12 +24,12 @@
* function.
*/
struct test_args {
- uint32_t nr_irqs; /* number of KVM supported IRQs. */
+ u32 nr_irqs; /* number of KVM supported IRQs. */
bool eoi_split; /* 1 is eoir+dir, 0 is eoir only */
bool level_sensitive; /* 1 is level, 0 is edge */
int kvm_max_routes; /* output of KVM_CAP_IRQ_ROUTING */
bool kvm_supports_irqfd; /* output of KVM_CAP_IRQFD */
- uint32_t shared_data;
+ u32 shared_data;
};
/*
@@ -64,15 +64,15 @@ typedef enum {
struct kvm_inject_args {
kvm_inject_cmd cmd;
- uint32_t first_intid;
- uint32_t num;
+ u32 first_intid;
+ u32 num;
int level;
bool expect_failure;
};
/* Used on the guest side to perform the hypercall. */
-static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
- uint32_t num, int level, bool expect_failure);
+static void kvm_inject_call(kvm_inject_cmd cmd, u32 first_intid,
+ u32 num, int level, bool expect_failure);
/* Used on the host side to get the hypercall info. */
static void kvm_inject_get_call(struct kvm_vm *vm, struct ucall *uc,
@@ -133,8 +133,8 @@ static struct kvm_inject_desc set_active_fns[] = {
for_each_supported_inject_fn((args), (t), (f))
/* Shared between the guest main thread and the IRQ handlers. */
-volatile uint64_t irq_handled;
-volatile uint32_t irqnr_received[MAX_SPI + 1];
+volatile u64 irq_handled;
+volatile u32 irqnr_received[MAX_SPI + 1];
static void reset_stats(void)
{
@@ -145,25 +145,25 @@ static void reset_stats(void)
irqnr_received[i] = 0;
}
-static uint64_t gic_read_ap1r0(void)
+static u64 gic_read_ap1r0(void)
{
- uint64_t reg = read_sysreg_s(SYS_ICC_AP1R0_EL1);
+ u64 reg = read_sysreg_s(SYS_ICC_AP1R0_EL1);
dsb(sy);
return reg;
}
-static void gic_write_ap1r0(uint64_t val)
+static void gic_write_ap1r0(u64 val)
{
write_sysreg_s(val, SYS_ICC_AP1R0_EL1);
isb();
}
-static void guest_set_irq_line(uint32_t intid, uint32_t level);
+static void guest_set_irq_line(u32 intid, u32 level);
static void guest_irq_generic_handler(bool eoi_split, bool level_sensitive)
{
- uint32_t intid = gic_get_and_ack_irq();
+ u32 intid = gic_get_and_ack_irq();
if (intid == IAR_SPURIOUS)
return;
@@ -189,8 +189,8 @@ static void guest_irq_generic_handler(bool eoi_split, bool level_sensitive)
GUEST_ASSERT(!gic_irq_get_pending(intid));
}
-static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
- uint32_t num, int level, bool expect_failure)
+static void kvm_inject_call(kvm_inject_cmd cmd, u32 first_intid,
+ u32 num, int level, bool expect_failure)
{
struct kvm_inject_args args = {
.cmd = cmd,
@@ -204,7 +204,7 @@ static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
#define GUEST_ASSERT_IAR_EMPTY() \
do { \
- uint32_t _intid; \
+ u32 _intid; \
_intid = gic_get_and_ack_irq(); \
GUEST_ASSERT(_intid == IAR_SPURIOUS); \
} while (0)
@@ -237,13 +237,13 @@ static void reset_priorities(struct test_args *args)
gic_set_priority(i, IRQ_DEFAULT_PRIO_REG);
}
-static void guest_set_irq_line(uint32_t intid, uint32_t level)
+static void guest_set_irq_line(u32 intid, u32 level)
{
kvm_inject_call(KVM_SET_IRQ_LINE, intid, 1, level, false);
}
static void test_inject_fail(struct test_args *args,
- uint32_t intid, kvm_inject_cmd cmd)
+ u32 intid, kvm_inject_cmd cmd)
{
reset_stats();
@@ -255,10 +255,10 @@ static void test_inject_fail(struct test_args *args,
}
static void guest_inject(struct test_args *args,
- uint32_t first_intid, uint32_t num,
- kvm_inject_cmd cmd)
+ u32 first_intid, u32 num,
+ kvm_inject_cmd cmd)
{
- uint32_t i;
+ u32 i;
reset_stats();
@@ -292,10 +292,10 @@ static void guest_inject(struct test_args *args,
* deactivated yet.
*/
static void guest_restore_active(struct test_args *args,
- uint32_t first_intid, uint32_t num,
- kvm_inject_cmd cmd)
+ u32 first_intid, u32 num,
+ kvm_inject_cmd cmd)
{
- uint32_t prio, intid, ap1r;
+ u32 prio, intid, ap1r;
int i;
/*
@@ -342,9 +342,9 @@ static void guest_restore_active(struct test_args *args,
* This function should only be used in test_inject_preemption (with IRQs
* masked).
*/
-static uint32_t wait_for_and_activate_irq(void)
+static u32 wait_for_and_activate_irq(void)
{
- uint32_t intid;
+ u32 intid;
do {
asm volatile("wfi" : : : "memory");
@@ -360,11 +360,11 @@ static uint32_t wait_for_and_activate_irq(void)
* interrupts for the whole test.
*/
static void test_inject_preemption(struct test_args *args,
- uint32_t first_intid, int num,
+ u32 first_intid, int num,
const unsigned long *exclude,
kvm_inject_cmd cmd)
{
- uint32_t intid, prio, step = KVM_PRIO_STEPS;
+ u32 intid, prio, step = KVM_PRIO_STEPS;
int i;
/* Set the priorities of the first (KVM_NUM_PRIOS - 1) IRQs
@@ -379,7 +379,7 @@ static void test_inject_preemption(struct test_args *args,
local_irq_disable();
for (i = 0; i < num; i++) {
- uint32_t tmp;
+ u32 tmp;
intid = i + first_intid;
if (exclude && test_bit(i, exclude))
@@ -431,7 +431,7 @@ static void test_inject_preemption(struct test_args *args,
static void test_injection(struct test_args *args, struct kvm_inject_desc *f)
{
- uint32_t nr_irqs = args->nr_irqs;
+ u32 nr_irqs = args->nr_irqs;
if (f->sgi) {
guest_inject(args, MIN_SGI, 1, f->cmd);
@@ -451,7 +451,7 @@ static void test_injection(struct test_args *args, struct kvm_inject_desc *f)
static void test_injection_failure(struct test_args *args,
struct kvm_inject_desc *f)
{
- uint32_t bad_intid[] = { args->nr_irqs, 1020, 1024, 1120, 5120, ~0U, };
+ u32 bad_intid[] = { args->nr_irqs, 1020, 1024, 1120, 5120, ~0U, };
int i;
for (i = 0; i < ARRAY_SIZE(bad_intid); i++)
@@ -490,7 +490,7 @@ static void test_restore_active(struct test_args *args, struct kvm_inject_desc *
static void guest_code(struct test_args *args)
{
- uint32_t i, nr_irqs = args->nr_irqs;
+ u32 i, nr_irqs = args->nr_irqs;
bool level_sensitive = args->level_sensitive;
struct kvm_inject_desc *f, *inject_fns;
@@ -529,8 +529,8 @@ static void guest_code(struct test_args *args)
GUEST_DONE();
}
-static void kvm_irq_line_check(struct kvm_vm *vm, uint32_t intid, int level,
- struct test_args *test_args, bool expect_failure)
+static void kvm_irq_line_check(struct kvm_vm *vm, u32 intid, int level,
+ struct test_args *test_args, bool expect_failure)
{
int ret;
@@ -548,8 +548,8 @@ static void kvm_irq_line_check(struct kvm_vm *vm, uint32_t intid, int level,
}
}
-void kvm_irq_set_level_info_check(int gic_fd, uint32_t intid, int level,
- bool expect_failure)
+void kvm_irq_set_level_info_check(int gic_fd, u32 intid, int level,
+ bool expect_failure)
{
if (!expect_failure) {
kvm_irq_set_level_info(gic_fd, intid, level);
@@ -573,17 +573,18 @@ void kvm_irq_set_level_info_check(int gic_fd, uint32_t intid, int level,
}
static void kvm_set_gsi_routing_irqchip_check(struct kvm_vm *vm,
- uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
- bool expect_failure)
+ u32 intid, u32 num,
+ u32 kvm_max_routes,
+ bool expect_failure)
{
struct kvm_irq_routing *routing;
int ret;
- uint64_t i;
+ u64 i;
assert(num <= kvm_max_routes && kvm_max_routes <= KVM_MAX_IRQ_ROUTES);
routing = kvm_gsi_routing_create();
- for (i = intid; i < (uint64_t)intid + num; i++)
+ for (i = intid; i < (u64)intid + num; i++)
kvm_gsi_routing_irqchip_add(routing, i - MIN_SPI, i - MIN_SPI);
if (!expect_failure) {
@@ -591,7 +592,7 @@ static void kvm_set_gsi_routing_irqchip_check(struct kvm_vm *vm,
} else {
ret = _kvm_gsi_routing_write(vm, routing);
/* The kernel only checks e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS */
- if (((uint64_t)intid + num - 1 - MIN_SPI) >= KVM_IRQCHIP_NUM_PINS)
+ if (((u64)intid + num - 1 - MIN_SPI) >= KVM_IRQCHIP_NUM_PINS)
TEST_ASSERT(ret != 0 && errno == EINVAL,
"Bad intid %u did not cause KVM_SET_GSI_ROUTING "
"error: rc: %i errno: %i", intid, ret, errno);
@@ -602,7 +603,7 @@ static void kvm_set_gsi_routing_irqchip_check(struct kvm_vm *vm,
}
}
-static void kvm_irq_write_ispendr_check(int gic_fd, uint32_t intid,
+static void kvm_irq_write_ispendr_check(int gic_fd, u32 intid,
struct kvm_vcpu *vcpu,
bool expect_failure)
{
@@ -618,13 +619,13 @@ static void kvm_irq_write_ispendr_check(int gic_fd, uint32_t intid,
}
static void kvm_routing_and_irqfd_check(struct kvm_vm *vm,
- uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
- bool expect_failure)
+ u32 intid, u32 num, u32 kvm_max_routes,
+ bool expect_failure)
{
int fd[MAX_SPI];
- uint64_t val;
+ u64 val;
int ret, f;
- uint64_t i;
+ u64 i;
/*
* There is no way to try injecting an SGI or PPI as the interface
@@ -643,29 +644,29 @@ static void kvm_routing_and_irqfd_check(struct kvm_vm *vm,
* that no actual interrupt was injected for those cases.
*/
- for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)
+ for (f = 0, i = intid; i < (u64)intid + num; i++, f++)
fd[f] = kvm_new_eventfd();
- for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
- assert(i <= (uint64_t)UINT_MAX);
+ for (f = 0, i = intid; i < (u64)intid + num; i++, f++) {
+ assert(i <= (u64)UINT_MAX);
kvm_assign_irqfd(vm, i - MIN_SPI, fd[f]);
}
- for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
+ for (f = 0, i = intid; i < (u64)intid + num; i++, f++) {
val = 1;
- ret = write(fd[f], &val, sizeof(uint64_t));
- TEST_ASSERT(ret == sizeof(uint64_t),
+ ret = write(fd[f], &val, sizeof(u64));
+ TEST_ASSERT(ret == sizeof(u64),
__KVM_SYSCALL_ERROR("write()", ret));
}
- for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)
+ for (f = 0, i = intid; i < (u64)intid + num; i++, f++)
kvm_close(fd[f]);
}
/* handles the valid case: intid=0xffffffff num=1 */
#define for_each_intid(first, num, tmp, i) \
for ((tmp) = (i) = (first); \
- (tmp) < (uint64_t)(first) + (uint64_t)(num); \
+ (tmp) < (u64)(first) + (u64)(num); \
(tmp)++, (i)++)
static void run_guest_cmd(struct kvm_vcpu *vcpu, int gic_fd,
@@ -673,13 +674,13 @@ static void run_guest_cmd(struct kvm_vcpu *vcpu, int gic_fd,
struct test_args *test_args)
{
kvm_inject_cmd cmd = inject_args->cmd;
- uint32_t intid = inject_args->first_intid;
- uint32_t num = inject_args->num;
+ u32 intid = inject_args->first_intid;
+ u32 num = inject_args->num;
int level = inject_args->level;
bool expect_failure = inject_args->expect_failure;
struct kvm_vm *vm = vcpu->vm;
- uint64_t tmp;
- uint32_t i;
+ u64 tmp;
+ u32 i;
/* handles the valid case: intid=0xffffffff num=1 */
assert(intid < UINT_MAX - num || num == 1);
@@ -731,7 +732,7 @@ static void kvm_inject_get_call(struct kvm_vm *vm, struct ucall *uc,
struct kvm_inject_args *args)
{
struct kvm_inject_args *kvm_args_hva;
- vm_vaddr_t kvm_args_gva;
+ gva_t kvm_args_gva;
kvm_args_gva = uc->args[1];
kvm_args_hva = (struct kvm_inject_args *)addr_gva2hva(vm, kvm_args_gva);
@@ -745,14 +746,14 @@ static void print_args(struct test_args *args)
args->eoi_split);
}
-static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
+static void test_vgic(u32 nr_irqs, bool level_sensitive, bool eoi_split)
{
struct ucall uc;
int gic_fd;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct kvm_inject_args inject_args;
- vm_vaddr_t args_gva;
+ gva_t args_gva;
struct test_args args = {
.nr_irqs = nr_irqs,
@@ -770,7 +771,7 @@ static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
vcpu_init_descriptor_tables(vcpu);
/* Setup the guest args page (so it gets the args). */
- args_gva = vm_vaddr_alloc_page(vm);
+ args_gva = vm_alloc_page(vm);
memcpy(addr_gva2hva(vm, args_gva), &args, sizeof(args));
vcpu_args_set(vcpu, 1, args_gva);
@@ -810,7 +811,7 @@ static void guest_code_asym_dir(struct test_args *args, int cpuid)
gic_set_priority_mask(CPU_PRIO_MASK);
if (cpuid == 0) {
- uint32_t intid;
+ u32 intid;
local_irq_disable();
@@ -848,7 +849,7 @@ static void guest_code_asym_dir(struct test_args *args, int cpuid)
static void guest_code_group_en(struct test_args *args, int cpuid)
{
- uint32_t intid;
+ u32 intid;
gic_init(GIC_V3, 2);
@@ -896,7 +897,7 @@ static void guest_code_group_en(struct test_args *args, int cpuid)
static void guest_code_timer_spi(struct test_args *args, int cpuid)
{
- uint32_t intid;
+ u32 intid;
u64 val;
gic_init(GIC_V3, 2);
@@ -986,7 +987,7 @@ static void test_vgic_two_cpus(void *gcode)
struct kvm_vcpu *vcpus[2];
struct test_args args = {};
struct kvm_vm *vm;
- vm_vaddr_t args_gva;
+ gva_t args_gva;
int gic_fd, ret;
vm = vm_create_with_vcpus(2, gcode, vcpus);
@@ -996,7 +997,7 @@ static void test_vgic_two_cpus(void *gcode)
vcpu_init_descriptor_tables(vcpus[1]);
/* Setup the guest args page (so it gets the args). */
- args_gva = vm_vaddr_alloc_page(vm);
+ args_gva = vm_alloc_page(vm);
memcpy(addr_gva2hva(vm, args_gva), &args, sizeof(args));
vcpu_args_set(vcpus[0], 2, args_gva, 0);
vcpu_args_set(vcpus[1], 2, args_gva, 1);
@@ -1033,7 +1034,7 @@ static void help(const char *name)
int main(int argc, char **argv)
{
- uint32_t nr_irqs = 64;
+ u32 nr_irqs = 64;
bool default_args = true;
bool level_sensitive = false;
int opt;
diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
index e857a60..d64d434 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
@@ -23,7 +23,7 @@
#define GIC_LPI_OFFSET 8192
static size_t nr_iterations = 1000;
-static vm_paddr_t gpa_base;
+static gpa_t gpa_base;
static struct kvm_vm *vm;
static struct kvm_vcpu **vcpus;
@@ -35,14 +35,14 @@ static struct test_data {
u32 nr_devices;
u32 nr_event_ids;
- vm_paddr_t device_table;
- vm_paddr_t collection_table;
- vm_paddr_t cmdq_base;
+ gpa_t device_table;
+ gpa_t collection_table;
+ gpa_t cmdq_base;
void *cmdq_base_va;
- vm_paddr_t itt_tables;
+ gpa_t itt_tables;
- vm_paddr_t lpi_prop_table;
- vm_paddr_t lpi_pend_tables;
+ gpa_t lpi_prop_table;
+ gpa_t lpi_pend_tables;
} test_data = {
.nr_cpus = 1,
.nr_devices = 1,
@@ -73,7 +73,7 @@ static void guest_setup_its_mappings(void)
/* Round-robin the LPIs to all of the vCPUs in the VM */
coll_id = 0;
for (device_id = 0; device_id < nr_devices; device_id++) {
- vm_paddr_t itt_base = test_data.itt_tables + (device_id * SZ_64K);
+ gpa_t itt_base = test_data.itt_tables + (device_id * SZ_64K);
its_send_mapd_cmd(test_data.cmdq_base_va, device_id,
itt_base, SZ_64K, true);
@@ -188,7 +188,7 @@ static void setup_test_data(void)
size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K);
u32 nr_devices = test_data.nr_devices;
u32 nr_cpus = test_data.nr_cpus;
- vm_paddr_t cmdq_base;
+ gpa_t cmdq_base;
test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k,
gpa_base,
@@ -224,7 +224,7 @@ static void setup_gic(void)
static void signal_lpi(u32 device_id, u32 event_id)
{
- vm_paddr_t db_addr = GITS_BASE_GPA + GITS_TRANSLATER;
+ gpa_t db_addr = GITS_BASE_GPA + GITS_TRANSLATER;
struct kvm_msi msi = {
.address_lo = db_addr,
diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c
index 3ce6cf3..d785b66 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_v5.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c
@@ -17,10 +17,10 @@
struct vm_gic {
struct kvm_vm *vm;
int gic_fd;
- uint32_t gic_dev_type;
+ u32 gic_dev_type;
};
-static uint64_t max_phys_size;
+static u64 max_phys_size;
#define GUEST_CMD_IRQ_CDIA 10
#define GUEST_CMD_IRQ_DIEOI 11
@@ -96,7 +96,7 @@ static void vm_gic_destroy(struct vm_gic *v)
kvm_vm_free(v->vm);
}
-static void test_vgic_v5_ppis(uint32_t gic_dev_type)
+static void test_vgic_v5_ppis(u32 gic_dev_type)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct ucall uc;
@@ -173,7 +173,7 @@ static void test_vgic_v5_ppis(uint32_t gic_dev_type)
/*
* Returns 0 if it's possible to create GIC device of a given type (V5).
*/
-int test_kvm_device(uint32_t gic_dev_type)
+int test_kvm_device(u32 gic_dev_type)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
struct vm_gic v;
@@ -199,7 +199,7 @@ int test_kvm_device(uint32_t gic_dev_type)
return 0;
}
-void run_tests(uint32_t gic_dev_type)
+void run_tests(u32 gic_dev_type)
{
pr_info("Test VGICv5 PPIs\n");
test_vgic_v5_ppis(gic_dev_type);
diff --git a/tools/testing/selftests/kvm/arm64/vpmu_counter_access.c b/tools/testing/selftests/kvm/arm64/vpmu_counter_access.c
index ae36325..2222339 100644
--- a/tools/testing/selftests/kvm/arm64/vpmu_counter_access.c
+++ b/tools/testing/selftests/kvm/arm64/vpmu_counter_access.c
@@ -33,20 +33,20 @@ struct vpmu_vm {
static struct vpmu_vm vpmu_vm;
struct pmreg_sets {
- uint64_t set_reg_id;
- uint64_t clr_reg_id;
+ u64 set_reg_id;
+ u64 clr_reg_id;
};
#define PMREG_SET(set, clr) {.set_reg_id = set, .clr_reg_id = clr}
-static uint64_t get_pmcr_n(uint64_t pmcr)
+static u64 get_pmcr_n(u64 pmcr)
{
return FIELD_GET(ARMV8_PMU_PMCR_N, pmcr);
}
-static uint64_t get_counters_mask(uint64_t n)
+static u64 get_counters_mask(u64 n)
{
- uint64_t mask = BIT(ARMV8_PMU_CYCLE_IDX);
+ u64 mask = BIT(ARMV8_PMU_CYCLE_IDX);
if (n)
mask |= GENMASK(n - 1, 0);
@@ -89,7 +89,7 @@ static inline void write_sel_evtyper(int sel, unsigned long val)
static void pmu_disable_reset(void)
{
- uint64_t pmcr = read_sysreg(pmcr_el0);
+ u64 pmcr = read_sysreg(pmcr_el0);
/* Reset all counters, disabling them */
pmcr &= ~ARMV8_PMU_PMCR_E;
@@ -169,7 +169,7 @@ struct pmc_accessor pmc_accessors[] = {
#define GUEST_ASSERT_BITMAP_REG(regname, mask, set_expected) \
{ \
- uint64_t _tval = read_sysreg(regname); \
+ u64 _tval = read_sysreg(regname); \
\
if (set_expected) \
__GUEST_ASSERT((_tval & mask), \
@@ -185,7 +185,7 @@ struct pmc_accessor pmc_accessors[] = {
* Check if @mask bits in {PMCNTEN,PMINTEN,PMOVS}{SET,CLR} registers
* are set or cleared as specified in @set_expected.
*/
-static void check_bitmap_pmu_regs(uint64_t mask, bool set_expected)
+static void check_bitmap_pmu_regs(u64 mask, bool set_expected)
{
GUEST_ASSERT_BITMAP_REG(pmcntenset_el0, mask, set_expected);
GUEST_ASSERT_BITMAP_REG(pmcntenclr_el0, mask, set_expected);
@@ -207,7 +207,7 @@ static void check_bitmap_pmu_regs(uint64_t mask, bool set_expected)
*/
static void test_bitmap_pmu_regs(int pmc_idx, bool set_op)
{
- uint64_t pmcr_n, test_bit = BIT(pmc_idx);
+ u64 pmcr_n, test_bit = BIT(pmc_idx);
bool set_expected = false;
if (set_op) {
@@ -232,7 +232,7 @@ static void test_bitmap_pmu_regs(int pmc_idx, bool set_op)
*/
static void test_access_pmc_regs(struct pmc_accessor *acc, int pmc_idx)
{
- uint64_t write_data, read_data;
+ u64 write_data, read_data;
/* Disable all PMCs and reset all PMCs to zero. */
pmu_disable_reset();
@@ -287,11 +287,11 @@ static void test_access_pmc_regs(struct pmc_accessor *acc, int pmc_idx)
}
#define INVALID_EC (-1ul)
-uint64_t expected_ec = INVALID_EC;
+u64 expected_ec = INVALID_EC;
static void guest_sync_handler(struct ex_regs *regs)
{
- uint64_t esr, ec;
+ u64 esr, ec;
esr = read_sysreg(esr_el1);
ec = ESR_ELx_EC(esr);
@@ -351,9 +351,9 @@ static void test_access_invalid_pmc_regs(struct pmc_accessor *acc, int pmc_idx)
* if reading/writing PMU registers for implemented or unimplemented
* counters works as expected.
*/
-static void guest_code(uint64_t expected_pmcr_n)
+static void guest_code(u64 expected_pmcr_n)
{
- uint64_t pmcr, pmcr_n, unimp_mask;
+ u64 pmcr, pmcr_n, unimp_mask;
int i, pmc;
__GUEST_ASSERT(expected_pmcr_n <= ARMV8_PMU_MAX_GENERAL_COUNTERS,
@@ -402,12 +402,12 @@ static void guest_code(uint64_t expected_pmcr_n)
static void create_vpmu_vm(void *guest_code)
{
struct kvm_vcpu_init init;
- uint8_t pmuver, ec;
- uint64_t dfr0, irq = 23;
+ u8 pmuver, ec;
+ u64 dfr0, irq = 23;
struct kvm_device_attr irq_attr = {
.group = KVM_ARM_VCPU_PMU_V3_CTRL,
.attr = KVM_ARM_VCPU_PMU_V3_IRQ,
- .addr = (uint64_t)&irq,
+ .addr = (u64)&irq,
};
/* The test creates the vpmu_vm multiple times. Ensure a clean state */
@@ -443,7 +443,7 @@ static void destroy_vpmu_vm(void)
kvm_vm_free(vpmu_vm.vm);
}
-static void run_vcpu(struct kvm_vcpu *vcpu, uint64_t pmcr_n)
+static void run_vcpu(struct kvm_vcpu *vcpu, u64 pmcr_n)
{
struct ucall uc;
@@ -489,9 +489,9 @@ static void test_create_vpmu_vm_with_nr_counters(unsigned int nr_counters, bool
* Create a guest with one vCPU, set the PMCR_EL0.N for the vCPU to @pmcr_n,
* and run the test.
*/
-static void run_access_test(uint64_t pmcr_n)
+static void run_access_test(u64 pmcr_n)
{
- uint64_t sp;
+ u64 sp;
struct kvm_vcpu *vcpu;
struct kvm_vcpu_init init;
@@ -514,7 +514,7 @@ static void run_access_test(uint64_t pmcr_n)
aarch64_vcpu_setup(vcpu, &init);
vcpu_init_descriptor_tables(vcpu);
vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1), sp);
- vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code);
+ vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (u64)guest_code);
run_vcpu(vcpu, pmcr_n);
@@ -531,12 +531,12 @@ static struct pmreg_sets validity_check_reg_sets[] = {
* Create a VM, and check if KVM handles the userspace accesses of
* the PMU register sets in @validity_check_reg_sets[] correctly.
*/
-static void run_pmregs_validity_test(uint64_t pmcr_n)
+static void run_pmregs_validity_test(u64 pmcr_n)
{
int i;
struct kvm_vcpu *vcpu;
- uint64_t set_reg_id, clr_reg_id, reg_val;
- uint64_t valid_counters_mask, max_counters_mask;
+ u64 set_reg_id, clr_reg_id, reg_val;
+ u64 valid_counters_mask, max_counters_mask;
test_create_vpmu_vm_with_nr_counters(pmcr_n, false);
vcpu = vpmu_vm.vcpu;
@@ -588,7 +588,7 @@ static void run_pmregs_validity_test(uint64_t pmcr_n)
* the vCPU to @pmcr_n, which is larger than the host value.
* The attempt should fail as @pmcr_n is too big to set for the vCPU.
*/
-static void run_error_test(uint64_t pmcr_n)
+static void run_error_test(u64 pmcr_n)
{
pr_debug("Error test with pmcr_n %lu (larger than the host)\n", pmcr_n);
@@ -600,9 +600,9 @@ static void run_error_test(uint64_t pmcr_n)
* Return the default number of implemented PMU event counters excluding
* the cycle counter (i.e. PMCR_EL0.N value) for the guest.
*/
-static uint64_t get_pmcr_n_limit(void)
+static u64 get_pmcr_n_limit(void)
{
- uint64_t pmcr;
+ u64 pmcr;
create_vpmu_vm(guest_code);
pmcr = vcpu_get_reg(vpmu_vm.vcpu, KVM_ARM64_SYS_REG(SYS_PMCR_EL0));
@@ -624,7 +624,7 @@ static bool kvm_supports_nr_counters_attr(void)
int main(void)
{
- uint64_t i, pmcr_n;
+ u64 i, pmcr_n;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_PMU_V3));
TEST_REQUIRE(kvm_supports_vgic_v3());
diff --git a/tools/testing/selftests/kvm/coalesced_io_test.c b/tools/testing/selftests/kvm/coalesced_io_test.c
index 60cb254..df4ed5e 100644
--- a/tools/testing/selftests/kvm/coalesced_io_test.c
+++ b/tools/testing/selftests/kvm/coalesced_io_test.c
@@ -14,16 +14,16 @@
struct kvm_coalesced_io {
struct kvm_coalesced_mmio_ring *ring;
- uint32_t ring_size;
- uint64_t mmio_gpa;
- uint64_t *mmio;
+ u32 ring_size;
+ u64 mmio_gpa;
+ u64 *mmio;
/*
* x86-only, but define pio_port for all architectures to minimize the
* amount of #ifdeffery and complexity, without having to sacrifice
* verbose error messages.
*/
- uint8_t pio_port;
+ u8 pio_port;
};
static struct kvm_coalesced_io kvm_builtin_io_ring;
@@ -70,13 +70,13 @@ static void guest_code(struct kvm_coalesced_io *io)
static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu,
struct kvm_coalesced_io *io,
- uint32_t ring_start,
- uint32_t expected_exit)
+ u32 ring_start,
+ u32 expected_exit)
{
const bool want_pio = expected_exit == KVM_EXIT_IO;
struct kvm_coalesced_mmio_ring *ring = io->ring;
struct kvm_run *run = vcpu->run;
- uint32_t pio_value;
+ u32 pio_value;
WRITE_ONCE(ring->first, ring_start);
WRITE_ONCE(ring->last, ring_start);
@@ -88,13 +88,13 @@ static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu,
* data_offset is garbage, e.g. an MMIO gpa.
*/
if (run->exit_reason == KVM_EXIT_IO)
- pio_value = *(uint32_t *)((void *)run + run->io.data_offset);
+ pio_value = *(u32 *)((void *)run + run->io.data_offset);
else
pio_value = 0;
TEST_ASSERT((!want_pio && (run->exit_reason == KVM_EXIT_MMIO && run->mmio.is_write &&
run->mmio.phys_addr == io->mmio_gpa && run->mmio.len == 8 &&
- *(uint64_t *)run->mmio.data == io->mmio_gpa + io->ring_size - 1)) ||
+ *(u64 *)run->mmio.data == io->mmio_gpa + io->ring_size - 1)) ||
(want_pio && (run->exit_reason == KVM_EXIT_IO && run->io.port == io->pio_port &&
run->io.direction == KVM_EXIT_IO_OUT && run->io.count == 1 &&
pio_value == io->pio_port + io->ring_size - 1)),
@@ -105,14 +105,14 @@ static void vcpu_run_and_verify_io_exit(struct kvm_vcpu *vcpu,
want_pio ? (unsigned long long)io->pio_port : io->mmio_gpa,
(want_pio ? io->pio_port : io->mmio_gpa) + io->ring_size - 1, run->exit_reason,
run->exit_reason == KVM_EXIT_MMIO ? "MMIO" : run->exit_reason == KVM_EXIT_IO ? "PIO" : "other",
- run->mmio.phys_addr, run->mmio.is_write, run->mmio.len, *(uint64_t *)run->mmio.data,
+ run->mmio.phys_addr, run->mmio.is_write, run->mmio.len, *(u64 *)run->mmio.data,
run->io.port, run->io.direction, run->io.size, run->io.count, pio_value);
}
static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu,
struct kvm_coalesced_io *io,
- uint32_t ring_start,
- uint32_t expected_exit)
+ u32 ring_start,
+ u32 expected_exit)
{
struct kvm_coalesced_mmio_ring *ring = io->ring;
int i;
@@ -124,18 +124,18 @@ static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu,
ring->first, ring->last, io->ring_size, ring_start);
for (i = 0; i < io->ring_size - 1; i++) {
- uint32_t idx = (ring->first + i) % io->ring_size;
+ u32 idx = (ring->first + i) % io->ring_size;
struct kvm_coalesced_mmio *entry = &ring->coalesced_mmio[idx];
#ifdef __x86_64__
if (i & 1)
TEST_ASSERT(entry->phys_addr == io->pio_port &&
entry->len == 4 && entry->pio &&
- *(uint32_t *)entry->data == io->pio_port + i,
+ *(u32 *)entry->data == io->pio_port + i,
"Wanted 4-byte port I/O 0x%x = 0x%x in entry %u, got %u-byte %s 0x%llx = 0x%x",
io->pio_port, io->pio_port + i, i,
entry->len, entry->pio ? "PIO" : "MMIO",
- entry->phys_addr, *(uint32_t *)entry->data);
+ entry->phys_addr, *(u32 *)entry->data);
else
#endif
TEST_ASSERT(entry->phys_addr == io->mmio_gpa &&
@@ -143,12 +143,12 @@ static void vcpu_run_and_verify_coalesced_io(struct kvm_vcpu *vcpu,
"Wanted 8-byte MMIO to 0x%lx = %lx in entry %u, got %u-byte %s 0x%llx = 0x%lx",
io->mmio_gpa, io->mmio_gpa + i, i,
entry->len, entry->pio ? "PIO" : "MMIO",
- entry->phys_addr, *(uint64_t *)entry->data);
+ entry->phys_addr, *(u64 *)entry->data);
}
}
static void test_coalesced_io(struct kvm_vcpu *vcpu,
- struct kvm_coalesced_io *io, uint32_t ring_start)
+ struct kvm_coalesced_io *io, u32 ring_start)
{
struct kvm_coalesced_mmio_ring *ring = io->ring;
@@ -219,11 +219,11 @@ int main(int argc, char *argv[])
* the MMIO GPA identity mapped in the guest.
*/
.mmio_gpa = 4ull * SZ_1G,
- .mmio = (uint64_t *)(4ull * SZ_1G),
+ .mmio = (u64 *)(4ull * SZ_1G),
.pio_port = 0x80,
};
- virt_map(vm, (uint64_t)kvm_builtin_io_ring.mmio, kvm_builtin_io_ring.mmio_gpa, 1);
+ virt_map(vm, (u64)kvm_builtin_io_ring.mmio, kvm_builtin_io_ring.mmio_gpa, 1);
sync_global_to_guest(vm, kvm_builtin_io_ring);
vcpu_args_set(vcpu, 1, &kvm_builtin_io_ring);
diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c
index 0202b78..302c492 100644
--- a/tools/testing/selftests/kvm/demand_paging_test.c
+++ b/tools/testing/selftests/kvm/demand_paging_test.c
@@ -24,7 +24,7 @@
#ifdef __NR_userfaultfd
static int nr_vcpus = 1;
-static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
+static u64 guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
static size_t demand_paging_size;
static char *guest_data_prototype;
@@ -58,7 +58,7 @@ static int handle_uffd_page_request(int uffd_mode, int uffd,
struct uffd_msg *msg)
{
pid_t tid = syscall(__NR_gettid);
- uint64_t addr = msg->arg.pagefault.address;
+ u64 addr = msg->arg.pagefault.address;
struct timespec start;
struct timespec ts_diff;
int r;
@@ -68,7 +68,7 @@ static int handle_uffd_page_request(int uffd_mode, int uffd,
if (uffd_mode == UFFDIO_REGISTER_MODE_MISSING) {
struct uffdio_copy copy;
- copy.src = (uint64_t)guest_data_prototype;
+ copy.src = (u64)guest_data_prototype;
copy.dst = addr;
copy.len = demand_paging_size;
copy.mode = 0;
@@ -138,7 +138,7 @@ struct test_params {
bool partition_vcpu_memory_access;
};
-static void prefault_mem(void *alias, uint64_t len)
+static void prefault_mem(void *alias, u64 len)
{
size_t p;
@@ -154,7 +154,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
struct memstress_vcpu_args *vcpu_args;
struct test_params *p = arg;
struct uffd_desc **uffd_descs = NULL;
- uint64_t uffd_region_size;
+ u64 uffd_region_size;
struct timespec start;
struct timespec ts_diff;
double vcpu_paging_rate;
diff --git a/tools/testing/selftests/kvm/dirty_log_perf_test.c b/tools/testing/selftests/kvm/dirty_log_perf_test.c
index 0a1ea1d..ef779fa 100644
--- a/tools/testing/selftests/kvm/dirty_log_perf_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_perf_test.c
@@ -24,7 +24,7 @@
#define TEST_HOST_LOOP_N 2UL
static int nr_vcpus = 1;
-static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
+static u64 guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
static bool run_vcpus_while_disabling_dirty_logging;
/* Host variables */
@@ -37,7 +37,7 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
{
struct kvm_vcpu *vcpu = vcpu_args->vcpu;
int vcpu_idx = vcpu_args->vcpu_idx;
- uint64_t pages_count = 0;
+ u64 pages_count = 0;
struct kvm_run *run;
struct timespec start;
struct timespec ts_diff;
@@ -93,11 +93,11 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
struct test_params {
unsigned long iterations;
- uint64_t phys_offset;
+ u64 phys_offset;
bool partition_vcpu_memory_access;
enum vm_mem_backing_src_type backing_src;
int slots;
- uint32_t write_percent;
+ u32 write_percent;
bool random_access;
};
@@ -106,9 +106,9 @@ static void run_test(enum vm_guest_mode mode, void *arg)
struct test_params *p = arg;
struct kvm_vm *vm;
unsigned long **bitmaps;
- uint64_t guest_num_pages;
- uint64_t host_num_pages;
- uint64_t pages_per_slot;
+ u64 guest_num_pages;
+ u64 host_num_pages;
+ u64 pages_per_slot;
struct timespec start;
struct timespec ts_diff;
struct timespec get_dirty_log_total = (struct timespec){0};
diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c
index 7627b32..12446a4 100644
--- a/tools/testing/selftests/kvm/dirty_log_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_test.c
@@ -74,11 +74,11 @@
* the host. READ/WRITE_ONCE() should also be used with anything
* that may change.
*/
-static uint64_t host_page_size;
-static uint64_t guest_page_size;
-static uint64_t guest_num_pages;
-static uint64_t iteration;
-static uint64_t nr_writes;
+static u64 host_page_size;
+static u64 guest_page_size;
+static u64 guest_num_pages;
+static u64 iteration;
+static u64 nr_writes;
static bool vcpu_stop;
/*
@@ -86,13 +86,13 @@ static bool vcpu_stop;
* This will be set to the topmost valid physical address minus
* the test memory size.
*/
-static uint64_t guest_test_phys_mem;
+static u64 guest_test_phys_mem;
/*
* Guest virtual memory offset of the testing memory slot.
* Must not conflict with identity mapped test code.
*/
-static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
+static u64 guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
/*
* Continuously write to the first 8 bytes of a random pages within
@@ -100,10 +100,10 @@ static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
*/
static void guest_code(void)
{
- uint64_t addr;
+ u64 addr;
#ifdef __s390x__
- uint64_t i;
+ u64 i;
/*
* On s390x, all pages of a 1M segment are initially marked as dirty
@@ -113,7 +113,7 @@ static void guest_code(void)
*/
for (i = 0; i < guest_num_pages; i++) {
addr = guest_test_virt_mem + i * guest_page_size;
- vcpu_arch_put_guest(*(uint64_t *)addr, READ_ONCE(iteration));
+ vcpu_arch_put_guest(*(u64 *)addr, READ_ONCE(iteration));
nr_writes++;
}
#endif
@@ -125,7 +125,7 @@ static void guest_code(void)
* guest_page_size;
addr = align_down(addr, host_page_size);
- vcpu_arch_put_guest(*(uint64_t *)addr, READ_ONCE(iteration));
+ vcpu_arch_put_guest(*(u64 *)addr, READ_ONCE(iteration));
nr_writes++;
}
@@ -138,11 +138,11 @@ static bool host_quit;
/* Points to the test VM memory region on which we track dirty logs */
static void *host_test_mem;
-static uint64_t host_num_pages;
+static u64 host_num_pages;
/* For statistics only */
-static uint64_t host_dirty_count;
-static uint64_t host_clear_count;
+static u64 host_dirty_count;
+static u64 host_clear_count;
/* Whether dirty ring reset is requested, or finished */
static sem_t sem_vcpu_stop;
@@ -169,7 +169,7 @@ static bool dirty_ring_vcpu_ring_full;
* dirty gfn we've collected, so that if a mismatch of data found later in the
* verifying process, we let it pass.
*/
-static uint64_t dirty_ring_last_page = -1ULL;
+static u64 dirty_ring_last_page = -1ULL;
/*
* In addition to the above, it is possible (especially if this
@@ -213,7 +213,7 @@ static uint64_t dirty_ring_last_page = -1ULL;
* and also don't fail when it is reported in the next iteration, together with
* an outdated iteration count.
*/
-static uint64_t dirty_ring_prev_iteration_last_page;
+static u64 dirty_ring_prev_iteration_last_page;
enum log_mode_t {
/* Only use KVM_GET_DIRTY_LOG for logging */
@@ -236,7 +236,7 @@ static enum log_mode_t host_log_mode_option = LOG_MODE_ALL;
/* Logging mode for current run */
static enum log_mode_t host_log_mode;
static pthread_t vcpu_thread;
-static uint32_t test_dirty_ring_count = TEST_DIRTY_RING_COUNT;
+static u32 test_dirty_ring_count = TEST_DIRTY_RING_COUNT;
static bool clear_log_supported(void)
{
@@ -255,15 +255,15 @@ static void clear_log_create_vm_done(struct kvm_vm *vm)
}
static void dirty_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
- void *bitmap, uint32_t num_pages,
- uint32_t *unused)
+ void *bitmap, u32 num_pages,
+ u32 *unused)
{
kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
}
static void clear_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
- void *bitmap, uint32_t num_pages,
- uint32_t *unused)
+ void *bitmap, u32 num_pages,
+ u32 *unused)
{
kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
kvm_vm_clear_dirty_log(vcpu->vm, slot, bitmap, 0, num_pages);
@@ -297,8 +297,8 @@ static bool dirty_ring_supported(void)
static void dirty_ring_create_vm_done(struct kvm_vm *vm)
{
- uint64_t pages;
- uint32_t limit;
+ u64 pages;
+ u32 limit;
/*
* We rely on vcpu exit due to full dirty ring state. Adjust
@@ -333,12 +333,12 @@ static inline void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn)
smp_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET);
}
-static uint32_t dirty_ring_collect_one(struct kvm_dirty_gfn *dirty_gfns,
- int slot, void *bitmap,
- uint32_t num_pages, uint32_t *fetch_index)
+static u32 dirty_ring_collect_one(struct kvm_dirty_gfn *dirty_gfns,
+ int slot, void *bitmap,
+ u32 num_pages, u32 *fetch_index)
{
struct kvm_dirty_gfn *cur;
- uint32_t count = 0;
+ u32 count = 0;
while (true) {
cur = &dirty_gfns[*fetch_index % test_dirty_ring_count];
@@ -359,10 +359,10 @@ static uint32_t dirty_ring_collect_one(struct kvm_dirty_gfn *dirty_gfns,
}
static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
- void *bitmap, uint32_t num_pages,
- uint32_t *ring_buf_idx)
+ void *bitmap, u32 num_pages,
+ u32 *ring_buf_idx)
{
- uint32_t count, cleared;
+ u32 count, cleared;
/* Only have one vcpu */
count = dirty_ring_collect_one(vcpu_map_dirty_ring(vcpu),
@@ -404,8 +404,8 @@ struct log_mode {
void (*create_vm_done)(struct kvm_vm *vm);
/* Hook to collect the dirty pages into the bitmap provided */
void (*collect_dirty_pages) (struct kvm_vcpu *vcpu, int slot,
- void *bitmap, uint32_t num_pages,
- uint32_t *ring_buf_idx);
+ void *bitmap, u32 num_pages,
+ u32 *ring_buf_idx);
/* Hook to call when after each vcpu run */
void (*after_vcpu_run)(struct kvm_vcpu *vcpu);
} log_modes[LOG_MODE_NUM] = {
@@ -459,8 +459,8 @@ static void log_mode_create_vm_done(struct kvm_vm *vm)
}
static void log_mode_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
- void *bitmap, uint32_t num_pages,
- uint32_t *ring_buf_idx)
+ void *bitmap, u32 num_pages,
+ u32 *ring_buf_idx)
{
struct log_mode *mode = &log_modes[host_log_mode];
@@ -494,11 +494,11 @@ static void *vcpu_worker(void *data)
static void vm_dirty_log_verify(enum vm_guest_mode mode, unsigned long **bmap)
{
- uint64_t page, nr_dirty_pages = 0, nr_clean_pages = 0;
- uint64_t step = vm_num_host_pages(mode, 1);
+ u64 page, nr_dirty_pages = 0, nr_clean_pages = 0;
+ u64 step = vm_num_host_pages(mode, 1);
for (page = 0; page < host_num_pages; page += step) {
- uint64_t val = *(uint64_t *)(host_test_mem + page * host_page_size);
+ u64 val = *(u64 *)(host_test_mem + page * host_page_size);
bool bmap0_dirty = __test_and_clear_bit_le(page, bmap[0]);
/*
@@ -575,7 +575,7 @@ static void vm_dirty_log_verify(enum vm_guest_mode mode, unsigned long **bmap)
}
static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu,
- uint64_t extra_mem_pages, void *guest_code)
+ u64 extra_mem_pages, void *guest_code)
{
struct kvm_vm *vm;
@@ -592,7 +592,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu,
struct test_params {
unsigned long iterations;
unsigned long interval;
- uint64_t phys_offset;
+ u64 phys_offset;
};
static void run_test(enum vm_guest_mode mode, void *arg)
@@ -601,7 +601,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
unsigned long *bmap[2];
- uint32_t ring_buf_idx = 0;
+ u32 ring_buf_idx = 0;
int sem_val;
if (!log_mode_supported()) {
@@ -667,7 +667,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
/* Cache the HVA pointer of the region */
- host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
+ host_test_mem = addr_gpa2hva(vm, (gpa_t)guest_test_phys_mem);
/* Export the shared variables to the guest */
sync_global_to_guest(vm, host_page_size);
diff --git a/tools/testing/selftests/kvm/get-reg-list.c b/tools/testing/selftests/kvm/get-reg-list.c
index f4644c9..216f106 100644
--- a/tools/testing/selftests/kvm/get-reg-list.c
+++ b/tools/testing/selftests/kvm/get-reg-list.c
@@ -216,7 +216,7 @@ static void run_test(struct vcpu_reg_list *c)
* since we don't know the capabilities of any new registers.
*/
for_each_present_blessed_reg(i) {
- uint8_t addr[2048 / 8];
+ u8 addr[2048 / 8];
struct kvm_one_reg reg = {
.id = reg_list->reg[i],
.addr = (__u64)&addr,
diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c
index ec7644a..832ef4d 100644
--- a/tools/testing/selftests/kvm/guest_memfd_test.c
+++ b/tools/testing/selftests/kvm/guest_memfd_test.c
@@ -14,10 +14,10 @@
#include <linux/bitmap.h>
#include <linux/falloc.h>
#include <linux/sizes.h>
-#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include "kvm_syscalls.h"
#include "kvm_util.h"
#include "numaif.h"
#include "test_util.h"
@@ -171,7 +171,7 @@ static void test_numa_allocation(int fd, size_t total_size)
kvm_munmap(mem, total_size);
}
-static void test_collapse(int fd, uint64_t flags)
+static void test_collapse(int fd, u64 flags)
{
const size_t pmd_size = get_trans_hugepagesz();
void *reserved_addr;
@@ -346,7 +346,7 @@ static void test_invalid_punch_hole(int fd, size_t total_size)
}
static void test_create_guest_memfd_invalid_sizes(struct kvm_vm *vm,
- uint64_t guest_memfd_flags)
+ u64 guest_memfd_flags)
{
size_t size;
int fd;
@@ -389,8 +389,8 @@ static void test_create_guest_memfd_multiple(struct kvm_vm *vm)
static void test_guest_memfd_flags(struct kvm_vm *vm)
{
- uint64_t valid_flags = vm_check_cap(vm, KVM_CAP_GUEST_MEMFD_FLAGS);
- uint64_t flag;
+ u64 valid_flags = vm_check_cap(vm, KVM_CAP_GUEST_MEMFD_FLAGS);
+ u64 flag;
int fd;
for (flag = BIT(0); flag; flag <<= 1) {
@@ -419,7 +419,7 @@ do { \
#define gmem_test(__test, __vm, __flags) \
__gmem_test(__test, __vm, __flags, page_size * 4)
-static void __test_guest_memfd(struct kvm_vm *vm, uint64_t flags)
+static void __test_guest_memfd(struct kvm_vm *vm, u64 flags)
{
test_create_guest_memfd_multiple(vm);
test_create_guest_memfd_invalid_sizes(vm, flags);
@@ -452,7 +452,7 @@ static void __test_guest_memfd(struct kvm_vm *vm, uint64_t flags)
static void test_guest_memfd(unsigned long vm_type)
{
struct kvm_vm *vm = vm_create_barebones_type(vm_type);
- uint64_t flags;
+ u64 flags;
test_guest_memfd_flags(vm);
@@ -470,7 +470,7 @@ static void test_guest_memfd(unsigned long vm_type)
kvm_vm_free(vm);
}
-static void guest_code(uint8_t *mem, uint64_t size)
+static void guest_code(u8 *mem, u64 size)
{
size_t i;
@@ -489,12 +489,12 @@ static void test_guest_memfd_guest(void)
* the guest's code, stack, and page tables, and low memory contains
* the PCI hole and other MMIO regions that need to be avoided.
*/
- const uint64_t gpa = SZ_4G;
+ const gpa_t gpa = SZ_4G;
const int slot = 1;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint8_t *mem;
+ u8 *mem;
size_t size;
int fd, i;
@@ -510,7 +510,12 @@ static void test_guest_memfd_guest(void)
"Default VM type should support INIT_SHARED, supported flags = 0x%x",
vm_check_cap(vm, KVM_CAP_GUEST_MEMFD_FLAGS));
- size = vm->page_size;
+ /*
+ * Use the max of the host or guest page size for all operations, as
+ * KVM requires guest_memfd files and memslots to be sized to multiples
+ * of the host page size.
+ */
+ size = max_t(size_t, vm->page_size, page_size);
fd = vm_create_guest_memfd(vm, size, GUEST_MEMFD_FLAG_MMAP |
GUEST_MEMFD_FLAG_INIT_SHARED);
vm_set_user_memory_region2(vm, slot, KVM_MEM_GUEST_MEMFD, gpa, size, NULL, fd, 0);
@@ -519,7 +524,7 @@ static void test_guest_memfd_guest(void)
memset(mem, 0xaa, size);
kvm_munmap(mem, size);
- virt_pg_map(vm, gpa, gpa);
+ virt_map(vm, gpa, gpa, size / vm->page_size);
vcpu_args_set(vcpu, 2, gpa, size);
vcpu_run(vcpu);
diff --git a/tools/testing/selftests/kvm/guest_print_test.c b/tools/testing/selftests/kvm/guest_print_test.c
index bcf5828..79d3fc3 100644
--- a/tools/testing/selftests/kvm/guest_print_test.c
+++ b/tools/testing/selftests/kvm/guest_print_test.c
@@ -16,22 +16,22 @@
#include "ucall_common.h"
struct guest_vals {
- uint64_t a;
- uint64_t b;
- uint64_t type;
+ u64 a;
+ u64 b;
+ u64 type;
};
static struct guest_vals vals;
/* GUEST_PRINTF()/GUEST_ASSERT_FMT() does not support float or double. */
#define TYPE_LIST \
-TYPE(test_type_i64, I64, "%ld", int64_t) \
-TYPE(test_type_u64, U64u, "%lu", uint64_t) \
-TYPE(test_type_x64, U64x, "0x%lx", uint64_t) \
-TYPE(test_type_X64, U64X, "0x%lX", uint64_t) \
-TYPE(test_type_u32, U32u, "%u", uint32_t) \
-TYPE(test_type_x32, U32x, "0x%x", uint32_t) \
-TYPE(test_type_X32, U32X, "0x%X", uint32_t) \
+TYPE(test_type_i64, I64, "%ld", s64) \
+TYPE(test_type_u64, U64u, "%lu", u64) \
+TYPE(test_type_x64, U64x, "0x%lx", u64) \
+TYPE(test_type_X64, U64X, "0x%lX", u64) \
+TYPE(test_type_u32, U32u, "%u", u32) \
+TYPE(test_type_x32, U32x, "0x%x", u32) \
+TYPE(test_type_X32, U32X, "0x%X", u32) \
TYPE(test_type_int, INT, "%d", int) \
TYPE(test_type_char, CHAR, "%c", char) \
TYPE(test_type_str, STR, "'%s'", const char *) \
@@ -56,7 +56,7 @@ static void fn(struct kvm_vcpu *vcpu, T a, T b) \
\
snprintf(expected_printf, UCALL_BUFFER_LEN, PRINTF_FMT_##ext, a, b); \
snprintf(expected_assert, UCALL_BUFFER_LEN, ASSERT_FMT_##ext, a, b); \
- vals = (struct guest_vals){ (uint64_t)a, (uint64_t)b, TYPE_##ext }; \
+ vals = (struct guest_vals){ (u64)a, (u64)b, TYPE_##ext }; \
sync_global_to_guest(vcpu->vm, vals); \
run_test(vcpu, expected_printf, expected_assert); \
}
diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c
index 94bd6ed2..3147f5c 100644
--- a/tools/testing/selftests/kvm/hardware_disable_test.c
+++ b/tools/testing/selftests/kvm/hardware_disable_test.c
@@ -80,7 +80,7 @@ static inline void check_join(pthread_t thread, void **retval)
TEST_ASSERT(r == 0, "%s: failed to join thread", __func__);
}
-static void run_test(uint32_t run)
+static void run_test(u32 run)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -88,7 +88,7 @@ static void run_test(uint32_t run)
pthread_t threads[VCPU_NUM];
pthread_t throw_away;
void *b;
- uint32_t i, j;
+ u32 i, j;
CPU_ZERO(&cpu_set);
for (i = 0; i < VCPU_NUM; i++)
@@ -149,7 +149,7 @@ void wait_for_child_setup(pid_t pid)
int main(int argc, char **argv)
{
- uint32_t i;
+ u32 i;
int s, r;
pid_t pid;
diff --git a/tools/testing/selftests/kvm/include/arm64/arch_timer.h b/tools/testing/selftests/kvm/include/arm64/arch_timer.h
index e2c4e9f..a5836d4 100644
--- a/tools/testing/selftests/kvm/include/arm64/arch_timer.h
+++ b/tools/testing/selftests/kvm/include/arm64/arch_timer.h
@@ -18,20 +18,20 @@ enum arch_timer {
#define CTL_ISTATUS (1 << 2)
#define msec_to_cycles(msec) \
- (timer_get_cntfrq() * (uint64_t)(msec) / 1000)
+ (timer_get_cntfrq() * (u64)(msec) / 1000)
#define usec_to_cycles(usec) \
- (timer_get_cntfrq() * (uint64_t)(usec) / 1000000)
+ (timer_get_cntfrq() * (u64)(usec) / 1000000)
#define cycles_to_usec(cycles) \
- ((uint64_t)(cycles) * 1000000 / timer_get_cntfrq())
+ ((u64)(cycles) * 1000000 / timer_get_cntfrq())
-static inline uint32_t timer_get_cntfrq(void)
+static inline u32 timer_get_cntfrq(void)
{
return read_sysreg(cntfrq_el0);
}
-static inline uint64_t timer_get_cntct(enum arch_timer timer)
+static inline u64 timer_get_cntct(enum arch_timer timer)
{
isb();
@@ -48,7 +48,7 @@ static inline uint64_t timer_get_cntct(enum arch_timer timer)
return 0;
}
-static inline void timer_set_cval(enum arch_timer timer, uint64_t cval)
+static inline void timer_set_cval(enum arch_timer timer, u64 cval)
{
switch (timer) {
case VIRTUAL:
@@ -64,7 +64,7 @@ static inline void timer_set_cval(enum arch_timer timer, uint64_t cval)
isb();
}
-static inline uint64_t timer_get_cval(enum arch_timer timer)
+static inline u64 timer_get_cval(enum arch_timer timer)
{
switch (timer) {
case VIRTUAL:
@@ -79,7 +79,7 @@ static inline uint64_t timer_get_cval(enum arch_timer timer)
return 0;
}
-static inline void timer_set_tval(enum arch_timer timer, int32_t tval)
+static inline void timer_set_tval(enum arch_timer timer, s32 tval)
{
switch (timer) {
case VIRTUAL:
@@ -95,7 +95,7 @@ static inline void timer_set_tval(enum arch_timer timer, int32_t tval)
isb();
}
-static inline int32_t timer_get_tval(enum arch_timer timer)
+static inline s32 timer_get_tval(enum arch_timer timer)
{
isb();
switch (timer) {
@@ -111,7 +111,7 @@ static inline int32_t timer_get_tval(enum arch_timer timer)
return 0;
}
-static inline void timer_set_ctl(enum arch_timer timer, uint32_t ctl)
+static inline void timer_set_ctl(enum arch_timer timer, u32 ctl)
{
switch (timer) {
case VIRTUAL:
@@ -127,7 +127,7 @@ static inline void timer_set_ctl(enum arch_timer timer, uint32_t ctl)
isb();
}
-static inline uint32_t timer_get_ctl(enum arch_timer timer)
+static inline u32 timer_get_ctl(enum arch_timer timer)
{
switch (timer) {
case VIRTUAL:
@@ -142,15 +142,15 @@ static inline uint32_t timer_get_ctl(enum arch_timer timer)
return 0;
}
-static inline void timer_set_next_cval_ms(enum arch_timer timer, uint32_t msec)
+static inline void timer_set_next_cval_ms(enum arch_timer timer, u32 msec)
{
- uint64_t now_ct = timer_get_cntct(timer);
- uint64_t next_ct = now_ct + msec_to_cycles(msec);
+ u64 now_ct = timer_get_cntct(timer);
+ u64 next_ct = now_ct + msec_to_cycles(msec);
timer_set_cval(timer, next_ct);
}
-static inline void timer_set_next_tval_ms(enum arch_timer timer, uint32_t msec)
+static inline void timer_set_next_tval_ms(enum arch_timer timer, u32 msec)
{
timer_set_tval(timer, msec_to_cycles(msec));
}
diff --git a/tools/testing/selftests/kvm/include/arm64/delay.h b/tools/testing/selftests/kvm/include/arm64/delay.h
index 329e4f5..6a5d463 100644
--- a/tools/testing/selftests/kvm/include/arm64/delay.h
+++ b/tools/testing/selftests/kvm/include/arm64/delay.h
@@ -8,10 +8,10 @@
#include "arch_timer.h"
-static inline void __delay(uint64_t cycles)
+static inline void __delay(u64 cycles)
{
enum arch_timer timer = VIRTUAL;
- uint64_t start = timer_get_cntct(timer);
+ u64 start = timer_get_cntct(timer);
while ((timer_get_cntct(timer) - start) < cycles)
cpu_relax();
diff --git a/tools/testing/selftests/kvm/include/arm64/gic.h b/tools/testing/selftests/kvm/include/arm64/gic.h
index cc7a7f3..6157450 100644
--- a/tools/testing/selftests/kvm/include/arm64/gic.h
+++ b/tools/testing/selftests/kvm/include/arm64/gic.h
@@ -48,8 +48,8 @@ void gic_set_dir(unsigned int intid);
* split is true, EOI drops the priority and deactivates the interrupt.
*/
void gic_set_eoi_split(bool split);
-void gic_set_priority_mask(uint64_t mask);
-void gic_set_priority(uint32_t intid, uint32_t prio);
+void gic_set_priority_mask(u64 mask);
+void gic_set_priority(u32 intid, u32 prio);
void gic_irq_set_active(unsigned int intid);
void gic_irq_clear_active(unsigned int intid);
bool gic_irq_get_active(unsigned int intid);
@@ -59,7 +59,7 @@ bool gic_irq_get_pending(unsigned int intid);
void gic_irq_set_config(unsigned int intid, bool is_edge);
void gic_irq_set_group(unsigned int intid, bool group);
-void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,
- vm_paddr_t pend_table);
+void gic_rdist_enable_lpis(gpa_t cfg_table, size_t cfg_table_size,
+ gpa_t pend_table);
#endif /* SELFTEST_KVM_GIC_H */
diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v3_its.h b/tools/testing/selftests/kvm/include/arm64/gic_v3_its.h
index 58feef3..a43a407 100644
--- a/tools/testing/selftests/kvm/include/arm64/gic_v3_its.h
+++ b/tools/testing/selftests/kvm/include/arm64/gic_v3_its.h
@@ -5,11 +5,10 @@
#include <linux/sizes.h>
-void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz,
- vm_paddr_t device_tbl, size_t device_tbl_sz,
- vm_paddr_t cmdq, size_t cmdq_size);
+void its_init(gpa_t coll_tbl, size_t coll_tbl_sz, gpa_t device_tbl,
+ size_t device_tbl_sz, gpa_t cmdq, size_t cmdq_size);
-void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base,
+void its_send_mapd_cmd(void *cmdq_base, u32 device_id, gpa_t itt_base,
size_t itt_size, bool valid);
void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid);
void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,
diff --git a/tools/testing/selftests/kvm/include/arm64/processor.h b/tools/testing/selftests/kvm/include/arm64/processor.h
index ac97a1c..b8a902b 100644
--- a/tools/testing/selftests/kvm/include/arm64/processor.h
+++ b/tools/testing/selftests/kvm/include/arm64/processor.h
@@ -128,7 +128,7 @@
#define PTE_ADDR_51_50_LPA2_SHIFT 8
void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init);
-struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
+struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, u32 vcpu_id,
struct kvm_vcpu_init *init, void *guest_code);
struct ex_regs {
@@ -167,8 +167,8 @@ enum {
(v) == VECTOR_SYNC_LOWER_64 || \
(v) == VECTOR_SYNC_LOWER_32)
-void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k,
- uint32_t *ipa16k, uint32_t *ipa64k);
+void aarch64_get_supported_page_sizes(u32 ipa, u32 *ipa4k,
+ u32 *ipa16k, u32 *ipa64k);
void vm_init_descriptor_tables(struct kvm_vm *vm);
void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu);
@@ -179,8 +179,8 @@ void vm_install_exception_handler(struct kvm_vm *vm,
void vm_install_sync_handler(struct kvm_vm *vm,
int vector, int ec, handler_fn handler);
-uint64_t *virt_get_pte_hva_at_level(struct kvm_vm *vm, vm_vaddr_t gva, int level);
-uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva);
+u64 *virt_get_pte_hva_at_level(struct kvm_vm *vm, gva_t gva, int level);
+u64 *virt_get_pte_hva(struct kvm_vm *vm, gva_t gva);
static inline void cpu_relax(void)
{
@@ -287,9 +287,9 @@ struct arm_smccc_res {
* @res: pointer to write the return values from registers x0-x3
*
*/
-void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
- uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5,
- uint64_t arg6, struct arm_smccc_res *res);
+void smccc_hvc(u32 function_id, u64 arg0, u64 arg1,
+ u64 arg2, u64 arg3, u64 arg4, u64 arg5,
+ u64 arg6, struct arm_smccc_res *res);
/**
* smccc_smc - Invoke a SMCCC function using the smc conduit
@@ -298,9 +298,9 @@ void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
* @res: pointer to write the return values from registers x0-x3
*
*/
-void smccc_smc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
- uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5,
- uint64_t arg6, struct arm_smccc_res *res);
+void smccc_smc(u32 function_id, u64 arg0, u64 arg1,
+ u64 arg2, u64 arg3, u64 arg4, u64 arg5,
+ u64 arg6, struct arm_smccc_res *res);
/* Execute a Wait For Interrupt instruction. */
void wfi(void);
diff --git a/tools/testing/selftests/kvm/include/arm64/ucall.h b/tools/testing/selftests/kvm/include/arm64/ucall.h
index 4ec801f3..2210d3d 100644
--- a/tools/testing/selftests/kvm/include/arm64/ucall.h
+++ b/tools/testing/selftests/kvm/include/arm64/ucall.h
@@ -10,9 +10,9 @@
* ucall_exit_mmio_addr holds per-VM values (global data is duplicated by each
* VM), it must not be accessed from host code.
*/
-extern vm_vaddr_t *ucall_exit_mmio_addr;
+extern gva_t *ucall_exit_mmio_addr;
-static inline void ucall_arch_do_ucall(vm_vaddr_t uc)
+static inline void ucall_arch_do_ucall(gva_t uc)
{
WRITE_ONCE(*ucall_exit_mmio_addr, uc);
}
diff --git a/tools/testing/selftests/kvm/include/arm64/vgic.h b/tools/testing/selftests/kvm/include/arm64/vgic.h
index 688becc..1f8b043 100644
--- a/tools/testing/selftests/kvm/include/arm64/vgic.h
+++ b/tools/testing/selftests/kvm/include/arm64/vgic.h
@@ -11,27 +11,27 @@
#include "kvm_util.h"
#define REDIST_REGION_ATTR_ADDR(count, base, flags, index) \
- (((uint64_t)(count) << 52) | \
- ((uint64_t)((base) >> 16) << 16) | \
- ((uint64_t)(flags) << 12) | \
+ (((u64)(count) << 52) | \
+ ((u64)((base) >> 16) << 16) | \
+ ((u64)(flags) << 12) | \
index)
bool kvm_supports_vgic_v3(void);
-int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs);
+int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, u32 nr_irqs);
void __vgic_v3_init(int fd);
-int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs);
+int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, u32 nr_irqs);
#define VGIC_MAX_RESERVED 1023
-void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
-int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
+void kvm_irq_set_level_info(int gic_fd, u32 intid, int level);
+int _kvm_irq_set_level_info(int gic_fd, u32 intid, int level);
-void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
-int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
+void kvm_arm_irq_line(struct kvm_vm *vm, u32 intid, int level);
+int _kvm_arm_irq_line(struct kvm_vm *vm, u32 intid, int level);
/* The vcpu arg only applies to private interrupts. */
-void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu);
-void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu);
+void kvm_irq_write_ispendr(int gic_fd, u32 intid, struct kvm_vcpu *vcpu);
+void kvm_irq_write_isactiver(int gic_fd, u32 intid, struct kvm_vcpu *vcpu);
#define KVM_IRQCHIP_NUM_PINS (1020 - 32)
diff --git a/tools/testing/selftests/kvm/include/kvm_syscalls.h b/tools/testing/selftests/kvm/include/kvm_syscalls.h
index 843c990..067a4c9 100644
--- a/tools/testing/selftests/kvm/include/kvm_syscalls.h
+++ b/tools/testing/selftests/kvm/include/kvm_syscalls.h
@@ -2,8 +2,18 @@
#ifndef SELFTEST_KVM_SYSCALLS_H
#define SELFTEST_KVM_SYSCALLS_H
+/*
+ * Include both the kernel and libc versions of mman.h. The kernel provides
+ * the most up-to-date flags and definitions, while libc provides the syscall
+ * wrappers tests expect.
+ */
+#include <linux/mman.h>
+
+#include <sys/mman.h>
#include <sys/syscall.h>
+#include <test_util.h>
+
#define MAP_ARGS0(m,...)
#define MAP_ARGS1(m,t,a,...) m(t,a)
#define MAP_ARGS2(m,t,a,...) m(t,a), MAP_ARGS1(m,__VA_ARGS__)
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index f861242..2ecaaa0 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -58,7 +58,7 @@ struct kvm_binary_stats {
struct kvm_vcpu {
struct list_head list;
- uint32_t id;
+ u32 id;
int fd;
struct kvm_vm *vm;
struct kvm_run *run;
@@ -70,8 +70,8 @@ struct kvm_vcpu {
#endif
struct kvm_binary_stats stats;
struct kvm_dirty_gfn *dirty_gfns;
- uint32_t fetch_index;
- uint32_t dirty_gfns_count;
+ u32 fetch_index;
+ u32 dirty_gfns_count;
};
struct userspace_mem_regions {
@@ -90,7 +90,7 @@ enum kvm_mem_region_type {
struct kvm_mmu {
bool pgd_created;
- uint64_t pgd;
+ u64 pgd;
int pgtable_levels;
struct kvm_mmu_arch arch;
@@ -105,16 +105,16 @@ struct kvm_vm {
unsigned int page_shift;
unsigned int pa_bits;
unsigned int va_bits;
- uint64_t max_gfn;
+ u64 max_gfn;
struct list_head vcpus;
struct userspace_mem_regions regions;
struct sparsebit *vpages_valid;
struct sparsebit *vpages_mapped;
bool has_irqchip;
- vm_paddr_t ucall_mmio_addr;
- vm_vaddr_t handlers;
- uint32_t dirty_ring_size;
- uint64_t gpa_tag_mask;
+ gpa_t ucall_mmio_addr;
+ gva_t handlers;
+ u32 dirty_ring_size;
+ gpa_t gpa_tag_mask;
/*
* "mmu" is the guest's stage-1, with a short name because the vast
@@ -132,7 +132,7 @@ struct kvm_vm {
* allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
* memslot.
*/
- uint32_t memslots[NR_MEM_REGIONS];
+ u32 memslots[NR_MEM_REGIONS];
};
struct vcpu_reg_sublist {
@@ -164,7 +164,7 @@ struct vcpu_reg_list {
else
struct userspace_mem_region *
-memslot2region(struct kvm_vm *vm, uint32_t memslot);
+memslot2region(struct kvm_vm *vm, u32 memslot);
static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
enum kvm_mem_region_type type)
@@ -213,13 +213,13 @@ enum vm_guest_mode {
};
struct vm_shape {
- uint32_t type;
- uint8_t mode;
- uint8_t pad0;
- uint16_t pad1;
+ u32 type;
+ u8 mode;
+ u8 pad0;
+ u16 pad1;
};
-kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t));
+kvm_static_assert(sizeof(struct vm_shape) == sizeof(u64));
#define VM_TYPE_DEFAULT 0
@@ -404,21 +404,22 @@ static inline int vm_check_cap(struct kvm_vm *vm, long cap)
return ret;
}
-static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
+static inline int __vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0)
{
struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
}
-static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
+
+static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0)
{
struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
}
-static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
- uint64_t size, uint64_t attributes)
+static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa,
+ u64 size, u64 attributes)
{
struct kvm_memory_attributes attr = {
.attributes = attributes,
@@ -438,35 +439,35 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
}
-static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,
- uint64_t size)
+static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa,
+ u64 size)
{
vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
}
-static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,
- uint64_t size)
+static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa,
+ u64 size)
{
vm_set_memory_attributes(vm, gpa, size, 0);
}
-void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,
+void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size,
bool punch_hole);
-static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,
- uint64_t size)
+static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, gpa_t gpa,
+ u64 size)
{
vm_guest_mem_fallocate(vm, gpa, size, true);
}
-static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,
- uint64_t size)
+static inline void vm_guest_mem_allocate(struct kvm_vm *vm, gpa_t gpa,
+ u64 size)
{
vm_guest_mem_fallocate(vm, gpa, size, false);
}
-void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
-const char *vm_guest_mode_string(uint32_t i);
+void vm_enable_dirty_ring(struct kvm_vm *vm, u32 ring_size);
+const char *vm_guest_mode_string(u32 i);
void kvm_vm_free(struct kvm_vm *vmp);
void kvm_vm_restart(struct kvm_vm *vmp);
@@ -474,7 +475,7 @@ void kvm_vm_release(struct kvm_vm *vmp);
void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
int kvm_memfd_alloc(size_t size, bool hugepages);
-void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
+void vm_dump(FILE *stream, struct kvm_vm *vm, u8 indent);
static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
{
@@ -484,7 +485,7 @@ static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
}
static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
- uint64_t first_page, uint32_t num_pages)
+ u64 first_page, u32 num_pages)
{
struct kvm_clear_dirty_log args = {
.dirty_bitmap = log,
@@ -496,14 +497,14 @@ static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log
vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);
}
-static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
+static inline u32 kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
{
return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);
}
static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm,
- uint64_t address,
- uint64_t size, bool pio)
+ u64 address,
+ u64 size, bool pio)
{
struct kvm_coalesced_mmio_zone zone = {
.addr = address,
@@ -515,8 +516,8 @@ static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm,
}
static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm,
- uint64_t address,
- uint64_t size, bool pio)
+ u64 address,
+ u64 size, bool pio)
{
struct kvm_coalesced_mmio_zone zone = {
.addr = address,
@@ -535,8 +536,8 @@ static inline int vm_get_stats_fd(struct kvm_vm *vm)
return fd;
}
-static inline int __kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
- uint32_t flags)
+static inline int __kvm_irqfd(struct kvm_vm *vm, u32 gsi, int eventfd,
+ u32 flags)
{
struct kvm_irqfd irqfd = {
.fd = eventfd,
@@ -548,20 +549,19 @@ static inline int __kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
return __vm_ioctl(vm, KVM_IRQFD, &irqfd);
}
-static inline void kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
- uint32_t flags)
+static inline void kvm_irqfd(struct kvm_vm *vm, u32 gsi, int eventfd, u32 flags)
{
int ret = __kvm_irqfd(vm, gsi, eventfd, flags);
TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_IRQFD, ret, vm);
}
-static inline void kvm_assign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
+static inline void kvm_assign_irqfd(struct kvm_vm *vm, u32 gsi, int eventfd)
{
kvm_irqfd(vm, gsi, eventfd, 0);
}
-static inline void kvm_deassign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
+static inline void kvm_deassign_irqfd(struct kvm_vm *vm, u32 gsi, int eventfd)
{
kvm_irqfd(vm, gsi, eventfd, KVM_IRQFD_FLAG_DEASSIGN);
}
@@ -610,15 +610,15 @@ static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc
}
void read_stat_data(int stats_fd, struct kvm_stats_header *header,
- struct kvm_stats_desc *desc, uint64_t *data,
+ struct kvm_stats_desc *desc, u64 *data,
size_t max_elements);
void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
- uint64_t *data, size_t max_elements);
+ u64 *data, size_t max_elements);
#define __get_stat(stats, stat) \
({ \
- uint64_t data; \
+ u64 data; \
\
kvm_get_stat(stats, #stat, &data, 1); \
data; \
@@ -664,8 +664,8 @@ static inline bool is_smt_on(void)
void vm_create_irqchip(struct kvm_vm *vm);
-static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
- uint64_t flags)
+static inline int __vm_create_guest_memfd(struct kvm_vm *vm, u64 size,
+ u64 flags)
{
struct kvm_create_guest_memfd guest_memfd = {
.size = size,
@@ -675,8 +675,8 @@ static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
}
-static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
- uint64_t flags)
+static inline int vm_create_guest_memfd(struct kvm_vm *vm, u64 size,
+ u64 flags)
{
int fd = __vm_create_guest_memfd(vm, size, flags);
@@ -684,24 +684,23 @@ static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
return fd;
}
-void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva);
-int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva);
-void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva,
- uint32_t guest_memfd, uint64_t guest_memfd_offset);
-int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva,
- uint32_t guest_memfd, uint64_t guest_memfd_offset);
+void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva);
+int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva);
+void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva,
+ u32 guest_memfd, u64 guest_memfd_offset);
+int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva,
+ u32 guest_memfd, u64 guest_memfd_offset);
void vm_userspace_mem_region_add(struct kvm_vm *vm,
enum vm_mem_backing_src_type src_type,
- uint64_t gpa, uint32_t slot, uint64_t npages,
- uint32_t flags);
+ gpa_t gpa, u32 slot, u64 npages, u32 flags);
void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
- uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags,
- int guest_memfd_fd, uint64_t guest_memfd_offset);
+ gpa_t gpa, u32 slot, u64 npages, u32 flags,
+ int guest_memfd_fd, u64 guest_memfd_offset);
#ifndef vm_arch_has_protected_memory
static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
@@ -710,36 +709,34 @@ static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
}
#endif
-void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
-void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot);
-void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
-void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
-struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
-void vm_populate_vaddr_bitmap(struct kvm_vm *vm);
-vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
-vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
-vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
- enum kvm_mem_region_type type);
-vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
- vm_vaddr_t vaddr_min,
- enum kvm_mem_region_type type);
-vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
-vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
- enum kvm_mem_region_type type);
-vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
+void vm_mem_region_set_flags(struct kvm_vm *vm, u32 slot, u32 flags);
+void vm_mem_region_reload(struct kvm_vm *vm, u32 slot);
+void vm_mem_region_move(struct kvm_vm *vm, u32 slot, u64 new_gpa);
+void vm_mem_region_delete(struct kvm_vm *vm, u32 slot);
+struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, u32 vcpu_id);
+void vm_populate_gva_bitmap(struct kvm_vm *vm);
+gva_t vm_unused_gva_gap(struct kvm_vm *vm, size_t sz, gva_t min_gva);
+gva_t vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva);
+gva_t __vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva,
+ enum kvm_mem_region_type type);
+gva_t vm_alloc_shared(struct kvm_vm *vm, size_t sz, gva_t min_gva,
+ enum kvm_mem_region_type type);
+gva_t vm_alloc_pages(struct kvm_vm *vm, int nr_pages);
+gva_t __vm_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type);
+gva_t vm_alloc_page(struct kvm_vm *vm);
-void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
+void virt_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa,
unsigned int npages);
-void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);
-void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);
-vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
-void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);
+void *addr_gpa2hva(struct kvm_vm *vm, gpa_t gpa);
+void *addr_gva2hva(struct kvm_vm *vm, gva_t gva);
+gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
+void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa);
#ifndef vcpu_arch_put_guest
#define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0)
#endif
-static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa)
+static inline gpa_t vm_untag_gpa(struct kvm_vm *vm, gpa_t gpa)
{
return gpa & ~vm->gpa_tag_mask;
}
@@ -755,8 +752,8 @@ static inline int __vcpu_run(struct kvm_vcpu *vcpu)
void vcpu_run_complete_io(struct kvm_vcpu *vcpu);
struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);
-static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,
- uint64_t arg0)
+static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, u32 cap,
+ u64 arg0)
{
struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
@@ -811,31 +808,34 @@ static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);
}
-static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
+static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, u64 id, void *addr)
{
- struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
+ struct kvm_one_reg reg = { .id = id, .addr = (u64)addr };
return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®);
}
-static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+
+static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, u64 id, u64 val)
{
- struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
+ struct kvm_one_reg reg = { .id = id, .addr = (u64)&val };
return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®);
}
-static inline uint64_t vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id)
+
+static inline u64 vcpu_get_reg(struct kvm_vcpu *vcpu, u64 id)
{
- uint64_t val;
- struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
+ u64 val;
+ struct kvm_one_reg reg = { .id = id, .addr = (u64)&val };
TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®);
return val;
}
-static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+
+static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u64 id, u64 val)
{
- struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
+ struct kvm_one_reg reg = { .id = id, .addr = (u64)&val };
TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
@@ -880,75 +880,75 @@ static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)
return fd;
}
-int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);
+int __kvm_has_device_attr(int dev_fd, u32 group, u64 attr);
-static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
+static inline void kvm_has_device_attr(int dev_fd, u32 group, u64 attr)
{
int ret = __kvm_has_device_attr(dev_fd, group, attr);
TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
}
-int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);
+int __kvm_device_attr_get(int dev_fd, u32 group, u64 attr, void *val);
-static inline void kvm_device_attr_get(int dev_fd, uint32_t group,
- uint64_t attr, void *val)
+static inline void kvm_device_attr_get(int dev_fd, u32 group,
+ u64 attr, void *val)
{
int ret = __kvm_device_attr_get(dev_fd, group, attr, val);
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));
}
-int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);
+int __kvm_device_attr_set(int dev_fd, u32 group, u64 attr, void *val);
-static inline void kvm_device_attr_set(int dev_fd, uint32_t group,
- uint64_t attr, void *val)
+static inline void kvm_device_attr_set(int dev_fd, u32 group,
+ u64 attr, void *val)
{
int ret = __kvm_device_attr_set(dev_fd, group, attr, val);
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));
}
-static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr)
+static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr)
{
return __kvm_has_device_attr(vcpu->fd, group, attr);
}
-static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr)
+static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr)
{
kvm_has_device_attr(vcpu->fd, group, attr);
}
-static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr, void *val)
+static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr, void *val)
{
return __kvm_device_attr_get(vcpu->fd, group, attr, val);
}
-static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr, void *val)
+static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr, void *val)
{
kvm_device_attr_get(vcpu->fd, group, attr, val);
}
-static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr, void *val)
+static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr, void *val)
{
return __kvm_device_attr_set(vcpu->fd, group, attr, val);
}
-static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
- uint64_t attr, void *val)
+static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, u32 group,
+ u64 attr, void *val)
{
kvm_device_attr_set(vcpu->fd, group, attr, val);
}
-int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);
-int __kvm_create_device(struct kvm_vm *vm, uint64_t type);
+int __kvm_test_create_device(struct kvm_vm *vm, u64 type);
+int __kvm_create_device(struct kvm_vm *vm, u64 type);
-static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)
+static inline int kvm_create_device(struct kvm_vm *vm, u64 type)
{
int fd = __kvm_create_device(vm, type);
@@ -964,7 +964,7 @@ void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
* Input Args:
* vcpu - vCPU
* num - number of arguments
- * ... - arguments, each of type uint64_t
+ * ... - arguments, each of type u64
*
* Output Args: None
*
@@ -972,40 +972,38 @@ void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
*
* Sets the first @num input parameters for the function at @vcpu's entry point,
* per the C calling convention of the architecture, to the values given as
- * variable args. Each of the variable args is expected to be of type uint64_t.
+ * variable args. Each of the variable args is expected to be of type u64.
* The maximum @num can be is specific to the architecture.
*/
void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
-void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
-int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
+void kvm_irq_line(struct kvm_vm *vm, u32 irq, int level);
+int _kvm_irq_line(struct kvm_vm *vm, u32 irq, int level);
#define KVM_MAX_IRQ_ROUTES 4096
struct kvm_irq_routing *kvm_gsi_routing_create(void);
void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
- uint32_t gsi, uint32_t pin);
+ u32 gsi, u32 pin);
int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
const char *exit_reason_str(unsigned int exit_reason);
-vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
- uint32_t memslot);
-vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
- vm_paddr_t paddr_min, uint32_t memslot,
- bool protected);
-vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
+gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot);
+gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa,
+ u32 memslot, bool protected);
+gpa_t vm_alloc_page_table(struct kvm_vm *vm);
-static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
- vm_paddr_t paddr_min, uint32_t memslot)
+static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
+ gpa_t min_gpa, u32 memslot)
{
/*
* By default, allocate memory as protected for VMs that support
* protected memory, as the majority of memory for such VMs is
* protected, i.e. using shared memory is effectively opt-in.
*/
- return __vm_phy_pages_alloc(vm, num, paddr_min, memslot,
+ return __vm_phy_pages_alloc(vm, num, min_gpa, memslot,
vm_arch_has_protected_memory(vm));
}
@@ -1016,8 +1014,8 @@ static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
* calculate the amount of memory needed for per-vCPU data, e.g. stacks.
*/
struct kvm_vm *____vm_create(struct vm_shape shape);
-struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
- uint64_t nr_extra_pages);
+struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
+ u64 nr_extra_pages);
static inline struct kvm_vm *vm_create_barebones(void)
{
@@ -1034,16 +1032,16 @@ static inline struct kvm_vm *vm_create_barebones_type(unsigned long type)
return ____vm_create(shape);
}
-static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
+static inline struct kvm_vm *vm_create(u32 nr_runnable_vcpus)
{
return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);
}
-struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
- uint64_t extra_mem_pages,
+struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, u32 nr_vcpus,
+ u64 extra_mem_pages,
void *guest_code, struct kvm_vcpu *vcpus[]);
-static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
+static inline struct kvm_vm *vm_create_with_vcpus(u32 nr_vcpus,
void *guest_code,
struct kvm_vcpu *vcpus[])
{
@@ -1054,7 +1052,7 @@ static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
struct kvm_vcpu **vcpu,
- uint64_t extra_mem_pages,
+ u64 extra_mem_pages,
void *guest_code);
/*
@@ -1062,7 +1060,7 @@ struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
* additional pages of guest memory. Returns the VM and vCPU (via out param).
*/
static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
- uint64_t extra_mem_pages,
+ u64 extra_mem_pages,
void *guest_code)
{
return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,
@@ -1084,7 +1082,7 @@ static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape
struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
-void kvm_set_files_rlimit(uint32_t nr_vcpus);
+void kvm_set_files_rlimit(u32 nr_vcpus);
int __pin_task_to_cpu(pthread_t task, int cpu);
@@ -1115,7 +1113,7 @@ static inline int pin_self_to_any_cpu(void)
}
void kvm_print_vcpu_pinning_help(void);
-void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
+void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[],
int nr_vcpus);
unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
@@ -1131,12 +1129,12 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
}
#define sync_global_to_guest(vm, g) ({ \
- typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \
+ typeof(g) *_p = addr_gva2hva(vm, (gva_t)&(g)); \
memcpy(_p, &(g), sizeof(g)); \
})
#define sync_global_from_guest(vm, g) ({ \
- typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \
+ typeof(g) *_p = addr_gva2hva(vm, (gva_t)&(g)); \
memcpy(&(g), _p, sizeof(g)); \
})
@@ -1147,7 +1145,7 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
* undesirable to change the host's copy of the global.
*/
#define write_guest_global(vm, g, val) ({ \
- typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \
+ typeof(g) *_p = addr_gva2hva(vm, (gva_t)&(g)); \
typeof(g) _val = val; \
\
memcpy(_p, &(_val), sizeof(g)); \
@@ -1156,10 +1154,10 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);
void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,
- uint8_t indent);
+ u8 indent);
static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
- uint8_t indent)
+ u8 indent)
{
vcpu_arch_dump(stream, vcpu, indent);
}
@@ -1171,10 +1169,10 @@ static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
* vm - Virtual Machine
* vcpu_id - The id of the VCPU to add to the VM.
*/
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id);
void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code);
-static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
+static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, u32 vcpu_id,
void *guest_code)
{
struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id);
@@ -1185,10 +1183,10 @@ static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
}
/* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */
-struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);
+struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, u32 vcpu_id);
static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,
- uint32_t vcpu_id)
+ u32 vcpu_id)
{
return vm_arch_vcpu_recreate(vm, vcpu_id);
}
@@ -1203,27 +1201,15 @@ static inline void virt_pgd_alloc(struct kvm_vm *vm)
}
/*
- * VM Virtual Page Map
- *
- * Input Args:
- * vm - Virtual Machine
- * vaddr - VM Virtual Address
- * paddr - VM Physical Address
- * memslot - Memory region slot for new virtual translation tables
- *
- * Output Args: None
- *
- * Return: None
- *
* Within @vm, creates a virtual translation for the page starting
- * at @vaddr to the page starting at @paddr.
+ * at @gva to the page starting at @gpa.
*/
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa);
-static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+static inline void virt_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
- virt_arch_pg_map(vm, vaddr, paddr);
- sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
+ virt_arch_pg_map(vm, gva, gpa);
+ sparsebit_set(vm->vpages_mapped, gva >> vm->page_shift);
}
@@ -1242,9 +1228,9 @@ static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr
* Returns the VM physical address of the translated VM virtual
* address given by @gva.
*/
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva);
-static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+static inline gpa_t addr_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
return addr_arch_gva2gpa(vm, gva);
}
@@ -1264,9 +1250,9 @@ static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
* Dumps to the FILE stream given by @stream, the contents of all the
* virtual translation tables for the VM given by @vm.
*/
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent);
-static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+static inline void virt_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
virt_arch_dump(stream, vm, indent);
}
@@ -1277,7 +1263,7 @@ static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)
return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);
}
-static inline uint64_t vm_page_align(struct kvm_vm *vm, uint64_t v)
+static inline u64 vm_page_align(struct kvm_vm *vm, u64 v)
{
return (v + vm->page_size - 1) & ~(vm->page_size - 1);
}
@@ -1293,9 +1279,9 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus);
void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm);
void kvm_arch_vm_release(struct kvm_vm *vm);
-bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr);
+bool vm_is_gpa_protected(struct kvm_vm *vm, gpa_t gpa);
-uint32_t guest_get_vcpuid(void);
+u32 guest_get_vcpuid(void);
bool kvm_arch_has_default_irqchip(void);
diff --git a/tools/testing/selftests/kvm/include/kvm_util_types.h b/tools/testing/selftests/kvm/include/kvm_util_types.h
index 0366e9b..ed0087e 100644
--- a/tools/testing/selftests/kvm/include/kvm_util_types.h
+++ b/tools/testing/selftests/kvm/include/kvm_util_types.h
@@ -2,6 +2,8 @@
#ifndef SELFTEST_KVM_UTIL_TYPES_H
#define SELFTEST_KVM_UTIL_TYPES_H
+#include <linux/types.h>
+
/*
* Provide a version of static_assert() that is guaranteed to have an optional
* message param. _GNU_SOURCE is defined for all KVM selftests, _GNU_SOURCE
@@ -14,9 +16,9 @@
#define __kvm_static_assert(expr, msg, ...) _Static_assert(expr, msg)
#define kvm_static_assert(expr, ...) __kvm_static_assert(expr, ##__VA_ARGS__, #expr)
-typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */
-typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */
+typedef u64 gpa_t; /* Virtual Machine (Guest) physical address */
+typedef u64 gva_t; /* Virtual Machine (Guest) virtual address */
-#define INVALID_GPA (~(uint64_t)0)
+#define INVALID_GPA (~(u64)0)
#endif /* SELFTEST_KVM_UTIL_TYPES_H */
diff --git a/tools/testing/selftests/kvm/include/loongarch/arch_timer.h b/tools/testing/selftests/kvm/include/loongarch/arch_timer.h
index 2ed106b..3888aee 100644
--- a/tools/testing/selftests/kvm/include/loongarch/arch_timer.h
+++ b/tools/testing/selftests/kvm/include/loongarch/arch_timer.h
@@ -70,9 +70,9 @@ static inline void timer_set_next_cmp_ms(unsigned int msec, bool period)
csr_write(val, LOONGARCH_CSR_TCFG);
}
-static inline void __delay(uint64_t cycles)
+static inline void __delay(u64 cycles)
{
- uint64_t start = timer_get_cycles();
+ u64 start = timer_get_cycles();
while ((timer_get_cycles() - start) < cycles)
cpu_relax();
diff --git a/tools/testing/selftests/kvm/include/loongarch/ucall.h b/tools/testing/selftests/kvm/include/loongarch/ucall.h
index 4ec801f3..2210d3d 100644
--- a/tools/testing/selftests/kvm/include/loongarch/ucall.h
+++ b/tools/testing/selftests/kvm/include/loongarch/ucall.h
@@ -10,9 +10,9 @@
* ucall_exit_mmio_addr holds per-VM values (global data is duplicated by each
* VM), it must not be accessed from host code.
*/
-extern vm_vaddr_t *ucall_exit_mmio_addr;
+extern gva_t *ucall_exit_mmio_addr;
-static inline void ucall_arch_do_ucall(vm_vaddr_t uc)
+static inline void ucall_arch_do_ucall(gva_t uc)
{
WRITE_ONCE(*ucall_exit_mmio_addr, uc);
}
diff --git a/tools/testing/selftests/kvm/include/memstress.h b/tools/testing/selftests/kvm/include/memstress.h
index 9071eb6..0d1d623 100644
--- a/tools/testing/selftests/kvm/include/memstress.h
+++ b/tools/testing/selftests/kvm/include/memstress.h
@@ -20,9 +20,9 @@
#define MEMSTRESS_MEM_SLOT_INDEX 1
struct memstress_vcpu_args {
- uint64_t gpa;
- uint64_t gva;
- uint64_t pages;
+ gpa_t gpa;
+ gva_t gva;
+ u64 pages;
/* Only used by the host userspace part of the vCPU thread */
struct kvm_vcpu *vcpu;
@@ -32,11 +32,11 @@ struct memstress_vcpu_args {
struct memstress_args {
struct kvm_vm *vm;
/* The starting address and size of the guest test region. */
- uint64_t gpa;
- uint64_t size;
- uint64_t guest_page_size;
- uint32_t random_seed;
- uint32_t write_percent;
+ gpa_t gpa;
+ u64 size;
+ u64 guest_page_size;
+ u32 random_seed;
+ u32 write_percent;
/* Run vCPUs in L2 instead of L1, if the architecture supports it. */
bool nested;
@@ -45,7 +45,7 @@ struct memstress_args {
/* True if all vCPUs are pinned to pCPUs */
bool pin_vcpus;
/* The vCPU=>pCPU pinning map. Only valid if pin_vcpus is true. */
- uint32_t vcpu_to_pcpu[KVM_MAX_VCPUS];
+ u32 vcpu_to_pcpu[KVM_MAX_VCPUS];
/* Test is done, stop running vCPUs. */
bool stop_vcpus;
@@ -56,27 +56,27 @@ struct memstress_args {
extern struct memstress_args memstress_args;
struct kvm_vm *memstress_create_vm(enum vm_guest_mode mode, int nr_vcpus,
- uint64_t vcpu_memory_bytes, int slots,
+ u64 vcpu_memory_bytes, int slots,
enum vm_mem_backing_src_type backing_src,
bool partition_vcpu_memory_access);
void memstress_destroy_vm(struct kvm_vm *vm);
-void memstress_set_write_percent(struct kvm_vm *vm, uint32_t write_percent);
+void memstress_set_write_percent(struct kvm_vm *vm, u32 write_percent);
void memstress_set_random_access(struct kvm_vm *vm, bool random_access);
void memstress_start_vcpu_threads(int vcpus, void (*vcpu_fn)(struct memstress_vcpu_args *));
void memstress_join_vcpu_threads(int vcpus);
-void memstress_guest_code(uint32_t vcpu_id);
+void memstress_guest_code(u32 vcpu_id);
-uint64_t memstress_nested_pages(int nr_vcpus);
+u64 memstress_nested_pages(int nr_vcpus);
void memstress_setup_nested(struct kvm_vm *vm, int nr_vcpus, struct kvm_vcpu *vcpus[]);
void memstress_enable_dirty_logging(struct kvm_vm *vm, int slots);
void memstress_disable_dirty_logging(struct kvm_vm *vm, int slots);
void memstress_get_dirty_log(struct kvm_vm *vm, unsigned long *bitmaps[], int slots);
void memstress_clear_dirty_log(struct kvm_vm *vm, unsigned long *bitmaps[],
- int slots, uint64_t pages_per_slot);
-unsigned long **memstress_alloc_bitmaps(int slots, uint64_t pages_per_slot);
+ int slots, u64 pages_per_slot);
+unsigned long **memstress_alloc_bitmaps(int slots, u64 pages_per_slot);
void memstress_free_bitmaps(unsigned long *bitmaps[], int slots);
#endif /* SELFTEST_KVM_MEMSTRESS_H */
diff --git a/tools/testing/selftests/kvm/include/riscv/arch_timer.h b/tools/testing/selftests/kvm/include/riscv/arch_timer.h
index 225d81d..28ffc01 100644
--- a/tools/testing/selftests/kvm/include/riscv/arch_timer.h
+++ b/tools/testing/selftests/kvm/include/riscv/arch_timer.h
@@ -14,25 +14,25 @@
static unsigned long timer_freq;
#define msec_to_cycles(msec) \
- ((timer_freq) * (uint64_t)(msec) / 1000)
+ ((timer_freq) * (u64)(msec) / 1000)
#define usec_to_cycles(usec) \
- ((timer_freq) * (uint64_t)(usec) / 1000000)
+ ((timer_freq) * (u64)(usec) / 1000000)
#define cycles_to_usec(cycles) \
- ((uint64_t)(cycles) * 1000000 / (timer_freq))
+ ((u64)(cycles) * 1000000 / (timer_freq))
-static inline uint64_t timer_get_cycles(void)
+static inline u64 timer_get_cycles(void)
{
return csr_read(CSR_TIME);
}
-static inline void timer_set_cmp(uint64_t cval)
+static inline void timer_set_cmp(u64 cval)
{
csr_write(CSR_STIMECMP, cval);
}
-static inline uint64_t timer_get_cmp(void)
+static inline u64 timer_get_cmp(void)
{
return csr_read(CSR_STIMECMP);
}
@@ -47,17 +47,17 @@ static inline void timer_irq_disable(void)
csr_clear(CSR_SIE, IE_TIE);
}
-static inline void timer_set_next_cmp_ms(uint32_t msec)
+static inline void timer_set_next_cmp_ms(u32 msec)
{
- uint64_t now_ct = timer_get_cycles();
- uint64_t next_ct = now_ct + msec_to_cycles(msec);
+ u64 now_ct = timer_get_cycles();
+ u64 next_ct = now_ct + msec_to_cycles(msec);
timer_set_cmp(next_ct);
}
-static inline void __delay(uint64_t cycles)
+static inline void __delay(u64 cycles)
{
- uint64_t start = timer_get_cycles();
+ u64 start = timer_get_cycles();
while ((timer_get_cycles() - start) < cycles)
cpu_relax();
diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h
index 4dade8c..e3acf2a 100644
--- a/tools/testing/selftests/kvm/include/riscv/processor.h
+++ b/tools/testing/selftests/kvm/include/riscv/processor.h
@@ -25,8 +25,7 @@
#define GET_RM(insn) (((insn) & INSN_MASK_FUNCT3) >> INSN_SHIFT_FUNCT3)
#define GET_CSR_NUM(insn) (((insn) & INSN_CSR_MASK) >> INSN_CSR_SHIFT)
-static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype,
- uint64_t idx, uint64_t size)
+static inline u64 __kvm_reg_id(u64 type, u64 subtype, u64 idx, u64 size)
{
return KVM_REG_RISCV | type | subtype | idx | size;
}
@@ -62,14 +61,14 @@ static inline uint64_t __kvm_reg_id(uint64_t type, uint64_t subtype,
KVM_REG_RISCV_SBI_SINGLE, \
idx, KVM_REG_SIZE_ULONG)
-bool __vcpu_has_ext(struct kvm_vcpu *vcpu, uint64_t ext);
+bool __vcpu_has_ext(struct kvm_vcpu *vcpu, u64 ext);
-static inline bool __vcpu_has_isa_ext(struct kvm_vcpu *vcpu, uint64_t isa_ext)
+static inline bool __vcpu_has_isa_ext(struct kvm_vcpu *vcpu, u64 isa_ext)
{
return __vcpu_has_ext(vcpu, RISCV_ISA_EXT_REG(isa_ext));
}
-static inline bool __vcpu_has_sbi_ext(struct kvm_vcpu *vcpu, uint64_t sbi_ext)
+static inline bool __vcpu_has_sbi_ext(struct kvm_vcpu *vcpu, u64 sbi_ext)
{
return __vcpu_has_ext(vcpu, RISCV_SBI_EXT_REG(sbi_ext));
}
diff --git a/tools/testing/selftests/kvm/include/riscv/ucall.h b/tools/testing/selftests/kvm/include/riscv/ucall.h
index a695ae3..2de7c6a 100644
--- a/tools/testing/selftests/kvm/include/riscv/ucall.h
+++ b/tools/testing/selftests/kvm/include/riscv/ucall.h
@@ -7,11 +7,11 @@
#define UCALL_EXIT_REASON KVM_EXIT_RISCV_SBI
-static inline void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
}
-static inline void ucall_arch_do_ucall(vm_vaddr_t uc)
+static inline void ucall_arch_do_ucall(gva_t uc)
{
sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT,
KVM_RISCV_SELFTESTS_SBI_UCALL,
diff --git a/tools/testing/selftests/kvm/include/s390/diag318_test_handler.h b/tools/testing/selftests/kvm/include/s390/diag318_test_handler.h
index b0ed713..6deaf18 100644
--- a/tools/testing/selftests/kvm/include/s390/diag318_test_handler.h
+++ b/tools/testing/selftests/kvm/include/s390/diag318_test_handler.h
@@ -8,6 +8,6 @@
#ifndef SELFTEST_KVM_DIAG318_TEST_HANDLER
#define SELFTEST_KVM_DIAG318_TEST_HANDLER
-uint64_t get_diag318_info(void);
+u64 get_diag318_info(void);
#endif
diff --git a/tools/testing/selftests/kvm/include/s390/facility.h b/tools/testing/selftests/kvm/include/s390/facility.h
index 00a1ced..41a2657 100644
--- a/tools/testing/selftests/kvm/include/s390/facility.h
+++ b/tools/testing/selftests/kvm/include/s390/facility.h
@@ -16,7 +16,7 @@
/* alt_stfle_fac_list[16] + stfle_fac_list[16] */
#define NB_STFL_DOUBLEWORDS 32
-extern uint64_t stfl_doublewords[NB_STFL_DOUBLEWORDS];
+extern u64 stfl_doublewords[NB_STFL_DOUBLEWORDS];
extern bool stfle_flag;
static inline bool test_bit_inv(unsigned long nr, const unsigned long *ptr)
@@ -24,7 +24,7 @@ static inline bool test_bit_inv(unsigned long nr, const unsigned long *ptr)
return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}
-static inline void stfle(uint64_t *fac, unsigned int nb_doublewords)
+static inline void stfle(u64 *fac, unsigned int nb_doublewords)
{
register unsigned long r0 asm("0") = nb_doublewords - 1;
diff --git a/tools/testing/selftests/kvm/include/s390/ucall.h b/tools/testing/selftests/kvm/include/s390/ucall.h
index 8035a87..3907d62 100644
--- a/tools/testing/selftests/kvm/include/s390/ucall.h
+++ b/tools/testing/selftests/kvm/include/s390/ucall.h
@@ -6,11 +6,11 @@
#define UCALL_EXIT_REASON KVM_EXIT_S390_SIEIC
-static inline void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
}
-static inline void ucall_arch_do_ucall(vm_vaddr_t uc)
+static inline void ucall_arch_do_ucall(gva_t uc)
{
/* Exit via DIAGNOSE 0x501 (normally used for breakpoints) */
asm volatile ("diag 0,%0,0x501" : : "a"(uc) : "memory");
diff --git a/tools/testing/selftests/kvm/include/sparsebit.h b/tools/testing/selftests/kvm/include/sparsebit.h
index bc76076..e027e57 100644
--- a/tools/testing/selftests/kvm/include/sparsebit.h
+++ b/tools/testing/selftests/kvm/include/sparsebit.h
@@ -6,7 +6,7 @@
*
* Header file that describes API to the sparsebit library.
* This library provides a memory efficient means of storing
- * the settings of bits indexed via a uint64_t. Memory usage
+ * the settings of bits indexed via a u64. Memory usage
* is reasonable, significantly less than (2^64 / 8) bytes, as
* long as bits that are mostly set or mostly cleared are close
* to each other. This library is efficient in memory usage
@@ -25,8 +25,8 @@ extern "C" {
#endif
struct sparsebit;
-typedef uint64_t sparsebit_idx_t;
-typedef uint64_t sparsebit_num_t;
+typedef u64 sparsebit_idx_t;
+typedef u64 sparsebit_num_t;
struct sparsebit *sparsebit_alloc(void);
void sparsebit_free(struct sparsebit **sbitp);
diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h
index b4872ba..a56271c 100644
--- a/tools/testing/selftests/kvm/include/test_util.h
+++ b/tools/testing/selftests/kvm/include/test_util.h
@@ -19,9 +19,11 @@
#include <errno.h>
#include <unistd.h>
#include <fcntl.h>
-#include <sys/mman.h>
#include "kselftest.h"
+#include <linux/mman.h>
+#include <linux/types.h>
+
#define msecs_to_usecs(msec) ((msec) * 1000ULL)
static inline __printf(1, 2) int _no_printf(const char *format, ...) { return 0; }
@@ -99,25 +101,25 @@ do { \
size_t parse_size(const char *size);
-int64_t timespec_to_ns(struct timespec ts);
-struct timespec timespec_add_ns(struct timespec ts, int64_t ns);
+s64 timespec_to_ns(struct timespec ts);
+struct timespec timespec_add_ns(struct timespec ts, s64 ns);
struct timespec timespec_add(struct timespec ts1, struct timespec ts2);
struct timespec timespec_sub(struct timespec ts1, struct timespec ts2);
struct timespec timespec_elapsed(struct timespec start);
struct timespec timespec_div(struct timespec ts, int divisor);
struct guest_random_state {
- uint32_t seed;
+ u32 seed;
};
-extern uint32_t guest_random_seed;
+extern u32 guest_random_seed;
extern struct guest_random_state guest_rng;
-struct guest_random_state new_guest_random_state(uint32_t seed);
-uint32_t guest_random_u32(struct guest_random_state *state);
+struct guest_random_state new_guest_random_state(u32 seed);
+u32 guest_random_u32(struct guest_random_state *state);
static inline bool __guest_random_bool(struct guest_random_state *state,
- uint8_t percent)
+ u8 percent)
{
return (guest_random_u32(state) % 100) < percent;
}
@@ -127,9 +129,9 @@ static inline bool guest_random_bool(struct guest_random_state *state)
return __guest_random_bool(state, 50);
}
-static inline uint64_t guest_random_u64(struct guest_random_state *state)
+static inline u64 guest_random_u64(struct guest_random_state *state)
{
- return ((uint64_t)guest_random_u32(state) << 32) | guest_random_u32(state);
+ return ((u64)guest_random_u32(state) << 32) | guest_random_u32(state);
}
enum vm_mem_backing_src_type {
@@ -158,7 +160,7 @@ enum vm_mem_backing_src_type {
struct vm_mem_backing_src_alias {
const char *name;
- uint32_t flag;
+ u32 flag;
};
#define MIN_RUN_DELAY_NS 200000UL
@@ -166,9 +168,9 @@ struct vm_mem_backing_src_alias {
bool thp_configured(void);
size_t get_trans_hugepagesz(void);
size_t get_def_hugetlb_pagesz(void);
-const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i);
-size_t get_backing_src_pagesz(uint32_t i);
-bool is_backing_src_hugetlb(uint32_t i);
+const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i);
+size_t get_backing_src_pagesz(u32 i);
+bool is_backing_src_hugetlb(u32 i);
void backing_src_help(const char *flag);
enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name);
long get_run_delay(void);
@@ -189,18 +191,18 @@ static inline bool backing_src_can_be_huge(enum vm_mem_backing_src_type t)
}
/* Aligns x up to the next multiple of size. Size must be a power of 2. */
-static inline uint64_t align_up(uint64_t x, uint64_t size)
+static inline u64 align_up(u64 x, u64 size)
{
- uint64_t mask = size - 1;
+ u64 mask = size - 1;
TEST_ASSERT(size != 0 && !(size & (size - 1)),
"size not a power of 2: %lu", size);
return ((x + mask) & ~mask);
}
-static inline uint64_t align_down(uint64_t x, uint64_t size)
+static inline u64 align_down(u64 x, u64 size)
{
- uint64_t x_aligned_up = align_up(x, size);
+ u64 x_aligned_up = align_up(x, size);
if (x == x_aligned_up)
return x;
@@ -215,7 +217,7 @@ static inline void *align_ptr_up(void *x, size_t size)
int atoi_paranoid(const char *num_str);
-static inline uint32_t atoi_positive(const char *name, const char *num_str)
+static inline u32 atoi_positive(const char *name, const char *num_str)
{
int num = atoi_paranoid(num_str);
@@ -223,7 +225,7 @@ static inline uint32_t atoi_positive(const char *name, const char *num_str)
return num;
}
-static inline uint32_t atoi_non_negative(const char *name, const char *num_str)
+static inline u32 atoi_non_negative(const char *name, const char *num_str)
{
int num = atoi_paranoid(num_str);
diff --git a/tools/testing/selftests/kvm/include/timer_test.h b/tools/testing/selftests/kvm/include/timer_test.h
index 9b6edaa..b7d5d2c 100644
--- a/tools/testing/selftests/kvm/include/timer_test.h
+++ b/tools/testing/selftests/kvm/include/timer_test.h
@@ -18,21 +18,21 @@
/* Timer test cmdline parameters */
struct test_args {
- uint32_t nr_vcpus;
- uint32_t nr_iter;
- uint32_t timer_period_ms;
- uint32_t migration_freq_ms;
- uint32_t timer_err_margin_us;
+ u32 nr_vcpus;
+ u32 nr_iter;
+ u32 timer_period_ms;
+ u32 migration_freq_ms;
+ u32 timer_err_margin_us;
/* Members of struct kvm_arm_counter_offset */
- uint64_t counter_offset;
- uint64_t reserved;
+ u64 counter_offset;
+ u64 reserved;
};
/* Shared variables between host and guest */
struct test_vcpu_shared_data {
- uint32_t nr_iter;
+ u32 nr_iter;
int guest_stage;
- uint64_t xcnt;
+ u64 xcnt;
};
extern struct test_args test_args;
diff --git a/tools/testing/selftests/kvm/include/ucall_common.h b/tools/testing/selftests/kvm/include/ucall_common.h
index d9d6581..cbdcb0a5 100644
--- a/tools/testing/selftests/kvm/include/ucall_common.h
+++ b/tools/testing/selftests/kvm/include/ucall_common.h
@@ -21,26 +21,26 @@ enum {
#define UCALL_BUFFER_LEN 1024
struct ucall {
- uint64_t cmd;
- uint64_t args[UCALL_MAX_ARGS];
+ u64 cmd;
+ u64 args[UCALL_MAX_ARGS];
char buffer[UCALL_BUFFER_LEN];
/* Host virtual address of this struct. */
struct ucall *hva;
};
-void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa);
-void ucall_arch_do_ucall(vm_vaddr_t uc);
+void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa);
+void ucall_arch_do_ucall(gva_t uc);
void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu);
-void ucall(uint64_t cmd, int nargs, ...);
-__printf(2, 3) void ucall_fmt(uint64_t cmd, const char *fmt, ...);
-__printf(5, 6) void ucall_assert(uint64_t cmd, const char *exp,
+void ucall(u64 cmd, int nargs, ...);
+__printf(2, 3) void ucall_fmt(u64 cmd, const char *fmt, ...);
+__printf(5, 6) void ucall_assert(u64 cmd, const char *exp,
const char *file, unsigned int line,
const char *fmt, ...);
-uint64_t get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc);
-void ucall_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa);
-int ucall_nr_pages_required(uint64_t page_size);
+u64 get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc);
+void ucall_init(struct kvm_vm *vm, gpa_t mmio_gpa);
+int ucall_nr_pages_required(u64 page_size);
/*
* Perform userspace call without any associated data. This bare call avoids
@@ -48,7 +48,7 @@ int ucall_nr_pages_required(uint64_t page_size);
* the full ucall() are problematic and/or unwanted. Note, this will come out
* as UCALL_NONE on the backend.
*/
-#define GUEST_UCALL_NONE() ucall_arch_do_ucall((vm_vaddr_t)NULL)
+#define GUEST_UCALL_NONE() ucall_arch_do_ucall((gva_t)NULL)
#define GUEST_SYNC_ARGS(stage, arg1, arg2, arg3, arg4) \
ucall(UCALL_SYNC, 6, "hello", stage, arg1, arg2, arg3, arg4)
diff --git a/tools/testing/selftests/kvm/include/userfaultfd_util.h b/tools/testing/selftests/kvm/include/userfaultfd_util.h
index 60f7f9d..0bc1dc1 100644
--- a/tools/testing/selftests/kvm/include/userfaultfd_util.h
+++ b/tools/testing/selftests/kvm/include/userfaultfd_util.h
@@ -25,7 +25,7 @@ struct uffd_reader_args {
struct uffd_desc {
int uffd;
- uint64_t num_readers;
+ u64 num_readers;
/* Holds the write ends of the pipes for killing the readers. */
int *pipefds;
pthread_t *readers;
@@ -33,8 +33,8 @@ struct uffd_desc {
};
struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
- void *hva, uint64_t len,
- uint64_t num_readers,
+ void *hva, u64 len,
+ u64 num_readers,
uffd_handler_t handler);
void uffd_stop_demand_paging(struct uffd_desc *uffd);
diff --git a/tools/testing/selftests/kvm/include/x86/apic.h b/tools/testing/selftests/kvm/include/x86/apic.h
index 5ca6bac..31887bd 100644
--- a/tools/testing/selftests/kvm/include/x86/apic.h
+++ b/tools/testing/selftests/kvm/include/x86/apic.h
@@ -79,42 +79,42 @@ void apic_disable(void);
void xapic_enable(void);
void x2apic_enable(void);
-static inline uint32_t get_bsp_flag(void)
+static inline u32 get_bsp_flag(void)
{
return rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_BSP;
}
-static inline uint32_t xapic_read_reg(unsigned int reg)
+static inline u32 xapic_read_reg(unsigned int reg)
{
- return ((volatile uint32_t *)APIC_DEFAULT_GPA)[reg >> 2];
+ return ((volatile u32 *)APIC_DEFAULT_GPA)[reg >> 2];
}
-static inline void xapic_write_reg(unsigned int reg, uint32_t val)
+static inline void xapic_write_reg(unsigned int reg, u32 val)
{
- ((volatile uint32_t *)APIC_DEFAULT_GPA)[reg >> 2] = val;
+ ((volatile u32 *)APIC_DEFAULT_GPA)[reg >> 2] = val;
}
-static inline uint64_t x2apic_read_reg(unsigned int reg)
+static inline u64 x2apic_read_reg(unsigned int reg)
{
return rdmsr(APIC_BASE_MSR + (reg >> 4));
}
-static inline uint8_t x2apic_write_reg_safe(unsigned int reg, uint64_t value)
+static inline u8 x2apic_write_reg_safe(unsigned int reg, u64 value)
{
return wrmsr_safe(APIC_BASE_MSR + (reg >> 4), value);
}
-static inline void x2apic_write_reg(unsigned int reg, uint64_t value)
+static inline void x2apic_write_reg(unsigned int reg, u64 value)
{
- uint8_t fault = x2apic_write_reg_safe(reg, value);
+ u8 fault = x2apic_write_reg_safe(reg, value);
__GUEST_ASSERT(!fault, "Unexpected fault 0x%x on WRMSR(%x) = %lx\n",
fault, APIC_BASE_MSR + (reg >> 4), value);
}
-static inline void x2apic_write_reg_fault(unsigned int reg, uint64_t value)
+static inline void x2apic_write_reg_fault(unsigned int reg, u64 value)
{
- uint8_t fault = x2apic_write_reg_safe(reg, value);
+ u8 fault = x2apic_write_reg_safe(reg, value);
__GUEST_ASSERT(fault == GP_VECTOR,
"Wanted #GP on WRMSR(%x) = %lx, got 0x%x\n",
diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h
index 5a74bb3..be79bda 100644
--- a/tools/testing/selftests/kvm/include/x86/evmcs.h
+++ b/tools/testing/selftests/kvm/include/x86/evmcs.h
@@ -10,9 +10,9 @@
#include "hyperv.h"
#include "vmx.h"
-#define u16 uint16_t
-#define u32 uint32_t
-#define u64 uint64_t
+#define u16 u16
+#define u32 u32
+#define u64 u64
#define EVMCS_VERSION 1
@@ -245,7 +245,7 @@ static inline void evmcs_enable(void)
enable_evmcs = true;
}
-static inline int evmcs_vmptrld(uint64_t vmcs_pa, void *vmcs)
+static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs)
{
current_vp_assist->current_nested_vmcs = vmcs_pa;
current_vp_assist->enlighten_vmentry = 1;
@@ -265,7 +265,7 @@ static inline bool load_evmcs(struct hyperv_test_pages *hv)
return true;
}
-static inline int evmcs_vmptrst(uint64_t *value)
+static inline int evmcs_vmptrst(u64 *value)
{
*value = current_vp_assist->current_nested_vmcs &
~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
@@ -273,7 +273,7 @@ static inline int evmcs_vmptrst(uint64_t *value)
return 0;
}
-static inline int evmcs_vmread(uint64_t encoding, uint64_t *value)
+static inline int evmcs_vmread(u64 encoding, u64 *value)
{
switch (encoding) {
case GUEST_RIP:
@@ -672,7 +672,7 @@ static inline int evmcs_vmread(uint64_t encoding, uint64_t *value)
return 0;
}
-static inline int evmcs_vmwrite(uint64_t encoding, uint64_t value)
+static inline int evmcs_vmwrite(u64 encoding, u64 value)
{
switch (encoding) {
case GUEST_RIP:
@@ -1226,9 +1226,9 @@ static inline int evmcs_vmlaunch(void)
"pop %%rbp;"
: [ret]"=&a"(ret)
: [host_rsp]"r"
- ((uint64_t)¤t_evmcs->host_rsp),
+ ((u64)¤t_evmcs->host_rsp),
[host_rip]"r"
- ((uint64_t)¤t_evmcs->host_rip)
+ ((u64)¤t_evmcs->host_rip)
: "memory", "cc", "rbx", "r8", "r9", "r10",
"r11", "r12", "r13", "r14", "r15");
return ret;
@@ -1265,9 +1265,9 @@ static inline int evmcs_vmresume(void)
"pop %%rbp;"
: [ret]"=&a"(ret)
: [host_rsp]"r"
- ((uint64_t)¤t_evmcs->host_rsp),
+ ((u64)¤t_evmcs->host_rsp),
[host_rip]"r"
- ((uint64_t)¤t_evmcs->host_rip)
+ ((u64)¤t_evmcs->host_rip)
: "memory", "cc", "rbx", "r8", "r9", "r10",
"r11", "r12", "r13", "r14", "r15");
return ret;
diff --git a/tools/testing/selftests/kvm/include/x86/hyperv.h b/tools/testing/selftests/kvm/include/x86/hyperv.h
index f13e532..78003f5 100644
--- a/tools/testing/selftests/kvm/include/x86/hyperv.h
+++ b/tools/testing/selftests/kvm/include/x86/hyperv.h
@@ -254,12 +254,12 @@
* Issue a Hyper-V hypercall. Returns exception vector raised or 0, 'hv_status'
* is set to the hypercall status (if no exception occurred).
*/
-static inline uint8_t __hyperv_hypercall(u64 control, vm_vaddr_t input_address,
- vm_vaddr_t output_address,
- uint64_t *hv_status)
+static inline u8 __hyperv_hypercall(u64 control, gva_t input_address,
+ gva_t output_address,
+ u64 *hv_status)
{
- uint64_t error_code;
- uint8_t vector;
+ u64 error_code;
+ u8 vector;
/* Note both the hypercall and the "asm safe" clobber r9-r11. */
asm volatile("mov %[output_address], %%r8\n\t"
@@ -274,11 +274,11 @@ static inline uint8_t __hyperv_hypercall(u64 control, vm_vaddr_t input_address,
}
/* Issue a Hyper-V hypercall and assert that it succeeded. */
-static inline void hyperv_hypercall(u64 control, vm_vaddr_t input_address,
- vm_vaddr_t output_address)
+static inline void hyperv_hypercall(u64 control, gva_t input_address,
+ gva_t output_address)
{
- uint64_t hv_status;
- uint8_t vector;
+ u64 hv_status;
+ u8 vector;
vector = __hyperv_hypercall(control, input_address, output_address, &hv_status);
@@ -327,27 +327,27 @@ struct hv_vp_assist_page {
extern struct hv_vp_assist_page *current_vp_assist;
-int enable_vp_assist(uint64_t vp_assist_pa, void *vp_assist);
+int enable_vp_assist(u64 vp_assist_pa, void *vp_assist);
struct hyperv_test_pages {
/* VP assist page */
void *vp_assist_hva;
- uint64_t vp_assist_gpa;
+ u64 vp_assist_gpa;
void *vp_assist;
/* Partition assist page */
void *partition_assist_hva;
- uint64_t partition_assist_gpa;
+ u64 partition_assist_gpa;
void *partition_assist;
/* Enlightened VMCS */
void *enlightened_vmcs_hva;
- uint64_t enlightened_vmcs_gpa;
+ u64 enlightened_vmcs_gpa;
void *enlightened_vmcs;
};
struct hyperv_test_pages *vcpu_alloc_hyperv_test_pages(struct kvm_vm *vm,
- vm_vaddr_t *p_hv_pages_gva);
+ gva_t *p_hv_pages_gva);
/* HV_X64_MSR_TSC_INVARIANT_CONTROL bits */
#define HV_INVARIANT_TSC_EXPOSED BIT_ULL(0)
diff --git a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
index be35d26..c33ab6e 100644
--- a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
+++ b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
@@ -11,19 +11,19 @@
extern bool is_forced_emulation_enabled;
struct pte_masks {
- uint64_t present;
- uint64_t writable;
- uint64_t user;
- uint64_t readable;
- uint64_t executable;
- uint64_t accessed;
- uint64_t dirty;
- uint64_t huge;
- uint64_t nx;
- uint64_t c;
- uint64_t s;
+ u64 present;
+ u64 writable;
+ u64 user;
+ u64 readable;
+ u64 executable;
+ u64 accessed;
+ u64 dirty;
+ u64 huge;
+ u64 nx;
+ u64 c;
+ u64 s;
- uint64_t always_set;
+ u64 always_set;
};
struct kvm_mmu_arch {
@@ -33,12 +33,12 @@ struct kvm_mmu_arch {
struct kvm_mmu;
struct kvm_vm_arch {
- vm_vaddr_t gdt;
- vm_vaddr_t tss;
- vm_vaddr_t idt;
+ gva_t gdt;
+ gva_t tss;
+ gva_t idt;
- uint64_t c_bit;
- uint64_t s_bit;
+ u64 c_bit;
+ u64 s_bit;
int sev_fd;
bool is_pt_protected;
};
@@ -62,7 +62,7 @@ do { \
: "+m" (mem) \
: "r" (val) : "memory"); \
} else { \
- uint64_t __old = READ_ONCE(mem); \
+ u64 __old = READ_ONCE(mem); \
\
__asm__ __volatile__(KVM_FEP LOCK_PREFIX "cmpxchg %[new], %[ptr]" \
: [ptr] "+m" (mem), [old] "+a" (__old) \
diff --git a/tools/testing/selftests/kvm/include/x86/pmu.h b/tools/testing/selftests/kvm/include/x86/pmu.h
index 72575ea..98537cc 100644
--- a/tools/testing/selftests/kvm/include/x86/pmu.h
+++ b/tools/testing/selftests/kvm/include/x86/pmu.h
@@ -6,8 +6,8 @@
#define SELFTEST_KVM_PMU_H
#include <stdbool.h>
-#include <stdint.h>
+#include <linux/types.h>
#include <linux/bits.h>
#define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
@@ -104,14 +104,15 @@ enum amd_pmu_zen_events {
NR_AMD_ZEN_EVENTS,
};
-extern const uint64_t intel_pmu_arch_events[];
-extern const uint64_t amd_pmu_zen_events[];
+extern const u64 intel_pmu_arch_events[];
+extern const u64 amd_pmu_zen_events[];
enum pmu_errata {
INSTRUCTIONS_RETIRED_OVERCOUNT,
BRANCHES_RETIRED_OVERCOUNT,
};
-extern uint64_t pmu_errata_mask;
+
+extern u64 pmu_errata_mask;
void kvm_init_pmu_errata(void);
diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h
index d8634a76..77f576ee 100644
--- a/tools/testing/selftests/kvm/include/x86/processor.h
+++ b/tools/testing/selftests/kvm/include/x86/processor.h
@@ -23,7 +23,7 @@ extern bool host_cpu_is_intel;
extern bool host_cpu_is_amd;
extern bool host_cpu_is_hygon;
extern bool host_cpu_is_amd_compatible;
-extern uint64_t guest_tsc_khz;
+extern u64 guest_tsc_khz;
#ifndef MAX_NR_CPUID_ENTRIES
#define MAX_NR_CPUID_ENTRIES 100
@@ -399,17 +399,17 @@ struct gpr64_regs {
};
struct desc64 {
- uint16_t limit0;
- uint16_t base0;
+ u16 limit0;
+ u16 base0;
unsigned base1:8, type:4, s:1, dpl:2, p:1;
unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8;
- uint32_t base3;
- uint32_t zero1;
+ u32 base3;
+ u32 zero1;
} __attribute__((packed));
struct desc_ptr {
- uint16_t size;
- uint64_t address;
+ u16 size;
+ u64 address;
} __attribute__((packed));
struct kvm_x86_state {
@@ -427,18 +427,18 @@ struct kvm_x86_state {
struct kvm_msrs msrs;
};
-static inline uint64_t get_desc64_base(const struct desc64 *desc)
+static inline u64 get_desc64_base(const struct desc64 *desc)
{
- return (uint64_t)desc->base3 << 32 |
- (uint64_t)desc->base2 << 24 |
- (uint64_t)desc->base1 << 16 |
- (uint64_t)desc->base0;
+ return (u64)desc->base3 << 32 |
+ (u64)desc->base2 << 24 |
+ (u64)desc->base1 << 16 |
+ (u64)desc->base0;
}
-static inline uint64_t rdtsc(void)
+static inline u64 rdtsc(void)
{
- uint32_t eax, edx;
- uint64_t tsc_val;
+ u32 eax, edx;
+ u64 tsc_val;
/*
* The lfence is to wait (on Intel CPUs) until all previous
* instructions have been executed. If software requires RDTSC to be
@@ -446,39 +446,39 @@ static inline uint64_t rdtsc(void)
* execute LFENCE immediately after RDTSC
*/
__asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
- tsc_val = ((uint64_t)edx) << 32 | eax;
+ tsc_val = ((u64)edx) << 32 | eax;
return tsc_val;
}
-static inline uint64_t rdtscp(uint32_t *aux)
+static inline u64 rdtscp(u32 *aux)
{
- uint32_t eax, edx;
+ u32 eax, edx;
__asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux));
- return ((uint64_t)edx) << 32 | eax;
+ return ((u64)edx) << 32 | eax;
}
-static inline uint64_t rdmsr(uint32_t msr)
+static inline u64 rdmsr(u32 msr)
{
- uint32_t a, d;
+ u32 a, d;
__asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory");
- return a | ((uint64_t) d << 32);
+ return a | ((u64)d << 32);
}
-static inline void wrmsr(uint32_t msr, uint64_t value)
+static inline void wrmsr(u32 msr, u64 value)
{
- uint32_t a = value;
- uint32_t d = value >> 32;
+ u32 a = value;
+ u32 d = value >> 32;
__asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory");
}
-static inline uint16_t inw(uint16_t port)
+static inline u16 inw(u16 port)
{
- uint16_t tmp;
+ u16 tmp;
__asm__ __volatile__("in %%dx, %%ax"
: /* output */ "=a" (tmp)
@@ -487,120 +487,120 @@ static inline uint16_t inw(uint16_t port)
return tmp;
}
-static inline uint16_t get_es(void)
+static inline u16 get_es(void)
{
- uint16_t es;
+ u16 es;
__asm__ __volatile__("mov %%es, %[es]"
: /* output */ [es]"=rm"(es));
return es;
}
-static inline uint16_t get_cs(void)
+static inline u16 get_cs(void)
{
- uint16_t cs;
+ u16 cs;
__asm__ __volatile__("mov %%cs, %[cs]"
: /* output */ [cs]"=rm"(cs));
return cs;
}
-static inline uint16_t get_ss(void)
+static inline u16 get_ss(void)
{
- uint16_t ss;
+ u16 ss;
__asm__ __volatile__("mov %%ss, %[ss]"
: /* output */ [ss]"=rm"(ss));
return ss;
}
-static inline uint16_t get_ds(void)
+static inline u16 get_ds(void)
{
- uint16_t ds;
+ u16 ds;
__asm__ __volatile__("mov %%ds, %[ds]"
: /* output */ [ds]"=rm"(ds));
return ds;
}
-static inline uint16_t get_fs(void)
+static inline u16 get_fs(void)
{
- uint16_t fs;
+ u16 fs;
__asm__ __volatile__("mov %%fs, %[fs]"
: /* output */ [fs]"=rm"(fs));
return fs;
}
-static inline uint16_t get_gs(void)
+static inline u16 get_gs(void)
{
- uint16_t gs;
+ u16 gs;
__asm__ __volatile__("mov %%gs, %[gs]"
: /* output */ [gs]"=rm"(gs));
return gs;
}
-static inline uint16_t get_tr(void)
+static inline u16 get_tr(void)
{
- uint16_t tr;
+ u16 tr;
__asm__ __volatile__("str %[tr]"
: /* output */ [tr]"=rm"(tr));
return tr;
}
-static inline uint64_t get_cr0(void)
+static inline u64 get_cr0(void)
{
- uint64_t cr0;
+ u64 cr0;
__asm__ __volatile__("mov %%cr0, %[cr0]"
: /* output */ [cr0]"=r"(cr0));
return cr0;
}
-static inline void set_cr0(uint64_t val)
+static inline void set_cr0(u64 val)
{
__asm__ __volatile__("mov %0, %%cr0" : : "r" (val) : "memory");
}
-static inline uint64_t get_cr3(void)
+static inline u64 get_cr3(void)
{
- uint64_t cr3;
+ u64 cr3;
__asm__ __volatile__("mov %%cr3, %[cr3]"
: /* output */ [cr3]"=r"(cr3));
return cr3;
}
-static inline void set_cr3(uint64_t val)
+static inline void set_cr3(u64 val)
{
__asm__ __volatile__("mov %0, %%cr3" : : "r" (val) : "memory");
}
-static inline uint64_t get_cr4(void)
+static inline u64 get_cr4(void)
{
- uint64_t cr4;
+ u64 cr4;
__asm__ __volatile__("mov %%cr4, %[cr4]"
: /* output */ [cr4]"=r"(cr4));
return cr4;
}
-static inline void set_cr4(uint64_t val)
+static inline void set_cr4(u64 val)
{
__asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory");
}
-static inline uint64_t get_cr8(void)
+static inline u64 get_cr8(void)
{
- uint64_t cr8;
+ u64 cr8;
__asm__ __volatile__("mov %%cr8, %[cr8]" : [cr8]"=r"(cr8));
return cr8;
}
-static inline void set_cr8(uint64_t val)
+static inline void set_cr8(u64 val)
{
__asm__ __volatile__("mov %0, %%cr8" : : "r" (val) : "memory");
}
@@ -651,14 +651,14 @@ static inline struct desc_ptr get_idt(void)
return idt;
}
-static inline void outl(uint16_t port, uint32_t value)
+static inline void outl(u16 port, u32 value)
{
__asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
}
-static inline void __cpuid(uint32_t function, uint32_t index,
- uint32_t *eax, uint32_t *ebx,
- uint32_t *ecx, uint32_t *edx)
+static inline void __cpuid(u32 function, u32 index,
+ u32 *eax, u32 *ebx,
+ u32 *ecx, u32 *edx)
{
*eax = function;
*ecx = index;
@@ -672,35 +672,35 @@ static inline void __cpuid(uint32_t function, uint32_t index,
: "memory");
}
-static inline void cpuid(uint32_t function,
- uint32_t *eax, uint32_t *ebx,
- uint32_t *ecx, uint32_t *edx)
+static inline void cpuid(u32 function,
+ u32 *eax, u32 *ebx,
+ u32 *ecx, u32 *edx)
{
return __cpuid(function, 0, eax, ebx, ecx, edx);
}
-static inline uint32_t this_cpu_fms(void)
+static inline u32 this_cpu_fms(void)
{
- uint32_t eax, ebx, ecx, edx;
+ u32 eax, ebx, ecx, edx;
cpuid(1, &eax, &ebx, &ecx, &edx);
return eax;
}
-static inline uint32_t this_cpu_family(void)
+static inline u32 this_cpu_family(void)
{
return x86_family(this_cpu_fms());
}
-static inline uint32_t this_cpu_model(void)
+static inline u32 this_cpu_model(void)
{
return x86_model(this_cpu_fms());
}
static inline bool this_cpu_vendor_string_is(const char *vendor)
{
- const uint32_t *chunk = (const uint32_t *)vendor;
- uint32_t eax, ebx, ecx, edx;
+ const u32 *chunk = (const u32 *)vendor;
+ u32 eax, ebx, ecx, edx;
cpuid(0, &eax, &ebx, &ecx, &edx);
return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);
@@ -724,10 +724,9 @@ static inline bool this_cpu_is_hygon(void)
return this_cpu_vendor_string_is("HygonGenuine");
}
-static inline uint32_t __this_cpu_has(uint32_t function, uint32_t index,
- uint8_t reg, uint8_t lo, uint8_t hi)
+static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi)
{
- uint32_t gprs[4];
+ u32 gprs[4];
__cpuid(function, index,
&gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX],
@@ -742,7 +741,7 @@ static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature)
feature.reg, feature.bit, feature.bit);
}
-static inline uint32_t this_cpu_property(struct kvm_x86_cpu_property property)
+static inline u32 this_cpu_property(struct kvm_x86_cpu_property property)
{
return __this_cpu_has(property.function, property.index,
property.reg, property.lo_bit, property.hi_bit);
@@ -750,7 +749,7 @@ static inline uint32_t this_cpu_property(struct kvm_x86_cpu_property property)
static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property)
{
- uint32_t max_leaf;
+ u32 max_leaf;
switch (property.function & 0xc0000000) {
case 0:
@@ -770,7 +769,7 @@ static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property)
static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature)
{
- uint32_t nr_bits;
+ u32 nr_bits;
if (feature.f.reg == KVM_CPUID_EBX) {
nr_bits = this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);
@@ -782,13 +781,13 @@ static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature)
return nr_bits > feature.f.bit || this_cpu_has(feature.f);
}
-static __always_inline uint64_t this_cpu_supported_xcr0(void)
+static __always_inline u64 this_cpu_supported_xcr0(void)
{
if (!this_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))
return 0;
return this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |
- ((uint64_t)this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
+ ((u64)this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
}
typedef u32 __attribute__((vector_size(16))) sse128_t;
@@ -867,7 +866,7 @@ static inline void cpu_relax(void)
static inline void udelay(unsigned long usec)
{
- uint64_t start, now, cycles;
+ u64 start, now, cycles;
GUEST_ASSERT(guest_tsc_khz);
cycles = guest_tsc_khz / 1000 * usec;
@@ -898,8 +897,8 @@ void kvm_x86_state_cleanup(struct kvm_x86_state *state);
const struct kvm_msr_list *kvm_get_msr_index_list(void);
const struct kvm_msr_list *kvm_get_feature_msr_index_list(void);
-bool kvm_msr_is_in_save_restore_list(uint32_t msr_index);
-uint64_t kvm_get_feature_msr(uint64_t msr_index);
+bool kvm_msr_is_in_save_restore_list(u32 msr_index);
+u64 kvm_get_feature_msr(u64 msr_index);
static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu,
struct kvm_msrs *msrs)
@@ -954,20 +953,20 @@ static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs)
}
const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
- uint32_t function, uint32_t index);
+ u32 function, u32 index);
const struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
-static inline uint32_t kvm_cpu_fms(void)
+static inline u32 kvm_cpu_fms(void)
{
return get_cpuid_entry(kvm_get_supported_cpuid(), 0x1, 0)->eax;
}
-static inline uint32_t kvm_cpu_family(void)
+static inline u32 kvm_cpu_family(void)
{
return x86_family(kvm_cpu_fms());
}
-static inline uint32_t kvm_cpu_model(void)
+static inline u32 kvm_cpu_model(void)
{
return x86_model(kvm_cpu_fms());
}
@@ -980,17 +979,17 @@ static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature)
return kvm_cpuid_has(kvm_get_supported_cpuid(), feature);
}
-uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
- struct kvm_x86_cpu_property property);
+u32 kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
+ struct kvm_x86_cpu_property property);
-static inline uint32_t kvm_cpu_property(struct kvm_x86_cpu_property property)
+static inline u32 kvm_cpu_property(struct kvm_x86_cpu_property property)
{
return kvm_cpuid_property(kvm_get_supported_cpuid(), property);
}
static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property)
{
- uint32_t max_leaf;
+ u32 max_leaf;
switch (property.function & 0xc0000000) {
case 0:
@@ -1010,7 +1009,7 @@ static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property)
static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature)
{
- uint32_t nr_bits;
+ u32 nr_bits;
if (feature.f.reg == KVM_CPUID_EBX) {
nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);
@@ -1022,13 +1021,13 @@ static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature)
return nr_bits > feature.f.bit || kvm_cpu_has(feature.f);
}
-static __always_inline uint64_t kvm_cpu_supported_xcr0(void)
+static __always_inline u64 kvm_cpu_supported_xcr0(void)
{
if (!kvm_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))
return 0;
return kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |
- ((uint64_t)kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
+ ((u64)kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
}
static inline size_t kvm_cpuid2_size(int nr_entries)
@@ -1062,8 +1061,8 @@ static inline void vcpu_get_cpuid(struct kvm_vcpu *vcpu)
}
static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
- uint32_t function,
- uint32_t index)
+ u32 function,
+ u32 index)
{
TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first (or equivalent)");
@@ -1074,7 +1073,7 @@ static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *v
}
static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
- uint32_t function)
+ u32 function)
{
return __vcpu_get_cpuid_entry(vcpu, function, 0);
}
@@ -1104,10 +1103,10 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)
void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
struct kvm_x86_cpu_property property,
- uint32_t value);
-void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr);
+ u32 value);
+void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr);
-void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function);
+void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function);
static inline bool vcpu_cpuid_has(struct kvm_vcpu *vcpu,
struct kvm_x86_cpu_feature feature)
@@ -1135,8 +1134,8 @@ static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu,
vcpu_set_or_clear_cpuid_feature(vcpu, feature, false);
}
-uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index);
-int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value);
+u64 vcpu_get_msr(struct kvm_vcpu *vcpu, u64 msr_index);
+int _vcpu_set_msr(struct kvm_vcpu *vcpu, u64 msr_index, u64 msr_value);
/*
* Assert on an MSR access(es) and pretty print the MSR name when possible.
@@ -1161,14 +1160,14 @@ do { \
* is changing, etc. This is NOT an exhaustive list! The intent is to filter
* out MSRs that are not durable _and_ that a selftest wants to write.
*/
-static inline bool is_durable_msr(uint32_t msr)
+static inline bool is_durable_msr(u32 msr)
{
return msr != MSR_IA32_TSC;
}
#define vcpu_set_msr(vcpu, msr, val) \
do { \
- uint64_t r, v = val; \
+ u64 r, v = val; \
\
TEST_ASSERT_MSR(_vcpu_set_msr(vcpu, msr, v) == 1, \
"KVM_SET_MSRS failed on %s, value = 0x%lx", msr, #msr, v); \
@@ -1182,28 +1181,28 @@ void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits);
void kvm_init_vm_address_properties(struct kvm_vm *vm);
struct ex_regs {
- uint64_t rax, rcx, rdx, rbx;
- uint64_t rbp, rsi, rdi;
- uint64_t r8, r9, r10, r11;
- uint64_t r12, r13, r14, r15;
- uint64_t vector;
- uint64_t error_code;
- uint64_t rip;
- uint64_t cs;
- uint64_t rflags;
+ u64 rax, rcx, rdx, rbx;
+ u64 rbp, rsi, rdi;
+ u64 r8, r9, r10, r11;
+ u64 r12, r13, r14, r15;
+ u64 vector;
+ u64 error_code;
+ u64 rip;
+ u64 cs;
+ u64 rflags;
};
struct idt_entry {
- uint16_t offset0;
- uint16_t selector;
- uint16_t ist : 3;
- uint16_t : 5;
- uint16_t type : 4;
- uint16_t : 1;
- uint16_t dpl : 2;
- uint16_t p : 1;
- uint16_t offset1;
- uint32_t offset2; uint32_t reserved;
+ u16 offset0;
+ u16 selector;
+ u16 ist : 3;
+ u16 : 5;
+ u16 type : 4;
+ u16 : 1;
+ u16 dpl : 2;
+ u16 p : 1;
+ u16 offset1;
+ u32 offset2; u32 reserved;
};
void vm_install_exception_handler(struct kvm_vm *vm, int vector,
@@ -1262,8 +1261,8 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
#define kvm_asm_safe(insn, inputs...) \
({ \
- uint64_t ign_error_code; \
- uint8_t vector; \
+ u64 ign_error_code; \
+ u8 vector; \
\
asm volatile(KVM_ASM_SAFE(insn) \
: KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \
@@ -1274,7 +1273,7 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
#define kvm_asm_safe_ec(insn, error_code, inputs...) \
({ \
- uint8_t vector; \
+ u8 vector; \
\
asm volatile(KVM_ASM_SAFE(insn) \
: KVM_ASM_SAFE_OUTPUTS(vector, error_code) \
@@ -1285,8 +1284,8 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
#define kvm_asm_safe_fep(insn, inputs...) \
({ \
- uint64_t ign_error_code; \
- uint8_t vector; \
+ u64 ign_error_code; \
+ u8 vector; \
\
asm volatile(KVM_ASM_SAFE_FEP(insn) \
: KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \
@@ -1297,7 +1296,7 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
#define kvm_asm_safe_ec_fep(insn, error_code, inputs...) \
({ \
- uint8_t vector; \
+ u8 vector; \
\
asm volatile(KVM_ASM_SAFE_FEP(insn) \
: KVM_ASM_SAFE_OUTPUTS(vector, error_code) \
@@ -1307,11 +1306,11 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
})
#define BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP) \
-static inline uint8_t insn##_safe ##_fep(uint32_t idx, uint64_t *val) \
+static inline u8 insn##_safe ##_fep(u32 idx, u64 *val) \
{ \
- uint64_t error_code; \
- uint8_t vector; \
- uint32_t a, d; \
+ u64 error_code; \
+ u8 vector; \
+ u32 a, d; \
\
asm volatile(KVM_ASM_SAFE##_FEP(#insn) \
: "=a"(a), "=d"(d), \
@@ -1319,7 +1318,7 @@ static inline uint8_t insn##_safe ##_fep(uint32_t idx, uint64_t *val) \
: "c"(idx) \
: KVM_ASM_SAFE_CLOBBERS); \
\
- *val = (uint64_t)a | ((uint64_t)d << 32); \
+ *val = (u64)a | ((u64)d << 32); \
return vector; \
}
@@ -1335,12 +1334,12 @@ BUILD_READ_U64_SAFE_HELPERS(rdmsr)
BUILD_READ_U64_SAFE_HELPERS(rdpmc)
BUILD_READ_U64_SAFE_HELPERS(xgetbv)
-static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val)
+static inline u8 wrmsr_safe(u32 msr, u64 val)
{
return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));
}
-static inline uint8_t xsetbv_safe(uint32_t index, uint64_t value)
+static inline u8 xsetbv_safe(u32 index, u64 value)
{
u32 eax = value;
u32 edx = value >> 32;
@@ -1395,23 +1394,20 @@ static inline bool kvm_is_lbrv_enabled(void)
return !!get_kvm_amd_param_integer("lbrv");
}
-uint64_t *vm_get_pte(struct kvm_vm *vm, uint64_t vaddr);
+u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva);
-uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
- uint64_t a3);
-uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
-void xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
+u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3);
+u64 __xen_hypercall(u64 nr, u64 a0, void *a1);
+void xen_hypercall(u64 nr, u64 a0, void *a1);
-static inline uint64_t __kvm_hypercall_map_gpa_range(uint64_t gpa,
- uint64_t size, uint64_t flags)
+static inline u64 __kvm_hypercall_map_gpa_range(gpa_t gpa, u64 size, u64 flags)
{
return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0);
}
-static inline void kvm_hypercall_map_gpa_range(uint64_t gpa, uint64_t size,
- uint64_t flags)
+static inline void kvm_hypercall_map_gpa_range(gpa_t gpa, u64 size, u64 flags)
{
- uint64_t ret = __kvm_hypercall_map_gpa_range(gpa, size, flags);
+ u64 ret = __kvm_hypercall_map_gpa_range(gpa, size, flags);
GUEST_ASSERT(!ret);
}
@@ -1456,7 +1452,7 @@ static inline void cli(void)
asm volatile ("cli");
}
-void __vm_xsave_require_permission(uint64_t xfeature, const char *name);
+void __vm_xsave_require_permission(u64 xfeature, const char *name);
#define vm_xsave_require_permission(xfeature) \
__vm_xsave_require_permission(xfeature, #xfeature)
@@ -1511,17 +1507,17 @@ enum pg_level {
void tdp_mmu_init(struct kvm_vm *vm, int pgtable_levels,
struct pte_masks *pte_masks);
-void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, uint64_t vaddr,
- uint64_t paddr, int level);
-void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
- uint64_t nr_bytes, int level);
+void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva,
+ gpa_t gpa, int level);
+void virt_map_level(struct kvm_vm *vm, gva_t gva, gpa_t gpa,
+ u64 nr_bytes, int level);
void vm_enable_tdp(struct kvm_vm *vm);
bool kvm_cpu_has_tdp(void);
-void tdp_map(struct kvm_vm *vm, uint64_t nested_paddr, uint64_t paddr, uint64_t size);
+void tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size);
void tdp_identity_map_default_memslots(struct kvm_vm *vm);
-void tdp_identity_map_1g(struct kvm_vm *vm, uint64_t addr, uint64_t size);
-uint64_t *tdp_get_pte(struct kvm_vm *vm, uint64_t l2_gpa);
+void tdp_identity_map_1g(struct kvm_vm *vm, u64 addr, u64 size);
+u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa);
/*
* Basic CPU control in CR0
diff --git a/tools/testing/selftests/kvm/include/x86/sev.h b/tools/testing/selftests/kvm/include/x86/sev.h
index 008b416..1af44c1 100644
--- a/tools/testing/selftests/kvm/include/x86/sev.h
+++ b/tools/testing/selftests/kvm/include/x86/sev.h
@@ -46,16 +46,16 @@ static inline bool is_sev_vm(struct kvm_vm *vm)
return is_sev_es_vm(vm) || vm->type == KVM_X86_SEV_VM;
}
-void sev_vm_launch(struct kvm_vm *vm, uint32_t policy);
-void sev_vm_launch_measure(struct kvm_vm *vm, uint8_t *measurement);
+void sev_vm_launch(struct kvm_vm *vm, u32 policy);
+void sev_vm_launch_measure(struct kvm_vm *vm, u8 *measurement);
void sev_vm_launch_finish(struct kvm_vm *vm);
-void snp_vm_launch_start(struct kvm_vm *vm, uint64_t policy);
+void snp_vm_launch_start(struct kvm_vm *vm, u64 policy);
void snp_vm_launch_update(struct kvm_vm *vm);
void snp_vm_launch_finish(struct kvm_vm *vm);
-struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code,
+struct kvm_vm *vm_sev_create_with_one_vcpu(u32 type, void *guest_code,
struct kvm_vcpu **cpu);
-void vm_sev_launch(struct kvm_vm *vm, uint64_t policy, uint8_t *measurement);
+void vm_sev_launch(struct kvm_vm *vm, u64 policy, u8 *measurement);
kvm_static_assert(SEV_RET_SUCCESS == 0);
@@ -85,7 +85,7 @@ static inline u64 snp_default_policy(void)
unsigned long raw; \
} sev_cmd = { .c = { \
.id = (cmd), \
- .data = (uint64_t)(arg), \
+ .data = (u64)(arg), \
.sev_fd = (vm)->arch.sev_fd, \
} }; \
\
@@ -120,8 +120,8 @@ static inline void sev_register_encrypted_memory(struct kvm_vm *vm,
vm_ioctl(vm, KVM_MEMORY_ENCRYPT_REG_REGION, &range);
}
-static inline void sev_launch_update_data(struct kvm_vm *vm, vm_paddr_t gpa,
- uint64_t size)
+static inline void sev_launch_update_data(struct kvm_vm *vm, gpa_t gpa,
+ u64 size)
{
struct kvm_sev_launch_update_data update_data = {
.uaddr = (unsigned long)addr_gpa2hva(vm, gpa),
@@ -131,8 +131,8 @@ static inline void sev_launch_update_data(struct kvm_vm *vm, vm_paddr_t gpa,
vm_sev_ioctl(vm, KVM_SEV_LAUNCH_UPDATE_DATA, &update_data);
}
-static inline void snp_launch_update_data(struct kvm_vm *vm, vm_paddr_t gpa,
- uint64_t hva, uint64_t size, uint8_t type)
+static inline void snp_launch_update_data(struct kvm_vm *vm, gpa_t gpa,
+ u64 hva, u64 size, u8 type)
{
struct kvm_sev_snp_launch_update update_data = {
.uaddr = hva,
diff --git a/tools/testing/selftests/kvm/include/x86/smm.h b/tools/testing/selftests/kvm/include/x86/smm.h
index 19337c3..2d1afa0 100644
--- a/tools/testing/selftests/kvm/include/x86/smm.h
+++ b/tools/testing/selftests/kvm/include/x86/smm.h
@@ -8,8 +8,7 @@
#define SMRAM_MEMSLOT ((1 << 16) | 1)
#define SMRAM_PAGES (SMRAM_SIZE / PAGE_SIZE)
-void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
- uint64_t smram_gpa,
+void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa,
const void *smi_handler, size_t handler_size);
void inject_smi(struct kvm_vcpu *vcpu);
diff --git a/tools/testing/selftests/kvm/include/x86/svm_util.h b/tools/testing/selftests/kvm/include/x86/svm_util.h
index 5d7c4253..6c013eb 100644
--- a/tools/testing/selftests/kvm/include/x86/svm_util.h
+++ b/tools/testing/selftests/kvm/include/x86/svm_util.h
@@ -16,20 +16,20 @@ struct svm_test_data {
/* VMCB */
struct vmcb *vmcb; /* gva */
void *vmcb_hva;
- uint64_t vmcb_gpa;
+ u64 vmcb_gpa;
/* host state-save area */
struct vmcb_save_area *save_area; /* gva */
void *save_area_hva;
- uint64_t save_area_gpa;
+ u64 save_area_gpa;
/* MSR-Bitmap */
void *msr; /* gva */
void *msr_hva;
- uint64_t msr_gpa;
+ u64 msr_gpa;
/* NPT */
- uint64_t ncr3_gpa;
+ u64 ncr3_gpa;
};
static inline void vmmcall(void)
@@ -56,9 +56,9 @@ static inline void vmmcall(void)
"clgi\n" \
)
-struct svm_test_data *vcpu_alloc_svm(struct kvm_vm *vm, vm_vaddr_t *p_svm_gva);
+struct svm_test_data *vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva);
void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp);
-void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa);
+void run_guest(struct vmcb *vmcb, u64 vmcb_gpa);
static inline bool kvm_cpu_has_npt(void)
{
diff --git a/tools/testing/selftests/kvm/include/x86/ucall.h b/tools/testing/selftests/kvm/include/x86/ucall.h
index d3825dc..0e49500 100644
--- a/tools/testing/selftests/kvm/include/x86/ucall.h
+++ b/tools/testing/selftests/kvm/include/x86/ucall.h
@@ -6,7 +6,7 @@
#define UCALL_EXIT_REASON KVM_EXIT_IO
-static inline void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
}
diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h
index 92b9187..90fffaf 100644
--- a/tools/testing/selftests/kvm/include/x86/vmx.h
+++ b/tools/testing/selftests/kvm/include/x86/vmx.h
@@ -285,16 +285,16 @@ enum vmcs_field {
};
struct vmx_msr_entry {
- uint32_t index;
- uint32_t reserved;
- uint64_t value;
+ u32 index;
+ u32 reserved;
+ u64 value;
} __attribute__ ((aligned(16)));
#include "evmcs.h"
-static inline int vmxon(uint64_t phys)
+static inline int vmxon(u64 phys)
{
- uint8_t ret;
+ u8 ret;
__asm__ __volatile__ ("vmxon %[pa]; setna %[ret]"
: [ret]"=rm"(ret)
@@ -309,9 +309,9 @@ static inline void vmxoff(void)
__asm__ __volatile__("vmxoff");
}
-static inline int vmclear(uint64_t vmcs_pa)
+static inline int vmclear(u64 vmcs_pa)
{
- uint8_t ret;
+ u8 ret;
__asm__ __volatile__ ("vmclear %[pa]; setna %[ret]"
: [ret]"=rm"(ret)
@@ -321,9 +321,9 @@ static inline int vmclear(uint64_t vmcs_pa)
return ret;
}
-static inline int vmptrld(uint64_t vmcs_pa)
+static inline int vmptrld(u64 vmcs_pa)
{
- uint8_t ret;
+ u8 ret;
if (enable_evmcs)
return -1;
@@ -336,10 +336,10 @@ static inline int vmptrld(uint64_t vmcs_pa)
return ret;
}
-static inline int vmptrst(uint64_t *value)
+static inline int vmptrst(u64 *value)
{
- uint64_t tmp;
- uint8_t ret;
+ u64 tmp;
+ u8 ret;
if (enable_evmcs)
return evmcs_vmptrst(value);
@@ -356,9 +356,9 @@ static inline int vmptrst(uint64_t *value)
* A wrapper around vmptrst that ignores errors and returns zero if the
* vmptrst instruction fails.
*/
-static inline uint64_t vmptrstz(void)
+static inline u64 vmptrstz(void)
{
- uint64_t value = 0;
+ u64 value = 0;
vmptrst(&value);
return value;
}
@@ -391,8 +391,8 @@ static inline int vmlaunch(void)
"pop %%rcx;"
"pop %%rbp;"
: [ret]"=&a"(ret)
- : [host_rsp]"r"((uint64_t)HOST_RSP),
- [host_rip]"r"((uint64_t)HOST_RIP)
+ : [host_rsp]"r"((u64)HOST_RSP),
+ [host_rip]"r"((u64)HOST_RIP)
: "memory", "cc", "rbx", "r8", "r9", "r10",
"r11", "r12", "r13", "r14", "r15");
return ret;
@@ -426,8 +426,8 @@ static inline int vmresume(void)
"pop %%rcx;"
"pop %%rbp;"
: [ret]"=&a"(ret)
- : [host_rsp]"r"((uint64_t)HOST_RSP),
- [host_rip]"r"((uint64_t)HOST_RIP)
+ : [host_rsp]"r"((u64)HOST_RSP),
+ [host_rip]"r"((u64)HOST_RIP)
: "memory", "cc", "rbx", "r8", "r9", "r10",
"r11", "r12", "r13", "r14", "r15");
return ret;
@@ -447,10 +447,10 @@ static inline void vmcall(void)
"r10", "r11", "r12", "r13", "r14", "r15");
}
-static inline int vmread(uint64_t encoding, uint64_t *value)
+static inline int vmread(u64 encoding, u64 *value)
{
- uint64_t tmp;
- uint8_t ret;
+ u64 tmp;
+ u8 ret;
if (enable_evmcs)
return evmcs_vmread(encoding, value);
@@ -468,16 +468,16 @@ static inline int vmread(uint64_t encoding, uint64_t *value)
* A wrapper around vmread that ignores errors and returns zero if the
* vmread instruction fails.
*/
-static inline uint64_t vmreadz(uint64_t encoding)
+static inline u64 vmreadz(u64 encoding)
{
- uint64_t value = 0;
+ u64 value = 0;
vmread(encoding, &value);
return value;
}
-static inline int vmwrite(uint64_t encoding, uint64_t value)
+static inline int vmwrite(u64 encoding, u64 value)
{
- uint8_t ret;
+ u8 ret;
if (enable_evmcs)
return evmcs_vmwrite(encoding, value);
@@ -490,41 +490,41 @@ static inline int vmwrite(uint64_t encoding, uint64_t value)
return ret;
}
-static inline uint32_t vmcs_revision(void)
+static inline u32 vmcs_revision(void)
{
return rdmsr(MSR_IA32_VMX_BASIC);
}
struct vmx_pages {
void *vmxon_hva;
- uint64_t vmxon_gpa;
+ u64 vmxon_gpa;
void *vmxon;
void *vmcs_hva;
- uint64_t vmcs_gpa;
+ u64 vmcs_gpa;
void *vmcs;
void *msr_hva;
- uint64_t msr_gpa;
+ u64 msr_gpa;
void *msr;
void *shadow_vmcs_hva;
- uint64_t shadow_vmcs_gpa;
+ u64 shadow_vmcs_gpa;
void *shadow_vmcs;
void *vmread_hva;
- uint64_t vmread_gpa;
+ u64 vmread_gpa;
void *vmread;
void *vmwrite_hva;
- uint64_t vmwrite_gpa;
+ u64 vmwrite_gpa;
void *vmwrite;
void *apic_access_hva;
- uint64_t apic_access_gpa;
+ u64 apic_access_gpa;
void *apic_access;
- uint64_t eptp_gpa;
+ u64 eptp_gpa;
};
union vmx_basic {
@@ -550,7 +550,7 @@ union vmx_ctrl_msr {
};
};
-struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva);
+struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva);
bool prepare_for_vmx_operation(struct vmx_pages *vmx);
void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp);
bool load_vmcs(struct vmx_pages *vmx);
diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c
index c60a24a..fc5242f 100644
--- a/tools/testing/selftests/kvm/kvm_page_table_test.c
+++ b/tools/testing/selftests/kvm/kvm_page_table_test.c
@@ -46,12 +46,12 @@ static const char * const test_stage_string[] = {
struct test_args {
struct kvm_vm *vm;
- uint64_t guest_test_virt_mem;
- uint64_t host_page_size;
- uint64_t host_num_pages;
- uint64_t large_page_size;
- uint64_t large_num_pages;
- uint64_t host_pages_per_lpage;
+ u64 guest_test_virt_mem;
+ u64 host_page_size;
+ u64 host_num_pages;
+ u64 large_page_size;
+ u64 large_num_pages;
+ u64 host_pages_per_lpage;
enum vm_mem_backing_src_type src_type;
struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
};
@@ -63,7 +63,7 @@ struct test_args {
static enum test_stage guest_test_stage;
/* Host variables */
-static uint32_t nr_vcpus = 1;
+static u32 nr_vcpus = 1;
static struct test_args test_args;
static enum test_stage *current_stage;
static bool host_quit;
@@ -77,19 +77,19 @@ static sem_t test_stage_completed;
* This will be set to the topmost valid physical address minus
* the test memory size.
*/
-static uint64_t guest_test_phys_mem;
+static u64 guest_test_phys_mem;
/*
* Guest virtual memory offset of the testing memory slot.
* Must not conflict with identity mapped test code.
*/
-static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
+static u64 guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
static void guest_code(bool do_write)
{
struct test_args *p = &test_args;
enum test_stage *current_stage = &guest_test_stage;
- uint64_t addr;
+ u64 addr;
int i, j;
while (true) {
@@ -113,9 +113,9 @@ static void guest_code(bool do_write)
case KVM_CREATE_MAPPINGS:
for (i = 0; i < p->large_num_pages; i++) {
if (do_write)
- *(uint64_t *)addr = 0x0123456789ABCDEF;
+ *(u64 *)addr = 0x0123456789ABCDEF;
else
- READ_ONCE(*(uint64_t *)addr);
+ READ_ONCE(*(u64 *)addr);
addr += p->large_page_size;
}
@@ -131,7 +131,7 @@ static void guest_code(bool do_write)
case KVM_UPDATE_MAPPINGS:
if (p->src_type == VM_MEM_SRC_ANONYMOUS) {
for (i = 0; i < p->host_num_pages; i++) {
- *(uint64_t *)addr = 0x0123456789ABCDEF;
+ *(u64 *)addr = 0x0123456789ABCDEF;
addr += p->host_page_size;
}
break;
@@ -142,7 +142,7 @@ static void guest_code(bool do_write)
* Write to the first host page in each large
* page region, and triger break of large pages.
*/
- *(uint64_t *)addr = 0x0123456789ABCDEF;
+ *(u64 *)addr = 0x0123456789ABCDEF;
/*
* Access the middle host pages in each large
@@ -152,7 +152,7 @@ static void guest_code(bool do_write)
*/
addr += p->large_page_size / 2;
for (j = 0; j < p->host_pages_per_lpage / 2; j++) {
- READ_ONCE(*(uint64_t *)addr);
+ READ_ONCE(*(u64 *)addr);
addr += p->host_page_size;
}
}
@@ -167,7 +167,7 @@ static void guest_code(bool do_write)
*/
case KVM_ADJUST_MAPPINGS:
for (i = 0; i < p->host_num_pages; i++) {
- READ_ONCE(*(uint64_t *)addr);
+ READ_ONCE(*(u64 *)addr);
addr += p->host_page_size;
}
break;
@@ -227,8 +227,8 @@ static void *vcpu_worker(void *data)
}
struct test_params {
- uint64_t phys_offset;
- uint64_t test_mem_size;
+ u64 phys_offset;
+ u64 test_mem_size;
enum vm_mem_backing_src_type src_type;
};
@@ -237,12 +237,12 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
int ret;
struct test_params *p = arg;
enum vm_mem_backing_src_type src_type = p->src_type;
- uint64_t large_page_size = get_backing_src_pagesz(src_type);
- uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;
- uint64_t host_page_size = getpagesize();
- uint64_t test_mem_size = p->test_mem_size;
- uint64_t guest_num_pages;
- uint64_t alignment;
+ u64 large_page_size = get_backing_src_pagesz(src_type);
+ u64 guest_page_size = vm_guest_mode_params[mode].page_size;
+ u64 host_page_size = getpagesize();
+ u64 test_mem_size = p->test_mem_size;
+ u64 guest_num_pages;
+ u64 alignment;
void *host_test_mem;
struct kvm_vm *vm;
@@ -281,7 +281,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
/* Cache the HVA pointer of the region */
- host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
+ host_test_mem = addr_gpa2hva(vm, (gpa_t)guest_test_phys_mem);
/* Export shared structure test_args to guest */
sync_global_to_guest(vm, test_args);
@@ -292,7 +292,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
ret = sem_init(&test_stage_completed, 0, 0);
TEST_ASSERT(ret == 0, "Error in sem_init");
- current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));
+ current_stage = addr_gva2hva(vm, (gva_t)(&guest_test_stage));
*current_stage = NUM_TEST_STAGES;
pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
@@ -304,7 +304,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
pr_info("Guest physical test memory offset: 0x%lx\n",
guest_test_phys_mem);
pr_info("Host virtual test memory offset: 0x%lx\n",
- (uint64_t)host_test_mem);
+ (u64)host_test_mem);
pr_info("Number of testing vCPUs: %d\n", nr_vcpus);
return vm;
diff --git a/tools/testing/selftests/kvm/lib/arm64/gic.c b/tools/testing/selftests/kvm/lib/arm64/gic.c
index b023868..011dfe1 100644
--- a/tools/testing/selftests/kvm/lib/arm64/gic.c
+++ b/tools/testing/selftests/kvm/lib/arm64/gic.c
@@ -50,7 +50,7 @@ static void gic_dist_init(enum gic_type type, unsigned int nr_cpus)
void gic_init(enum gic_type type, unsigned int nr_cpus)
{
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
GUEST_ASSERT(type < GIC_TYPE_MAX);
GUEST_ASSERT(nr_cpus);
@@ -73,7 +73,7 @@ void gic_irq_disable(unsigned int intid)
unsigned int gic_get_and_ack_irq(void)
{
- uint64_t irqstat;
+ u64 irqstat;
unsigned int intid;
GUEST_ASSERT(gic_common_ops);
@@ -102,7 +102,7 @@ void gic_set_eoi_split(bool split)
gic_common_ops->gic_set_eoi_split(split);
}
-void gic_set_priority_mask(uint64_t pmr)
+void gic_set_priority_mask(u64 pmr)
{
GUEST_ASSERT(gic_common_ops);
gic_common_ops->gic_set_priority_mask(pmr);
diff --git a/tools/testing/selftests/kvm/lib/arm64/gic_private.h b/tools/testing/selftests/kvm/lib/arm64/gic_private.h
index b6a7e30..6d393f5 100644
--- a/tools/testing/selftests/kvm/lib/arm64/gic_private.h
+++ b/tools/testing/selftests/kvm/lib/arm64/gic_private.h
@@ -12,20 +12,20 @@ struct gic_common_ops {
void (*gic_cpu_init)(unsigned int cpu);
void (*gic_irq_enable)(unsigned int intid);
void (*gic_irq_disable)(unsigned int intid);
- uint64_t (*gic_read_iar)(void);
- void (*gic_write_eoir)(uint32_t irq);
- void (*gic_write_dir)(uint32_t irq);
+ u64 (*gic_read_iar)(void);
+ void (*gic_write_eoir)(u32 irq);
+ void (*gic_write_dir)(u32 irq);
void (*gic_set_eoi_split)(bool split);
- void (*gic_set_priority_mask)(uint64_t mask);
- void (*gic_set_priority)(uint32_t intid, uint32_t prio);
- void (*gic_irq_set_active)(uint32_t intid);
- void (*gic_irq_clear_active)(uint32_t intid);
- bool (*gic_irq_get_active)(uint32_t intid);
- void (*gic_irq_set_pending)(uint32_t intid);
- void (*gic_irq_clear_pending)(uint32_t intid);
- bool (*gic_irq_get_pending)(uint32_t intid);
- void (*gic_irq_set_config)(uint32_t intid, bool is_edge);
- void (*gic_irq_set_group)(uint32_t intid, bool group);
+ void (*gic_set_priority_mask)(u64 mask);
+ void (*gic_set_priority)(u32 intid, u32 prio);
+ void (*gic_irq_set_active)(u32 intid);
+ void (*gic_irq_clear_active)(u32 intid);
+ bool (*gic_irq_get_active)(u32 intid);
+ void (*gic_irq_set_pending)(u32 intid);
+ void (*gic_irq_clear_pending)(u32 intid);
+ bool (*gic_irq_get_pending)(u32 intid);
+ void (*gic_irq_set_config)(u32 intid, bool is_edge);
+ void (*gic_irq_set_group)(u32 intid, bool group);
};
extern const struct gic_common_ops gicv3_ops;
diff --git a/tools/testing/selftests/kvm/lib/arm64/gic_v3.c b/tools/testing/selftests/kvm/lib/arm64/gic_v3.c
index 50754a2..a99a53a 100644
--- a/tools/testing/selftests/kvm/lib/arm64/gic_v3.c
+++ b/tools/testing/selftests/kvm/lib/arm64/gic_v3.c
@@ -50,13 +50,13 @@ static void gicv3_gicd_wait_for_rwp(void)
}
}
-static inline volatile void *gicr_base_cpu(uint32_t cpu)
+static inline volatile void *gicr_base_cpu(u32 cpu)
{
/* Align all the redistributors sequentially */
return GICR_BASE_GVA + cpu * SZ_64K * 2;
}
-static void gicv3_gicr_wait_for_rwp(uint32_t cpu)
+static void gicv3_gicr_wait_for_rwp(u32 cpu)
{
unsigned int count = 100000; /* 1s */
@@ -66,7 +66,7 @@ static void gicv3_gicr_wait_for_rwp(uint32_t cpu)
}
}
-static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)
+static void gicv3_wait_for_rwp(u32 cpu_or_dist)
{
if (cpu_or_dist & DIST_BIT)
gicv3_gicd_wait_for_rwp();
@@ -91,34 +91,34 @@ static enum gicv3_intid_range get_intid_range(unsigned int intid)
return INVALID_RANGE;
}
-static uint64_t gicv3_read_iar(void)
+static u64 gicv3_read_iar(void)
{
- uint64_t irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
+ u64 irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
dsb(sy);
return irqstat;
}
-static void gicv3_write_eoir(uint32_t irq)
+static void gicv3_write_eoir(u32 irq)
{
write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
isb();
}
-static void gicv3_write_dir(uint32_t irq)
+static void gicv3_write_dir(u32 irq)
{
write_sysreg_s(irq, SYS_ICC_DIR_EL1);
isb();
}
-static void gicv3_set_priority_mask(uint64_t mask)
+static void gicv3_set_priority_mask(u64 mask)
{
write_sysreg_s(mask, SYS_ICC_PMR_EL1);
}
static void gicv3_set_eoi_split(bool split)
{
- uint32_t val;
+ u32 val;
/*
* All other fields are read-only, so no need to read CTLR first. In
@@ -129,29 +129,29 @@ static void gicv3_set_eoi_split(bool split)
isb();
}
-uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)
+u32 gicv3_reg_readl(u32 cpu_or_dist, u64 offset)
{
volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA
: sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));
return readl(base + offset);
}
-void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)
+void gicv3_reg_writel(u32 cpu_or_dist, u64 offset, u32 reg_val)
{
volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA
: sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));
writel(reg_val, base + offset);
}
-uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)
+u32 gicv3_getl_fields(u32 cpu_or_dist, u64 offset, u32 mask)
{
return gicv3_reg_readl(cpu_or_dist, offset) & mask;
}
-void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
- uint32_t mask, uint32_t reg_val)
+void gicv3_setl_fields(u32 cpu_or_dist, u64 offset,
+ u32 mask, u32 reg_val)
{
- uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
+ u32 tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
tmp |= (reg_val & mask);
gicv3_reg_writel(cpu_or_dist, offset, tmp);
@@ -165,14 +165,14 @@ void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
* map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
* marked as "Reserved" in the Distributor map.
*/
-static void gicv3_access_reg(uint32_t intid, uint64_t offset,
- uint32_t reg_bits, uint32_t bits_per_field,
- bool write, uint32_t *val)
+static void gicv3_access_reg(u32 intid, u64 offset,
+ u32 reg_bits, u32 bits_per_field,
+ bool write, u32 *val)
{
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
enum gicv3_intid_range intid_range = get_intid_range(intid);
- uint32_t fields_per_reg, index, mask, shift;
- uint32_t cpu_or_dist;
+ u32 fields_per_reg, index, mask, shift;
+ u32 cpu_or_dist;
GUEST_ASSERT(bits_per_field <= reg_bits);
GUEST_ASSERT(!write || *val < (1U << bits_per_field));
@@ -197,32 +197,32 @@ static void gicv3_access_reg(uint32_t intid, uint64_t offset,
*val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
}
-static void gicv3_write_reg(uint32_t intid, uint64_t offset,
- uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)
+static void gicv3_write_reg(u32 intid, u64 offset,
+ u32 reg_bits, u32 bits_per_field, u32 val)
{
gicv3_access_reg(intid, offset, reg_bits,
bits_per_field, true, &val);
}
-static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,
- uint32_t reg_bits, uint32_t bits_per_field)
+static u32 gicv3_read_reg(u32 intid, u64 offset,
+ u32 reg_bits, u32 bits_per_field)
{
- uint32_t val;
+ u32 val;
gicv3_access_reg(intid, offset, reg_bits,
bits_per_field, false, &val);
return val;
}
-static void gicv3_set_priority(uint32_t intid, uint32_t prio)
+static void gicv3_set_priority(u32 intid, u32 prio)
{
gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
}
/* Sets the intid to be level-sensitive or edge-triggered. */
-static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
+static void gicv3_irq_set_config(u32 intid, bool is_edge)
{
- uint32_t val;
+ u32 val;
/* N/A for private interrupts. */
GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
@@ -230,57 +230,57 @@ static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
}
-static void gicv3_irq_enable(uint32_t intid)
+static void gicv3_irq_enable(u32 intid)
{
bool is_spi = get_intid_range(intid) == SPI_RANGE;
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
}
-static void gicv3_irq_disable(uint32_t intid)
+static void gicv3_irq_disable(u32 intid)
{
bool is_spi = get_intid_range(intid) == SPI_RANGE;
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
}
-static void gicv3_irq_set_active(uint32_t intid)
+static void gicv3_irq_set_active(u32 intid)
{
gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
}
-static void gicv3_irq_clear_active(uint32_t intid)
+static void gicv3_irq_clear_active(u32 intid)
{
gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
}
-static bool gicv3_irq_get_active(uint32_t intid)
+static bool gicv3_irq_get_active(u32 intid)
{
return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
}
-static void gicv3_irq_set_pending(uint32_t intid)
+static void gicv3_irq_set_pending(u32 intid)
{
gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
}
-static void gicv3_irq_clear_pending(uint32_t intid)
+static void gicv3_irq_clear_pending(u32 intid)
{
gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
}
-static bool gicv3_irq_get_pending(uint32_t intid)
+static bool gicv3_irq_get_pending(u32 intid)
{
return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
}
static void gicv3_enable_redist(volatile void *redist_base)
{
- uint32_t val = readl(redist_base + GICR_WAKER);
+ u32 val = readl(redist_base + GICR_WAKER);
unsigned int count = 100000; /* 1s */
val &= ~GICR_WAKER_ProcessorSleep;
@@ -293,10 +293,10 @@ static void gicv3_enable_redist(volatile void *redist_base)
}
}
-static void gicv3_set_group(uint32_t intid, bool grp)
+static void gicv3_set_group(u32 intid, bool grp)
{
- uint32_t cpu_or_dist;
- uint32_t val;
+ u32 cpu_or_dist;
+ u32 val;
cpu_or_dist = (get_intid_range(intid) == SPI_RANGE) ? DIST_BIT : guest_get_vcpuid();
val = gicv3_reg_readl(cpu_or_dist, GICD_IGROUPR + (intid / 32) * 4);
@@ -424,8 +424,8 @@ const struct gic_common_ops gicv3_ops = {
.gic_irq_set_group = gicv3_set_group,
};
-void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,
- vm_paddr_t pend_table)
+void gic_rdist_enable_lpis(gpa_t cfg_table, size_t cfg_table_size,
+ gpa_t pend_table)
{
volatile void *rdist_base = gicr_base_cpu(guest_get_vcpuid());
diff --git a/tools/testing/selftests/kvm/lib/arm64/gic_v3_its.c b/tools/testing/selftests/kvm/lib/arm64/gic_v3_its.c
index 7f9fdcf..1188b57 100644
--- a/tools/testing/selftests/kvm/lib/arm64/gic_v3_its.c
+++ b/tools/testing/selftests/kvm/lib/arm64/gic_v3_its.c
@@ -54,7 +54,7 @@ static unsigned long its_find_baser(unsigned int type)
return -1;
}
-static void its_install_table(unsigned int type, vm_paddr_t base, size_t size)
+static void its_install_table(unsigned int type, gpa_t base, size_t size)
{
unsigned long offset = its_find_baser(type);
u64 baser;
@@ -69,7 +69,7 @@ static void its_install_table(unsigned int type, vm_paddr_t base, size_t size)
its_write_u64(offset, baser);
}
-static void its_install_cmdq(vm_paddr_t base, size_t size)
+static void its_install_cmdq(gpa_t base, size_t size)
{
u64 cbaser;
@@ -82,9 +82,8 @@ static void its_install_cmdq(vm_paddr_t base, size_t size)
its_write_u64(GITS_CBASER, cbaser);
}
-void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz,
- vm_paddr_t device_tbl, size_t device_tbl_sz,
- vm_paddr_t cmdq, size_t cmdq_size)
+void its_init(gpa_t coll_tbl, size_t coll_tbl_sz, gpa_t device_tbl,
+ size_t device_tbl_sz, gpa_t cmdq, size_t cmdq_size)
{
u32 ctlr;
@@ -204,7 +203,7 @@ static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd)
}
}
-void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base,
+void its_send_mapd_cmd(void *cmdq_base, u32 device_id, gpa_t itt_base,
size_t itt_size, bool valid)
{
struct its_cmd_block cmd = {};
diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c
index 43ea40e..01325bf 100644
--- a/tools/testing/selftests/kvm/lib/arm64/processor.c
+++ b/tools/testing/selftests/kvm/lib/arm64/processor.c
@@ -19,20 +19,20 @@
#define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000
-static vm_vaddr_t exception_handlers;
+static gva_t exception_handlers;
-static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva)
+static u64 pgd_index(struct kvm_vm *vm, gva_t gva)
{
unsigned int shift = (vm->mmu.pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
- uint64_t mask = (1UL << (vm->va_bits - shift)) - 1;
+ u64 mask = (1UL << (vm->va_bits - shift)) - 1;
return (gva >> shift) & mask;
}
-static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)
+static u64 pud_index(struct kvm_vm *vm, gva_t gva)
{
unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift;
- uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
+ u64 mask = (1UL << (vm->page_shift - 3)) - 1;
TEST_ASSERT(vm->mmu.pgtable_levels == 4,
"Mode %d does not have 4 page table levels", vm->mode);
@@ -40,10 +40,10 @@ static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)
return (gva >> shift) & mask;
}
-static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)
+static u64 pmd_index(struct kvm_vm *vm, gva_t gva)
{
unsigned int shift = (vm->page_shift - 3) + vm->page_shift;
- uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
+ u64 mask = (1UL << (vm->page_shift - 3)) - 1;
TEST_ASSERT(vm->mmu.pgtable_levels >= 3,
"Mode %d does not have >= 3 page table levels", vm->mode);
@@ -51,9 +51,9 @@ static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)
return (gva >> shift) & mask;
}
-static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva)
+static u64 pte_index(struct kvm_vm *vm, gva_t gva)
{
- uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
+ u64 mask = (1UL << (vm->page_shift - 3)) - 1;
return (gva >> vm->page_shift) & mask;
}
@@ -63,9 +63,9 @@ static inline bool use_lpa2_pte_format(struct kvm_vm *vm)
(vm->pa_bits > 48 || vm->va_bits > 48);
}
-static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs)
+static u64 addr_pte(struct kvm_vm *vm, u64 pa, u64 attrs)
{
- uint64_t pte;
+ u64 pte;
if (use_lpa2_pte_format(vm)) {
pte = pa & PTE_ADDR_MASK_LPA2(vm->page_shift);
@@ -81,9 +81,9 @@ static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs)
return pte;
}
-static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte)
+static u64 pte_addr(struct kvm_vm *vm, u64 pte)
{
- uint64_t pa;
+ u64 pa;
if (use_lpa2_pte_format(vm)) {
pa = pte & PTE_ADDR_MASK_LPA2(vm->page_shift);
@@ -97,13 +97,13 @@ static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte)
return pa;
}
-static uint64_t ptrs_per_pgd(struct kvm_vm *vm)
+static u64 ptrs_per_pgd(struct kvm_vm *vm)
{
unsigned int shift = (vm->mmu.pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
return 1 << (vm->va_bits - shift);
}
-static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm)
+static u64 __maybe_unused ptrs_per_pte(struct kvm_vm *vm)
{
return 1 << (vm->page_shift - 3);
}
@@ -121,47 +121,46 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
vm->mmu.pgd_created = true;
}
-static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
- uint64_t flags)
+static void _virt_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa,
+ u64 flags)
{
- uint8_t attr_idx = flags & (PTE_ATTRINDX_MASK >> PTE_ATTRINDX_SHIFT);
- uint64_t pg_attr;
- uint64_t *ptep;
+ u8 attr_idx = flags & (PTE_ATTRINDX_MASK >> PTE_ATTRINDX_SHIFT);
+ u64 pg_attr;
+ u64 *ptep;
- TEST_ASSERT((vaddr % vm->page_size) == 0,
+ TEST_ASSERT((gva % vm->page_size) == 0,
"Virtual address not on page boundary,\n"
- " vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
- (vaddr >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx", vaddr);
- TEST_ASSERT((paddr % vm->page_size) == 0,
- "Physical address not on page boundary,\n"
- " paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
- TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
- "Physical address beyond beyond maximum supported,\n"
- " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
- paddr, vm->max_gfn, vm->page_size);
+ " gva: 0x%lx vm->page_size: 0x%x", gva, vm->page_size);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
+ TEST_ASSERT((gpa % vm->page_size) == 0,
+ "Physical address not on page boundary,\n"
+ " gpa: 0x%lx vm->page_size: 0x%x", gpa, vm->page_size);
+ TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
+ "Physical address beyond beyond maximum supported,\n"
+ " gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
+ gpa, vm->max_gfn, vm->page_size);
- ptep = addr_gpa2hva(vm, vm->mmu.pgd) + pgd_index(vm, vaddr) * 8;
+ ptep = addr_gpa2hva(vm, vm->mmu.pgd) + pgd_index(vm, gva) * 8;
if (!*ptep)
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
PGD_TYPE_TABLE | PTE_VALID);
switch (vm->mmu.pgtable_levels) {
case 4:
- ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;
+ ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8;
if (!*ptep)
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
PUD_TYPE_TABLE | PTE_VALID);
/* fall through */
case 3:
- ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;
+ ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8;
if (!*ptep)
*ptep = addr_pte(vm, vm_alloc_page_table(vm),
PMD_TYPE_TABLE | PTE_VALID);
/* fall through */
case 2:
- ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;
+ ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8;
break;
default:
TEST_FAIL("Page table levels must be 2, 3, or 4");
@@ -171,19 +170,19 @@ static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
if (!use_lpa2_pte_format(vm))
pg_attr |= PTE_SHARED;
- *ptep = addr_pte(vm, paddr, pg_attr);
+ *ptep = addr_pte(vm, gpa, pg_attr);
}
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
- uint64_t attr_idx = MT_NORMAL;
+ u64 attr_idx = MT_NORMAL;
- _virt_pg_map(vm, vaddr, paddr, attr_idx);
+ _virt_pg_map(vm, gva, gpa, attr_idx);
}
-uint64_t *virt_get_pte_hva_at_level(struct kvm_vm *vm, vm_vaddr_t gva, int level)
+u64 *virt_get_pte_hva_at_level(struct kvm_vm *vm, gva_t gva, int level)
{
- uint64_t *ptep;
+ u64 *ptep;
if (!vm->mmu.pgd_created)
goto unmapped_gva;
@@ -225,23 +224,23 @@ uint64_t *virt_get_pte_hva_at_level(struct kvm_vm *vm, vm_vaddr_t gva, int level
exit(EXIT_FAILURE);
}
-uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva)
+u64 *virt_get_pte_hva(struct kvm_vm *vm, gva_t gva)
{
return virt_get_pte_hva_at_level(vm, gva, 3);
}
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
- uint64_t *ptep = virt_get_pte_hva(vm, gva);
+ u64 *ptep = virt_get_pte_hva(vm, gva);
return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
}
-static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
+static void pte_dump(FILE *stream, struct kvm_vm *vm, u8 indent, u64 page, int level)
{
#ifdef DEBUG
static const char * const type[] = { "", "pud", "pmd", "pte" };
- uint64_t pte, *ptep;
+ u64 pte, *ptep;
if (level == 4)
return;
@@ -256,10 +255,10 @@ static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t p
#endif
}
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
int level = 4 - (vm->mmu.pgtable_levels - 1);
- uint64_t pgd, *ptep;
+ u64 pgd, *ptep;
if (!vm->mmu.pgd_created)
return;
@@ -298,7 +297,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
{
struct kvm_vcpu_init default_init = { .target = -1, };
struct kvm_vm *vm = vcpu->vm;
- uint64_t sctlr_el1, tcr_el1, ttbr0_el1;
+ u64 sctlr_el1, tcr_el1, ttbr0_el1;
if (!init) {
kvm_get_default_vcpu_target(vm, &default_init);
@@ -397,9 +396,9 @@ void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init)
HCR_EL2_RW | HCR_EL2_TGE | HCR_EL2_E2H);
}
-void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
+void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
{
- uint64_t pstate, pc;
+ u64 pstate, pc;
pstate = vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pstate));
pc = vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc));
@@ -410,29 +409,29 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
{
- vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code);
+ vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (u64)guest_code);
}
-static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
+static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, u32 vcpu_id,
struct kvm_vcpu_init *init)
{
size_t stack_size;
- uint64_t stack_vaddr;
+ gva_t stack_gva;
struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
vm->page_size;
- stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
- DEFAULT_ARM64_GUEST_STACK_VADDR_MIN,
- MEM_REGION_DATA);
+ stack_gva = __vm_alloc(vm, stack_size,
+ DEFAULT_ARM64_GUEST_STACK_VADDR_MIN,
+ MEM_REGION_DATA);
aarch64_vcpu_setup(vcpu, init);
- vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1), stack_vaddr + stack_size);
+ vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_SP_EL1), stack_gva + stack_size);
return vcpu;
}
-struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
+struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, u32 vcpu_id,
struct kvm_vcpu_init *init, void *guest_code)
{
struct kvm_vcpu *vcpu = __aarch64_vcpu_add(vm, vcpu_id, init);
@@ -442,7 +441,7 @@ struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
return vcpu;
}
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
return __aarch64_vcpu_add(vm, vcpu_id, NULL);
}
@@ -459,13 +458,13 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
for (i = 0; i < num; i++) {
vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.regs[i]),
- va_arg(ap, uint64_t));
+ va_arg(ap, u64));
}
va_end(ap);
}
-void kvm_exit_unexpected_exception(int vector, uint64_t ec, bool valid_ec)
+void kvm_exit_unexpected_exception(int vector, u64 ec, bool valid_ec)
{
ucall(UCALL_UNHANDLED, 3, vector, ec, valid_ec);
while (1)
@@ -498,7 +497,7 @@ void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu)
{
extern char vectors;
- vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_VBAR_EL1), (uint64_t)&vectors);
+ vcpu_set_reg(vcpu, ctxt_reg_alias(vcpu, SYS_VBAR_EL1), (u64)&vectors);
}
void route_exception(struct ex_regs *regs, int vector)
@@ -536,10 +535,10 @@ void route_exception(struct ex_regs *regs, int vector)
void vm_init_descriptor_tables(struct kvm_vm *vm)
{
- vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
- vm->page_size, MEM_REGION_DATA);
+ vm->handlers = __vm_alloc(vm, sizeof(struct handlers), vm->page_size,
+ MEM_REGION_DATA);
- *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
+ *(gva_t *)addr_gva2hva(vm, (gva_t)(&exception_handlers)) = vm->handlers;
}
void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec,
@@ -563,13 +562,13 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector,
handlers->exception_handlers[vector][0] = handler;
}
-uint32_t guest_get_vcpuid(void)
+u32 guest_get_vcpuid(void)
{
return read_sysreg(tpidr_el1);
}
-static uint32_t max_ipa_for_page_size(uint32_t vm_ipa, uint32_t gran,
- uint32_t not_sup_val, uint32_t ipa52_min_val)
+static u32 max_ipa_for_page_size(u32 vm_ipa, u32 gran,
+ u32 not_sup_val, u32 ipa52_min_val)
{
if (gran == not_sup_val)
return 0;
@@ -579,16 +578,16 @@ static uint32_t max_ipa_for_page_size(uint32_t vm_ipa, uint32_t gran,
return min(vm_ipa, 48U);
}
-void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k,
- uint32_t *ipa16k, uint32_t *ipa64k)
+void aarch64_get_supported_page_sizes(u32 ipa, u32 *ipa4k,
+ u32 *ipa16k, u32 *ipa64k)
{
struct kvm_vcpu_init preferred_init;
int kvm_fd, vm_fd, vcpu_fd, err;
- uint64_t val;
- uint32_t gran;
+ u64 val;
+ u32 gran;
struct kvm_one_reg reg = {
.id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1),
- .addr = (uint64_t)&val,
+ .addr = (u64)&val,
};
kvm_fd = open_kvm_dev_path_or_exit();
@@ -646,17 +645,17 @@ void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k,
: "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7")
-void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
- uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5,
- uint64_t arg6, struct arm_smccc_res *res)
+void smccc_hvc(u32 function_id, u64 arg0, u64 arg1,
+ u64 arg2, u64 arg3, u64 arg4, u64 arg5,
+ u64 arg6, struct arm_smccc_res *res)
{
__smccc_call(hvc, function_id, arg0, arg1, arg2, arg3, arg4, arg5,
arg6, res);
}
-void smccc_smc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
- uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5,
- uint64_t arg6, struct arm_smccc_res *res)
+void smccc_smc(u32 function_id, u64 arg0, u64 arg1,
+ u64 arg2, u64 arg3, u64 arg4, u64 arg5,
+ u64 arg6, struct arm_smccc_res *res)
{
__smccc_call(smc, function_id, arg0, arg1, arg2, arg3, arg4, arg5,
arg6, res);
@@ -671,7 +670,7 @@ void kvm_selftest_arch_init(void)
guest_modes_append_default();
}
-void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
+void vm_populate_gva_bitmap(struct kvm_vm *vm)
{
/*
* arm64 selftests use only TTBR0_EL1, meaning that the valid VA space
diff --git a/tools/testing/selftests/kvm/lib/arm64/ucall.c b/tools/testing/selftests/kvm/lib/arm64/ucall.c
index ddab0ce8..e0550ad 100644
--- a/tools/testing/selftests/kvm/lib/arm64/ucall.c
+++ b/tools/testing/selftests/kvm/lib/arm64/ucall.c
@@ -6,17 +6,17 @@
*/
#include "kvm_util.h"
-vm_vaddr_t *ucall_exit_mmio_addr;
+gva_t *ucall_exit_mmio_addr;
-void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
- vm_vaddr_t mmio_gva = vm_vaddr_unused_gap(vm, vm->page_size, KVM_UTIL_MIN_VADDR);
+ gva_t mmio_gva = vm_unused_gva_gap(vm, vm->page_size, KVM_UTIL_MIN_VADDR);
virt_map(vm, mmio_gva, mmio_gpa, 1);
vm->ucall_mmio_addr = mmio_gpa;
- write_guest_global(vm, ucall_exit_mmio_addr, (vm_vaddr_t *)mmio_gva);
+ write_guest_global(vm, ucall_exit_mmio_addr, (gva_t *)mmio_gva);
}
void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu)
@@ -25,9 +25,9 @@ void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu)
if (run->exit_reason == KVM_EXIT_MMIO &&
run->mmio.phys_addr == vcpu->vm->ucall_mmio_addr) {
- TEST_ASSERT(run->mmio.is_write && run->mmio.len == sizeof(uint64_t),
+ TEST_ASSERT(run->mmio.is_write && run->mmio.len == sizeof(u64),
"Unexpected ucall exit mmio address access");
- return (void *)(*((uint64_t *)run->mmio.data));
+ return (void *)(*((u64 *)run->mmio.data));
}
return NULL;
diff --git a/tools/testing/selftests/kvm/lib/arm64/vgic.c b/tools/testing/selftests/kvm/lib/arm64/vgic.c
index d0f7bd0..4ecebf3 100644
--- a/tools/testing/selftests/kvm/lib/arm64/vgic.c
+++ b/tools/testing/selftests/kvm/lib/arm64/vgic.c
@@ -41,10 +41,10 @@ bool kvm_supports_vgic_v3(void)
* redistributor regions of the guest. Since it depends on the number of
* vCPUs for the VM, it must be called after all the vCPUs have been created.
*/
-int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs)
+int __vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, u32 nr_irqs)
{
int gic_fd;
- uint64_t attr;
+ u64 attr;
unsigned int nr_gic_pages;
/* Distributor setup */
@@ -77,7 +77,7 @@ void __vgic_v3_init(int fd)
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
}
-int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs)
+int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, u32 nr_irqs)
{
unsigned int nr_vcpus_created = 0;
struct list_head *iter;
@@ -104,11 +104,11 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs)
}
/* should only work for level sensitive interrupts */
-int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
+int _kvm_irq_set_level_info(int gic_fd, u32 intid, int level)
{
- uint64_t attr = 32 * (intid / 32);
- uint64_t index = intid % 32;
- uint64_t val;
+ u64 attr = 32 * (intid / 32);
+ u64 index = intid % 32;
+ u64 val;
int ret;
ret = __kvm_device_attr_get(gic_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
@@ -122,16 +122,16 @@ int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
return ret;
}
-void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
+void kvm_irq_set_level_info(int gic_fd, u32 intid, int level)
{
int ret = _kvm_irq_set_level_info(gic_fd, intid, level);
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO, ret));
}
-int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
+int _kvm_arm_irq_line(struct kvm_vm *vm, u32 intid, int level)
{
- uint32_t irq = intid & KVM_ARM_IRQ_NUM_MASK;
+ u32 irq = intid & KVM_ARM_IRQ_NUM_MASK;
TEST_ASSERT(!INTID_IS_SGI(intid), "KVM_IRQ_LINE's interface itself "
"doesn't allow injecting SGIs. There's no mask for it.");
@@ -144,23 +144,23 @@ int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
return _kvm_irq_line(vm, irq, level);
}
-void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
+void kvm_arm_irq_line(struct kvm_vm *vm, u32 intid, int level)
{
int ret = _kvm_arm_irq_line(vm, intid, level);
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));
}
-static void vgic_poke_irq(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu,
- uint64_t reg_off)
+static void vgic_poke_irq(int gic_fd, u32 intid, struct kvm_vcpu *vcpu,
+ u64 reg_off)
{
- uint64_t reg = intid / 32;
- uint64_t index = intid % 32;
- uint64_t attr = reg_off + reg * 4;
- uint64_t val;
+ u64 reg = intid / 32;
+ u64 index = intid % 32;
+ u64 attr = reg_off + reg * 4;
+ u64 val;
bool intid_is_private = INTID_IS_SGI(intid) || INTID_IS_PPI(intid);
- uint32_t group = intid_is_private ? KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
+ u32 group = intid_is_private ? KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
: KVM_DEV_ARM_VGIC_GRP_DIST_REGS;
if (intid_is_private) {
@@ -183,12 +183,12 @@ static void vgic_poke_irq(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu,
kvm_device_attr_set(gic_fd, group, attr, &val);
}
-void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu)
+void kvm_irq_write_ispendr(int gic_fd, u32 intid, struct kvm_vcpu *vcpu)
{
vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISPENDR);
}
-void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, struct kvm_vcpu *vcpu)
+void kvm_irq_write_isactiver(int gic_fd, u32 intid, struct kvm_vcpu *vcpu)
{
vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISACTIVER);
}
diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c
index b496906..8be0d09 100644
--- a/tools/testing/selftests/kvm/lib/assert.c
+++ b/tools/testing/selftests/kvm/lib/assert.c
@@ -6,11 +6,14 @@
*/
#include "test_util.h"
-#include <execinfo.h>
+
#include <sys/syscall.h>
#include "kselftest.h"
+#ifdef __GLIBC__
+#include <execinfo.h>
+
/* Dumps the current stack trace to stderr. */
static void __attribute__((noinline)) test_dump_stack(void);
static void test_dump_stack(void)
@@ -57,6 +60,9 @@ static void test_dump_stack(void)
system(cmd);
#pragma GCC diagnostic pop
}
+#else
+static void test_dump_stack(void) {}
+#endif
static pid_t _gettid(void)
{
diff --git a/tools/testing/selftests/kvm/lib/elf.c b/tools/testing/selftests/kvm/lib/elf.c
index f34d926d..1924a98 100644
--- a/tools/testing/selftests/kvm/lib/elf.c
+++ b/tools/testing/selftests/kvm/lib/elf.c
@@ -7,7 +7,7 @@
#include "test_util.h"
-#include <bits/endian.h>
+#include <endian.h>
#include <linux/elf.h>
#include "kvm_util.h"
@@ -156,21 +156,20 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename)
TEST_ASSERT(phdr.p_memsz > 0, "Unexpected loadable segment "
"memsize of 0,\n"
" phdr index: %u p_memsz: 0x%" PRIx64,
- n1, (uint64_t) phdr.p_memsz);
- vm_vaddr_t seg_vstart = align_down(phdr.p_vaddr, vm->page_size);
- vm_vaddr_t seg_vend = phdr.p_vaddr + phdr.p_memsz - 1;
+ n1, (u64)phdr.p_memsz);
+ gva_t seg_vstart = align_down(phdr.p_vaddr, vm->page_size);
+ gva_t seg_vend = phdr.p_vaddr + phdr.p_memsz - 1;
seg_vend |= vm->page_size - 1;
size_t seg_size = seg_vend - seg_vstart + 1;
- vm_vaddr_t vaddr = __vm_vaddr_alloc(vm, seg_size, seg_vstart,
- MEM_REGION_CODE);
- TEST_ASSERT(vaddr == seg_vstart, "Unable to allocate "
+ gva_t gva = __vm_alloc(vm, seg_size, seg_vstart, MEM_REGION_CODE);
+ TEST_ASSERT(gva == seg_vstart, "Unable to allocate "
"virtual memory for segment at requested min addr,\n"
" segment idx: %u\n"
" seg_vstart: 0x%lx\n"
- " vaddr: 0x%lx",
- n1, seg_vstart, vaddr);
- memset(addr_gva2hva(vm, vaddr), 0, seg_size);
+ " gva: 0x%lx",
+ n1, seg_vstart, gva);
+ memset(addr_gva2hva(vm, gva), 0, seg_size);
/* TODO(lhuemill): Set permissions of each memory segment
* based on the least-significant 3 bits of phdr.p_flags.
*/
diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c
index ce30996..7a96c43 100644
--- a/tools/testing/selftests/kvm/lib/guest_modes.c
+++ b/tools/testing/selftests/kvm/lib/guest_modes.c
@@ -20,7 +20,7 @@ void guest_modes_append_default(void)
#ifdef __aarch64__
{
unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE);
- uint32_t ipa4k, ipa16k, ipa64k;
+ u32 ipa4k, ipa16k, ipa64k;
int i;
aarch64_get_supported_page_sizes(limit, &ipa4k, &ipa16k, &ipa64k);
diff --git a/tools/testing/selftests/kvm/lib/guest_sprintf.c b/tools/testing/selftests/kvm/lib/guest_sprintf.c
index 7462751..7a33965 100644
--- a/tools/testing/selftests/kvm/lib/guest_sprintf.c
+++ b/tools/testing/selftests/kvm/lib/guest_sprintf.c
@@ -35,8 +35,8 @@ static int skip_atoi(const char **s)
({ \
int __res; \
\
- __res = ((uint64_t) n) % (uint32_t) base; \
- n = ((uint64_t) n) / (uint32_t) base; \
+ __res = ((u64)n) % (u32)base; \
+ n = ((u64)n) / (u32)base; \
__res; \
})
@@ -119,7 +119,7 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args)
{
char *str, *end;
const char *s;
- uint64_t num;
+ u64 num;
int i, base;
int len;
@@ -216,7 +216,7 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args)
while (--field_width > 0)
APPEND_BUFFER_SAFE(str, end, ' ');
APPEND_BUFFER_SAFE(str, end,
- (uint8_t)va_arg(args, int));
+ (u8)va_arg(args, int));
while (--field_width > 0)
APPEND_BUFFER_SAFE(str, end, ' ');
continue;
@@ -240,7 +240,7 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args)
flags |= SPECIAL | SMALL | ZEROPAD;
}
str = number(str, end,
- (uint64_t)va_arg(args, void *), 16,
+ (u64)va_arg(args, void *), 16,
field_width, precision, flags);
continue;
@@ -284,15 +284,15 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args)
continue;
}
if (qualifier == 'l')
- num = va_arg(args, uint64_t);
+ num = va_arg(args, u64);
else if (qualifier == 'h') {
- num = (uint16_t)va_arg(args, int);
+ num = (u16)va_arg(args, int);
if (flags & SIGN)
- num = (int16_t)num;
+ num = (s16)num;
} else if (flags & SIGN)
num = va_arg(args, int);
else
- num = va_arg(args, uint32_t);
+ num = va_arg(args, u32);
str = number(str, end, num, base, field_width, precision, flags);
}
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index f5e0765..e08967e 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -5,13 +5,13 @@
* Copyright (C) 2018, Google LLC.
*/
#include "test_util.h"
+#include "kvm_syscalls.h"
#include "kvm_util.h"
#include "processor.h"
#include "ucall_common.h"
#include <assert.h>
#include <sched.h>
-#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/stat.h>
@@ -20,9 +20,9 @@
#define KVM_UTIL_MIN_PFN 2
-uint32_t guest_random_seed;
+u32 guest_random_seed;
struct guest_random_state guest_rng;
-static uint32_t last_guest_seed;
+static u32 last_guest_seed;
static size_t vcpu_mmap_sz(void);
@@ -165,7 +165,7 @@ unsigned int kvm_check_cap(long cap)
return (unsigned int)ret;
}
-void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
+void vm_enable_dirty_ring(struct kvm_vm *vm, u32 ring_size)
{
if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))
vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);
@@ -189,7 +189,7 @@ static void vm_open(struct kvm_vm *vm)
vm->stats.fd = -1;
}
-const char *vm_guest_mode_string(uint32_t i)
+const char *vm_guest_mode_string(u32 i)
{
static const char * const strings[] = {
[VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
@@ -267,7 +267,7 @@ _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params)
* based on the MSB of the VA. On architectures with this behavior
* the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1].
*/
-__weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
+__weak void vm_populate_gva_bitmap(struct kvm_vm *vm)
{
sparsebit_set_num(vm->vpages_valid,
0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
@@ -385,7 +385,7 @@ struct kvm_vm *____vm_create(struct vm_shape shape)
/* Limit to VA-bit canonical virtual addresses. */
vm->vpages_valid = sparsebit_alloc();
- vm_vaddr_populate_bitmap(vm);
+ vm_populate_gva_bitmap(vm);
/* Limit physical addresses to PA-bits. */
vm->max_gfn = vm_compute_max_gfn(vm);
@@ -396,12 +396,12 @@ struct kvm_vm *____vm_create(struct vm_shape shape)
return vm;
}
-static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
- uint32_t nr_runnable_vcpus,
- uint64_t extra_mem_pages)
+static u64 vm_nr_pages_required(enum vm_guest_mode mode,
+ u32 nr_runnable_vcpus,
+ u64 extra_mem_pages)
{
- uint64_t page_size = vm_guest_mode_params[mode].page_size;
- uint64_t nr_pages;
+ u64 page_size = vm_guest_mode_params[mode].page_size;
+ u64 nr_pages;
TEST_ASSERT(nr_runnable_vcpus,
"Use vm_create_barebones() for VMs that _never_ have vCPUs");
@@ -435,7 +435,7 @@ static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
return vm_adjust_num_guest_pages(mode, nr_pages);
}
-void kvm_set_files_rlimit(uint32_t nr_vcpus)
+void kvm_set_files_rlimit(u32 nr_vcpus)
{
/*
* Each vCPU will open two file descriptors: the vCPU itself and the
@@ -476,10 +476,10 @@ static bool is_guest_memfd_required(struct vm_shape shape)
#endif
}
-struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
- uint64_t nr_extra_pages)
+struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
+ u64 nr_extra_pages)
{
- uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
+ u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
nr_extra_pages);
struct userspace_mem_region *slot0;
struct kvm_vm *vm;
@@ -546,8 +546,8 @@ struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
* extra_mem_pages is only used to calculate the maximum page table size,
* no real memory allocation for non-slot0 memory in this function.
*/
-struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
- uint64_t extra_mem_pages,
+struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, u32 nr_vcpus,
+ u64 extra_mem_pages,
void *guest_code, struct kvm_vcpu *vcpus[])
{
struct kvm_vm *vm;
@@ -566,7 +566,7 @@ struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
struct kvm_vcpu **vcpu,
- uint64_t extra_mem_pages,
+ u64 extra_mem_pages,
void *guest_code)
{
struct kvm_vcpu *vcpus[1];
@@ -614,7 +614,7 @@ void kvm_vm_restart(struct kvm_vm *vmp)
}
__weak struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm,
- uint32_t vcpu_id)
+ u32 vcpu_id)
{
return __vm_vcpu_add(vm, vcpu_id);
}
@@ -636,9 +636,9 @@ int __pin_task_to_cpu(pthread_t task, int cpu)
return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset);
}
-static uint32_t parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
+static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
{
- uint32_t pcpu = atoi_non_negative("CPU number", cpu_str);
+ u32 pcpu = atoi_non_negative("CPU number", cpu_str);
TEST_ASSERT(CPU_ISSET(pcpu, allowed_mask),
"Not allowed to run on pCPU '%d', check cgroups?", pcpu);
@@ -662,7 +662,7 @@ void kvm_print_vcpu_pinning_help(void)
" (default: no pinning)\n", name, name);
}
-void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
+void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[],
int nr_vcpus)
{
cpu_set_t allowed_mask;
@@ -715,15 +715,15 @@ void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
* region exists.
*/
static struct userspace_mem_region *
-userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
+userspace_mem_region_find(struct kvm_vm *vm, u64 start, u64 end)
{
struct rb_node *node;
for (node = vm->regions.gpa_tree.rb_node; node; ) {
struct userspace_mem_region *region =
container_of(node, struct userspace_mem_region, gpa_node);
- uint64_t existing_start = region->region.guest_phys_addr;
- uint64_t existing_end = region->region.guest_phys_addr
+ u64 existing_start = region->region.guest_phys_addr;
+ u64 existing_end = region->region.guest_phys_addr
+ region->region.memory_size - 1;
if (start <= existing_end && end >= existing_start)
return region;
@@ -918,8 +918,8 @@ static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
}
-int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva)
+int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva)
{
struct kvm_userspace_memory_region region = {
.slot = slot,
@@ -932,8 +932,8 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags
return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion);
}
-void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva)
+void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva)
{
int ret = __vm_set_user_memory_region(vm, slot, flags, gpa, size, hva);
@@ -945,9 +945,9 @@ void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
__TEST_REQUIRE(kvm_has_cap(KVM_CAP_USER_MEMORY2), \
"KVM selftests now require KVM_SET_USER_MEMORY_REGION2 (introduced in v6.8)")
-int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva,
- uint32_t guest_memfd, uint64_t guest_memfd_offset)
+int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva,
+ u32 guest_memfd, u64 guest_memfd_offset)
{
struct kvm_userspace_memory_region2 region = {
.slot = slot,
@@ -964,9 +964,9 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flag
return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, ®ion);
}
-void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
- uint64_t gpa, uint64_t size, void *hva,
- uint32_t guest_memfd, uint64_t guest_memfd_offset)
+void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
+ gpa_t gpa, u64 size, void *hva,
+ u32 guest_memfd, u64 guest_memfd_offset)
{
int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
guest_memfd, guest_memfd_offset);
@@ -978,8 +978,8 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags
/* FIXME: This thing needs to be ripped apart and rewritten. */
void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
- uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags,
- int guest_memfd, uint64_t guest_memfd_offset)
+ gpa_t gpa, u32 slot, u64 npages, u32 flags,
+ int guest_memfd, u64 guest_memfd_offset)
{
int ret;
struct userspace_mem_region *region;
@@ -1016,8 +1016,8 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
" requested gpa: 0x%lx npages: 0x%lx page_size: 0x%x\n"
" existing gpa: 0x%lx size: 0x%lx",
gpa, npages, vm->page_size,
- (uint64_t) region->region.guest_phys_addr,
- (uint64_t) region->region.memory_size);
+ (u64)region->region.guest_phys_addr,
+ (u64)region->region.memory_size);
/* Confirm no region with the requested slot already exists. */
hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
@@ -1027,11 +1027,11 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
TEST_FAIL("A mem region with the requested slot "
"already exists.\n"
- " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
- " existing slot: %u paddr: 0x%lx size: 0x%lx",
+ " requested slot: %u gpa: 0x%lx npages: 0x%lx\n"
+ " existing slot: %u gpa: 0x%lx size: 0x%lx",
slot, gpa, npages, region->region.slot,
- (uint64_t) region->region.guest_phys_addr,
- (uint64_t) region->region.memory_size);
+ (u64)region->region.guest_phys_addr,
+ (u64)region->region.memory_size);
}
/* Allocate and initialize new mem region structure. */
@@ -1085,7 +1085,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
if (flags & KVM_MEM_GUEST_MEMFD) {
if (guest_memfd < 0) {
- uint32_t guest_memfd_flags = 0;
+ u32 guest_memfd_flags = 0;
TEST_ASSERT(!guest_memfd_offset,
"Offset must be zero when creating new guest_memfd");
guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
@@ -1141,8 +1141,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
void vm_userspace_mem_region_add(struct kvm_vm *vm,
enum vm_mem_backing_src_type src_type,
- uint64_t gpa, uint32_t slot, uint64_t npages,
- uint32_t flags)
+ gpa_t gpa, u32 slot, u64 npages, u32 flags)
{
vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0);
}
@@ -1163,7 +1162,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
* memory slot ID).
*/
struct userspace_mem_region *
-memslot2region(struct kvm_vm *vm, uint32_t memslot)
+memslot2region(struct kvm_vm *vm, u32 memslot)
{
struct userspace_mem_region *region;
@@ -1194,7 +1193,7 @@ memslot2region(struct kvm_vm *vm, uint32_t memslot)
* Sets the flags of the memory region specified by the value of slot,
* to the values given by flags.
*/
-void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
+void vm_mem_region_set_flags(struct kvm_vm *vm, u32 slot, u32 flags)
{
int ret;
struct userspace_mem_region *region;
@@ -1210,7 +1209,7 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
ret, errno, slot, flags);
}
-void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot)
+void vm_mem_region_reload(struct kvm_vm *vm, u32 slot)
{
struct userspace_mem_region *region = memslot2region(vm, slot);
struct kvm_userspace_memory_region2 tmp = region->region;
@@ -1234,7 +1233,7 @@ void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot)
*
* Change the gpa of a memory region.
*/
-void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
+void vm_mem_region_move(struct kvm_vm *vm, u32 slot, u64 new_gpa)
{
struct userspace_mem_region *region;
int ret;
@@ -1263,7 +1262,7 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
*
* Delete a memory region.
*/
-void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
+void vm_mem_region_delete(struct kvm_vm *vm, u32 slot)
{
struct userspace_mem_region *region = memslot2region(vm, slot);
@@ -1273,18 +1272,18 @@ void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
__vm_mem_region_delete(vm, region);
}
-void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
+void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size,
bool punch_hole)
{
const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
struct userspace_mem_region *region;
- uint64_t end = base + size;
- uint64_t gpa, len;
+ u64 end = base + size;
+ gpa_t gpa, len;
off_t fd_offset;
int ret;
for (gpa = base; gpa < end; gpa += len) {
- uint64_t offset;
+ u64 offset;
region = userspace_mem_region_find(vm, gpa, gpa);
TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
@@ -1292,7 +1291,7 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
offset = gpa - region->region.guest_phys_addr;
fd_offset = region->region.guest_memfd_offset + offset;
- len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);
+ len = min_t(u64, end - gpa, region->region.memory_size - offset);
ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",
@@ -1317,7 +1316,7 @@ static size_t vcpu_mmap_sz(void)
return ret;
}
-static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)
+static bool vcpu_exists(struct kvm_vm *vm, u32 vcpu_id)
{
struct kvm_vcpu *vcpu;
@@ -1333,7 +1332,7 @@ static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)
* Adds a virtual CPU to the VM specified by vm with the ID given by vcpu_id.
* No additional vCPU setup is done. Returns the vCPU.
*/
-struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
struct kvm_vcpu *vcpu;
@@ -1367,33 +1366,18 @@ struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
}
/*
- * VM Virtual Address Unused Gap
- *
- * Input Args:
- * vm - Virtual Machine
- * sz - Size (bytes)
- * vaddr_min - Minimum Virtual Address
- *
- * Output Args: None
- *
- * Return:
- * Lowest virtual address at or above vaddr_min, with at least
- * sz unused bytes. TEST_ASSERT failure if no area of at least
- * size sz is available.
- *
- * Within the VM specified by vm, locates the lowest starting virtual
- * address >= vaddr_min, that has at least sz unallocated bytes. A
+ * Within the VM specified by @vm, locates the lowest starting guest virtual
+ * address >= @min_gva, that has at least @sz unallocated bytes. A
* TEST_ASSERT failure occurs for invalid input or no area of at least
- * sz unallocated bytes >= vaddr_min is available.
+ * @sz unallocated bytes >= @min_gva is available.
*/
-vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
- vm_vaddr_t vaddr_min)
+gva_t vm_unused_gva_gap(struct kvm_vm *vm, size_t sz, gva_t min_gva)
{
- uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
+ u64 pages = (sz + vm->page_size - 1) >> vm->page_shift;
/* Determine lowest permitted virtual page index. */
- uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
- if ((pgidx_start * vm->page_size) < vaddr_min)
+ u64 pgidx_start = (min_gva + vm->page_size - 1) >> vm->page_shift;
+ if ((pgidx_start * vm->page_size) < min_gva)
goto no_va_found;
/* Loop over section with enough valid virtual page indexes. */
@@ -1430,7 +1414,7 @@ vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
} while (pgidx_start != 0);
no_va_found:
- TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
+ TEST_FAIL("No gva of specified pages available, pages: 0x%lx", pages);
/* NOT REACHED */
return -1;
@@ -1452,145 +1436,91 @@ vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
return pgidx_start * vm->page_size;
}
-static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz,
- vm_vaddr_t vaddr_min,
- enum kvm_mem_region_type type,
- bool protected)
+static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva,
+ enum kvm_mem_region_type type, bool protected)
{
- uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
+ u64 pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
virt_pgd_alloc(vm);
- vm_paddr_t paddr = __vm_phy_pages_alloc(vm, pages,
- KVM_UTIL_MIN_PFN * vm->page_size,
- vm->memslots[type], protected);
+ gpa_t gpa = __vm_phy_pages_alloc(vm, pages,
+ KVM_UTIL_MIN_PFN * vm->page_size,
+ vm->memslots[type], protected);
/*
* Find an unused range of virtual page addresses of at least
* pages in length.
*/
- vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
+ gva_t gva_start = vm_unused_gva_gap(vm, sz, min_gva);
/* Map the virtual pages. */
- for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
- pages--, vaddr += vm->page_size, paddr += vm->page_size) {
+ for (gva_t gva = gva_start; pages > 0;
+ pages--, gva += vm->page_size, gpa += vm->page_size) {
- virt_pg_map(vm, vaddr, paddr);
+ virt_pg_map(vm, gva, gpa);
}
- return vaddr_start;
+ return gva_start;
}
-vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
- enum kvm_mem_region_type type)
+gva_t __vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva,
+ enum kvm_mem_region_type type)
{
- return ____vm_vaddr_alloc(vm, sz, vaddr_min, type,
- vm_arch_has_protected_memory(vm));
+ return ____vm_alloc(vm, sz, min_gva, type,
+ vm_arch_has_protected_memory(vm));
}
-vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
- vm_vaddr_t vaddr_min,
- enum kvm_mem_region_type type)
+gva_t vm_alloc_shared(struct kvm_vm *vm, size_t sz, gva_t min_gva,
+ enum kvm_mem_region_type type)
{
- return ____vm_vaddr_alloc(vm, sz, vaddr_min, type, false);
+ return ____vm_alloc(vm, sz, min_gva, type, false);
}
/*
- * VM Virtual Address Allocate
- *
- * Input Args:
- * vm - Virtual Machine
- * sz - Size in bytes
- * vaddr_min - Minimum starting virtual address
- *
- * Output Args: None
- *
- * Return:
- * Starting guest virtual address
- *
- * Allocates at least sz bytes within the virtual address space of the vm
- * given by vm. The allocated bytes are mapped to a virtual address >=
- * the address given by vaddr_min. Note that each allocation uses a
- * a unique set of pages, with the minimum real allocation being at least
- * a page. The allocated physical space comes from the TEST_DATA memory region.
+ * Allocates at least sz bytes within the virtual address space of the VM
+ * given by @vm. The allocated bytes are mapped to a virtual address >= the
+ * address given by @min_gva. Note that each allocation uses a a unique set
+ * of pages, with the minimum real allocation being at least a page. The
+ * allocated physical space comes from the TEST_DATA memory region.
*/
-vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
+gva_t vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva)
{
- return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
+ return __vm_alloc(vm, sz, min_gva, MEM_REGION_TEST_DATA);
+}
+
+gva_t vm_alloc_pages(struct kvm_vm *vm, int nr_pages)
+{
+ return vm_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
+}
+
+gva_t __vm_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
+{
+ return __vm_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
+}
+
+gva_t vm_alloc_page(struct kvm_vm *vm)
+{
+ return vm_alloc_pages(vm, 1);
}
/*
- * VM Virtual Address Allocate Pages
+ * Map a range of VM virtual address to the VM's physical address.
*
- * Input Args:
- * vm - Virtual Machine
- *
- * Output Args: None
- *
- * Return:
- * Starting guest virtual address
- *
- * Allocates at least N system pages worth of bytes within the virtual address
- * space of the vm.
+ * Within the VM given by @vm, creates a virtual translation for @npages
+ * starting at @gva to the page range starting at @gpa.
*/
-vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages)
-{
- return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
-}
-
-vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
-{
- return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
-}
-
-/*
- * VM Virtual Address Allocate Page
- *
- * Input Args:
- * vm - Virtual Machine
- *
- * Output Args: None
- *
- * Return:
- * Starting guest virtual address
- *
- * Allocates at least one system page worth of bytes within the virtual address
- * space of the vm.
- */
-vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm)
-{
- return vm_vaddr_alloc_pages(vm, 1);
-}
-
-/*
- * Map a range of VM virtual address to the VM's physical address
- *
- * Input Args:
- * vm - Virtual Machine
- * vaddr - Virtuall address to map
- * paddr - VM Physical Address
- * npages - The number of pages to map
- *
- * Output Args: None
- *
- * Return: None
- *
- * Within the VM given by @vm, creates a virtual translation for
- * @npages starting at @vaddr to the page range starting at @paddr.
- */
-void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
- unsigned int npages)
+void virt_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa, unsigned int npages)
{
size_t page_size = vm->page_size;
size_t size = npages * page_size;
- TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
- TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
+ TEST_ASSERT(gva + size > gva, "Vaddr overflow");
+ TEST_ASSERT(gpa + size > gpa, "Paddr overflow");
while (npages--) {
- virt_pg_map(vm, vaddr, paddr);
+ virt_pg_map(vm, gva, gpa);
- vaddr += page_size;
- paddr += page_size;
+ gva += page_size;
+ gpa += page_size;
}
}
@@ -1611,7 +1541,7 @@ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
* address providing the memory to the vm physical address is returned.
* A TEST_ASSERT failure occurs if no region containing gpa exists.
*/
-void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
+void *addr_gpa2hva(struct kvm_vm *vm, gpa_t gpa)
{
struct userspace_mem_region *region;
@@ -1644,7 +1574,7 @@ void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
* VM physical address is returned. A TEST_ASSERT failure occurs if no
* region containing hva exists.
*/
-vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
+gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
{
struct rb_node *node;
@@ -1655,7 +1585,7 @@ vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
if (hva >= region->host_mem) {
if (hva <= (region->host_mem
+ region->region.memory_size - 1))
- return (vm_paddr_t)((uintptr_t)
+ return (gpa_t)((uintptr_t)
region->region.guest_phys_addr
+ (hva - (uintptr_t)region->host_mem));
@@ -1687,7 +1617,7 @@ vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
* memory without mapping said memory in the guest's address space. And, for
* userfaultfd-based demand paging, to do so without triggering userfaults.
*/
-void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)
+void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa)
{
struct userspace_mem_region *region;
uintptr_t offset;
@@ -1781,8 +1711,8 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)
void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
{
- uint32_t page_size = getpagesize();
- uint32_t size = vcpu->vm->dirty_ring_size;
+ u32 page_size = getpagesize();
+ u32 size = vcpu->vm->dirty_ring_size;
TEST_ASSERT(size > 0, "Should enable dirty ring first");
@@ -1811,7 +1741,7 @@ void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
* Device Ioctl
*/
-int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
+int __kvm_has_device_attr(int dev_fd, u32 group, u64 attr)
{
struct kvm_device_attr attribute = {
.group = group,
@@ -1822,7 +1752,7 @@ int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
}
-int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)
+int __kvm_test_create_device(struct kvm_vm *vm, u64 type)
{
struct kvm_create_device create_dev = {
.type = type,
@@ -1832,7 +1762,7 @@ int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)
return __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);
}
-int __kvm_create_device(struct kvm_vm *vm, uint64_t type)
+int __kvm_create_device(struct kvm_vm *vm, u64 type)
{
struct kvm_create_device create_dev = {
.type = type,
@@ -1846,7 +1776,7 @@ int __kvm_create_device(struct kvm_vm *vm, uint64_t type)
return err ? : create_dev.fd;
}
-int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)
+int __kvm_device_attr_get(int dev_fd, u32 group, u64 attr, void *val)
{
struct kvm_device_attr kvmattr = {
.group = group,
@@ -1858,7 +1788,7 @@ int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)
return __kvm_ioctl(dev_fd, KVM_GET_DEVICE_ATTR, &kvmattr);
}
-int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)
+int __kvm_device_attr_set(int dev_fd, u32 group, u64 attr, void *val)
{
struct kvm_device_attr kvmattr = {
.group = group,
@@ -1874,7 +1804,7 @@ int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)
* IRQ related functions.
*/
-int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
+int _kvm_irq_line(struct kvm_vm *vm, u32 irq, int level)
{
struct kvm_irq_level irq_level = {
.irq = irq,
@@ -1884,7 +1814,7 @@ int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
return __vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
}
-void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
+void kvm_irq_line(struct kvm_vm *vm, u32 irq, int level)
{
int ret = _kvm_irq_line(vm, irq, level);
@@ -1906,7 +1836,7 @@ struct kvm_irq_routing *kvm_gsi_routing_create(void)
}
void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
- uint32_t gsi, uint32_t pin)
+ u32 gsi, u32 pin)
{
int i;
@@ -1956,7 +1886,7 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
* Dumps the current state of the VM given by vm, to the FILE stream
* given by stream.
*/
-void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void vm_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
int ctr;
struct userspace_mem_region *region;
@@ -1969,8 +1899,8 @@ void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
"host_virt: %p\n", indent + 2, "",
- (uint64_t) region->region.guest_phys_addr,
- (uint64_t) region->region.memory_size,
+ (u64)region->region.guest_phys_addr,
+ (u64)region->region.memory_size,
region->host_mem);
fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
sparsebit_dump(stream, region->unused_phy_pages, 0);
@@ -2077,7 +2007,7 @@ const char *exit_reason_str(unsigned int exit_reason)
* Input Args:
* vm - Virtual Machine
* num - number of pages
- * paddr_min - Physical address minimum
+ * min_gpa - Physical address minimum
* memslot - Memory region to allocate page from
* protected - True if the pages will be used as protected/private memory
*
@@ -2087,29 +2017,29 @@ const char *exit_reason_str(unsigned int exit_reason)
* Starting physical address
*
* Within the VM specified by vm, locates a range of available physical
- * pages at or above paddr_min. If found, the pages are marked as in use
+ * pages at or above min_gpa. If found, the pages are marked as in use
* and their base address is returned. A TEST_ASSERT failure occurs if
- * not enough pages are available at or above paddr_min.
+ * not enough pages are available at or above min_gpa.
*/
-vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
- vm_paddr_t paddr_min, uint32_t memslot,
- bool protected)
+gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
+ gpa_t min_gpa, u32 memslot,
+ bool protected)
{
struct userspace_mem_region *region;
sparsebit_idx_t pg, base;
TEST_ASSERT(num > 0, "Must allocate at least one page");
- TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
+ TEST_ASSERT((min_gpa % vm->page_size) == 0, "Min physical address "
"not divisible by page size.\n"
- " paddr_min: 0x%lx page_size: 0x%x",
- paddr_min, vm->page_size);
+ " min_gpa: 0x%lx page_size: 0x%x",
+ min_gpa, vm->page_size);
region = memslot2region(vm, memslot);
TEST_ASSERT(!protected || region->protected_phy_pages,
"Region doesn't support protected memory");
- base = pg = paddr_min >> vm->page_shift;
+ base = pg = min_gpa >> vm->page_shift;
do {
for (; pg < base + num; ++pg) {
if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
@@ -2121,8 +2051,8 @@ vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
if (pg == 0) {
fprintf(stderr, "No guest physical page available, "
- "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
- paddr_min, vm->page_size, memslot);
+ "min_gpa: 0x%lx page_size: 0x%x memslot: %u\n",
+ min_gpa, vm->page_size, memslot);
fputs("---- vm dump ----\n", stderr);
vm_dump(stderr, vm, 2);
abort();
@@ -2137,13 +2067,12 @@ vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
return base * vm->page_size;
}
-vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
- uint32_t memslot)
+gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot)
{
- return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
+ return vm_phy_pages_alloc(vm, 1, min_gpa, memslot);
}
-vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
+gpa_t vm_alloc_page_table(struct kvm_vm *vm)
{
return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
vm->memslots[MEM_REGION_PT]);
@@ -2161,7 +2090,7 @@ vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
* Return:
* Equivalent host virtual address
*/
-void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
+void *addr_gva2hva(struct kvm_vm *vm, gva_t gva)
{
return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
}
@@ -2259,7 +2188,7 @@ struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
* Read the data values of a specified stat from the binary stats interface.
*/
void read_stat_data(int stats_fd, struct kvm_stats_header *header,
- struct kvm_stats_desc *desc, uint64_t *data,
+ struct kvm_stats_desc *desc, u64 *data,
size_t max_elements)
{
size_t nr_elements = min_t(ssize_t, desc->size, max_elements);
@@ -2280,7 +2209,7 @@ void read_stat_data(int stats_fd, struct kvm_stats_header *header,
}
void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
- uint64_t *data, size_t max_elements)
+ u64 *data, size_t max_elements)
{
struct kvm_stats_desc *desc;
size_t size_desc;
@@ -2357,7 +2286,7 @@ void __attribute((constructor)) kvm_selftest_init(void)
kvm_selftest_arch_init();
}
-bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)
+bool vm_is_gpa_protected(struct kvm_vm *vm, gpa_t gpa)
{
sparsebit_idx_t pg = 0;
struct userspace_mem_region *region;
@@ -2365,10 +2294,10 @@ bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)
if (!vm_arch_has_protected_memory(vm))
return false;
- region = userspace_mem_region_find(vm, paddr, paddr);
- TEST_ASSERT(region, "No vm physical memory at 0x%lx", paddr);
+ region = userspace_mem_region_find(vm, gpa, gpa);
+ TEST_ASSERT(region, "No vm physical memory at 0x%lx", gpa);
- pg = paddr >> vm->page_shift;
+ pg = gpa >> vm->page_shift;
return sparsebit_is_set(region->protected_phy_pages, pg);
}
diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c
index ee4ad3b..64d91fb 100644
--- a/tools/testing/selftests/kvm/lib/loongarch/processor.c
+++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c
@@ -12,32 +12,32 @@
#define LOONGARCH_PAGE_TABLE_PHYS_MIN 0x200000
#define LOONGARCH_GUEST_STACK_VADDR_MIN 0x200000
-static vm_paddr_t invalid_pgtable[4];
-static vm_vaddr_t exception_handlers;
+static gpa_t invalid_pgtable[4];
+static gva_t exception_handlers;
-static uint64_t virt_pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)
+static u64 virt_pte_index(struct kvm_vm *vm, gva_t gva, int level)
{
unsigned int shift;
- uint64_t mask;
+ u64 mask;
shift = level * (vm->page_shift - 3) + vm->page_shift;
mask = (1UL << (vm->page_shift - 3)) - 1;
return (gva >> shift) & mask;
}
-static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)
+static u64 pte_addr(struct kvm_vm *vm, u64 entry)
{
return entry & ~((0x1UL << vm->page_shift) - 1);
}
-static uint64_t ptrs_per_pte(struct kvm_vm *vm)
+static u64 ptrs_per_pte(struct kvm_vm *vm)
{
return 1 << (vm->page_shift - 3);
}
-static void virt_set_pgtable(struct kvm_vm *vm, vm_paddr_t table, vm_paddr_t child)
+static void virt_set_pgtable(struct kvm_vm *vm, gpa_t table, gpa_t child)
{
- uint64_t *ptep;
+ u64 *ptep;
int i, ptrs_per_pte;
ptep = addr_gpa2hva(vm, table);
@@ -49,7 +49,7 @@ static void virt_set_pgtable(struct kvm_vm *vm, vm_paddr_t table, vm_paddr_t chi
void virt_arch_pgd_alloc(struct kvm_vm *vm)
{
int i;
- vm_paddr_t child, table;
+ gpa_t child, table;
if (vm->mmu.pgd_created)
return;
@@ -67,16 +67,16 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
vm->mmu.pgd_created = true;
}
-static int virt_pte_none(uint64_t *ptep, int level)
+static int virt_pte_none(u64 *ptep, int level)
{
return *ptep == invalid_pgtable[level];
}
-static uint64_t *virt_populate_pte(struct kvm_vm *vm, vm_vaddr_t gva, int alloc)
+static u64 *virt_populate_pte(struct kvm_vm *vm, gva_t gva, int alloc)
{
int level;
- uint64_t *ptep;
- vm_paddr_t child;
+ u64 *ptep;
+ gpa_t child;
if (!vm->mmu.pgd_created)
goto unmapped_gva;
@@ -106,43 +106,42 @@ static uint64_t *virt_populate_pte(struct kvm_vm *vm, vm_vaddr_t gva, int alloc)
exit(EXIT_FAILURE);
}
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
- uint64_t *ptep;
+ u64 *ptep;
ptep = virt_populate_pte(vm, gva, 0);
- TEST_ASSERT(*ptep != 0, "Virtual address vaddr: 0x%lx not mapped\n", gva);
+ TEST_ASSERT(*ptep != 0, "Virtual address gva: 0x%lx not mapped\n", gva);
return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
}
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
- uint32_t prot_bits;
- uint64_t *ptep;
+ u32 prot_bits;
+ u64 *ptep;
- TEST_ASSERT((vaddr % vm->page_size) == 0,
+ TEST_ASSERT((gva % vm->page_size) == 0,
"Virtual address not on page boundary,\n"
- "vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
- (vaddr >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx", vaddr);
- TEST_ASSERT((paddr % vm->page_size) == 0,
+ "gva: 0x%lx vm->page_size: 0x%x", gva, vm->page_size);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
+ TEST_ASSERT((gpa % vm->page_size) == 0,
"Physical address not on page boundary,\n"
- "paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
- TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
+ "gpa: 0x%lx vm->page_size: 0x%x", gpa, vm->page_size);
+ TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
"Physical address beyond maximum supported,\n"
- "paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
- paddr, vm->max_gfn, vm->page_size);
+ "gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
+ gpa, vm->max_gfn, vm->page_size);
- ptep = virt_populate_pte(vm, vaddr, 1);
+ ptep = virt_populate_pte(vm, gva, 1);
prot_bits = _PAGE_PRESENT | __READABLE | __WRITEABLE | _CACHE_CC | _PAGE_USER;
- WRITE_ONCE(*ptep, paddr | prot_bits);
+ WRITE_ONCE(*ptep, gpa | prot_bits);
}
-static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
+static void pte_dump(FILE *stream, struct kvm_vm *vm, u8 indent, u64 page, int level)
{
- uint64_t pte, *ptep;
+ u64 pte, *ptep;
static const char * const type[] = { "pte", "pmd", "pud", "pgd"};
if (level < 0)
@@ -158,7 +157,7 @@ static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t p
}
}
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
int level;
@@ -169,7 +168,7 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
pte_dump(stream, vm, indent, vm->mmu.pgd, level);
}
-void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
+void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
{
}
@@ -206,8 +205,9 @@ void vm_init_descriptor_tables(struct kvm_vm *vm)
{
void *addr;
- vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
- LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
+ vm->handlers = __vm_alloc(vm, sizeof(struct handlers),
+ LOONGARCH_GUEST_STACK_VADDR_MIN,
+ MEM_REGION_DATA);
addr = addr_gva2hva(vm, vm->handlers);
memset(addr, 0, vm->page_size);
@@ -223,7 +223,7 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector, handler_fn hand
handlers->exception_handlers[vector] = handler;
}
-uint32_t guest_get_vcpuid(void)
+u32 guest_get_vcpuid(void)
{
return csr_read(LOONGARCH_CSR_CPUID);
}
@@ -241,36 +241,36 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
va_start(ap, num);
for (i = 0; i < num; i++)
- regs.gpr[i + 4] = va_arg(ap, uint64_t);
+ regs.gpr[i + 4] = va_arg(ap, u64);
va_end(ap);
vcpu_regs_set(vcpu, ®s);
}
-static void loongarch_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+static void loongarch_set_reg(struct kvm_vcpu *vcpu, u64 id, u64 val)
{
__vcpu_set_reg(vcpu, id, val);
}
-static void loongarch_set_cpucfg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+static void loongarch_set_cpucfg(struct kvm_vcpu *vcpu, u64 id, u64 val)
{
- uint64_t cfgid;
+ u64 cfgid;
cfgid = KVM_REG_LOONGARCH_CPUCFG | KVM_REG_SIZE_U64 | 8 * id;
__vcpu_set_reg(vcpu, cfgid, val);
}
-static void loongarch_get_csr(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
+static void loongarch_get_csr(struct kvm_vcpu *vcpu, u64 id, void *addr)
{
- uint64_t csrid;
+ u64 csrid;
csrid = KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | 8 * id;
__vcpu_get_reg(vcpu, csrid, addr);
}
-static void loongarch_set_csr(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+static void loongarch_set_csr(struct kvm_vcpu *vcpu, u64 id, u64 val)
{
- uint64_t csrid;
+ u64 csrid;
csrid = KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | 8 * id;
__vcpu_set_reg(vcpu, csrid, val);
@@ -354,8 +354,8 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu)
loongarch_set_csr(vcpu, LOONGARCH_CSR_STLBPGSIZE, PS_DEFAULT_SIZE);
/* LOONGARCH_CSR_KS1 is used for exception stack */
- val = __vm_vaddr_alloc(vm, vm->page_size,
- LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
+ val = __vm_alloc(vm, vm->page_size, LOONGARCH_GUEST_STACK_VADDR_MIN,
+ MEM_REGION_DATA);
TEST_ASSERT(val != 0, "No memory for exception stack");
val = val + vm->page_size;
loongarch_set_csr(vcpu, LOONGARCH_CSR_KS1, val);
@@ -369,23 +369,23 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu)
loongarch_set_csr(vcpu, LOONGARCH_CSR_TMID, vcpu->id);
}
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
size_t stack_size;
- uint64_t stack_vaddr;
+ u64 stack_gva;
struct kvm_regs regs;
struct kvm_vcpu *vcpu;
vcpu = __vm_vcpu_add(vm, vcpu_id);
stack_size = vm->page_size;
- stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
- LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
- TEST_ASSERT(stack_vaddr != 0, "No memory for vm stack");
+ stack_gva = __vm_alloc(vm, stack_size,
+ LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
+ TEST_ASSERT(stack_gva != 0, "No memory for vm stack");
loongarch_vcpu_setup(vcpu);
/* Setup guest general purpose registers */
vcpu_regs_get(vcpu, ®s);
- regs.gpr[3] = stack_vaddr + stack_size;
+ regs.gpr[3] = stack_gva + stack_size;
vcpu_regs_set(vcpu, ®s);
return vcpu;
@@ -397,6 +397,6 @@ void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
/* Setup guest PC register */
vcpu_regs_get(vcpu, ®s);
- regs.pc = (uint64_t)guest_code;
+ regs.pc = (u64)guest_code;
vcpu_regs_set(vcpu, ®s);
}
diff --git a/tools/testing/selftests/kvm/lib/loongarch/ucall.c b/tools/testing/selftests/kvm/lib/loongarch/ucall.c
index fc6cbb5..cd49a344 100644
--- a/tools/testing/selftests/kvm/lib/loongarch/ucall.c
+++ b/tools/testing/selftests/kvm/lib/loongarch/ucall.c
@@ -9,17 +9,17 @@
* ucall_exit_mmio_addr holds per-VM values (global data is duplicated by each
* VM), it must not be accessed from host code.
*/
-vm_vaddr_t *ucall_exit_mmio_addr;
+gva_t *ucall_exit_mmio_addr;
-void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
- vm_vaddr_t mmio_gva = vm_vaddr_unused_gap(vm, vm->page_size, KVM_UTIL_MIN_VADDR);
+ gva_t mmio_gva = vm_unused_gva_gap(vm, vm->page_size, KVM_UTIL_MIN_VADDR);
virt_map(vm, mmio_gva, mmio_gpa, 1);
vm->ucall_mmio_addr = mmio_gpa;
- write_guest_global(vm, ucall_exit_mmio_addr, (vm_vaddr_t *)mmio_gva);
+ write_guest_global(vm, ucall_exit_mmio_addr, (gva_t *)mmio_gva);
}
void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu)
@@ -28,10 +28,10 @@ void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu)
if (run->exit_reason == KVM_EXIT_MMIO &&
run->mmio.phys_addr == vcpu->vm->ucall_mmio_addr) {
- TEST_ASSERT(run->mmio.is_write && run->mmio.len == sizeof(uint64_t),
+ TEST_ASSERT(run->mmio.is_write && run->mmio.len == sizeof(u64),
"Unexpected ucall exit mmio address access");
- return (void *)(*((uint64_t *)run->mmio.data));
+ return (void *)(*((u64 *)run->mmio.data));
}
return NULL;
diff --git a/tools/testing/selftests/kvm/lib/memstress.c b/tools/testing/selftests/kvm/lib/memstress.c
index 1ea735d..6dcd1591 100644
--- a/tools/testing/selftests/kvm/lib/memstress.c
+++ b/tools/testing/selftests/kvm/lib/memstress.c
@@ -16,7 +16,7 @@ struct memstress_args memstress_args;
* Guest virtual memory offset of the testing memory slot.
* Must not conflict with identity mapped test code.
*/
-static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
+static u64 guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
struct vcpu_thread {
/* The index of the vCPU. */
@@ -44,15 +44,15 @@ static struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
* Continuously write to the first 8 bytes of each page in the
* specified region.
*/
-void memstress_guest_code(uint32_t vcpu_idx)
+void memstress_guest_code(u32 vcpu_idx)
{
struct memstress_args *args = &memstress_args;
struct memstress_vcpu_args *vcpu_args = &args->vcpu_args[vcpu_idx];
struct guest_random_state rand_state;
- uint64_t gva;
- uint64_t pages;
- uint64_t addr;
- uint64_t page;
+ gva_t gva;
+ u64 pages;
+ u64 addr;
+ u64 page;
int i;
rand_state = new_guest_random_state(guest_random_seed + vcpu_idx);
@@ -76,9 +76,9 @@ void memstress_guest_code(uint32_t vcpu_idx)
addr = gva + (page * args->guest_page_size);
if (__guest_random_bool(&rand_state, args->write_percent))
- *(uint64_t *)addr = 0x0123456789ABCDEF;
+ *(u64 *)addr = 0x0123456789ABCDEF;
else
- READ_ONCE(*(uint64_t *)addr);
+ READ_ONCE(*(u64 *)addr);
}
GUEST_SYNC(1);
@@ -87,7 +87,7 @@ void memstress_guest_code(uint32_t vcpu_idx)
void memstress_setup_vcpus(struct kvm_vm *vm, int nr_vcpus,
struct kvm_vcpu *vcpus[],
- uint64_t vcpu_memory_bytes,
+ u64 vcpu_memory_bytes,
bool partition_vcpu_memory_access)
{
struct memstress_args *args = &memstress_args;
@@ -122,15 +122,15 @@ void memstress_setup_vcpus(struct kvm_vm *vm, int nr_vcpus,
}
struct kvm_vm *memstress_create_vm(enum vm_guest_mode mode, int nr_vcpus,
- uint64_t vcpu_memory_bytes, int slots,
+ u64 vcpu_memory_bytes, int slots,
enum vm_mem_backing_src_type backing_src,
bool partition_vcpu_memory_access)
{
struct memstress_args *args = &memstress_args;
struct kvm_vm *vm;
- uint64_t guest_num_pages, slot0_pages = 0;
- uint64_t backing_src_pagesz = get_backing_src_pagesz(backing_src);
- uint64_t region_end_gfn;
+ u64 guest_num_pages, slot0_pages = 0;
+ u64 backing_src_pagesz = get_backing_src_pagesz(backing_src);
+ u64 region_end_gfn;
int i;
pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
@@ -202,8 +202,8 @@ struct kvm_vm *memstress_create_vm(enum vm_guest_mode mode, int nr_vcpus,
/* Add extra memory slots for testing */
for (i = 0; i < slots; i++) {
- uint64_t region_pages = guest_num_pages / slots;
- vm_paddr_t region_start = args->gpa + region_pages * args->guest_page_size * i;
+ u64 region_pages = guest_num_pages / slots;
+ gpa_t region_start = args->gpa + region_pages * args->guest_page_size * i;
vm_userspace_mem_region_add(vm, backing_src, region_start,
MEMSTRESS_MEM_SLOT_INDEX + i,
@@ -232,7 +232,7 @@ void memstress_destroy_vm(struct kvm_vm *vm)
kvm_vm_free(vm);
}
-void memstress_set_write_percent(struct kvm_vm *vm, uint32_t write_percent)
+void memstress_set_write_percent(struct kvm_vm *vm, u32 write_percent)
{
memstress_args.write_percent = write_percent;
sync_global_to_guest(vm, memstress_args.write_percent);
@@ -244,7 +244,7 @@ void memstress_set_random_access(struct kvm_vm *vm, bool random_access)
sync_global_to_guest(vm, memstress_args.random_access);
}
-uint64_t __weak memstress_nested_pages(int nr_vcpus)
+u64 __weak memstress_nested_pages(int nr_vcpus)
{
return 0;
}
@@ -349,7 +349,7 @@ void memstress_get_dirty_log(struct kvm_vm *vm, unsigned long *bitmaps[], int sl
}
void memstress_clear_dirty_log(struct kvm_vm *vm, unsigned long *bitmaps[],
- int slots, uint64_t pages_per_slot)
+ int slots, u64 pages_per_slot)
{
int i;
@@ -360,7 +360,7 @@ void memstress_clear_dirty_log(struct kvm_vm *vm, unsigned long *bitmaps[],
}
}
-unsigned long **memstress_alloc_bitmaps(int slots, uint64_t pages_per_slot)
+unsigned long **memstress_alloc_bitmaps(int slots, u64 pages_per_slot)
{
unsigned long **bitmaps;
int i;
diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c
index 067c6b2..ded5429 100644
--- a/tools/testing/selftests/kvm/lib/riscv/processor.c
+++ b/tools/testing/selftests/kvm/lib/riscv/processor.c
@@ -15,9 +15,9 @@
#define DEFAULT_RISCV_GUEST_STACK_VADDR_MIN 0xac0000
-static vm_vaddr_t exception_handlers;
+static gva_t exception_handlers;
-bool __vcpu_has_ext(struct kvm_vcpu *vcpu, uint64_t ext)
+bool __vcpu_has_ext(struct kvm_vcpu *vcpu, u64 ext)
{
unsigned long value = 0;
int ret;
@@ -27,32 +27,32 @@ bool __vcpu_has_ext(struct kvm_vcpu *vcpu, uint64_t ext)
return !ret && !!value;
}
-static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)
+static u64 pte_addr(struct kvm_vm *vm, u64 entry)
{
return ((entry & PGTBL_PTE_ADDR_MASK) >> PGTBL_PTE_ADDR_SHIFT) <<
PGTBL_PAGE_SIZE_SHIFT;
}
-static uint64_t ptrs_per_pte(struct kvm_vm *vm)
+static u64 ptrs_per_pte(struct kvm_vm *vm)
{
- return PGTBL_PAGE_SIZE / sizeof(uint64_t);
+ return PGTBL_PAGE_SIZE / sizeof(u64);
}
-static uint64_t pte_index_mask[] = {
+static u64 pte_index_mask[] = {
PGTBL_L0_INDEX_MASK,
PGTBL_L1_INDEX_MASK,
PGTBL_L2_INDEX_MASK,
PGTBL_L3_INDEX_MASK,
};
-static uint32_t pte_index_shift[] = {
+static u32 pte_index_shift[] = {
PGTBL_L0_INDEX_SHIFT,
PGTBL_L1_INDEX_SHIFT,
PGTBL_L2_INDEX_SHIFT,
PGTBL_L3_INDEX_SHIFT,
};
-static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)
+static u64 pte_index(struct kvm_vm *vm, gva_t gva, int level)
{
TEST_ASSERT(level > -1,
"Negative page table level (%d) not possible", level);
@@ -75,26 +75,25 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
vm->mmu.pgd_created = true;
}
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
- uint64_t *ptep, next_ppn;
+ u64 *ptep, next_ppn;
int level = vm->mmu.pgtable_levels - 1;
- TEST_ASSERT((vaddr % vm->page_size) == 0,
+ TEST_ASSERT((gva % vm->page_size) == 0,
"Virtual address not on page boundary,\n"
- " vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
- (vaddr >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx", vaddr);
- TEST_ASSERT((paddr % vm->page_size) == 0,
+ " gva: 0x%lx vm->page_size: 0x%x", gva, vm->page_size);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
+ TEST_ASSERT((gpa % vm->page_size) == 0,
"Physical address not on page boundary,\n"
- " paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
- TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
+ " gpa: 0x%lx vm->page_size: 0x%x", gpa, vm->page_size);
+ TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
"Physical address beyond maximum supported,\n"
- " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
- paddr, vm->max_gfn, vm->page_size);
+ " gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
+ gpa, vm->max_gfn, vm->page_size);
- ptep = addr_gpa2hva(vm, vm->mmu.pgd) + pte_index(vm, vaddr, level) * 8;
+ ptep = addr_gpa2hva(vm, vm->mmu.pgd) + pte_index(vm, gva, level) * 8;
if (!*ptep) {
next_ppn = vm_alloc_page_table(vm) >> PGTBL_PAGE_SIZE_SHIFT;
*ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) |
@@ -104,7 +103,7 @@ void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
while (level > -1) {
ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) +
- pte_index(vm, vaddr, level) * 8;
+ pte_index(vm, gva, level) * 8;
if (!*ptep && level > 0) {
next_ppn = vm_alloc_page_table(vm) >>
PGTBL_PAGE_SIZE_SHIFT;
@@ -114,14 +113,14 @@ void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
level--;
}
- paddr = paddr >> PGTBL_PAGE_SIZE_SHIFT;
- *ptep = (paddr << PGTBL_PTE_ADDR_SHIFT) |
+ gpa = gpa >> PGTBL_PAGE_SIZE_SHIFT;
+ *ptep = (gpa << PGTBL_PTE_ADDR_SHIFT) |
PGTBL_PTE_PERM_MASK | PGTBL_PTE_VALID_MASK;
}
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
- uint64_t *ptep;
+ u64 *ptep;
int level = vm->mmu.pgtable_levels - 1;
if (!vm->mmu.pgd_created)
@@ -148,12 +147,12 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
exit(1);
}
-static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent,
- uint64_t page, int level)
+static void pte_dump(FILE *stream, struct kvm_vm *vm, u8 indent,
+ u64 page, int level)
{
#ifdef DEBUG
static const char *const type[] = { "pte", "pmd", "pud", "p4d"};
- uint64_t pte, *ptep;
+ u64 pte, *ptep;
if (level < 0)
return;
@@ -170,11 +169,11 @@ static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent,
#endif
}
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
struct kvm_mmu *mmu = &vm->mmu;
int level = mmu->pgtable_levels - 1;
- uint64_t pgd, *ptep;
+ u64 pgd, *ptep;
if (!mmu->pgd_created)
return;
@@ -233,7 +232,7 @@ void riscv_vcpu_mmu_setup(struct kvm_vcpu *vcpu)
vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(satp), satp);
}
-void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
+void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
{
struct kvm_riscv_core core;
@@ -311,20 +310,20 @@ void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.pc), (unsigned long)guest_code);
}
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
int r;
size_t stack_size;
- unsigned long stack_vaddr;
+ unsigned long stack_gva;
unsigned long current_gp = 0;
struct kvm_mp_state mps;
struct kvm_vcpu *vcpu;
stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
vm->page_size;
- stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
- DEFAULT_RISCV_GUEST_STACK_VADDR_MIN,
- MEM_REGION_DATA);
+ stack_gva = __vm_alloc(vm, stack_size,
+ DEFAULT_RISCV_GUEST_STACK_VADDR_MIN,
+ MEM_REGION_DATA);
vcpu = __vm_vcpu_add(vm, vcpu_id);
riscv_vcpu_mmu_setup(vcpu);
@@ -344,7 +343,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.gp), current_gp);
/* Setup stack pointer and program counter of guest */
- vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.sp), stack_vaddr + stack_size);
+ vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.sp), stack_gva + stack_size);
/* Setup sscratch for guest_get_vcpuid() */
vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(sscratch), vcpu_id);
@@ -358,7 +357,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
{
va_list ap;
- uint64_t id = RISCV_CORE_REG(regs.a0);
+ u64 id = RISCV_CORE_REG(regs.a0);
int i;
TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n"
@@ -393,7 +392,7 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
id = RISCV_CORE_REG(regs.a7);
break;
}
- vcpu_set_reg(vcpu, id, va_arg(ap, uint64_t));
+ vcpu_set_reg(vcpu, id, va_arg(ap, u64));
}
va_end(ap);
@@ -449,10 +448,10 @@ void vcpu_init_vector_tables(struct kvm_vcpu *vcpu)
void vm_init_vector_tables(struct kvm_vm *vm)
{
- vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
- vm->page_size, MEM_REGION_DATA);
+ vm->handlers = __vm_alloc(vm, sizeof(struct handlers), vm->page_size,
+ MEM_REGION_DATA);
- *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
+ *(gva_t *)addr_gva2hva(vm, (gva_t)(&exception_handlers)) = vm->handlers;
}
void vm_install_exception_handler(struct kvm_vm *vm, int vector, exception_handler_fn handler)
@@ -470,7 +469,7 @@ void vm_install_interrupt_handler(struct kvm_vm *vm, exception_handler_fn handle
handlers->exception_handlers[1][0] = handler;
}
-uint32_t guest_get_vcpuid(void)
+u32 guest_get_vcpuid(void)
{
return csr_read(CSR_SSCRATCH);
}
@@ -544,10 +543,10 @@ void kvm_selftest_arch_init(void)
unsigned long riscv64_get_satp_mode(void)
{
int kvm_fd, vm_fd, vcpu_fd, err;
- uint64_t val;
+ u64 val;
struct kvm_one_reg reg = {
.id = RISCV_CONFIG_REG(satp_mode),
- .addr = (uint64_t)&val,
+ .addr = (u64)&val,
};
kvm_fd = open_kvm_dev_path_or_exit();
diff --git a/tools/testing/selftests/kvm/lib/s390/diag318_test_handler.c b/tools/testing/selftests/kvm/lib/s390/diag318_test_handler.c
index 2c432fa..f548047 100644
--- a/tools/testing/selftests/kvm/lib/s390/diag318_test_handler.c
+++ b/tools/testing/selftests/kvm/lib/s390/diag318_test_handler.c
@@ -13,7 +13,7 @@
static void guest_code(void)
{
- uint64_t diag318_info = 0x12345678;
+ u64 diag318_info = 0x12345678;
asm volatile ("diag %0,0,0x318\n" : : "d" (diag318_info));
}
@@ -23,13 +23,13 @@ static void guest_code(void)
* we create an ad-hoc VM here to handle the instruction then extract the
* necessary data. It is up to the caller to decide what to do with that data.
*/
-static uint64_t diag318_handler(void)
+static u64 diag318_handler(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct kvm_run *run;
- uint64_t reg;
- uint64_t diag318_info;
+ u64 reg;
+ u64 diag318_info;
vm = vm_create_with_one_vcpu(&vcpu, guest_code);
vcpu_run(vcpu);
@@ -51,9 +51,9 @@ static uint64_t diag318_handler(void)
return diag318_info;
}
-uint64_t get_diag318_info(void)
+u64 get_diag318_info(void)
{
- static uint64_t diag318_info;
+ static u64 diag318_info;
static bool printed_skip;
/*
diff --git a/tools/testing/selftests/kvm/lib/s390/facility.c b/tools/testing/selftests/kvm/lib/s390/facility.c
index d540812..9a77805 100644
--- a/tools/testing/selftests/kvm/lib/s390/facility.c
+++ b/tools/testing/selftests/kvm/lib/s390/facility.c
@@ -10,5 +10,5 @@
#include "facility.h"
-uint64_t stfl_doublewords[NB_STFL_DOUBLEWORDS];
+u64 stfl_doublewords[NB_STFL_DOUBLEWORDS];
bool stfle_flag;
diff --git a/tools/testing/selftests/kvm/lib/s390/processor.c b/tools/testing/selftests/kvm/lib/s390/processor.c
index 6a9a660..a9adb37 100644
--- a/tools/testing/selftests/kvm/lib/s390/processor.c
+++ b/tools/testing/selftests/kvm/lib/s390/processor.c
@@ -12,7 +12,7 @@
void virt_arch_pgd_alloc(struct kvm_vm *vm)
{
- vm_paddr_t paddr;
+ gpa_t gpa;
TEST_ASSERT(vm->page_size == PAGE_SIZE, "Unsupported page size: 0x%x",
vm->page_size);
@@ -20,12 +20,12 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
if (vm->mmu.pgd_created)
return;
- paddr = vm_phy_pages_alloc(vm, PAGES_PER_REGION,
+ gpa = vm_phy_pages_alloc(vm, PAGES_PER_REGION,
KVM_GUEST_PAGE_TABLE_MIN_PADDR,
vm->memslots[MEM_REGION_PT]);
- memset(addr_gpa2hva(vm, paddr), 0xff, PAGES_PER_REGION * vm->page_size);
+ memset(addr_gpa2hva(vm, gpa), 0xff, PAGES_PER_REGION * vm->page_size);
- vm->mmu.pgd = paddr;
+ vm->mmu.pgd = gpa;
vm->mmu.pgd_created = true;
}
@@ -34,9 +34,9 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm)
* a page table (ri == 4). Returns a suitable region/segment table entry
* which points to the freshly allocated pages.
*/
-static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri)
+static u64 virt_alloc_region(struct kvm_vm *vm, int ri)
{
- uint64_t taddr;
+ u64 taddr;
taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1,
KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
@@ -47,26 +47,24 @@ static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri)
| ((ri < 4 ? (PAGES_PER_REGION - 1) : 0) & REGION_ENTRY_LENGTH);
}
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa)
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
int ri, idx;
- uint64_t *entry;
+ u64 *entry;
TEST_ASSERT((gva % vm->page_size) == 0,
- "Virtual address not on page boundary,\n"
- " vaddr: 0x%lx vm->page_size: 0x%x",
- gva, vm->page_size);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
- (gva >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx",
- gva);
+ "Virtual address not on page boundary,\n"
+ " gva: 0x%lx vm->page_size: 0x%x",
+ gva, vm->page_size);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
TEST_ASSERT((gpa % vm->page_size) == 0,
"Physical address not on page boundary,\n"
- " paddr: 0x%lx vm->page_size: 0x%x",
+ " gpa: 0x%lx vm->page_size: 0x%x",
gva, vm->page_size);
TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
"Physical address beyond beyond maximum supported,\n"
- " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
+ " gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
gva, vm->max_gfn, vm->page_size);
/* Walk through region and segment tables */
@@ -86,10 +84,10 @@ void virt_arch_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa)
entry[idx] = gpa;
}
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
int ri, idx;
- uint64_t *entry;
+ u64 *entry;
TEST_ASSERT(vm->page_size == PAGE_SIZE, "Unsupported page size: 0x%x",
vm->page_size);
@@ -111,10 +109,10 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
return (entry[idx] & ~0xffful) + (gva & 0xffful);
}
-static void virt_dump_ptes(FILE *stream, struct kvm_vm *vm, uint8_t indent,
- uint64_t ptea_start)
+static void virt_dump_ptes(FILE *stream, struct kvm_vm *vm, u8 indent,
+ u64 ptea_start)
{
- uint64_t *pte, ptea;
+ u64 *pte, ptea;
for (ptea = ptea_start; ptea < ptea_start + 0x100 * 8; ptea += 8) {
pte = addr_gpa2hva(vm, ptea);
@@ -125,10 +123,10 @@ static void virt_dump_ptes(FILE *stream, struct kvm_vm *vm, uint8_t indent,
}
}
-static void virt_dump_region(FILE *stream, struct kvm_vm *vm, uint8_t indent,
- uint64_t reg_tab_addr)
+static void virt_dump_region(FILE *stream, struct kvm_vm *vm, u8 indent,
+ u64 reg_tab_addr)
{
- uint64_t addr, *entry;
+ u64 addr, *entry;
for (addr = reg_tab_addr; addr < reg_tab_addr + 0x400 * 8; addr += 8) {
entry = addr_gpa2hva(vm, addr);
@@ -147,7 +145,7 @@ static void virt_dump_region(FILE *stream, struct kvm_vm *vm, uint8_t indent,
}
}
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
if (!vm->mmu.pgd_created)
return;
@@ -160,10 +158,10 @@ void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
vcpu->run->psw_addr = (uintptr_t)guest_code;
}
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
size_t stack_size = DEFAULT_STACK_PGS * getpagesize();
- uint64_t stack_vaddr;
+ u64 stack_gva;
struct kvm_regs regs;
struct kvm_sregs sregs;
struct kvm_vcpu *vcpu;
@@ -171,15 +169,14 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
TEST_ASSERT(vm->page_size == PAGE_SIZE, "Unsupported page size: 0x%x",
vm->page_size);
- stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
- DEFAULT_GUEST_STACK_VADDR_MIN,
- MEM_REGION_DATA);
+ stack_gva = __vm_alloc(vm, stack_size, DEFAULT_GUEST_STACK_VADDR_MIN,
+ MEM_REGION_DATA);
vcpu = __vm_vcpu_add(vm, vcpu_id);
/* Setup guest registers */
vcpu_regs_get(vcpu, ®s);
- regs.gprs[15] = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize()) - 160;
+ regs.gprs[15] = stack_gva + (DEFAULT_STACK_PGS * getpagesize()) - 160;
vcpu_regs_set(vcpu, ®s);
vcpu_sregs_get(vcpu, &sregs);
@@ -206,13 +203,13 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
vcpu_regs_get(vcpu, ®s);
for (i = 0; i < num; i++)
- regs.gprs[i + 2] = va_arg(ap, uint64_t);
+ regs.gprs[i + 2] = va_arg(ap, u64);
vcpu_regs_set(vcpu, ®s);
va_end(ap);
}
-void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
+void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
{
fprintf(stream, "%*spstate: psw: 0x%.16llx:0x%.16llx\n",
indent, "", vcpu->run->psw_mask, vcpu->run->psw_addr);
diff --git a/tools/testing/selftests/kvm/lib/sparsebit.c b/tools/testing/selftests/kvm/lib/sparsebit.c
index a99188f..4d84500 100644
--- a/tools/testing/selftests/kvm/lib/sparsebit.c
+++ b/tools/testing/selftests/kvm/lib/sparsebit.c
@@ -76,11 +76,11 @@
* the use of a binary-search tree, where each node contains at least
* the following members:
*
- * typedef uint64_t sparsebit_idx_t;
- * typedef uint64_t sparsebit_num_t;
+ * typedef u64 sparsebit_idx_t;
+ * typedef u64 sparsebit_num_t;
*
* sparsebit_idx_t idx;
- * uint32_t mask;
+ * u32 mask;
* sparsebit_num_t num_after;
*
* The idx member contains the bit index of the first bit described by this
@@ -162,7 +162,7 @@
#define DUMP_LINE_MAX 100 /* Does not include indent amount */
-typedef uint32_t mask_t;
+typedef u32 mask_t;
#define MASK_BITS (sizeof(mask_t) * CHAR_BIT)
struct node {
@@ -2056,9 +2056,9 @@ unsigned char get8(void)
return ch;
}
-uint64_t get64(void)
+u64 get64(void)
{
- uint64_t x;
+ u64 x;
x = get8();
x = (x << 8) | get8();
@@ -2074,9 +2074,9 @@ int main(void)
{
s = sparsebit_alloc();
for (;;) {
- uint8_t op = get8() & 0xf;
- uint64_t first = get64();
- uint64_t last = get64();
+ u8 op = get8() & 0xf;
+ u64 first = get64();
+ u64 last = get64();
operate(op, first, last);
}
diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c
index 8a184858..bab1bd2 100644
--- a/tools/testing/selftests/kvm/lib/test_util.c
+++ b/tools/testing/selftests/kvm/lib/test_util.c
@@ -30,15 +30,15 @@ void __attribute__((used)) expect_sigbus_handler(int signum)
* Park-Miller LCG using standard constants.
*/
-struct guest_random_state new_guest_random_state(uint32_t seed)
+struct guest_random_state new_guest_random_state(u32 seed)
{
struct guest_random_state s = {.seed = seed};
return s;
}
-uint32_t guest_random_u32(struct guest_random_state *state)
+u32 guest_random_u32(struct guest_random_state *state)
{
- state->seed = (uint64_t)state->seed * 48271 % ((uint32_t)(1 << 31) - 1);
+ state->seed = (u64)state->seed * 48271 % ((u32)(1 << 31) - 1);
return state->seed;
}
@@ -83,12 +83,12 @@ size_t parse_size(const char *size)
return base << shift;
}
-int64_t timespec_to_ns(struct timespec ts)
+s64 timespec_to_ns(struct timespec ts)
{
- return (int64_t)ts.tv_nsec + 1000000000LL * (int64_t)ts.tv_sec;
+ return (s64)ts.tv_nsec + 1000000000LL * (s64)ts.tv_sec;
}
-struct timespec timespec_add_ns(struct timespec ts, int64_t ns)
+struct timespec timespec_add_ns(struct timespec ts, s64 ns)
{
struct timespec res;
@@ -101,15 +101,15 @@ struct timespec timespec_add_ns(struct timespec ts, int64_t ns)
struct timespec timespec_add(struct timespec ts1, struct timespec ts2)
{
- int64_t ns1 = timespec_to_ns(ts1);
- int64_t ns2 = timespec_to_ns(ts2);
+ s64 ns1 = timespec_to_ns(ts1);
+ s64 ns2 = timespec_to_ns(ts2);
return timespec_add_ns((struct timespec){0}, ns1 + ns2);
}
struct timespec timespec_sub(struct timespec ts1, struct timespec ts2)
{
- int64_t ns1 = timespec_to_ns(ts1);
- int64_t ns2 = timespec_to_ns(ts2);
+ s64 ns1 = timespec_to_ns(ts1);
+ s64 ns2 = timespec_to_ns(ts2);
return timespec_add_ns((struct timespec){0}, ns1 - ns2);
}
@@ -123,7 +123,7 @@ struct timespec timespec_elapsed(struct timespec start)
struct timespec timespec_div(struct timespec ts, int divisor)
{
- int64_t ns = timespec_to_ns(ts) / divisor;
+ s64 ns = timespec_to_ns(ts) / divisor;
return timespec_add_ns((struct timespec){0}, ns);
}
@@ -225,7 +225,7 @@ size_t get_def_hugetlb_pagesz(void)
#define ANON_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS)
#define ANON_HUGE_FLAGS (ANON_FLAGS | MAP_HUGETLB)
-const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i)
+const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i)
{
static const struct vm_mem_backing_src_alias aliases[] = {
[VM_MEM_SRC_ANONYMOUS] = {
@@ -317,9 +317,9 @@ const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i)
#define MAP_HUGE_PAGE_SIZE(x) (1ULL << ((x >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK))
-size_t get_backing_src_pagesz(uint32_t i)
+size_t get_backing_src_pagesz(u32 i)
{
- uint32_t flag = vm_mem_backing_src_alias(i)->flag;
+ u32 flag = vm_mem_backing_src_alias(i)->flag;
switch (i) {
case VM_MEM_SRC_ANONYMOUS:
@@ -335,7 +335,7 @@ size_t get_backing_src_pagesz(uint32_t i)
}
}
-bool is_backing_src_hugetlb(uint32_t i)
+bool is_backing_src_hugetlb(u32 i)
{
return !!(vm_mem_backing_src_alias(i)->flag & MAP_HUGETLB);
}
diff --git a/tools/testing/selftests/kvm/lib/ucall_common.c b/tools/testing/selftests/kvm/lib/ucall_common.c
index 42151e57..029ce21 100644
--- a/tools/testing/selftests/kvm/lib/ucall_common.c
+++ b/tools/testing/selftests/kvm/lib/ucall_common.c
@@ -14,7 +14,7 @@ struct ucall_header {
struct ucall ucalls[KVM_MAX_VCPUS];
};
-int ucall_nr_pages_required(uint64_t page_size)
+int ucall_nr_pages_required(u64 page_size)
{
return align_up(sizeof(struct ucall_header), page_size) / page_size;
}
@@ -25,16 +25,16 @@ int ucall_nr_pages_required(uint64_t page_size)
*/
static struct ucall_header *ucall_pool;
-void ucall_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
+void ucall_init(struct kvm_vm *vm, gpa_t mmio_gpa)
{
struct ucall_header *hdr;
struct ucall *uc;
- vm_vaddr_t vaddr;
+ gva_t gva;
int i;
- vaddr = vm_vaddr_alloc_shared(vm, sizeof(*hdr), KVM_UTIL_MIN_VADDR,
- MEM_REGION_DATA);
- hdr = (struct ucall_header *)addr_gva2hva(vm, vaddr);
+ gva = vm_alloc_shared(vm, sizeof(*hdr), KVM_UTIL_MIN_VADDR,
+ MEM_REGION_DATA);
+ hdr = (struct ucall_header *)addr_gva2hva(vm, gva);
memset(hdr, 0, sizeof(*hdr));
for (i = 0; i < KVM_MAX_VCPUS; ++i) {
@@ -42,7 +42,7 @@ void ucall_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa)
uc->hva = uc;
}
- write_guest_global(vm, ucall_pool, (struct ucall_header *)vaddr);
+ write_guest_global(vm, ucall_pool, (struct ucall_header *)gva);
ucall_arch_init(vm, mmio_gpa);
}
@@ -79,7 +79,7 @@ static void ucall_free(struct ucall *uc)
clear_bit(uc - ucall_pool->ucalls, ucall_pool->in_use);
}
-void ucall_assert(uint64_t cmd, const char *exp, const char *file,
+void ucall_assert(u64 cmd, const char *exp, const char *file,
unsigned int line, const char *fmt, ...)
{
struct ucall *uc;
@@ -88,20 +88,20 @@ void ucall_assert(uint64_t cmd, const char *exp, const char *file,
uc = ucall_alloc();
uc->cmd = cmd;
- WRITE_ONCE(uc->args[GUEST_ERROR_STRING], (uint64_t)(exp));
- WRITE_ONCE(uc->args[GUEST_FILE], (uint64_t)(file));
+ WRITE_ONCE(uc->args[GUEST_ERROR_STRING], (u64)(exp));
+ WRITE_ONCE(uc->args[GUEST_FILE], (u64)(file));
WRITE_ONCE(uc->args[GUEST_LINE], line);
va_start(va, fmt);
guest_vsnprintf(uc->buffer, UCALL_BUFFER_LEN, fmt, va);
va_end(va);
- ucall_arch_do_ucall((vm_vaddr_t)uc->hva);
+ ucall_arch_do_ucall((gva_t)uc->hva);
ucall_free(uc);
}
-void ucall_fmt(uint64_t cmd, const char *fmt, ...)
+void ucall_fmt(u64 cmd, const char *fmt, ...)
{
struct ucall *uc;
va_list va;
@@ -113,12 +113,12 @@ void ucall_fmt(uint64_t cmd, const char *fmt, ...)
guest_vsnprintf(uc->buffer, UCALL_BUFFER_LEN, fmt, va);
va_end(va);
- ucall_arch_do_ucall((vm_vaddr_t)uc->hva);
+ ucall_arch_do_ucall((gva_t)uc->hva);
ucall_free(uc);
}
-void ucall(uint64_t cmd, int nargs, ...)
+void ucall(u64 cmd, int nargs, ...)
{
struct ucall *uc;
va_list va;
@@ -132,15 +132,15 @@ void ucall(uint64_t cmd, int nargs, ...)
va_start(va, nargs);
for (i = 0; i < nargs; ++i)
- WRITE_ONCE(uc->args[i], va_arg(va, uint64_t));
+ WRITE_ONCE(uc->args[i], va_arg(va, u64));
va_end(va);
- ucall_arch_do_ucall((vm_vaddr_t)uc->hva);
+ ucall_arch_do_ucall((gva_t)uc->hva);
ucall_free(uc);
}
-uint64_t get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc)
+u64 get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc)
{
struct ucall ucall;
void *addr;
diff --git a/tools/testing/selftests/kvm/lib/userfaultfd_util.c b/tools/testing/selftests/kvm/lib/userfaultfd_util.c
index 5bde176..ef8d76f 100644
--- a/tools/testing/selftests/kvm/lib/userfaultfd_util.c
+++ b/tools/testing/selftests/kvm/lib/userfaultfd_util.c
@@ -27,7 +27,7 @@ static void *uffd_handler_thread_fn(void *arg)
{
struct uffd_reader_args *reader_args = (struct uffd_reader_args *)arg;
int uffd = reader_args->uffd;
- int64_t pages = 0;
+ s64 pages = 0;
struct timespec start;
struct timespec ts_diff;
struct epoll_event evt;
@@ -100,8 +100,8 @@ static void *uffd_handler_thread_fn(void *arg)
}
struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
- void *hva, uint64_t len,
- uint64_t num_readers,
+ void *hva, u64 len,
+ u64 num_readers,
uffd_handler_t handler)
{
struct uffd_desc *uffd_desc;
@@ -109,7 +109,7 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
int uffd;
struct uffdio_api uffdio_api;
struct uffdio_register uffdio_register;
- uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
+ u64 expected_ioctls = ((u64)1) << _UFFDIO_COPY;
int ret, i;
PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
@@ -132,7 +132,7 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
/* In order to get minor faults, prefault via the alias. */
if (is_minor)
- expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
+ expected_ioctls = ((u64)1) << _UFFDIO_CONTINUE;
uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
TEST_ASSERT(uffd >= 0, "uffd creation failed, errno: %d", errno);
@@ -141,9 +141,9 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
uffdio_api.features = 0;
TEST_ASSERT(ioctl(uffd, UFFDIO_API, &uffdio_api) != -1,
"ioctl UFFDIO_API failed: %" PRIu64,
- (uint64_t)uffdio_api.api);
+ (u64)uffdio_api.api);
- uffdio_register.range.start = (uint64_t)hva;
+ uffdio_register.range.start = (u64)hva;
uffdio_register.range.len = len;
uffdio_register.mode = uffd_mode;
TEST_ASSERT(ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) != -1,
diff --git a/tools/testing/selftests/kvm/lib/x86/apic.c b/tools/testing/selftests/kvm/lib/x86/apic.c
index 89153a3..5182fd0 100644
--- a/tools/testing/selftests/kvm/lib/x86/apic.c
+++ b/tools/testing/selftests/kvm/lib/x86/apic.c
@@ -14,7 +14,7 @@ void apic_disable(void)
void xapic_enable(void)
{
- uint64_t val = rdmsr(MSR_IA32_APICBASE);
+ u64 val = rdmsr(MSR_IA32_APICBASE);
/* Per SDM: to enable xAPIC when in x2APIC must first disable APIC */
if (val & MSR_IA32_APICBASE_EXTD) {
diff --git a/tools/testing/selftests/kvm/lib/x86/hyperv.c b/tools/testing/selftests/kvm/lib/x86/hyperv.c
index 15bc8cd..d200c5c 100644
--- a/tools/testing/selftests/kvm/lib/x86/hyperv.c
+++ b/tools/testing/selftests/kvm/lib/x86/hyperv.c
@@ -76,23 +76,23 @@ bool kvm_hv_cpu_has(struct kvm_x86_cpu_feature feature)
}
struct hyperv_test_pages *vcpu_alloc_hyperv_test_pages(struct kvm_vm *vm,
- vm_vaddr_t *p_hv_pages_gva)
+ gva_t *p_hv_pages_gva)
{
- vm_vaddr_t hv_pages_gva = vm_vaddr_alloc_page(vm);
+ gva_t hv_pages_gva = vm_alloc_page(vm);
struct hyperv_test_pages *hv = addr_gva2hva(vm, hv_pages_gva);
/* Setup of a region of guest memory for the VP Assist page. */
- hv->vp_assist = (void *)vm_vaddr_alloc_page(vm);
+ hv->vp_assist = (void *)vm_alloc_page(vm);
hv->vp_assist_hva = addr_gva2hva(vm, (uintptr_t)hv->vp_assist);
hv->vp_assist_gpa = addr_gva2gpa(vm, (uintptr_t)hv->vp_assist);
/* Setup of a region of guest memory for the partition assist page. */
- hv->partition_assist = (void *)vm_vaddr_alloc_page(vm);
+ hv->partition_assist = (void *)vm_alloc_page(vm);
hv->partition_assist_hva = addr_gva2hva(vm, (uintptr_t)hv->partition_assist);
hv->partition_assist_gpa = addr_gva2gpa(vm, (uintptr_t)hv->partition_assist);
/* Setup of a region of guest memory for the enlightened VMCS. */
- hv->enlightened_vmcs = (void *)vm_vaddr_alloc_page(vm);
+ hv->enlightened_vmcs = (void *)vm_alloc_page(vm);
hv->enlightened_vmcs_hva = addr_gva2hva(vm, (uintptr_t)hv->enlightened_vmcs);
hv->enlightened_vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)hv->enlightened_vmcs);
@@ -100,9 +100,9 @@ struct hyperv_test_pages *vcpu_alloc_hyperv_test_pages(struct kvm_vm *vm,
return hv;
}
-int enable_vp_assist(uint64_t vp_assist_pa, void *vp_assist)
+int enable_vp_assist(u64 vp_assist_pa, void *vp_assist)
{
- uint64_t val = (vp_assist_pa & HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_MASK) |
+ u64 val = (vp_assist_pa & HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_MASK) |
HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
wrmsr(HV_X64_MSR_VP_ASSIST_PAGE, val);
diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c
index f53414b..61cf952 100644
--- a/tools/testing/selftests/kvm/lib/x86/memstress.c
+++ b/tools/testing/selftests/kvm/lib/x86/memstress.c
@@ -16,7 +16,7 @@
#include "svm_util.h"
#include "vmx.h"
-void memstress_l2_guest_code(uint64_t vcpu_id)
+void memstress_l2_guest_code(u64 vcpu_id)
{
memstress_guest_code(vcpu_id);
vmcall();
@@ -32,7 +32,7 @@ __asm__(
#define L2_GUEST_STACK_SIZE 64
-static void l1_vmx_code(struct vmx_pages *vmx, uint64_t vcpu_id)
+static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
unsigned long *rsp;
@@ -51,7 +51,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, uint64_t vcpu_id)
GUEST_DONE();
}
-static void l1_svm_code(struct svm_test_data *svm, uint64_t vcpu_id)
+static void l1_svm_code(struct svm_test_data *svm, u64 vcpu_id)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
unsigned long *rsp;
@@ -67,7 +67,7 @@ static void l1_svm_code(struct svm_test_data *svm, uint64_t vcpu_id)
}
-static void memstress_l1_guest_code(void *data, uint64_t vcpu_id)
+static void memstress_l1_guest_code(void *data, u64 vcpu_id)
{
if (this_cpu_has(X86_FEATURE_VMX))
l1_vmx_code(data, vcpu_id);
@@ -75,7 +75,7 @@ static void memstress_l1_guest_code(void *data, uint64_t vcpu_id)
l1_svm_code(data, vcpu_id);
}
-uint64_t memstress_nested_pages(int nr_vcpus)
+u64 memstress_nested_pages(int nr_vcpus)
{
/*
* 513 page tables is enough to identity-map 256 TiB of L2 with 1G
@@ -87,7 +87,7 @@ uint64_t memstress_nested_pages(int nr_vcpus)
static void memstress_setup_ept_mappings(struct kvm_vm *vm)
{
- uint64_t start, end;
+ u64 start, end;
/*
* Identity map the first 4G and the test region with 1G pages so that
@@ -104,7 +104,7 @@ static void memstress_setup_ept_mappings(struct kvm_vm *vm)
void memstress_setup_nested(struct kvm_vm *vm, int nr_vcpus, struct kvm_vcpu *vcpus[])
{
struct kvm_regs regs;
- vm_vaddr_t nested_gva;
+ gva_t nested_gva;
int vcpu_id;
TEST_REQUIRE(kvm_cpu_has_tdp());
diff --git a/tools/testing/selftests/kvm/lib/x86/pmu.c b/tools/testing/selftests/kvm/lib/x86/pmu.c
index 34cb57d..0851b74 100644
--- a/tools/testing/selftests/kvm/lib/x86/pmu.c
+++ b/tools/testing/selftests/kvm/lib/x86/pmu.c
@@ -11,7 +11,7 @@
#include "processor.h"
#include "pmu.h"
-const uint64_t intel_pmu_arch_events[] = {
+const u64 intel_pmu_arch_events[] = {
INTEL_ARCH_CPU_CYCLES,
INTEL_ARCH_INSTRUCTIONS_RETIRED,
INTEL_ARCH_REFERENCE_CYCLES,
@@ -28,7 +28,7 @@ const uint64_t intel_pmu_arch_events[] = {
};
kvm_static_assert(ARRAY_SIZE(intel_pmu_arch_events) == NR_INTEL_ARCH_EVENTS);
-const uint64_t amd_pmu_zen_events[] = {
+const u64 amd_pmu_zen_events[] = {
AMD_ZEN_CORE_CYCLES,
AMD_ZEN_INSTRUCTIONS_RETIRED,
AMD_ZEN_BRANCHES_RETIRED,
@@ -50,7 +50,7 @@ kvm_static_assert(ARRAY_SIZE(amd_pmu_zen_events) == NR_AMD_ZEN_EVENTS);
* be overcounted on these certain instructions, but for Clearwater Forest
* only "Instruction Retired" event is overcounted on these instructions.
*/
-static uint64_t get_pmu_errata(void)
+static u64 get_pmu_errata(void)
{
if (!this_cpu_is_intel())
return 0;
@@ -72,7 +72,7 @@ static uint64_t get_pmu_errata(void)
}
}
-uint64_t pmu_errata_mask;
+u64 pmu_errata_mask;
void kvm_init_pmu_errata(void)
{
diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c
index 01f0f97..b51467d 100644
--- a/tools/testing/selftests/kvm/lib/x86/processor.c
+++ b/tools/testing/selftests/kvm/lib/x86/processor.c
@@ -21,13 +21,13 @@
#define KERNEL_DS 0x10
#define KERNEL_TSS 0x18
-vm_vaddr_t exception_handlers;
+gva_t exception_handlers;
bool host_cpu_is_amd;
bool host_cpu_is_intel;
bool host_cpu_is_hygon;
bool host_cpu_is_amd_compatible;
bool is_forced_emulation_enabled;
-uint64_t guest_tsc_khz;
+u64 guest_tsc_khz;
const char *ex_str(int vector)
{
@@ -62,7 +62,7 @@ const char *ex_str(int vector)
}
}
-static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent)
+static void regs_dump(FILE *stream, struct kvm_regs *regs, u8 indent)
{
fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
"rcx: 0x%.16llx rdx: 0x%.16llx\n",
@@ -86,7 +86,7 @@ static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent)
}
static void segment_dump(FILE *stream, struct kvm_segment *segment,
- uint8_t indent)
+ u8 indent)
{
fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
"selector: 0x%.4x type: 0x%.2x\n",
@@ -103,7 +103,7 @@ static void segment_dump(FILE *stream, struct kvm_segment *segment,
}
static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
- uint8_t indent)
+ u8 indent)
{
fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
"padding: 0x%.4x 0x%.4x 0x%.4x\n",
@@ -111,7 +111,7 @@ static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
dtable->padding[0], dtable->padding[1], dtable->padding[2]);
}
-static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent)
+static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent)
{
unsigned int i;
@@ -207,37 +207,37 @@ void tdp_mmu_init(struct kvm_vm *vm, int pgtable_levels,
}
static void *virt_get_pte(struct kvm_vm *vm, struct kvm_mmu *mmu,
- uint64_t *parent_pte, uint64_t vaddr, int level)
+ u64 *parent_pte, gva_t gva, int level)
{
- uint64_t pt_gpa = PTE_GET_PA(*parent_pte);
- uint64_t *page_table = addr_gpa2hva(vm, pt_gpa);
- int index = (vaddr >> PG_LEVEL_SHIFT(level)) & 0x1ffu;
+ u64 pt_gpa = PTE_GET_PA(*parent_pte);
+ u64 *page_table = addr_gpa2hva(vm, pt_gpa);
+ int index = (gva >> PG_LEVEL_SHIFT(level)) & 0x1ffu;
TEST_ASSERT((*parent_pte == mmu->pgd) || is_present_pte(mmu, parent_pte),
"Parent PTE (level %d) not PRESENT for gva: 0x%08lx",
- level + 1, vaddr);
+ level + 1, gva);
return &page_table[index];
}
-static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
- struct kvm_mmu *mmu,
- uint64_t *parent_pte,
- uint64_t vaddr,
- uint64_t paddr,
- int current_level,
- int target_level)
+static u64 *virt_create_upper_pte(struct kvm_vm *vm,
+ struct kvm_mmu *mmu,
+ u64 *parent_pte,
+ gva_t gva,
+ gpa_t gpa,
+ int current_level,
+ int target_level)
{
- uint64_t *pte = virt_get_pte(vm, mmu, parent_pte, vaddr, current_level);
+ u64 *pte = virt_get_pte(vm, mmu, parent_pte, gva, current_level);
- paddr = vm_untag_gpa(vm, paddr);
+ gpa = vm_untag_gpa(vm, gpa);
if (!is_present_pte(mmu, pte)) {
*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |
PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |
PTE_ALWAYS_SET_MASK(mmu);
if (current_level == target_level)
- *pte |= PTE_HUGE_MASK(mmu) | (paddr & PHYSICAL_PAGE_MASK);
+ *pte |= PTE_HUGE_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);
else
*pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
} else {
@@ -247,39 +247,39 @@ static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
* this level.
*/
TEST_ASSERT(current_level != target_level,
- "Cannot create hugepage at level: %u, vaddr: 0x%lx",
- current_level, vaddr);
+ "Cannot create hugepage at level: %u, gva: 0x%lx",
+ current_level, gva);
TEST_ASSERT(!is_huge_pte(mmu, pte),
- "Cannot create page table at level: %u, vaddr: 0x%lx",
- current_level, vaddr);
+ "Cannot create page table at level: %u, gva: 0x%lx",
+ current_level, gva);
}
return pte;
}
-void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, uint64_t vaddr,
- uint64_t paddr, int level)
+void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva,
+ gpa_t gpa, int level)
{
- const uint64_t pg_size = PG_LEVEL_SIZE(level);
- uint64_t *pte = &mmu->pgd;
+ const u64 pg_size = PG_LEVEL_SIZE(level);
+ u64 *pte = &mmu->pgd;
int current_level;
TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
"Unknown or unsupported guest mode: 0x%x", vm->mode);
- TEST_ASSERT((vaddr % pg_size) == 0,
+ TEST_ASSERT((gva % pg_size) == 0,
"Virtual address not aligned,\n"
- "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx", vaddr);
- TEST_ASSERT((paddr % pg_size) == 0,
+ "gva: 0x%lx page size: 0x%lx", gva, pg_size);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
+ TEST_ASSERT((gpa % pg_size) == 0,
"Physical address not aligned,\n"
- " paddr: 0x%lx page size: 0x%lx", paddr, pg_size);
- TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
+ " gpa: 0x%lx page size: 0x%lx", gpa, pg_size);
+ TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
"Physical address beyond maximum supported,\n"
- " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
- paddr, vm->max_gfn, vm->page_size);
- TEST_ASSERT(vm_untag_gpa(vm, paddr) == paddr,
- "Unexpected bits in paddr: %lx", paddr);
+ " gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
+ gpa, vm->max_gfn, vm->page_size);
+ TEST_ASSERT(vm_untag_gpa(vm, gpa) == gpa,
+ "Unexpected bits in gpa: %lx", gpa);
TEST_ASSERT(!PTE_EXECUTABLE_MASK(mmu) || !PTE_NX_MASK(mmu),
"X and NX bit masks cannot be used simultaneously");
@@ -291,40 +291,40 @@ void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, uint64_t vaddr,
for (current_level = mmu->pgtable_levels;
current_level > PG_LEVEL_4K;
current_level--) {
- pte = virt_create_upper_pte(vm, mmu, pte, vaddr, paddr,
+ pte = virt_create_upper_pte(vm, mmu, pte, gva, gpa,
current_level, level);
if (is_huge_pte(mmu, pte))
return;
}
/* Fill in page table entry. */
- pte = virt_get_pte(vm, mmu, pte, vaddr, PG_LEVEL_4K);
+ pte = virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);
TEST_ASSERT(!is_present_pte(mmu, pte),
- "PTE already present for 4k page at vaddr: 0x%lx", vaddr);
+ "PTE already present for 4k page at gva: 0x%lx", gva);
*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |
PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |
- PTE_ALWAYS_SET_MASK(mmu) | (paddr & PHYSICAL_PAGE_MASK);
+ PTE_ALWAYS_SET_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);
/*
* Neither SEV nor TDX supports shared page tables, so only the final
* leaf PTE needs manually set the C/S-bit.
*/
- if (vm_is_gpa_protected(vm, paddr))
+ if (vm_is_gpa_protected(vm, gpa))
*pte |= PTE_C_BIT_MASK(mmu);
else
*pte |= PTE_S_BIT_MASK(mmu);
}
-void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
+void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
{
- __virt_pg_map(vm, &vm->mmu, vaddr, paddr, PG_LEVEL_4K);
+ __virt_pg_map(vm, &vm->mmu, gva, gpa, PG_LEVEL_4K);
}
-void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
- uint64_t nr_bytes, int level)
+void virt_map_level(struct kvm_vm *vm, gva_t gva, gpa_t gpa,
+ u64 nr_bytes, int level)
{
- uint64_t pg_size = PG_LEVEL_SIZE(level);
- uint64_t nr_pages = nr_bytes / pg_size;
+ u64 pg_size = PG_LEVEL_SIZE(level);
+ u64 nr_pages = nr_bytes / pg_size;
int i;
TEST_ASSERT(nr_bytes % pg_size == 0,
@@ -332,16 +332,16 @@ void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
nr_bytes, pg_size);
for (i = 0; i < nr_pages; i++) {
- __virt_pg_map(vm, &vm->mmu, vaddr, paddr, level);
- sparsebit_set_num(vm->vpages_mapped, vaddr >> vm->page_shift,
+ __virt_pg_map(vm, &vm->mmu, gva, gpa, level);
+ sparsebit_set_num(vm->vpages_mapped, gva >> vm->page_shift,
nr_bytes / PAGE_SIZE);
- vaddr += pg_size;
- paddr += pg_size;
+ gva += pg_size;
+ gpa += pg_size;
}
}
-static bool vm_is_target_pte(struct kvm_mmu *mmu, uint64_t *pte,
+static bool vm_is_target_pte(struct kvm_mmu *mmu, u64 *pte,
int *level, int current_level)
{
if (is_huge_pte(mmu, pte)) {
@@ -354,13 +354,13 @@ static bool vm_is_target_pte(struct kvm_mmu *mmu, uint64_t *pte,
return *level == current_level;
}
-static uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm,
- struct kvm_mmu *mmu,
- uint64_t vaddr,
- int *level)
+static u64 *__vm_get_page_table_entry(struct kvm_vm *vm,
+ struct kvm_mmu *mmu,
+ gva_t gva,
+ int *level)
{
int va_width = 12 + (mmu->pgtable_levels) * 9;
- uint64_t *pte = &mmu->pgd;
+ u64 *pte = &mmu->pgd;
int current_level;
TEST_ASSERT(!vm->arch.is_pt_protected,
@@ -371,49 +371,46 @@ static uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm,
TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
"Unknown or unsupported guest mode: 0x%x", vm->mode);
- TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
- (vaddr >> vm->page_shift)),
- "Invalid virtual address, vaddr: 0x%lx",
- vaddr);
+ TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
+ "Invalid virtual address, gva: 0x%lx", gva);
/*
- * Check that the vaddr is a sign-extended va_width value.
+ * Check that the gva is a sign-extended va_width value.
*/
- TEST_ASSERT(vaddr ==
- (((int64_t)vaddr << (64 - va_width) >> (64 - va_width))),
+ TEST_ASSERT(gva == (((s64)gva << (64 - va_width) >> (64 - va_width))),
"Canonical check failed. The virtual address is invalid.");
for (current_level = mmu->pgtable_levels;
current_level > PG_LEVEL_4K;
current_level--) {
- pte = virt_get_pte(vm, mmu, pte, vaddr, current_level);
+ pte = virt_get_pte(vm, mmu, pte, gva, current_level);
if (vm_is_target_pte(mmu, pte, level, current_level))
return pte;
}
- return virt_get_pte(vm, mmu, pte, vaddr, PG_LEVEL_4K);
+ return virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);
}
-uint64_t *tdp_get_pte(struct kvm_vm *vm, uint64_t l2_gpa)
+u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa)
{
int level = PG_LEVEL_4K;
return __vm_get_page_table_entry(vm, &vm->stage2_mmu, l2_gpa, &level);
}
-uint64_t *vm_get_pte(struct kvm_vm *vm, uint64_t vaddr)
+u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva)
{
int level = PG_LEVEL_4K;
- return __vm_get_page_table_entry(vm, &vm->mmu, vaddr, &level);
+ return __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);
}
-void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
{
struct kvm_mmu *mmu = &vm->mmu;
- uint64_t *pml4e, *pml4e_start;
- uint64_t *pdpe, *pdpe_start;
- uint64_t *pde, *pde_start;
- uint64_t *pte, *pte_start;
+ u64 *pml4e, *pml4e_start;
+ u64 *pdpe, *pdpe_start;
+ u64 *pde, *pde_start;
+ u64 *pte, *pte_start;
if (!mmu->pgd_created)
return;
@@ -423,8 +420,8 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
fprintf(stream, "%*s index hvaddr gpaddr "
"addr w exec dirty\n",
indent, "");
- pml4e_start = (uint64_t *) addr_gpa2hva(vm, mmu->pgd);
- for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
+ pml4e_start = (u64 *)addr_gpa2hva(vm, mmu->pgd);
+ for (u16 n1 = 0; n1 <= 0x1ffu; n1++) {
pml4e = &pml4e_start[n1];
if (!is_present_pte(mmu, pml4e))
continue;
@@ -436,7 +433,7 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
is_writable_pte(mmu, pml4e), is_nx_pte(mmu, pml4e));
pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
- for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
+ for (u16 n2 = 0; n2 <= 0x1ffu; n2++) {
pdpe = &pdpe_start[n2];
if (!is_present_pte(mmu, pdpe))
continue;
@@ -449,7 +446,7 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
is_nx_pte(mmu, pdpe));
pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
- for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
+ for (u16 n3 = 0; n3 <= 0x1ffu; n3++) {
pde = &pde_start[n3];
if (!is_present_pte(mmu, pde))
continue;
@@ -461,7 +458,7 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
is_nx_pte(mmu, pde));
pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
- for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
+ for (u16 n4 = 0; n4 <= 0x1ffu; n4++) {
pte = &pte_start[n4];
if (!is_present_pte(mmu, pte))
continue;
@@ -475,10 +472,10 @@ void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
is_writable_pte(mmu, pte),
is_nx_pte(mmu, pte),
is_dirty_pte(mmu, pte),
- ((uint64_t) n1 << 27)
- | ((uint64_t) n2 << 18)
- | ((uint64_t) n3 << 9)
- | ((uint64_t) n4));
+ ((u64)n1 << 27)
+ | ((u64)n2 << 18)
+ | ((u64)n3 << 9)
+ | ((u64)n4));
}
}
}
@@ -498,26 +495,24 @@ bool kvm_cpu_has_tdp(void)
return kvm_cpu_has_ept() || kvm_cpu_has_npt();
}
-void __tdp_map(struct kvm_vm *vm, uint64_t nested_paddr, uint64_t paddr,
- uint64_t size, int level)
+void __tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size, int level)
{
size_t page_size = PG_LEVEL_SIZE(level);
size_t npages = size / page_size;
- TEST_ASSERT(nested_paddr + size > nested_paddr, "Vaddr overflow");
- TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
+ TEST_ASSERT(l2_gpa + size > l2_gpa, "L2 GPA overflow");
+ TEST_ASSERT(gpa + size > gpa, "GPA overflow");
while (npages--) {
- __virt_pg_map(vm, &vm->stage2_mmu, nested_paddr, paddr, level);
- nested_paddr += page_size;
- paddr += page_size;
+ __virt_pg_map(vm, &vm->stage2_mmu, l2_gpa, gpa, level);
+ l2_gpa += page_size;
+ gpa += page_size;
}
}
-void tdp_map(struct kvm_vm *vm, uint64_t nested_paddr, uint64_t paddr,
- uint64_t size)
+void tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size)
{
- __tdp_map(vm, nested_paddr, paddr, size, PG_LEVEL_4K);
+ __tdp_map(vm, l2_gpa, gpa, size, PG_LEVEL_4K);
}
/* Prepare an identity extended page table that maps all the
@@ -525,7 +520,7 @@ void tdp_map(struct kvm_vm *vm, uint64_t nested_paddr, uint64_t paddr,
*/
void tdp_identity_map_default_memslots(struct kvm_vm *vm)
{
- uint32_t s, memslot = 0;
+ u32 s, memslot = 0;
sparsebit_idx_t i, last;
struct userspace_mem_region *region = memslot2region(vm, memslot);
@@ -540,13 +535,13 @@ void tdp_identity_map_default_memslots(struct kvm_vm *vm)
if (i > last)
break;
- tdp_map(vm, (uint64_t)i << vm->page_shift,
- (uint64_t)i << vm->page_shift, 1 << vm->page_shift);
+ tdp_map(vm, (u64)i << vm->page_shift,
+ (u64)i << vm->page_shift, 1 << vm->page_shift);
}
}
/* Identity map a region with 1GiB Pages. */
-void tdp_identity_map_1g(struct kvm_vm *vm, uint64_t addr, uint64_t size)
+void tdp_identity_map_1g(struct kvm_vm *vm, u64 addr, u64 size)
{
__tdp_map(vm, addr, addr, size, PG_LEVEL_1G);
}
@@ -618,10 +613,10 @@ static void kvm_seg_set_kernel_data_64bit(struct kvm_segment *segp)
segp->present = true;
}
-vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
{
int level = PG_LEVEL_NONE;
- uint64_t *pte = __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);
+ u64 *pte = __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);
TEST_ASSERT(is_present_pte(&vm->mmu, pte),
"Leaf PTE not PRESENT for gva: 0x%08lx", gva);
@@ -633,7 +628,7 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
return vm_untag_gpa(vm, PTE_GET_PA(*pte)) | (gva & ~HUGEPAGE_MASK(level));
}
-static void kvm_seg_set_tss_64bit(vm_vaddr_t base, struct kvm_segment *segp)
+static void kvm_seg_set_tss_64bit(gva_t base, struct kvm_segment *segp)
{
memset(segp, 0, sizeof(*segp));
segp->base = base;
@@ -746,16 +741,16 @@ static void vm_init_descriptor_tables(struct kvm_vm *vm)
struct kvm_segment seg;
int i;
- vm->arch.gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
- vm->arch.idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
- vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
- vm->arch.tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
+ vm->arch.gdt = __vm_alloc_page(vm, MEM_REGION_DATA);
+ vm->arch.idt = __vm_alloc_page(vm, MEM_REGION_DATA);
+ vm->handlers = __vm_alloc_page(vm, MEM_REGION_DATA);
+ vm->arch.tss = __vm_alloc_page(vm, MEM_REGION_DATA);
/* Handlers have the same address in both address spaces.*/
for (i = 0; i < NUM_INTERRUPTS; i++)
set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0, KERNEL_CS);
- *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
+ *(gva_t *)addr_gva2hva(vm, (gva_t)(&exception_handlers)) = vm->handlers;
kvm_seg_set_kernel_code_64bit(&seg);
kvm_seg_fill_gdt_64bit(vm, &seg);
@@ -770,9 +765,9 @@ static void vm_init_descriptor_tables(struct kvm_vm *vm)
void vm_install_exception_handler(struct kvm_vm *vm, int vector,
void (*handler)(struct ex_regs *))
{
- vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers);
+ gva_t *handlers = (gva_t *)addr_gva2hva(vm, vm->handlers);
- handlers[vector] = (vm_vaddr_t)handler;
+ handlers[vector] = (gva_t)handler;
}
void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
@@ -821,18 +816,17 @@ void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
vcpu_regs_set(vcpu, ®s);
}
-struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
{
struct kvm_mp_state mp_state;
struct kvm_regs regs;
- vm_vaddr_t stack_vaddr;
+ gva_t stack_gva;
struct kvm_vcpu *vcpu;
- stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
- DEFAULT_GUEST_STACK_VADDR_MIN,
- MEM_REGION_DATA);
+ stack_gva = __vm_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
+ DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
- stack_vaddr += DEFAULT_STACK_PGS * getpagesize();
+ stack_gva += DEFAULT_STACK_PGS * getpagesize();
/*
* Align stack to match calling sequence requirements in section "The
@@ -843,9 +837,9 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
* If this code is ever used to launch a vCPU with 32-bit entry point it
* may need to subtract 4 bytes instead of 8 bytes.
*/
- TEST_ASSERT(IS_ALIGNED(stack_vaddr, PAGE_SIZE),
- "__vm_vaddr_alloc() did not provide a page-aligned address");
- stack_vaddr -= 8;
+ TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE),
+ "__vm_alloc() did not provide a page-aligned address");
+ stack_gva -= 8;
vcpu = __vm_vcpu_add(vm, vcpu_id);
vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
@@ -855,7 +849,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
/* Setup guest general purpose registers */
vcpu_regs_get(vcpu, ®s);
regs.rflags = regs.rflags | 0x2;
- regs.rsp = stack_vaddr;
+ regs.rsp = stack_gva;
vcpu_regs_set(vcpu, ®s);
/* Setup the MP state */
@@ -872,7 +866,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
return vcpu;
}
-struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id)
+struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, u32 vcpu_id)
{
struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
@@ -907,9 +901,9 @@ const struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
return kvm_supported_cpuid;
}
-static uint32_t __kvm_cpu_has(const struct kvm_cpuid2 *cpuid,
- uint32_t function, uint32_t index,
- uint8_t reg, uint8_t lo, uint8_t hi)
+static u32 __kvm_cpu_has(const struct kvm_cpuid2 *cpuid,
+ u32 function, u32 index,
+ u8 reg, u8 lo, u8 hi)
{
const struct kvm_cpuid_entry2 *entry;
int i;
@@ -936,14 +930,14 @@ bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
feature.reg, feature.bit, feature.bit);
}
-uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
- struct kvm_x86_cpu_property property)
+u32 kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
+ struct kvm_x86_cpu_property property)
{
return __kvm_cpu_has(cpuid, property.function, property.index,
property.reg, property.lo_bit, property.hi_bit);
}
-uint64_t kvm_get_feature_msr(uint64_t msr_index)
+u64 kvm_get_feature_msr(u64 msr_index)
{
struct {
struct kvm_msrs header;
@@ -962,7 +956,7 @@ uint64_t kvm_get_feature_msr(uint64_t msr_index)
return buffer.entry.data;
}
-void __vm_xsave_require_permission(uint64_t xfeature, const char *name)
+void __vm_xsave_require_permission(u64 xfeature, const char *name)
{
int kvm_fd;
u64 bitmask;
@@ -1019,7 +1013,7 @@ void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid)
void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
struct kvm_x86_cpu_property property,
- uint32_t value)
+ u32 value)
{
struct kvm_cpuid_entry2 *entry;
@@ -1034,7 +1028,7 @@ void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
TEST_ASSERT_EQ(kvm_cpuid_property(vcpu->cpuid, property), value);
}
-void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function)
+void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function)
{
struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function);
@@ -1063,7 +1057,7 @@ void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
vcpu_set_cpuid(vcpu);
}
-uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index)
+u64 vcpu_get_msr(struct kvm_vcpu *vcpu, u64 msr_index)
{
struct {
struct kvm_msrs header;
@@ -1078,7 +1072,7 @@ uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index)
return buffer.entry.data;
}
-int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value)
+int _vcpu_set_msr(struct kvm_vcpu *vcpu, u64 msr_index, u64 msr_value)
{
struct {
struct kvm_msrs header;
@@ -1106,28 +1100,28 @@ void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
vcpu_regs_get(vcpu, ®s);
if (num >= 1)
- regs.rdi = va_arg(ap, uint64_t);
+ regs.rdi = va_arg(ap, u64);
if (num >= 2)
- regs.rsi = va_arg(ap, uint64_t);
+ regs.rsi = va_arg(ap, u64);
if (num >= 3)
- regs.rdx = va_arg(ap, uint64_t);
+ regs.rdx = va_arg(ap, u64);
if (num >= 4)
- regs.rcx = va_arg(ap, uint64_t);
+ regs.rcx = va_arg(ap, u64);
if (num >= 5)
- regs.r8 = va_arg(ap, uint64_t);
+ regs.r8 = va_arg(ap, u64);
if (num >= 6)
- regs.r9 = va_arg(ap, uint64_t);
+ regs.r9 = va_arg(ap, u64);
vcpu_regs_set(vcpu, ®s);
va_end(ap);
}
-void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
+void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
{
struct kvm_regs regs;
struct kvm_sregs sregs;
@@ -1196,7 +1190,7 @@ const struct kvm_msr_list *kvm_get_feature_msr_index_list(void)
return list;
}
-bool kvm_msr_is_in_save_restore_list(uint32_t msr_index)
+bool kvm_msr_is_in_save_restore_list(u32 msr_index)
{
const struct kvm_msr_list *list = kvm_get_msr_index_list();
int i;
@@ -1327,7 +1321,7 @@ void kvm_init_vm_address_properties(struct kvm_vm *vm)
}
const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
- uint32_t function, uint32_t index)
+ u32 function, u32 index)
{
int i;
@@ -1344,7 +1338,7 @@ const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
#define X86_HYPERCALL(inputs...) \
({ \
- uint64_t r; \
+ u64 r; \
\
asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t" \
"jnz 1f\n\t" \
@@ -1359,18 +1353,17 @@ const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
r; \
})
-uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
- uint64_t a3)
+u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3)
{
return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
}
-uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
+u64 __xen_hypercall(u64 nr, u64 a0, void *a1)
{
return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1));
}
-void xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
+void xen_hypercall(u64 nr, u64 a0, void *a1)
{
GUEST_ASSERT(!__xen_hypercall(nr, a0, a1));
}
@@ -1379,7 +1372,7 @@ unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
{
const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
unsigned long ht_gfn, max_gfn, max_pfn;
- uint8_t maxphyaddr, guest_maxphyaddr;
+ u8 maxphyaddr, guest_maxphyaddr;
/*
* Use "guest MAXPHYADDR" from KVM if it's available. Guest MAXPHYADDR
@@ -1453,8 +1446,7 @@ bool kvm_arch_has_default_irqchip(void)
return true;
}
-void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
- uint64_t smram_gpa,
+void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa,
const void *smi_handler, size_t handler_size)
{
vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa,
diff --git a/tools/testing/selftests/kvm/lib/x86/sev.c b/tools/testing/selftests/kvm/lib/x86/sev.c
index c3a9838..93f9169 100644
--- a/tools/testing/selftests/kvm/lib/x86/sev.c
+++ b/tools/testing/selftests/kvm/lib/x86/sev.c
@@ -15,10 +15,10 @@
* expression would cause us to quit the loop.
*/
static void encrypt_region(struct kvm_vm *vm, struct userspace_mem_region *region,
- uint8_t page_type, bool private)
+ u8 page_type, bool private)
{
const struct sparsebit *protected_phy_pages = region->protected_phy_pages;
- const vm_paddr_t gpa_base = region->region.guest_phys_addr;
+ const gpa_t gpa_base = region->region.guest_phys_addr;
const sparsebit_idx_t lowest_page_in_region = gpa_base >> vm->page_shift;
sparsebit_idx_t i, j;
@@ -29,15 +29,15 @@ static void encrypt_region(struct kvm_vm *vm, struct userspace_mem_region *regio
sev_register_encrypted_memory(vm, region);
sparsebit_for_each_set_range(protected_phy_pages, i, j) {
- const uint64_t size = (j - i + 1) * vm->page_size;
- const uint64_t offset = (i - lowest_page_in_region) * vm->page_size;
+ const u64 size = (j - i + 1) * vm->page_size;
+ const u64 offset = (i - lowest_page_in_region) * vm->page_size;
if (private)
vm_mem_set_private(vm, gpa_base + offset, size);
if (is_sev_snp_vm(vm))
snp_launch_update_data(vm, gpa_base + offset,
- (uint64_t)addr_gpa2hva(vm, gpa_base + offset),
+ (u64)addr_gpa2hva(vm, gpa_base + offset),
size, page_type);
else
sev_launch_update_data(vm, gpa_base + offset, size);
@@ -79,7 +79,7 @@ void snp_vm_init(struct kvm_vm *vm)
vm_sev_ioctl(vm, KVM_SEV_INIT2, &init);
}
-void sev_vm_launch(struct kvm_vm *vm, uint32_t policy)
+void sev_vm_launch(struct kvm_vm *vm, u32 policy)
{
struct kvm_sev_launch_start launch_start = {
.policy = policy,
@@ -103,7 +103,7 @@ void sev_vm_launch(struct kvm_vm *vm, uint32_t policy)
vm->arch.is_pt_protected = true;
}
-void sev_vm_launch_measure(struct kvm_vm *vm, uint8_t *measurement)
+void sev_vm_launch_measure(struct kvm_vm *vm, u8 *measurement)
{
struct kvm_sev_launch_measure launch_measure;
struct kvm_sev_guest_status guest_status;
@@ -131,7 +131,7 @@ void sev_vm_launch_finish(struct kvm_vm *vm)
TEST_ASSERT_EQ(status.state, SEV_GUEST_STATE_RUNNING);
}
-void snp_vm_launch_start(struct kvm_vm *vm, uint64_t policy)
+void snp_vm_launch_start(struct kvm_vm *vm, u64 policy)
{
struct kvm_sev_snp_launch_start launch_start = {
.policy = policy,
@@ -158,7 +158,7 @@ void snp_vm_launch_finish(struct kvm_vm *vm)
vm_sev_ioctl(vm, KVM_SEV_SNP_LAUNCH_FINISH, &launch_finish);
}
-struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code,
+struct kvm_vm *vm_sev_create_with_one_vcpu(u32 type, void *guest_code,
struct kvm_vcpu **cpu)
{
struct vm_shape shape = {
@@ -174,7 +174,7 @@ struct kvm_vm *vm_sev_create_with_one_vcpu(uint32_t type, void *guest_code,
return vm;
}
-void vm_sev_launch(struct kvm_vm *vm, uint64_t policy, uint8_t *measurement)
+void vm_sev_launch(struct kvm_vm *vm, u64 policy, u8 *measurement)
{
if (is_sev_snp_vm(vm)) {
vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, BIT(KVM_HC_MAP_GPA_RANGE));
diff --git a/tools/testing/selftests/kvm/lib/x86/svm.c b/tools/testing/selftests/kvm/lib/x86/svm.c
index eb20b00..3b01605 100644
--- a/tools/testing/selftests/kvm/lib/x86/svm.c
+++ b/tools/testing/selftests/kvm/lib/x86/svm.c
@@ -28,20 +28,20 @@ u64 rflags;
* Pointer to structure with the addresses of the SVM areas.
*/
struct svm_test_data *
-vcpu_alloc_svm(struct kvm_vm *vm, vm_vaddr_t *p_svm_gva)
+vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva)
{
- vm_vaddr_t svm_gva = vm_vaddr_alloc_page(vm);
+ gva_t svm_gva = vm_alloc_page(vm);
struct svm_test_data *svm = addr_gva2hva(vm, svm_gva);
- svm->vmcb = (void *)vm_vaddr_alloc_page(vm);
+ svm->vmcb = (void *)vm_alloc_page(vm);
svm->vmcb_hva = addr_gva2hva(vm, (uintptr_t)svm->vmcb);
svm->vmcb_gpa = addr_gva2gpa(vm, (uintptr_t)svm->vmcb);
- svm->save_area = (void *)vm_vaddr_alloc_page(vm);
+ svm->save_area = (void *)vm_alloc_page(vm);
svm->save_area_hva = addr_gva2hva(vm, (uintptr_t)svm->save_area);
svm->save_area_gpa = addr_gva2gpa(vm, (uintptr_t)svm->save_area);
- svm->msr = (void *)vm_vaddr_alloc_page(vm);
+ svm->msr = (void *)vm_alloc_page(vm);
svm->msr_hva = addr_gva2hva(vm, (uintptr_t)svm->msr);
svm->msr_gpa = addr_gva2gpa(vm, (uintptr_t)svm->msr);
memset(svm->msr_hva, 0, getpagesize());
@@ -84,14 +84,14 @@ void vm_enable_npt(struct kvm_vm *vm)
void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp)
{
struct vmcb *vmcb = svm->vmcb;
- uint64_t vmcb_gpa = svm->vmcb_gpa;
+ u64 vmcb_gpa = svm->vmcb_gpa;
struct vmcb_save_area *save = &vmcb->save;
struct vmcb_control_area *ctrl = &vmcb->control;
u32 data_seg_attr = 3 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK
| SVM_SELECTOR_DB_MASK | SVM_SELECTOR_G_MASK;
u32 code_seg_attr = 9 | SVM_SELECTOR_S_MASK | SVM_SELECTOR_P_MASK
| SVM_SELECTOR_L_MASK | SVM_SELECTOR_G_MASK;
- uint64_t efer;
+ u64 efer;
efer = rdmsr(MSR_EFER);
wrmsr(MSR_EFER, efer | EFER_SVME);
@@ -158,7 +158,7 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r
* for now. registers involved in LOAD/SAVE_GPR_C are eventually
* unmodified so they do not need to be in the clobber list.
*/
-void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa)
+void run_guest(struct vmcb *vmcb, u64 vmcb_gpa)
{
asm volatile (
"vmload %[vmcb_gpa]\n\t"
diff --git a/tools/testing/selftests/kvm/lib/x86/ucall.c b/tools/testing/selftests/kvm/lib/x86/ucall.c
index 1265cec..e7dd579 100644
--- a/tools/testing/selftests/kvm/lib/x86/ucall.c
+++ b/tools/testing/selftests/kvm/lib/x86/ucall.c
@@ -6,9 +6,9 @@
*/
#include "kvm_util.h"
-#define UCALL_PIO_PORT ((uint16_t)0x1000)
+#define UCALL_PIO_PORT ((u16)0x1000)
-void ucall_arch_do_ucall(vm_vaddr_t uc)
+void ucall_arch_do_ucall(gva_t uc)
{
/*
* FIXME: Revert this hack (the entire commit that added it) once nVMX
diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c
index c87b340..6764275 100644
--- a/tools/testing/selftests/kvm/lib/x86/vmx.c
+++ b/tools/testing/selftests/kvm/lib/x86/vmx.c
@@ -27,7 +27,7 @@ struct hv_vp_assist_page *current_vp_assist;
int vcpu_enable_evmcs(struct kvm_vcpu *vcpu)
{
- uint16_t evmcs_ver;
+ u16 evmcs_ver;
vcpu_enable_cap(vcpu, KVM_CAP_HYPERV_ENLIGHTENED_VMCS,
(unsigned long)&evmcs_ver);
@@ -79,39 +79,39 @@ void vm_enable_ept(struct kvm_vm *vm)
* Pointer to structure with the addresses of the VMX areas.
*/
struct vmx_pages *
-vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva)
+vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva)
{
- vm_vaddr_t vmx_gva = vm_vaddr_alloc_page(vm);
+ gva_t vmx_gva = vm_alloc_page(vm);
struct vmx_pages *vmx = addr_gva2hva(vm, vmx_gva);
/* Setup of a region of guest memory for the vmxon region. */
- vmx->vmxon = (void *)vm_vaddr_alloc_page(vm);
+ vmx->vmxon = (void *)vm_alloc_page(vm);
vmx->vmxon_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmxon);
vmx->vmxon_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmxon);
/* Setup of a region of guest memory for a vmcs. */
- vmx->vmcs = (void *)vm_vaddr_alloc_page(vm);
+ vmx->vmcs = (void *)vm_alloc_page(vm);
vmx->vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmcs);
vmx->vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmcs);
/* Setup of a region of guest memory for the MSR bitmap. */
- vmx->msr = (void *)vm_vaddr_alloc_page(vm);
+ vmx->msr = (void *)vm_alloc_page(vm);
vmx->msr_hva = addr_gva2hva(vm, (uintptr_t)vmx->msr);
vmx->msr_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->msr);
memset(vmx->msr_hva, 0, getpagesize());
/* Setup of a region of guest memory for the shadow VMCS. */
- vmx->shadow_vmcs = (void *)vm_vaddr_alloc_page(vm);
+ vmx->shadow_vmcs = (void *)vm_alloc_page(vm);
vmx->shadow_vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->shadow_vmcs);
vmx->shadow_vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->shadow_vmcs);
/* Setup of a region of guest memory for the VMREAD and VMWRITE bitmaps. */
- vmx->vmread = (void *)vm_vaddr_alloc_page(vm);
+ vmx->vmread = (void *)vm_alloc_page(vm);
vmx->vmread_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmread);
vmx->vmread_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmread);
memset(vmx->vmread_hva, 0, getpagesize());
- vmx->vmwrite = (void *)vm_vaddr_alloc_page(vm);
+ vmx->vmwrite = (void *)vm_alloc_page(vm);
vmx->vmwrite_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmwrite);
vmx->vmwrite_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmwrite);
memset(vmx->vmwrite_hva, 0, getpagesize());
@@ -125,8 +125,8 @@ vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva)
bool prepare_for_vmx_operation(struct vmx_pages *vmx)
{
- uint64_t feature_control;
- uint64_t required;
+ u64 feature_control;
+ u64 required;
unsigned long cr0;
unsigned long cr4;
@@ -160,7 +160,7 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx)
wrmsr(MSR_IA32_FEAT_CTL, feature_control | required);
/* Enter VMX root operation. */
- *(uint32_t *)(vmx->vmxon) = vmcs_revision();
+ *(u32 *)(vmx->vmxon) = vmcs_revision();
if (vmxon(vmx->vmxon_gpa))
return false;
@@ -170,7 +170,7 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx)
bool load_vmcs(struct vmx_pages *vmx)
{
/* Load a VMCS. */
- *(uint32_t *)(vmx->vmcs) = vmcs_revision();
+ *(u32 *)(vmx->vmcs) = vmcs_revision();
if (vmclear(vmx->vmcs_gpa))
return false;
@@ -178,14 +178,14 @@ bool load_vmcs(struct vmx_pages *vmx)
return false;
/* Setup shadow VMCS, do not load it yet. */
- *(uint32_t *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul;
+ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul;
if (vmclear(vmx->shadow_vmcs_gpa))
return false;
return true;
}
-static bool ept_vpid_cap_supported(uint64_t mask)
+static bool ept_vpid_cap_supported(u64 mask)
{
return rdmsr(MSR_IA32_VMX_EPT_VPID_CAP) & mask;
}
@@ -200,7 +200,7 @@ bool ept_1g_pages_supported(void)
*/
static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
{
- uint32_t sec_exec_ctl = 0;
+ u32 sec_exec_ctl = 0;
vmwrite(VIRTUAL_PROCESSOR_ID, 0);
vmwrite(POSTED_INTR_NV, 0);
@@ -208,7 +208,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
vmwrite(PIN_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PINBASED_CTLS));
if (vmx->eptp_gpa) {
- uint64_t eptp = vmx->eptp_gpa | EPTP_WB | EPTP_PWL_4;
+ u64 eptp = vmx->eptp_gpa | EPTP_WB | EPTP_PWL_4;
TEST_ASSERT((vmx->eptp_gpa & ~PHYSICAL_PAGE_MASK) == 0,
"Illegal bits set in vmx->eptp_gpa");
@@ -259,7 +259,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx)
*/
static inline void init_vmcs_host_state(void)
{
- uint32_t exit_controls = vmreadz(VM_EXIT_CONTROLS);
+ u32 exit_controls = vmreadz(VM_EXIT_CONTROLS);
vmwrite(HOST_ES_SELECTOR, get_es());
vmwrite(HOST_CS_SELECTOR, get_cs());
@@ -358,8 +358,8 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp)
vmwrite(GUEST_GDTR_BASE, vmreadz(HOST_GDTR_BASE));
vmwrite(GUEST_IDTR_BASE, vmreadz(HOST_IDTR_BASE));
vmwrite(GUEST_DR7, 0x400);
- vmwrite(GUEST_RSP, (uint64_t)rsp);
- vmwrite(GUEST_RIP, (uint64_t)rip);
+ vmwrite(GUEST_RSP, (u64)rsp);
+ vmwrite(GUEST_RIP, (u64)rip);
vmwrite(GUEST_RFLAGS, 2);
vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0);
vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP));
@@ -375,7 +375,7 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp)
bool kvm_cpu_has_ept(void)
{
- uint64_t ctrl;
+ u64 ctrl;
if (!kvm_cpu_has(X86_FEATURE_VMX))
return false;
@@ -390,7 +390,7 @@ bool kvm_cpu_has_ept(void)
void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm)
{
- vmx->apic_access = (void *)vm_vaddr_alloc_page(vm);
+ vmx->apic_access = (void *)vm_alloc_page(vm);
vmx->apic_access_hva = addr_gva2hva(vm, (uintptr_t)vmx->apic_access);
vmx->apic_access_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->apic_access);
}
diff --git a/tools/testing/selftests/kvm/loongarch/arch_timer.c b/tools/testing/selftests/kvm/loongarch/arch_timer.c
index 355ecac..a7279de 100644
--- a/tools/testing/selftests/kvm/loongarch/arch_timer.c
+++ b/tools/testing/selftests/kvm/loongarch/arch_timer.c
@@ -27,8 +27,8 @@ static void do_idle(void)
static void guest_irq_handler(struct ex_regs *regs)
{
unsigned int intid;
- uint32_t cpu = guest_get_vcpuid();
- uint64_t xcnt, val, cfg, xcnt_diff_us;
+ u32 cpu = guest_get_vcpuid();
+ u64 xcnt, val, cfg, xcnt_diff_us;
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
intid = !!(regs->estat & BIT(INT_TI));
@@ -62,10 +62,10 @@ static void guest_irq_handler(struct ex_regs *regs)
WRITE_ONCE(shared_data->nr_iter, shared_data->nr_iter + 1);
}
-static void guest_test_period_timer(uint32_t cpu)
+static void guest_test_period_timer(u32 cpu)
{
- uint32_t irq_iter, config_iter;
- uint64_t us;
+ u32 irq_iter, config_iter;
+ u64 us;
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
shared_data->nr_iter = test_args.nr_iter;
@@ -86,10 +86,10 @@ static void guest_test_period_timer(uint32_t cpu)
irq_iter);
}
-static void guest_test_oneshot_timer(uint32_t cpu)
+static void guest_test_oneshot_timer(u32 cpu)
{
- uint32_t irq_iter, config_iter;
- uint64_t us;
+ u32 irq_iter, config_iter;
+ u64 us;
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
shared_data->nr_iter = 0;
@@ -112,10 +112,10 @@ static void guest_test_oneshot_timer(uint32_t cpu)
}
}
-static void guest_test_emulate_timer(uint32_t cpu)
+static void guest_test_emulate_timer(u32 cpu)
{
- uint32_t config_iter;
- uint64_t xcnt_diff_us, us;
+ u32 config_iter;
+ u64 xcnt_diff_us, us;
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
local_irq_disable();
@@ -136,9 +136,9 @@ static void guest_test_emulate_timer(uint32_t cpu)
local_irq_enable();
}
-static void guest_time_count_test(uint32_t cpu)
+static void guest_time_count_test(u32 cpu)
{
- uint32_t config_iter;
+ u32 config_iter;
unsigned long start, end, prev, us;
/* Assuming that test case starts to run in 1 second */
@@ -165,7 +165,7 @@ static void guest_time_count_test(uint32_t cpu)
static void guest_code(void)
{
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
/* must run at first */
guest_time_count_test(cpu);
diff --git a/tools/testing/selftests/kvm/loongarch/pmu_test.c b/tools/testing/selftests/kvm/loongarch/pmu_test.c
index 88bb530..ec3fefb 100644
--- a/tools/testing/selftests/kvm/loongarch/pmu_test.c
+++ b/tools/testing/selftests/kvm/loongarch/pmu_test.c
@@ -15,7 +15,7 @@ static int pmu_irq_count;
/* Check PMU support */
static bool has_pmu_support(void)
{
- uint32_t cfg6;
+ u32 cfg6;
/* Read CPUCFG6 to check PMU */
cfg6 = read_cpucfg(LOONGARCH_CPUCFG6);
@@ -34,7 +34,7 @@ static bool has_pmu_support(void)
/* Dump PMU capabilities */
static void dump_pmu_caps(void)
{
- uint32_t cfg6;
+ u32 cfg6;
int nr_counters, counter_bits;
cfg6 = read_cpucfg(LOONGARCH_CPUCFG6);
@@ -51,8 +51,8 @@ static void dump_pmu_caps(void)
static void guest_pmu_base_test(void)
{
int i;
- uint32_t cfg6, pmnum;
- uint64_t cnt[4];
+ u32 cfg6, pmnum;
+ u64 cnt[4];
cfg6 = read_cpucfg(LOONGARCH_CPUCFG6);
pmnum = (cfg6 >> 4) & 0xf;
@@ -114,7 +114,7 @@ static void guest_irq_handler(struct ex_regs *regs)
static void guest_pmu_interrupt_test(void)
{
- uint64_t cnt;
+ u64 cnt;
csr_write(PMU_OVERFLOW - 1, LOONGARCH_CSR_PERFCNTR0);
csr_write(PMU_ENVENT_ENABLED | CSR_PERFCTRL_PMIE | LOONGARCH_PMU_EVENT_CYCLES, LOONGARCH_CSR_PERFCTRL0);
diff --git a/tools/testing/selftests/kvm/memslot_modification_stress_test.c b/tools/testing/selftests/kvm/memslot_modification_stress_test.c
index 3cdfa3b..9c7578a 100644
--- a/tools/testing/selftests/kvm/memslot_modification_stress_test.c
+++ b/tools/testing/selftests/kvm/memslot_modification_stress_test.c
@@ -30,7 +30,7 @@
static int nr_vcpus = 1;
-static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
+static u64 guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
{
@@ -55,10 +55,10 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
}
static void add_remove_memslot(struct kvm_vm *vm, useconds_t delay,
- uint64_t nr_modifications)
+ u64 nr_modifications)
{
- uint64_t pages = max_t(int, vm->page_size, getpagesize()) / vm->page_size;
- uint64_t gpa;
+ u64 pages = max_t(int, vm->page_size, getpagesize()) / vm->page_size;
+ gpa_t gpa;
int i;
/*
@@ -78,7 +78,7 @@ static void add_remove_memslot(struct kvm_vm *vm, useconds_t delay,
struct test_params {
useconds_t delay;
- uint64_t nr_iterations;
+ u64 nr_iterations;
bool partition_vcpu_memory_access;
bool disable_slot_zap_quirk;
};
diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c
index 5087d08..e977e97 100644
--- a/tools/testing/selftests/kvm/memslot_perf_test.c
+++ b/tools/testing/selftests/kvm/memslot_perf_test.c
@@ -15,7 +15,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
@@ -23,6 +22,7 @@
#include <linux/sizes.h>
#include <test_util.h>
+#include <kvm_syscalls.h>
#include <kvm_util.h>
#include <processor.h>
#include <ucall_common.h>
@@ -85,17 +85,17 @@ struct vm_data {
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
pthread_t vcpu_thread;
- uint32_t nslots;
- uint64_t npages;
- uint64_t pages_per_slot;
+ u32 nslots;
+ u64 npages;
+ u64 pages_per_slot;
void **hva_slots;
bool mmio_ok;
- uint64_t mmio_gpa_min;
- uint64_t mmio_gpa_max;
+ u64 mmio_gpa_min;
+ u64 mmio_gpa_max;
};
struct sync_area {
- uint32_t guest_page_size;
+ u32 guest_page_size;
atomic_bool start_flag;
atomic_bool exit_flag;
atomic_bool sync_flag;
@@ -186,12 +186,12 @@ static void wait_for_vcpu(void)
"sem_timedwait() failed: %d", errno);
}
-static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages)
+static void *vm_gpa2hva(struct vm_data *data, gpa_t gpa, u64 *rempages)
{
- uint64_t gpage, pgoffs;
- uint32_t slot, slotoffs;
+ gpa_t gpage, pgoffs;
+ u32 slot, slotoffs;
void *base;
- uint32_t guest_page_size = data->vm->page_size;
+ u32 guest_page_size = data->vm->page_size;
TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");
TEST_ASSERT(gpa < MEM_GPA + data->npages * guest_page_size,
@@ -200,11 +200,11 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages)
gpage = gpa / guest_page_size;
pgoffs = gpa % guest_page_size;
- slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);
+ slot = min(gpage / data->pages_per_slot, (u64)data->nslots - 1);
slotoffs = gpage - (slot * data->pages_per_slot);
if (rempages) {
- uint64_t slotpages;
+ u64 slotpages;
if (slot == data->nslots - 1)
slotpages = data->npages - slot * data->pages_per_slot;
@@ -217,12 +217,12 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages)
}
base = data->hva_slots[slot];
- return (uint8_t *)base + slotoffs * guest_page_size + pgoffs;
+ return (u8 *)base + slotoffs * guest_page_size + pgoffs;
}
-static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)
+static u64 vm_slot2gpa(struct vm_data *data, u32 slot)
{
- uint32_t guest_page_size = data->vm->page_size;
+ u32 guest_page_size = data->vm->page_size;
TEST_ASSERT(slot < data->nslots, "Too high slot number");
@@ -243,8 +243,8 @@ static struct vm_data *alloc_vm(void)
return data;
}
-static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
- uint64_t pages_per_slot, uint64_t rempages)
+static bool check_slot_pages(u32 host_page_size, u32 guest_page_size,
+ u64 pages_per_slot, u64 rempages)
{
if (!pages_per_slot)
return false;
@@ -259,11 +259,11 @@ static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
}
-static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
+static u64 get_max_slots(struct vm_data *data, u32 host_page_size)
{
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t mempages, pages_per_slot, rempages;
- uint64_t slots;
+ u32 guest_page_size = data->vm->page_size;
+ u64 mempages, pages_per_slot, rempages;
+ u64 slots;
mempages = data->npages;
slots = data->nslots;
@@ -281,13 +281,13 @@ static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
return 0;
}
-static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
- void *guest_code, uint64_t mem_size,
+static bool prepare_vm(struct vm_data *data, int nslots, u64 *maxslots,
+ void *guest_code, u64 mem_size,
struct timespec *slot_runtime)
{
- uint64_t mempages, rempages;
- uint64_t guest_addr;
- uint32_t slot, host_page_size, guest_page_size;
+ u64 mempages, rempages;
+ u64 guest_addr;
+ u32 slot, host_page_size, guest_page_size;
struct timespec tstart;
struct sync_area *sync;
@@ -317,7 +317,7 @@ static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
clock_gettime(CLOCK_MONOTONIC, &tstart);
for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
- uint64_t npages;
+ u64 npages;
npages = data->pages_per_slot;
if (slot == data->nslots)
@@ -331,8 +331,8 @@ static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
*slot_runtime = timespec_elapsed(tstart);
for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
- uint64_t npages;
- uint64_t gpa;
+ u64 npages;
+ gpa_t gpa;
npages = data->pages_per_slot;
if (slot == data->nslots)
@@ -448,7 +448,7 @@ static bool guest_perform_sync(void)
static void guest_code_test_memslot_move(void)
{
struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
- uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
+ u32 page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);
GUEST_SYNC(0);
@@ -460,7 +460,7 @@ static void guest_code_test_memslot_move(void)
for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;
ptr += page_size)
- *(uint64_t *)ptr = MEM_TEST_VAL_1;
+ *(u64 *)ptr = MEM_TEST_VAL_1;
/*
* No host sync here since the MMIO exits are so expensive
@@ -477,7 +477,7 @@ static void guest_code_test_memslot_move(void)
static void guest_code_test_memslot_map(void)
{
struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
- uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
+ u32 page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
GUEST_SYNC(0);
@@ -489,7 +489,7 @@ static void guest_code_test_memslot_map(void)
for (ptr = MEM_TEST_GPA;
ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
ptr += page_size)
- *(uint64_t *)ptr = MEM_TEST_VAL_1;
+ *(u64 *)ptr = MEM_TEST_VAL_1;
if (!guest_perform_sync())
break;
@@ -497,7 +497,7 @@ static void guest_code_test_memslot_map(void)
for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE;
ptr += page_size)
- *(uint64_t *)ptr = MEM_TEST_VAL_2;
+ *(u64 *)ptr = MEM_TEST_VAL_2;
if (!guest_perform_sync())
break;
@@ -526,13 +526,13 @@ static void guest_code_test_memslot_unmap(void)
*
* Just access a single page to be on the safe side.
*/
- *(uint64_t *)ptr = MEM_TEST_VAL_1;
+ *(u64 *)ptr = MEM_TEST_VAL_1;
if (!guest_perform_sync())
break;
ptr += MEM_TEST_UNMAP_SIZE / 2;
- *(uint64_t *)ptr = MEM_TEST_VAL_2;
+ *(u64 *)ptr = MEM_TEST_VAL_2;
if (!guest_perform_sync())
break;
@@ -544,7 +544,7 @@ static void guest_code_test_memslot_unmap(void)
static void guest_code_test_memslot_rw(void)
{
struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
- uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
+ u32 page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
GUEST_SYNC(0);
@@ -555,17 +555,17 @@ static void guest_code_test_memslot_rw(void)
for (ptr = MEM_TEST_GPA;
ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size)
- *(uint64_t *)ptr = MEM_TEST_VAL_1;
+ *(u64 *)ptr = MEM_TEST_VAL_1;
if (!guest_perform_sync())
break;
for (ptr = MEM_TEST_GPA + page_size / 2;
ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size) {
- uint64_t val = *(uint64_t *)ptr;
+ u64 val = *(u64 *)ptr;
GUEST_ASSERT_EQ(val, MEM_TEST_VAL_2);
- *(uint64_t *)ptr = 0;
+ *(u64 *)ptr = 0;
}
if (!guest_perform_sync())
@@ -577,10 +577,10 @@ static void guest_code_test_memslot_rw(void)
static bool test_memslot_move_prepare(struct vm_data *data,
struct sync_area *sync,
- uint64_t *maxslots, bool isactive)
+ u64 *maxslots, bool isactive)
{
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t movesrcgpa, movetestgpa;
+ u32 guest_page_size = data->vm->page_size;
+ u64 movesrcgpa, movetestgpa;
#ifdef __x86_64__
if (disable_slot_zap_quirk)
@@ -590,7 +590,7 @@ static bool test_memslot_move_prepare(struct vm_data *data,
movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
if (isactive) {
- uint64_t lastpages;
+ u64 lastpages;
vm_gpa2hva(data, movesrcgpa, &lastpages);
if (lastpages * guest_page_size < MEM_TEST_MOVE_SIZE / 2) {
@@ -613,21 +613,21 @@ static bool test_memslot_move_prepare(struct vm_data *data,
static bool test_memslot_move_prepare_active(struct vm_data *data,
struct sync_area *sync,
- uint64_t *maxslots)
+ u64 *maxslots)
{
return test_memslot_move_prepare(data, sync, maxslots, true);
}
static bool test_memslot_move_prepare_inactive(struct vm_data *data,
struct sync_area *sync,
- uint64_t *maxslots)
+ u64 *maxslots)
{
return test_memslot_move_prepare(data, sync, maxslots, false);
}
static void test_memslot_move_loop(struct vm_data *data, struct sync_area *sync)
{
- uint64_t movesrcgpa;
+ u64 movesrcgpa;
movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
vm_mem_region_move(data->vm, data->nslots - 1 + 1,
@@ -636,13 +636,13 @@ static void test_memslot_move_loop(struct vm_data *data, struct sync_area *sync)
}
static void test_memslot_do_unmap(struct vm_data *data,
- uint64_t offsp, uint64_t count)
+ u64 offsp, u64 count)
{
- uint64_t gpa, ctr;
- uint32_t guest_page_size = data->vm->page_size;
+ gpa_t gpa, ctr;
+ u32 guest_page_size = data->vm->page_size;
for (gpa = MEM_TEST_GPA + offsp * guest_page_size, ctr = 0; ctr < count; ) {
- uint64_t npages;
+ u64 npages;
void *hva;
int ret;
@@ -661,11 +661,11 @@ static void test_memslot_do_unmap(struct vm_data *data,
}
static void test_memslot_map_unmap_check(struct vm_data *data,
- uint64_t offsp, uint64_t valexp)
+ u64 offsp, u64 valexp)
{
- uint64_t gpa;
- uint64_t *val;
- uint32_t guest_page_size = data->vm->page_size;
+ gpa_t gpa;
+ u64 *val;
+ u32 guest_page_size = data->vm->page_size;
if (!map_unmap_verify)
return;
@@ -680,8 +680,8 @@ static void test_memslot_map_unmap_check(struct vm_data *data,
static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
{
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
+ u32 guest_page_size = data->vm->page_size;
+ u64 guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
/*
* Unmap the second half of the test area while guest writes to (maps)
@@ -718,11 +718,11 @@ static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
static void test_memslot_unmap_loop_common(struct vm_data *data,
struct sync_area *sync,
- uint64_t chunk)
+ u64 chunk)
{
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
- uint64_t ctr;
+ u32 guest_page_size = data->vm->page_size;
+ u64 guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
+ u64 ctr;
/*
* Wait for the guest to finish mapping page(s) in the first half
@@ -746,9 +746,9 @@ static void test_memslot_unmap_loop_common(struct vm_data *data,
static void test_memslot_unmap_loop(struct vm_data *data,
struct sync_area *sync)
{
- uint32_t host_page_size = getpagesize();
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t guest_chunk_pages = guest_page_size >= host_page_size ?
+ u32 host_page_size = getpagesize();
+ u32 guest_page_size = data->vm->page_size;
+ u64 guest_chunk_pages = guest_page_size >= host_page_size ?
1 : host_page_size / guest_page_size;
test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
@@ -757,27 +757,27 @@ static void test_memslot_unmap_loop(struct vm_data *data,
static void test_memslot_unmap_loop_chunked(struct vm_data *data,
struct sync_area *sync)
{
- uint32_t guest_page_size = data->vm->page_size;
- uint64_t guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
+ u32 guest_page_size = data->vm->page_size;
+ u64 guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
}
static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
{
- uint64_t gptr;
- uint32_t guest_page_size = data->vm->page_size;
+ u64 gptr;
+ u32 guest_page_size = data->vm->page_size;
for (gptr = MEM_TEST_GPA + guest_page_size / 2;
gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size)
- *(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
+ *(u64 *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
host_perform_sync(sync);
for (gptr = MEM_TEST_GPA;
gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size) {
- uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
- uint64_t val = *vptr;
+ u64 *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
+ u64 val = *vptr;
TEST_ASSERT(val == MEM_TEST_VAL_1,
"Guest written values should read back correctly (is %"PRIu64" @ %"PRIx64")",
@@ -790,21 +790,21 @@ static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
struct test_data {
const char *name;
- uint64_t mem_size;
+ u64 mem_size;
void (*guest_code)(void);
bool (*prepare)(struct vm_data *data, struct sync_area *sync,
- uint64_t *maxslots);
+ u64 *maxslots);
void (*loop)(struct vm_data *data, struct sync_area *sync);
};
-static bool test_execute(int nslots, uint64_t *maxslots,
+static bool test_execute(int nslots, u64 *maxslots,
unsigned int maxtime,
const struct test_data *tdata,
- uint64_t *nloops,
+ u64 *nloops,
struct timespec *slot_runtime,
struct timespec *guest_runtime)
{
- uint64_t mem_size = tdata->mem_size ? : MEM_SIZE;
+ u64 mem_size = tdata->mem_size ? : MEM_SIZE;
struct vm_data *data;
struct sync_area *sync;
struct timespec tstart;
@@ -924,8 +924,8 @@ static void help(char *name, struct test_args *targs)
static bool check_memory_sizes(void)
{
- uint32_t host_page_size = getpagesize();
- uint32_t guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
+ u32 host_page_size = getpagesize();
+ u32 guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
if (host_page_size > SZ_64K || guest_page_size > SZ_64K) {
pr_info("Unsupported page size on host (0x%x) or guest (0x%x)\n",
@@ -961,7 +961,7 @@ static bool check_memory_sizes(void)
static bool parse_args(int argc, char *argv[],
struct test_args *targs)
{
- uint32_t max_mem_slots;
+ u32 max_mem_slots;
int opt;
while ((opt = getopt(argc, argv, "hvdqs:f:e:l:r:")) != -1) {
@@ -1040,8 +1040,8 @@ static bool parse_args(int argc, char *argv[],
struct test_result {
struct timespec slot_runtime, guest_runtime, iter_runtime;
- int64_t slottimens, runtimens;
- uint64_t nloops;
+ s64 slottimens, runtimens;
+ u64 nloops;
};
static bool test_loop(const struct test_data *data,
@@ -1049,7 +1049,7 @@ static bool test_loop(const struct test_data *data,
struct test_result *rbestslottime,
struct test_result *rbestruntime)
{
- uint64_t maxslots;
+ u64 maxslots;
struct test_result result = {};
if (!test_execute(targs->nslots, &maxslots, targs->seconds, data,
diff --git a/tools/testing/selftests/kvm/mmu_stress_test.c b/tools/testing/selftests/kvm/mmu_stress_test.c
index 51c0705..54d2814 100644
--- a/tools/testing/selftests/kvm/mmu_stress_test.c
+++ b/tools/testing/selftests/kvm/mmu_stress_test.c
@@ -20,19 +20,19 @@
static bool mprotect_ro_done;
static bool all_vcpus_hit_ro_fault;
-static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)
+static void guest_code(u64 start_gpa, u64 end_gpa, u64 stride)
{
- uint64_t gpa;
+ gpa_t gpa;
int i;
for (i = 0; i < 2; i++) {
for (gpa = start_gpa; gpa < end_gpa; gpa += stride)
- vcpu_arch_put_guest(*((volatile uint64_t *)gpa), gpa);
+ vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa);
GUEST_SYNC(i);
}
for (gpa = start_gpa; gpa < end_gpa; gpa += stride)
- *((volatile uint64_t *)gpa);
+ *((volatile u64 *)gpa);
GUEST_SYNC(2);
/*
@@ -55,7 +55,7 @@ static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)
#elif defined(__aarch64__)
asm volatile("str %0, [%0]" :: "r" (gpa) : "memory");
#else
- vcpu_arch_put_guest(*((volatile uint64_t *)gpa), gpa);
+ vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa);
#endif
} while (!READ_ONCE(mprotect_ro_done) || !READ_ONCE(all_vcpus_hit_ro_fault));
@@ -68,7 +68,7 @@ static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)
#endif
for (gpa = start_gpa; gpa < end_gpa; gpa += stride)
- vcpu_arch_put_guest(*((volatile uint64_t *)gpa), gpa);
+ vcpu_arch_put_guest(*((volatile u64 *)gpa), gpa);
GUEST_SYNC(4);
GUEST_ASSERT(0);
@@ -76,8 +76,8 @@ static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)
struct vcpu_info {
struct kvm_vcpu *vcpu;
- uint64_t start_gpa;
- uint64_t end_gpa;
+ u64 start_gpa;
+ u64 end_gpa;
};
static int nr_vcpus;
@@ -203,10 +203,10 @@ static void *vcpu_worker(void *data)
}
static pthread_t *spawn_workers(struct kvm_vm *vm, struct kvm_vcpu **vcpus,
- uint64_t start_gpa, uint64_t end_gpa)
+ u64 start_gpa, u64 end_gpa)
{
struct vcpu_info *info;
- uint64_t gpa, nr_bytes;
+ gpa_t gpa, nr_bytes;
pthread_t *threads;
int i;
@@ -217,7 +217,7 @@ static pthread_t *spawn_workers(struct kvm_vm *vm, struct kvm_vcpu **vcpus,
TEST_ASSERT(info, "Failed to allocate vCPU gpa ranges");
nr_bytes = ((end_gpa - start_gpa) / nr_vcpus) &
- ~((uint64_t)vm->page_size - 1);
+ ~((u64)vm->page_size - 1);
TEST_ASSERT(nr_bytes, "C'mon, no way you have %d CPUs", nr_vcpus);
for (i = 0, gpa = start_gpa; i < nr_vcpus; i++, gpa += nr_bytes) {
@@ -278,11 +278,11 @@ int main(int argc, char *argv[])
* just below the 4gb boundary. This test could create memory at
* 1gb-3gb,but it's simpler to skip straight to 4gb.
*/
- const uint64_t start_gpa = SZ_4G;
+ const u64 start_gpa = SZ_4G;
const int first_slot = 1;
struct timespec time_start, time_run1, time_reset, time_run2, time_ro, time_rw;
- uint64_t max_gpa, gpa, slot_size, max_mem, i;
+ u64 max_gpa, gpa, slot_size, max_mem, i;
int max_slots, slot, opt, fd;
bool hugepages = false;
struct kvm_vcpu **vcpus;
@@ -347,7 +347,7 @@ int main(int argc, char *argv[])
/* Pre-fault the memory to avoid taking mmap_sem on guest page faults. */
for (i = 0; i < slot_size; i += vm->page_size)
- ((uint8_t *)mem)[i] = 0xaa;
+ ((u8 *)mem)[i] = 0xaa;
gpa = 0;
for (slot = first_slot; slot < max_slots; slot++) {
diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c
index f3de038..fcb57fd 100644
--- a/tools/testing/selftests/kvm/pre_fault_memory_test.c
+++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c
@@ -17,13 +17,13 @@
#define TEST_NPAGES (TEST_SIZE / PAGE_SIZE)
#define TEST_SLOT 10
-static void guest_code(uint64_t base_gva)
+static void guest_code(u64 base_gva)
{
- volatile uint64_t val __used;
+ volatile u64 val __used;
int i;
for (i = 0; i < TEST_NPAGES; i++) {
- uint64_t *src = (uint64_t *)(base_gva + i * PAGE_SIZE);
+ u64 *src = (u64 *)(base_gva + i * PAGE_SIZE);
val = *src;
}
@@ -33,8 +33,8 @@ static void guest_code(uint64_t base_gva)
struct slot_worker_data {
struct kvm_vm *vm;
- u64 gpa;
- uint32_t flags;
+ gpa_t gpa;
+ u32 flags;
bool worker_ready;
bool prefault_ready;
bool recreate_slot;
@@ -161,7 +161,7 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset,
static void __test_pre_fault_memory(unsigned long vm_type, bool private)
{
- uint64_t gpa, gva, alignment, guest_page_size;
+ gpa_t gpa, gva, alignment, guest_page_size;
const struct vm_shape shape = {
.mode = VM_MODE_DEFAULT,
.type = vm_type,
diff --git a/tools/testing/selftests/kvm/riscv/arch_timer.c b/tools/testing/selftests/kvm/riscv/arch_timer.c
index f962fef..d67c918 100644
--- a/tools/testing/selftests/kvm/riscv/arch_timer.c
+++ b/tools/testing/selftests/kvm/riscv/arch_timer.c
@@ -17,9 +17,9 @@ static int timer_irq = IRQ_S_TIMER;
static void guest_irq_handler(struct pt_regs *regs)
{
- uint64_t xcnt, xcnt_diff_us, cmp;
+ u64 xcnt, xcnt_diff_us, cmp;
unsigned int intid = regs->cause & ~CAUSE_IRQ_FLAG;
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
timer_irq_disable();
@@ -40,7 +40,7 @@ static void guest_irq_handler(struct pt_regs *regs)
static void guest_run(struct test_vcpu_shared_data *shared_data)
{
- uint32_t irq_iter, config_iter;
+ u32 irq_iter, config_iter;
shared_data->nr_iter = 0;
shared_data->guest_stage = 0;
@@ -66,7 +66,7 @@ static void guest_run(struct test_vcpu_shared_data *shared_data)
static void guest_code(void)
{
- uint32_t cpu = guest_get_vcpuid();
+ u32 cpu = guest_get_vcpuid();
struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
timer_irq_disable();
diff --git a/tools/testing/selftests/kvm/riscv/ebreak_test.c b/tools/testing/selftests/kvm/riscv/ebreak_test.c
index 739d17b..3f44b04 100644
--- a/tools/testing/selftests/kvm/riscv/ebreak_test.c
+++ b/tools/testing/selftests/kvm/riscv/ebreak_test.c
@@ -8,10 +8,10 @@
#include "kvm_util.h"
#include "ucall_common.h"
-#define LABEL_ADDRESS(v) ((uint64_t)&(v))
+#define LABEL_ADDRESS(v) ((u64)&(v))
extern unsigned char sw_bp_1, sw_bp_2;
-static uint64_t sw_bp_addr;
+static u64 sw_bp_addr;
static void guest_code(void)
{
@@ -37,7 +37,7 @@ int main(void)
{
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
- uint64_t pc;
+ u64 pc;
struct kvm_guest_debug debug = {
.control = KVM_GUESTDBG_ENABLE,
};
diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c
index 8d6b9514..8d6fdb5 100644
--- a/tools/testing/selftests/kvm/riscv/get-reg-list.c
+++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c
@@ -162,7 +162,7 @@ bool check_reject_set(int err)
}
static int override_vector_reg_size(struct kvm_vcpu *vcpu, struct vcpu_reg_sublist *s,
- uint64_t feature)
+ u64 feature)
{
unsigned long vlenb_reg = 0;
int rc;
@@ -197,7 +197,7 @@ void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)
{
unsigned long isa_ext_state[KVM_RISCV_ISA_EXT_MAX] = { 0 };
struct vcpu_reg_sublist *s;
- uint64_t feature;
+ u64 feature;
int rc;
for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++)
diff --git a/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c b/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c
index cec1621..e56a3dd 100644
--- a/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c
+++ b/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c
@@ -24,7 +24,7 @@ union sbi_pmu_ctr_info ctrinfo_arr[RISCV_MAX_PMU_COUNTERS];
/* Snapshot shared memory data */
#define PMU_SNAPSHOT_GPA_BASE BIT(30)
static void *snapshot_gva;
-static vm_paddr_t snapshot_gpa;
+static gpa_t snapshot_gpa;
static int vcpu_shared_irq_count;
static int counter_in_use;
@@ -86,7 +86,7 @@ unsigned long pmu_csr_read_num(int csr_num)
#undef switchcase_csr_read
}
-static inline void dummy_func_loop(uint64_t iter)
+static inline void dummy_func_loop(u64 iter)
{
int i = 0;
@@ -259,7 +259,7 @@ static inline void verify_sbi_requirement_assert(void)
__GUEST_ASSERT(0, "SBI implementation version doesn't support PMU Snapshot");
}
-static void snapshot_set_shmem(vm_paddr_t gpa, unsigned long flags)
+static void snapshot_set_shmem(gpa_t gpa, unsigned long flags)
{
unsigned long lo = (unsigned long)gpa;
#if __riscv_xlen == 32
@@ -610,7 +610,7 @@ static void test_vm_setup_snapshot_mem(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
virt_map(vm, PMU_SNAPSHOT_GPA_BASE, PMU_SNAPSHOT_GPA_BASE, 1);
snapshot_gva = (void *)(PMU_SNAPSHOT_GPA_BASE);
- snapshot_gpa = addr_gva2gpa(vcpu->vm, (vm_vaddr_t)snapshot_gva);
+ snapshot_gpa = addr_gva2gpa(vcpu->vm, (gva_t)snapshot_gva);
sync_global_to_guest(vcpu->vm, snapshot_gva);
sync_global_to_guest(vcpu->vm, snapshot_gpa);
}
diff --git a/tools/testing/selftests/kvm/s390/debug_test.c b/tools/testing/selftests/kvm/s390/debug_test.c
index ad80959..751c61c 100644
--- a/tools/testing/selftests/kvm/s390/debug_test.c
+++ b/tools/testing/selftests/kvm/s390/debug_test.c
@@ -17,7 +17,7 @@ asm("int_handler:\n"
"j .\n");
static struct kvm_vm *test_step_int_1(struct kvm_vcpu **vcpu, void *guest_code,
- size_t new_psw_off, uint64_t *new_psw)
+ size_t new_psw_off, u64 *new_psw)
{
struct kvm_guest_debug debug = {};
struct kvm_regs regs;
@@ -27,7 +27,7 @@ static struct kvm_vm *test_step_int_1(struct kvm_vcpu **vcpu, void *guest_code,
vm = vm_create_with_one_vcpu(vcpu, guest_code);
lowcore = addr_gpa2hva(vm, 0);
new_psw[0] = (*vcpu)->run->psw_mask;
- new_psw[1] = (uint64_t)int_handler;
+ new_psw[1] = (u64)int_handler;
memcpy(lowcore + new_psw_off, new_psw, 16);
vcpu_regs_get(*vcpu, ®s);
regs.gprs[2] = -1;
@@ -42,7 +42,7 @@ static struct kvm_vm *test_step_int_1(struct kvm_vcpu **vcpu, void *guest_code,
static void test_step_int(void *guest_code, size_t new_psw_off)
{
struct kvm_vcpu *vcpu;
- uint64_t new_psw[2];
+ u64 new_psw[2];
struct kvm_vm *vm;
vm = test_step_int_1(&vcpu, guest_code, new_psw_off, new_psw);
@@ -79,7 +79,7 @@ static void test_step_pgm_diag(void)
.u.pgm.code = PGM_SPECIFICATION,
};
struct kvm_vcpu *vcpu;
- uint64_t new_psw[2];
+ u64 new_psw[2];
struct kvm_vm *vm;
vm = test_step_int_1(&vcpu, test_step_pgm_diag_guest_code,
diff --git a/tools/testing/selftests/kvm/s390/irq_routing.c b/tools/testing/selftests/kvm/s390/irq_routing.c
index 7819a0a..f383928 100644
--- a/tools/testing/selftests/kvm/s390/irq_routing.c
+++ b/tools/testing/selftests/kvm/s390/irq_routing.c
@@ -27,7 +27,7 @@ static void test(void)
struct kvm_irq_routing *routing;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_paddr_t mem;
+ gpa_t mem;
int ret;
struct kvm_irq_routing_entry ue = {
diff --git a/tools/testing/selftests/kvm/s390/memop.c b/tools/testing/selftests/kvm/s390/memop.c
index 4374b4c..0244848 100644
--- a/tools/testing/selftests/kvm/s390/memop.c
+++ b/tools/testing/selftests/kvm/s390/memop.c
@@ -34,7 +34,7 @@ enum mop_access_mode {
struct mop_desc {
uintptr_t gaddr;
uintptr_t gaddr_v;
- uint64_t set_flags;
+ u64 set_flags;
unsigned int f_check : 1;
unsigned int f_inject : 1;
unsigned int f_key : 1;
@@ -42,19 +42,19 @@ struct mop_desc {
unsigned int _set_flags : 1;
unsigned int _sida_offset : 1;
unsigned int _ar : 1;
- uint32_t size;
+ u32 size;
enum mop_target target;
enum mop_access_mode mode;
void *buf;
- uint32_t sida_offset;
+ u32 sida_offset;
void *old;
- uint8_t old_value[16];
+ u8 old_value[16];
bool *cmpxchg_success;
- uint8_t ar;
- uint8_t key;
+ u8 ar;
+ u8 key;
};
-const uint8_t NO_KEY = 0xff;
+const u8 NO_KEY = 0xff;
static struct kvm_s390_mem_op ksmo_from_desc(struct mop_desc *desc)
{
@@ -85,7 +85,7 @@ static struct kvm_s390_mem_op ksmo_from_desc(struct mop_desc *desc)
ksmo.op = KVM_S390_MEMOP_ABSOLUTE_WRITE;
if (desc->mode == CMPXCHG) {
ksmo.op = KVM_S390_MEMOP_ABSOLUTE_CMPXCHG;
- ksmo.old_addr = (uint64_t)desc->old;
+ ksmo.old_addr = (u64)desc->old;
memcpy(desc->old_value, desc->old, desc->size);
}
break;
@@ -230,8 +230,8 @@ static void memop_ioctl(struct test_info info, struct kvm_s390_mem_op *ksmo,
#define CR0_FETCH_PROTECTION_OVERRIDE (1UL << (63 - 38))
#define CR0_STORAGE_PROTECTION_OVERRIDE (1UL << (63 - 39))
-static uint8_t __aligned(PAGE_SIZE) mem1[65536];
-static uint8_t __aligned(PAGE_SIZE) mem2[65536];
+static u8 __aligned(PAGE_SIZE) mem1[65536];
+static u8 __aligned(PAGE_SIZE) mem2[65536];
struct test_default {
struct kvm_vm *kvm_vm;
@@ -296,7 +296,7 @@ static void prepare_mem12(void)
TEST_ASSERT(!memcmp(p1, p2, size), "Memory contents do not match!")
static void default_write_read(struct test_info copy_cpu, struct test_info mop_cpu,
- enum mop_target mop_target, uint32_t size, uint8_t key)
+ enum mop_target mop_target, u32 size, u8 key)
{
prepare_mem12();
CHECK_N_DO(MOP, mop_cpu, mop_target, WRITE, mem1, size,
@@ -308,7 +308,7 @@ static void default_write_read(struct test_info copy_cpu, struct test_info mop_c
}
static void default_read(struct test_info copy_cpu, struct test_info mop_cpu,
- enum mop_target mop_target, uint32_t size, uint8_t key)
+ enum mop_target mop_target, u32 size, u8 key)
{
prepare_mem12();
CHECK_N_DO(MOP, mop_cpu, mop_target, WRITE, mem1, size, GADDR_V(mem1));
@@ -318,12 +318,12 @@ static void default_read(struct test_info copy_cpu, struct test_info mop_cpu,
ASSERT_MEM_EQ(mem1, mem2, size);
}
-static void default_cmpxchg(struct test_default *test, uint8_t key)
+static void default_cmpxchg(struct test_default *test, u8 key)
{
for (int size = 1; size <= 16; size *= 2) {
for (int offset = 0; offset < 16; offset += size) {
- uint8_t __aligned(16) new[16] = {};
- uint8_t __aligned(16) old[16];
+ u8 __aligned(16) new[16] = {};
+ u8 __aligned(16) old[16];
bool succ;
prepare_mem12();
@@ -400,7 +400,7 @@ static void test_copy_access_register(void)
kvm_vm_free(t.kvm_vm);
}
-static void set_storage_key_range(void *addr, size_t len, uint8_t key)
+static void set_storage_key_range(void *addr, size_t len, u8 key)
{
uintptr_t _addr, abs, i;
int not_mapped = 0;
@@ -483,13 +483,13 @@ static __uint128_t cut_to_size(int size, __uint128_t val)
{
switch (size) {
case 1:
- return (uint8_t)val;
+ return (u8)val;
case 2:
- return (uint16_t)val;
+ return (u16)val;
case 4:
- return (uint32_t)val;
+ return (u32)val;
case 8:
- return (uint64_t)val;
+ return (u64)val;
case 16:
return val;
}
@@ -501,10 +501,10 @@ static bool popcount_eq(__uint128_t a, __uint128_t b)
{
unsigned int count_a, count_b;
- count_a = __builtin_popcountl((uint64_t)(a >> 64)) +
- __builtin_popcountl((uint64_t)a);
- count_b = __builtin_popcountl((uint64_t)(b >> 64)) +
- __builtin_popcountl((uint64_t)b);
+ count_a = __builtin_popcountl((u64)(a >> 64)) +
+ __builtin_popcountl((u64)a);
+ count_b = __builtin_popcountl((u64)(b >> 64)) +
+ __builtin_popcountl((u64)b);
return count_a == count_b;
}
@@ -553,7 +553,7 @@ static __uint128_t permutate_bits(bool guest, int i, int size, __uint128_t old)
if (swap) {
int i, j;
__uint128_t new;
- uint8_t byte0, byte1;
+ u8 byte0, byte1;
rand = rand * 3 + 1;
i = rand % size;
@@ -585,28 +585,28 @@ static bool _cmpxchg(int size, void *target, __uint128_t *old_addr, __uint128_t
switch (size) {
case 4: {
- uint32_t old = *old_addr;
+ u32 old = *old_addr;
asm volatile ("cs %[old],%[new],%[address]"
: [old] "+d" (old),
- [address] "+Q" (*(uint32_t *)(target))
- : [new] "d" ((uint32_t)new)
+ [address] "+Q" (*(u32 *)(target))
+ : [new] "d" ((u32)new)
: "cc"
);
- ret = old == (uint32_t)*old_addr;
+ ret = old == (u32)*old_addr;
*old_addr = old;
return ret;
}
case 8: {
- uint64_t old = *old_addr;
+ u64 old = *old_addr;
asm volatile ("csg %[old],%[new],%[address]"
: [old] "+d" (old),
- [address] "+Q" (*(uint64_t *)(target))
- : [new] "d" ((uint64_t)new)
+ [address] "+Q" (*(u64 *)(target))
+ : [new] "d" ((u64)new)
: "cc"
);
- ret = old == (uint64_t)*old_addr;
+ ret = old == (u64)*old_addr;
*old_addr = old;
return ret;
}
@@ -811,10 +811,10 @@ static void test_errors_cmpxchg_key(void)
static void test_termination(void)
{
struct test_default t = test_default_init(guest_error_key);
- uint64_t prefix;
- uint64_t teid;
- uint64_t teid_mask = BIT(63 - 56) | BIT(63 - 60) | BIT(63 - 61);
- uint64_t psw[2];
+ u64 prefix;
+ u64 teid;
+ u64 teid_mask = BIT(63 - 56) | BIT(63 - 60) | BIT(63 - 61);
+ u64 psw[2];
HOST_SYNC(t.vcpu, STAGE_INITED);
HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);
@@ -855,7 +855,7 @@ static void test_errors_key_storage_prot_override(void)
kvm_vm_free(t.kvm_vm);
}
-const uint64_t last_page_addr = -PAGE_SIZE;
+const u64 last_page_addr = -PAGE_SIZE;
static void guest_copy_key_fetch_prot_override(void)
{
@@ -878,10 +878,10 @@ static void guest_copy_key_fetch_prot_override(void)
static void test_copy_key_fetch_prot_override(void)
{
struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);
- vm_vaddr_t guest_0_page, guest_last_page;
+ gva_t guest_0_page, guest_last_page;
- guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);
- guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
+ guest_0_page = vm_alloc(t.kvm_vm, PAGE_SIZE, 0);
+ guest_last_page = vm_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
if (guest_0_page != 0 || guest_last_page != last_page_addr) {
print_skip("did not allocate guest pages at required positions");
goto out;
@@ -917,10 +917,10 @@ static void test_copy_key_fetch_prot_override(void)
static void test_errors_key_fetch_prot_override_not_enabled(void)
{
struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);
- vm_vaddr_t guest_0_page, guest_last_page;
+ gva_t guest_0_page, guest_last_page;
- guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);
- guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
+ guest_0_page = vm_alloc(t.kvm_vm, PAGE_SIZE, 0);
+ guest_last_page = vm_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
if (guest_0_page != 0 || guest_last_page != last_page_addr) {
print_skip("did not allocate guest pages at required positions");
goto out;
@@ -938,10 +938,10 @@ static void test_errors_key_fetch_prot_override_not_enabled(void)
static void test_errors_key_fetch_prot_override_enabled(void)
{
struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);
- vm_vaddr_t guest_0_page, guest_last_page;
+ gva_t guest_0_page, guest_last_page;
- guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);
- guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
+ guest_0_page = vm_alloc(t.kvm_vm, PAGE_SIZE, 0);
+ guest_last_page = vm_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);
if (guest_0_page != 0 || guest_last_page != last_page_addr) {
print_skip("did not allocate guest pages at required positions");
goto out;
diff --git a/tools/testing/selftests/kvm/s390/resets.c b/tools/testing/selftests/kvm/s390/resets.c
index b58f75b..e3c7a2f 100644
--- a/tools/testing/selftests/kvm/s390/resets.c
+++ b/tools/testing/selftests/kvm/s390/resets.c
@@ -20,7 +20,7 @@
struct kvm_s390_irq buf[ARBITRARY_NON_ZERO_VCPU_ID + LOCAL_IRQS];
-static uint8_t regs_null[512];
+static u8 regs_null[512];
static void guest_code_initial(void)
{
@@ -57,9 +57,9 @@ static void guest_code_initial(void)
);
}
-static void test_one_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t value)
+static void test_one_reg(struct kvm_vcpu *vcpu, u64 id, u64 value)
{
- uint64_t eval_reg;
+ u64 eval_reg;
eval_reg = vcpu_get_reg(vcpu, id);
TEST_ASSERT(eval_reg == value, "value == 0x%lx", value);
diff --git a/tools/testing/selftests/kvm/s390/shared_zeropage_test.c b/tools/testing/selftests/kvm/s390/shared_zeropage_test.c
index bba0d9a..478381e 100644
--- a/tools/testing/selftests/kvm/s390/shared_zeropage_test.c
+++ b/tools/testing/selftests/kvm/s390/shared_zeropage_test.c
@@ -4,16 +4,15 @@
*
* Copyright (C) 2024, Red Hat, Inc.
*/
-#include <sys/mman.h>
-
#include <linux/fs.h>
#include "test_util.h"
+#include "kvm_syscalls.h"
#include "kvm_util.h"
#include "kselftest.h"
#include "ucall_common.h"
-static void set_storage_key(void *addr, uint8_t skey)
+static void set_storage_key(void *addr, u8 skey)
{
asm volatile("sske %0,%1" : : "d" (skey), "a" (addr));
}
diff --git a/tools/testing/selftests/kvm/s390/tprot.c b/tools/testing/selftests/kvm/s390/tprot.c
index 12d5e1c..d861798 100644
--- a/tools/testing/selftests/kvm/s390/tprot.c
+++ b/tools/testing/selftests/kvm/s390/tprot.c
@@ -4,8 +4,8 @@
*
* Copyright IBM Corp. 2021
*/
-#include <sys/mman.h>
#include "test_util.h"
+#include "kvm_syscalls.h"
#include "kvm_util.h"
#include "kselftest.h"
#include "ucall_common.h"
@@ -14,12 +14,12 @@
#define CR0_FETCH_PROTECTION_OVERRIDE (1UL << (63 - 38))
#define CR0_STORAGE_PROTECTION_OVERRIDE (1UL << (63 - 39))
-static __aligned(PAGE_SIZE) uint8_t pages[2][PAGE_SIZE];
-static uint8_t *const page_store_prot = pages[0];
-static uint8_t *const page_fetch_prot = pages[1];
+static __aligned(PAGE_SIZE) u8 pages[2][PAGE_SIZE];
+static u8 *const page_store_prot = pages[0];
+static u8 *const page_fetch_prot = pages[1];
/* Nonzero return value indicates that address not mapped */
-static int set_storage_key(void *addr, uint8_t key)
+static int set_storage_key(void *addr, u8 key)
{
int not_mapped = 0;
@@ -44,9 +44,9 @@ enum permission {
TRANSL_UNAVAIL = 3,
};
-static enum permission test_protection(void *addr, uint8_t key)
+static enum permission test_protection(void *addr, u8 key)
{
- uint64_t mask;
+ u64 mask;
asm volatile (
"tprot %[addr], 0(%[key])\n"
@@ -72,7 +72,7 @@ enum stage {
struct test {
enum stage stage;
void *addr;
- uint8_t key;
+ u8 key;
enum permission expected;
} tests[] = {
/*
@@ -146,7 +146,7 @@ static enum stage perform_next_stage(int *i, bool mapped_0)
/*
* Some fetch protection override tests require that page 0
* be mapped, however, when the hosts tries to map that page via
- * vm_vaddr_alloc, it may happen that some other page gets mapped
+ * vm_alloc, it may happen that some other page gets mapped
* instead.
* In order to skip these tests we detect this inside the guest
*/
@@ -207,7 +207,7 @@ int main(int argc, char *argv[])
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct kvm_run *run;
- vm_vaddr_t guest_0_page;
+ gva_t guest_0_page;
ksft_print_header();
ksft_set_plan(STAGE_END);
@@ -216,10 +216,10 @@ int main(int argc, char *argv[])
run = vcpu->run;
HOST_SYNC(vcpu, STAGE_INIT_SIMPLE);
- mprotect(addr_gva2hva(vm, (vm_vaddr_t)pages), PAGE_SIZE * 2, PROT_READ);
+ mprotect(addr_gva2hva(vm, (gva_t)pages), PAGE_SIZE * 2, PROT_READ);
HOST_SYNC(vcpu, TEST_SIMPLE);
- guest_0_page = vm_vaddr_alloc(vm, PAGE_SIZE, 0);
+ guest_0_page = vm_alloc(vm, PAGE_SIZE, 0);
if (guest_0_page != 0) {
/* Use NO_TAP so we don't get a PASS print */
HOST_SYNC_NO_TAP(vcpu, STAGE_INIT_FETCH_PROT_OVERRIDE);
@@ -229,7 +229,7 @@ int main(int argc, char *argv[])
HOST_SYNC(vcpu, STAGE_INIT_FETCH_PROT_OVERRIDE);
}
if (guest_0_page == 0)
- mprotect(addr_gva2hva(vm, (vm_vaddr_t)0), PAGE_SIZE, PROT_READ);
+ mprotect(addr_gva2hva(vm, (gva_t)0), PAGE_SIZE, PROT_READ);
run->s.regs.crs[0] |= CR0_FETCH_PROTECTION_OVERRIDE;
run->kvm_dirty_regs = KVM_SYNC_CRS;
HOST_SYNC(vcpu, TEST_FETCH_PROT_OVERRIDE);
diff --git a/tools/testing/selftests/kvm/s390/ucontrol_test.c b/tools/testing/selftests/kvm/s390/ucontrol_test.c
index 50bc1c3..b8c6f37 100644
--- a/tools/testing/selftests/kvm/s390/ucontrol_test.c
+++ b/tools/testing/selftests/kvm/s390/ucontrol_test.c
@@ -111,7 +111,7 @@ FIXTURE(uc_kvm)
uintptr_t base_hva;
uintptr_t code_hva;
int kvm_run_size;
- vm_paddr_t pgd;
+ gpa_t pgd;
void *vm_mem;
int vcpu_fd;
int kvm_fd;
@@ -269,7 +269,7 @@ TEST(uc_cap_hpage)
}
/* calculate host virtual addr from guest physical addr */
-static void *gpa2hva(FIXTURE_DATA(uc_kvm) *self, u64 gpa)
+static void *gpa2hva(FIXTURE_DATA(uc_kvm) *self, gpa_t gpa)
{
return (void *)(self->base_hva - self->base_gpa + gpa);
}
@@ -571,7 +571,7 @@ TEST_F(uc_kvm, uc_skey)
{
struct kvm_s390_sie_block *sie_block = self->sie_block;
struct kvm_sync_regs *sync_regs = &self->run->s.regs;
- u64 test_vaddr = VM_MEM_SIZE - (SZ_1M / 2);
+ u64 test_gva = VM_MEM_SIZE - (SZ_1M / 2);
struct kvm_run *run = self->run;
const u8 skeyvalue = 0x34;
@@ -583,7 +583,7 @@ TEST_F(uc_kvm, uc_skey)
/* set register content for test_skey_asm to access not mapped memory */
sync_regs->gprs[1] = skeyvalue;
sync_regs->gprs[5] = self->base_gpa;
- sync_regs->gprs[6] = test_vaddr;
+ sync_regs->gprs[6] = test_gva;
run->kvm_dirty_regs |= KVM_SYNC_GPRS;
/* DAT disabled + 64 bit mode */
diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c
index a398dc3..e639a9d 100644
--- a/tools/testing/selftests/kvm/set_memory_region_test.c
+++ b/tools/testing/selftests/kvm/set_memory_region_test.c
@@ -8,11 +8,11 @@
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
-#include <sys/mman.h>
#include <linux/compiler.h>
#include <test_util.h>
+#include <kvm_syscalls.h>
#include <kvm_util.h>
#include <processor.h>
@@ -30,19 +30,19 @@
#define MEM_REGION_GPA 0xc0000000
#define MEM_REGION_SLOT 10
-static const uint64_t MMIO_VAL = 0xbeefull;
+static const u64 MMIO_VAL = 0xbeefull;
-extern const uint64_t final_rip_start;
-extern const uint64_t final_rip_end;
+extern const u64 final_rip_start;
+extern const u64 final_rip_end;
static sem_t vcpu_ready;
-static inline uint64_t guest_spin_on_val(uint64_t spin_val)
+static inline u64 guest_spin_on_val(u64 spin_val)
{
- uint64_t val;
+ u64 val;
do {
- val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA));
+ val = READ_ONCE(*((u64 *)MEM_REGION_GPA));
} while (val == spin_val);
GUEST_SYNC(0);
@@ -54,7 +54,7 @@ static void *vcpu_worker(void *data)
struct kvm_vcpu *vcpu = data;
struct kvm_run *run = vcpu->run;
struct ucall uc;
- uint64_t cmd;
+ u64 cmd;
/*
* Loop until the guest is done. Re-enter the guest on all MMIO exits,
@@ -111,8 +111,8 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread,
void *guest_code)
{
struct kvm_vm *vm;
- uint64_t *hva;
- uint64_t gpa;
+ u64 *hva;
+ gpa_t gpa;
vm = vm_create_with_one_vcpu(vcpu, guest_code);
@@ -144,7 +144,7 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread,
static void guest_code_move_memory_region(void)
{
- uint64_t val;
+ u64 val;
GUEST_SYNC(0);
@@ -180,7 +180,7 @@ static void test_move_memory_region(bool disable_slot_zap_quirk)
pthread_t vcpu_thread;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint64_t *hva;
+ u64 *hva;
vm = spawn_vm(&vcpu, &vcpu_thread, guest_code_move_memory_region);
@@ -224,7 +224,7 @@ static void test_move_memory_region(bool disable_slot_zap_quirk)
static void guest_code_delete_memory_region(void)
{
struct desc_ptr idt;
- uint64_t val;
+ u64 val;
/*
* Clobber the IDT so that a #PF due to the memory region being deleted
@@ -345,8 +345,8 @@ static void test_zero_memory_regions(void)
static void test_invalid_memory_region_flags(void)
{
- uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES;
- const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD;
+ u32 supported_flags = KVM_MEM_LOG_DIRTY_PAGES;
+ const u32 v2_only_flags = KVM_MEM_GUEST_MEMFD;
struct kvm_vm *vm;
int r, i;
@@ -410,8 +410,8 @@ static void test_add_max_memory_regions(void)
{
int ret;
struct kvm_vm *vm;
- uint32_t max_mem_slots;
- uint32_t slot;
+ u32 max_mem_slots;
+ u32 slot;
void *mem, *mem_aligned, *mem_extra;
size_t alignment = 1;
@@ -434,16 +434,16 @@ static void test_add_max_memory_regions(void)
for (slot = 0; slot < max_mem_slots; slot++)
vm_set_user_memory_region(vm, slot, 0,
- ((uint64_t)slot * MEM_REGION_SIZE),
+ ((u64)slot * MEM_REGION_SIZE),
MEM_REGION_SIZE,
- mem_aligned + (uint64_t)slot * MEM_REGION_SIZE);
+ mem_aligned + (u64)slot * MEM_REGION_SIZE);
/* Check it cannot be added memory slots beyond the limit */
mem_extra = kvm_mmap(MEM_REGION_SIZE, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1);
ret = __vm_set_user_memory_region(vm, max_mem_slots, 0,
- (uint64_t)max_mem_slots * MEM_REGION_SIZE,
+ (u64)max_mem_slots * MEM_REGION_SIZE,
MEM_REGION_SIZE, mem_extra);
TEST_ASSERT(ret == -1 && errno == EINVAL,
"Adding one more memory slot should fail with EINVAL");
@@ -556,7 +556,7 @@ static void guest_code_mmio_during_vectoring(void)
set_idt(&idt_desc);
/* Generate a #GP by dereferencing a non-canonical address */
- *((uint8_t *)NONCANONICAL) = 0x1;
+ *((u8 *)NONCANONICAL) = 0x1;
GUEST_ASSERT(0);
}
diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c
index efe56a1..76fcdd1 100644
--- a/tools/testing/selftests/kvm/steal_time.c
+++ b/tools/testing/selftests/kvm/steal_time.c
@@ -25,7 +25,7 @@
#define ST_GPA_BASE (1 << 30)
static void *st_gva[NR_VCPUS];
-static uint64_t guest_stolen_time[NR_VCPUS];
+static u64 guest_stolen_time[NR_VCPUS];
#if defined(__x86_64__)
@@ -42,9 +42,9 @@ static void check_status(struct kvm_steal_time *st)
static void guest_code(int cpu)
{
struct kvm_steal_time *st = st_gva[cpu];
- uint32_t version;
+ u32 version;
- GUEST_ASSERT_EQ(rdmsr(MSR_KVM_STEAL_TIME), ((uint64_t)st_gva[cpu] | KVM_MSR_ENABLED));
+ GUEST_ASSERT_EQ(rdmsr(MSR_KVM_STEAL_TIME), ((u64)st_gva[cpu] | KVM_MSR_ENABLED));
memset(st, 0, sizeof(*st));
GUEST_SYNC(0);
@@ -67,7 +67,7 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
return kvm_cpu_has(X86_FEATURE_KVM_STEAL_TIME);
}
-static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
+static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
{
/* ST_GPA_BASE is identity mapped */
st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
@@ -76,7 +76,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED);
}
-static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx)
+static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
{
struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
@@ -118,12 +118,12 @@ static void check_steal_time_uapi(void)
#define PV_TIME_ST 0xc5000021
struct st_time {
- uint32_t rev;
- uint32_t attr;
- uint64_t st_time;
+ u32 rev;
+ u32 attr;
+ u64 st_time;
};
-static int64_t smccc(uint32_t func, uint64_t arg)
+static s64 smccc(u32 func, u64 arg)
{
struct arm_smccc_res res;
@@ -140,7 +140,7 @@ static void check_status(struct st_time *st)
static void guest_code(int cpu)
{
struct st_time *st;
- int64_t status;
+ s64 status;
status = smccc(SMCCC_ARCH_FEATURES, PV_TIME_FEATURES);
GUEST_ASSERT_EQ(status, 0);
@@ -175,15 +175,15 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
return !__vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev);
}
-static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
+static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
{
struct kvm_vm *vm = vcpu->vm;
- uint64_t st_ipa;
+ u64 st_ipa;
struct kvm_device_attr dev = {
.group = KVM_ARM_VCPU_PVTIME_CTRL,
.attr = KVM_ARM_VCPU_PVTIME_IPA,
- .addr = (uint64_t)&st_ipa,
+ .addr = (u64)&st_ipa,
};
/* ST_GPA_BASE is identity mapped */
@@ -194,7 +194,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
}
-static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx)
+static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
{
struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
@@ -208,7 +208,7 @@ static void check_steal_time_uapi(void)
{
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
- uint64_t st_ipa;
+ u64 st_ipa;
int ret;
vm = vm_create_with_one_vcpu(&vcpu, NULL);
@@ -216,10 +216,12 @@ static void check_steal_time_uapi(void)
struct kvm_device_attr dev = {
.group = KVM_ARM_VCPU_PVTIME_CTRL,
.attr = KVM_ARM_VCPU_PVTIME_IPA,
- .addr = (uint64_t)&st_ipa,
+ .addr = (u64)&st_ipa,
};
vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev);
+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, 1, 0);
+ virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, 1);
st_ipa = (ulong)ST_GPA_BASE | 1;
ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
@@ -239,17 +241,17 @@ static void check_steal_time_uapi(void)
/* SBI STA shmem must have 64-byte alignment */
#define STEAL_TIME_SIZE ((sizeof(struct sta_struct) + 63) & ~63)
-static vm_paddr_t st_gpa[NR_VCPUS];
+static gpa_t st_gpa[NR_VCPUS];
struct sta_struct {
- uint32_t sequence;
- uint32_t flags;
- uint64_t steal;
- uint8_t preempted;
- uint8_t pad[47];
+ u32 sequence;
+ u32 flags;
+ u64 steal;
+ u8 preempted;
+ u8 pad[47];
} __packed;
-static void sta_set_shmem(vm_paddr_t gpa, unsigned long flags)
+static void sta_set_shmem(gpa_t gpa, unsigned long flags)
{
unsigned long lo = (unsigned long)gpa;
#if __riscv_xlen == 32
@@ -272,7 +274,7 @@ static void check_status(struct sta_struct *st)
static void guest_code(int cpu)
{
struct sta_struct *st = st_gva[cpu];
- uint32_t sequence;
+ u32 sequence;
long out_val = 0;
bool probe;
@@ -297,7 +299,7 @@ static void guest_code(int cpu)
static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
{
- uint64_t id = RISCV_SBI_EXT_REG(KVM_RISCV_SBI_EXT_STA);
+ u64 id = RISCV_SBI_EXT_REG(KVM_RISCV_SBI_EXT_STA);
unsigned long enabled = vcpu_get_reg(vcpu, id);
TEST_ASSERT(enabled == 0 || enabled == 1, "Expected boolean result");
@@ -305,16 +307,16 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
return enabled;
}
-static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
+static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
{
/* ST_GPA_BASE is identity mapped */
st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
- st_gpa[i] = addr_gva2gpa(vcpu->vm, (vm_vaddr_t)st_gva[i]);
+ st_gpa[i] = addr_gva2gpa(vcpu->vm, (gva_t)st_gva[i]);
sync_global_to_guest(vcpu->vm, st_gva[i]);
sync_global_to_guest(vcpu->vm, st_gpa[i]);
}
-static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx)
+static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
{
struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
int i;
@@ -335,7 +337,7 @@ static void check_steal_time_uapi(void)
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
struct kvm_one_reg reg;
- uint64_t shmem;
+ u64 shmem;
int ret;
vm = vm_create_with_one_vcpu(&vcpu, NULL);
@@ -345,7 +347,7 @@ static void check_steal_time_uapi(void)
KVM_REG_RISCV_SBI_STATE |
KVM_REG_RISCV_SBI_STA |
KVM_REG_RISCV_SBI_STA_REG(shmem_lo);
- reg.addr = (uint64_t)&shmem;
+ reg.addr = (u64)&shmem;
shmem = ST_GPA_BASE + 1;
ret = __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®);
@@ -388,7 +390,7 @@ static void check_status(struct kvm_steal_time *st)
static void guest_code(int cpu)
{
- uint32_t version;
+ u32 version;
struct kvm_steal_time *st = st_gva[cpu];
memset(st, 0, sizeof(*st));
@@ -410,11 +412,11 @@ static void guest_code(int cpu)
static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
{
int err;
- uint64_t val;
+ u64 val;
struct kvm_device_attr attr = {
.group = KVM_LOONGARCH_VCPU_CPUCFG,
.attr = CPUCFG_KVM_FEATURE,
- .addr = (uint64_t)&val,
+ .addr = (u64)&val,
};
err = __vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &attr);
@@ -428,15 +430,15 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
return val & BIT(KVM_FEATURE_STEAL_TIME);
}
-static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
+static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
{
int err;
- uint64_t st_gpa;
+ u64 st_gpa;
struct kvm_vm *vm = vcpu->vm;
struct kvm_device_attr attr = {
.group = KVM_LOONGARCH_VCPU_PVTIME_CTRL,
.attr = KVM_LOONGARCH_VCPU_PVTIME_GPA,
- .addr = (uint64_t)&st_gpa,
+ .addr = (u64)&st_gpa,
};
/* ST_GPA_BASE is identity mapped */
@@ -451,7 +453,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, uint32_t i)
TEST_ASSERT(err == 0, "Fail to set PV stealtime GPA");
}
-static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx)
+static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
{
struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]);
@@ -461,6 +463,11 @@ static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpu_idx)
ksft_print_msg(" version: %d\n", st->version);
ksft_print_msg(" preempted: %d\n", st->preempted);
}
+
+static void check_steal_time_uapi(void)
+{
+
+}
#endif
static void *do_steal_time(void *arg)
diff --git a/tools/testing/selftests/kvm/system_counter_offset_test.c b/tools/testing/selftests/kvm/system_counter_offset_test.c
index 513d421..dc5e30b7 100644
--- a/tools/testing/selftests/kvm/system_counter_offset_test.c
+++ b/tools/testing/selftests/kvm/system_counter_offset_test.c
@@ -17,7 +17,7 @@
#ifdef __x86_64__
struct test_case {
- uint64_t tsc_offset;
+ u64 tsc_offset;
};
static struct test_case test_cases[] = {
@@ -39,12 +39,12 @@ static void setup_system_counter(struct kvm_vcpu *vcpu, struct test_case *test)
&test->tsc_offset);
}
-static uint64_t guest_read_system_counter(struct test_case *test)
+static u64 guest_read_system_counter(struct test_case *test)
{
return rdtsc();
}
-static uint64_t host_read_guest_system_counter(struct test_case *test)
+static u64 host_read_guest_system_counter(struct test_case *test)
{
return rdtsc() + test->tsc_offset;
}
@@ -69,9 +69,9 @@ static void guest_main(void)
}
}
-static void handle_sync(struct ucall *uc, uint64_t start, uint64_t end)
+static void handle_sync(struct ucall *uc, u64 start, u64 end)
{
- uint64_t obs = uc->args[2];
+ u64 obs = uc->args[2];
TEST_ASSERT(start <= obs && obs <= end,
"unexpected system counter value: %"PRIu64" expected range: [%"PRIu64", %"PRIu64"]",
@@ -88,7 +88,7 @@ static void handle_abort(struct ucall *uc)
static void enter_guest(struct kvm_vcpu *vcpu)
{
- uint64_t start, end;
+ u64 start, end;
struct ucall uc;
int i;
diff --git a/tools/testing/selftests/kvm/x86/amx_test.c b/tools/testing/selftests/kvm/x86/amx_test.c
index 37b1662..4e63da2 100644
--- a/tools/testing/selftests/kvm/x86/amx_test.c
+++ b/tools/testing/selftests/kvm/x86/amx_test.c
@@ -80,10 +80,10 @@ static inline void __tilerelease(void)
asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
}
-static inline void __xsavec(struct xstate *xstate, uint64_t rfbm)
+static inline void __xsavec(struct xstate *xstate, u64 rfbm)
{
- uint32_t rfbm_lo = rfbm;
- uint32_t rfbm_hi = rfbm >> 32;
+ u32 rfbm_lo = rfbm;
+ u32 rfbm_hi = rfbm >> 32;
asm volatile("xsavec (%%rdi)"
: : "D" (xstate), "a" (rfbm_lo), "d" (rfbm_hi)
@@ -236,7 +236,7 @@ int main(int argc, char *argv[])
struct kvm_x86_state *state;
struct kvm_x86_state *tile_state = NULL;
int xsave_restore_size;
- vm_vaddr_t amx_cfg, tiledata, xstate;
+ gva_t amx_cfg, tiledata, xstate;
struct ucall uc;
int ret;
@@ -263,15 +263,15 @@ int main(int argc, char *argv[])
vcpu_regs_get(vcpu, ®s1);
/* amx cfg for guest_code */
- amx_cfg = vm_vaddr_alloc_page(vm);
+ amx_cfg = vm_alloc_page(vm);
memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
/* amx tiledata for guest_code */
- tiledata = vm_vaddr_alloc_pages(vm, 2);
+ tiledata = vm_alloc_pages(vm, 2);
memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize());
/* XSAVE state for guest_code */
- xstate = vm_vaddr_alloc_pages(vm, DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
+ xstate = vm_alloc_pages(vm, DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
memset(addr_gva2hva(vm, xstate), 0, PAGE_SIZE * DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xstate);
diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
index 8b15a13..c916601 100644
--- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c
+++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
@@ -35,9 +35,9 @@ static int open_dev_msr(int cpu)
return open_path_or_exit(path, O_RDONLY);
}
-static uint64_t read_dev_msr(int msr_fd, uint32_t msr)
+static u64 read_dev_msr(int msr_fd, u32 msr)
{
- uint64_t data;
+ u64 data;
ssize_t rc;
rc = pread(msr_fd, &data, sizeof(data), msr);
@@ -107,8 +107,8 @@ static void guest_code(void *nested_test_data)
static void guest_no_aperfmperf(void)
{
- uint64_t msr_val;
- uint8_t vector;
+ u64 msr_val;
+ u8 vector;
vector = rdmsr_safe(MSR_IA32_APERF, &msr_val);
GUEST_ASSERT(vector == GP_VECTOR);
@@ -122,8 +122,8 @@ static void guest_no_aperfmperf(void)
int main(int argc, char *argv[])
{
const bool has_nested = kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX);
- uint64_t host_aperf_before, host_mperf_before;
- vm_vaddr_t nested_test_data_gva;
+ u64 host_aperf_before, host_mperf_before;
+ gva_t nested_test_data_gva;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
int msr_fd, cpu, i;
@@ -166,8 +166,8 @@ int main(int argc, char *argv[])
host_mperf_before = read_dev_msr(msr_fd, MSR_IA32_MPERF);
for (i = 0; i <= NUM_ITERATIONS * (1 + has_nested); i++) {
- uint64_t host_aperf_after, host_mperf_after;
- uint64_t guest_aperf, guest_mperf;
+ u64 host_aperf_after, host_mperf_after;
+ u64 guest_aperf, guest_mperf;
struct ucall uc;
vcpu_run(vcpu);
diff --git a/tools/testing/selftests/kvm/x86/apic_bus_clock_test.c b/tools/testing/selftests/kvm/x86/apic_bus_clock_test.c
index f8916bb..0c84c27 100644
--- a/tools/testing/selftests/kvm/x86/apic_bus_clock_test.c
+++ b/tools/testing/selftests/kvm/x86/apic_bus_clock_test.c
@@ -19,8 +19,8 @@
* timer frequency.
*/
static const struct {
- const uint32_t tdcr;
- const uint32_t divide_count;
+ const u32 tdcr;
+ const u32 divide_count;
} tdcrs[] = {
{0x0, 2},
{0x1, 4},
@@ -42,12 +42,12 @@ static void apic_enable(void)
xapic_enable();
}
-static uint32_t apic_read_reg(unsigned int reg)
+static u32 apic_read_reg(unsigned int reg)
{
return is_x2apic ? x2apic_read_reg(reg) : xapic_read_reg(reg);
}
-static void apic_write_reg(unsigned int reg, uint32_t val)
+static void apic_write_reg(unsigned int reg, u32 val)
{
if (is_x2apic)
x2apic_write_reg(reg, val);
@@ -55,12 +55,12 @@ static void apic_write_reg(unsigned int reg, uint32_t val)
xapic_write_reg(reg, val);
}
-static void apic_guest_code(uint64_t apic_hz, uint64_t delay_ms)
+static void apic_guest_code(u64 apic_hz, u64 delay_ms)
{
- uint64_t tsc_hz = guest_tsc_khz * 1000;
- const uint32_t tmict = ~0u;
- uint64_t tsc0, tsc1, freq;
- uint32_t tmcct;
+ u64 tsc_hz = guest_tsc_khz * 1000;
+ const u32 tmict = ~0u;
+ u64 tsc0, tsc1, freq;
+ u32 tmcct;
int i;
apic_enable();
@@ -121,7 +121,7 @@ static void test_apic_bus_clock(struct kvm_vcpu *vcpu)
}
}
-static void run_apic_bus_clock_test(uint64_t apic_hz, uint64_t delay_ms,
+static void run_apic_bus_clock_test(u64 apic_hz, u64 delay_ms,
bool x2apic)
{
struct kvm_vcpu *vcpu;
@@ -137,6 +137,10 @@ static void run_apic_bus_clock_test(uint64_t apic_hz, uint64_t delay_ms,
vm_enable_cap(vm, KVM_CAP_X86_APIC_BUS_CYCLES_NS,
NSEC_PER_SEC / apic_hz);
+ TEST_ASSERT_EQ(kvm_check_cap(KVM_CAP_X86_APIC_BUS_CYCLES_NS), 1);
+ TEST_ASSERT_EQ(vm_check_cap(vm, KVM_CAP_X86_APIC_BUS_CYCLES_NS),
+ NSEC_PER_SEC / apic_hz);
+
vcpu = vm_vcpu_add(vm, 0, apic_guest_code);
vcpu_args_set(vcpu, 2, apic_hz, delay_ms);
@@ -168,8 +172,8 @@ int main(int argc, char *argv[])
* Arbitrarilty default to 25MHz for the APIC bus frequency, which is
* different enough from the default 1GHz to be interesting.
*/
- uint64_t apic_hz = 25 * 1000 * 1000;
- uint64_t delay_ms = 100;
+ u64 apic_hz = 25 * 1000 * 1000;
+ u64 delay_ms = 100;
int opt;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_X86_APIC_BUS_CYCLES_NS));
diff --git a/tools/testing/selftests/kvm/x86/cpuid_test.c b/tools/testing/selftests/kvm/x86/cpuid_test.c
index f9ed149..ef0ddd2 100644
--- a/tools/testing/selftests/kvm/x86/cpuid_test.c
+++ b/tools/testing/selftests/kvm/x86/cpuid_test.c
@@ -140,10 +140,10 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage)
}
}
-struct kvm_cpuid2 *vcpu_alloc_cpuid(struct kvm_vm *vm, vm_vaddr_t *p_gva, struct kvm_cpuid2 *cpuid)
+struct kvm_cpuid2 *vcpu_alloc_cpuid(struct kvm_vm *vm, gva_t *p_gva, struct kvm_cpuid2 *cpuid)
{
int size = sizeof(*cpuid) + cpuid->nent * sizeof(cpuid->entries[0]);
- vm_vaddr_t gva = vm_vaddr_alloc(vm, size, KVM_UTIL_MIN_VADDR);
+ gva_t gva = vm_alloc(vm, size, KVM_UTIL_MIN_VADDR);
struct kvm_cpuid2 *guest_cpuids = addr_gva2hva(vm, gva);
memcpy(guest_cpuids, cpuid, size);
@@ -217,7 +217,7 @@ static void test_get_cpuid2(struct kvm_vcpu *vcpu)
int main(void)
{
struct kvm_vcpu *vcpu;
- vm_vaddr_t cpuid_gva;
+ gva_t cpuid_gva;
struct kvm_vm *vm;
int stage;
diff --git a/tools/testing/selftests/kvm/x86/debug_regs.c b/tools/testing/selftests/kvm/x86/debug_regs.c
index 2d814c1..0dfaf03 100644
--- a/tools/testing/selftests/kvm/x86/debug_regs.c
+++ b/tools/testing/selftests/kvm/x86/debug_regs.c
@@ -16,7 +16,7 @@
#define IRQ_VECTOR 0xAA
/* For testing data access debug BP */
-uint32_t guest_value;
+u32 guest_value;
extern unsigned char sw_bp, hw_bp, write_data, ss_start, bd_start;
@@ -86,7 +86,7 @@ int main(void)
struct kvm_run *run;
struct kvm_vm *vm;
struct ucall uc;
- uint64_t cmd;
+ u64 cmd;
int i;
/* Instruction lengths starting at ss_start */
int ss_size[6] = {
diff --git a/tools/testing/selftests/kvm/x86/dirty_log_page_splitting_test.c b/tools/testing/selftests/kvm/x86/dirty_log_page_splitting_test.c
index b0d2b04..388ba41 100644
--- a/tools/testing/selftests/kvm/x86/dirty_log_page_splitting_test.c
+++ b/tools/testing/selftests/kvm/x86/dirty_log_page_splitting_test.c
@@ -23,7 +23,7 @@
#define SLOTS 2
#define ITERATIONS 2
-static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
+static u64 guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
static enum vm_mem_backing_src_type backing_src = VM_MEM_SRC_ANONYMOUS_HUGETLB;
@@ -33,10 +33,10 @@ static int iteration;
static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
struct kvm_page_stats {
- uint64_t pages_4k;
- uint64_t pages_2m;
- uint64_t pages_1g;
- uint64_t hugepages;
+ u64 pages_4k;
+ u64 pages_2m;
+ u64 pages_1g;
+ u64 hugepages;
};
static void get_page_stats(struct kvm_vm *vm, struct kvm_page_stats *stats, const char *stage)
@@ -89,9 +89,9 @@ static void run_test(enum vm_guest_mode mode, void *unused)
{
struct kvm_vm *vm;
unsigned long **bitmaps;
- uint64_t guest_num_pages;
- uint64_t host_num_pages;
- uint64_t pages_per_slot;
+ u64 guest_num_pages;
+ u64 host_num_pages;
+ u64 pages_per_slot;
int i;
struct kvm_page_stats stats_populated;
struct kvm_page_stats stats_dirty_logging_enabled;
diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
index af7c901..5b3aef1 100644
--- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
+++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
@@ -29,13 +29,13 @@
* SMI handler: runs in real-address mode.
* Reports SMRAM_STAGE via port IO, then does RSM.
*/
-static uint8_t smi_handler[] = {
+static u8 smi_handler[] = {
0xb0, SMRAM_STAGE, /* mov $SMRAM_STAGE, %al */
0xe4, SYNC_PORT, /* in $SYNC_PORT, %al */
0x0f, 0xaa, /* rsm */
};
-static inline void sync_with_host(uint64_t phase)
+static inline void sync_with_host(u64 phase)
{
asm volatile("in $" XSTR(SYNC_PORT) ", %%al \n"
: "+a" (phase));
@@ -73,7 +73,7 @@ static void guest_code(struct vmx_pages *vmx_pages,
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva = 0, hv_pages_gva = 0;
+ gva_t vmx_pages_gva = 0, hv_pages_gva = 0;
struct hyperv_test_pages *hv;
struct hv_enlightened_vmcs *evmcs;
struct kvm_vcpu *vcpu;
diff --git a/tools/testing/selftests/kvm/x86/fastops_test.c b/tools/testing/selftests/kvm/x86/fastops_test.c
index 8926cfe..c0d30cc 100644
--- a/tools/testing/selftests/kvm/x86/fastops_test.c
+++ b/tools/testing/selftests/kvm/x86/fastops_test.c
@@ -15,7 +15,7 @@
"pop %[flags]\n\t"
#define flags_constraint(flags_val) [flags]"=r"(flags_val)
-#define bt_constraint(__bt_val) [bt_val]"rm"((uint32_t)__bt_val)
+#define bt_constraint(__bt_val) [bt_val]"rm"((u32)__bt_val)
#define guest_execute_fastop_1(FEP, insn, __val, __flags) \
({ \
@@ -28,17 +28,17 @@
#define guest_test_fastop_1(insn, type_t, __val) \
({ \
type_t val = __val, ex_val = __val, input = __val; \
- uint64_t flags, ex_flags; \
+ u64 flags, ex_flags; \
\
guest_execute_fastop_1("", insn, ex_val, ex_flags); \
guest_execute_fastop_1(KVM_FEP, insn, val, flags); \
\
__GUEST_ASSERT(val == ex_val, \
"Wanted 0x%lx for '%s 0x%lx', got 0x%lx", \
- (uint64_t)ex_val, insn, (uint64_t)input, (uint64_t)val); \
+ (u64)ex_val, insn, (u64)input, (u64)val); \
__GUEST_ASSERT(flags == ex_flags, \
"Wanted flags 0x%lx for '%s 0x%lx', got 0x%lx", \
- ex_flags, insn, (uint64_t)input, flags); \
+ ex_flags, insn, (u64)input, flags); \
})
#define guest_execute_fastop_2(FEP, insn, __input, __output, __flags) \
@@ -52,18 +52,18 @@
#define guest_test_fastop_2(insn, type_t, __val1, __val2) \
({ \
type_t input = __val1, input2 = __val2, output = __val2, ex_output = __val2; \
- uint64_t flags, ex_flags; \
+ u64 flags, ex_flags; \
\
guest_execute_fastop_2("", insn, input, ex_output, ex_flags); \
guest_execute_fastop_2(KVM_FEP, insn, input, output, flags); \
\
__GUEST_ASSERT(output == ex_output, \
"Wanted 0x%lx for '%s 0x%lx 0x%lx', got 0x%lx", \
- (uint64_t)ex_output, insn, (uint64_t)input, \
- (uint64_t)input2, (uint64_t)output); \
+ (u64)ex_output, insn, (u64)input, \
+ (u64)input2, (u64)output); \
__GUEST_ASSERT(flags == ex_flags, \
"Wanted flags 0x%lx for '%s 0x%lx, 0x%lx', got 0x%lx", \
- ex_flags, insn, (uint64_t)input, (uint64_t)input2, flags); \
+ ex_flags, insn, (u64)input, (u64)input2, flags); \
})
#define guest_execute_fastop_cl(FEP, insn, __shift, __output, __flags) \
@@ -77,25 +77,25 @@
#define guest_test_fastop_cl(insn, type_t, __val1, __val2) \
({ \
type_t output = __val2, ex_output = __val2, input = __val2; \
- uint8_t shift = __val1; \
- uint64_t flags, ex_flags; \
+ u8 shift = __val1; \
+ u64 flags, ex_flags; \
\
guest_execute_fastop_cl("", insn, shift, ex_output, ex_flags); \
guest_execute_fastop_cl(KVM_FEP, insn, shift, output, flags); \
\
__GUEST_ASSERT(output == ex_output, \
"Wanted 0x%lx for '%s 0x%x, 0x%lx', got 0x%lx", \
- (uint64_t)ex_output, insn, shift, (uint64_t)input, \
- (uint64_t)output); \
+ (u64)ex_output, insn, shift, (u64)input, \
+ (u64)output); \
__GUEST_ASSERT(flags == ex_flags, \
"Wanted flags 0x%lx for '%s 0x%x, 0x%lx', got 0x%lx", \
- ex_flags, insn, shift, (uint64_t)input, flags); \
+ ex_flags, insn, shift, (u64)input, flags); \
})
#define guest_execute_fastop_div(__KVM_ASM_SAFE, insn, __a, __d, __rm, __flags) \
({ \
- uint64_t ign_error_code; \
- uint8_t vector; \
+ u64 ign_error_code; \
+ u8 vector; \
\
__asm__ __volatile__(fastop(__KVM_ASM_SAFE(insn " %[denom]")) \
: "+a"(__a), "+d"(__d), flags_constraint(__flags), \
@@ -109,8 +109,8 @@
({ \
type_t _a = __val1, _d = __val1, rm = __val2; \
type_t a = _a, d = _d, ex_a = _a, ex_d = _d; \
- uint64_t flags, ex_flags; \
- uint8_t v, ex_v; \
+ u64 flags, ex_flags; \
+ u8 v, ex_v; \
\
ex_v = guest_execute_fastop_div(KVM_ASM_SAFE, insn, ex_a, ex_d, rm, ex_flags); \
v = guest_execute_fastop_div(KVM_ASM_SAFE_FEP, insn, a, d, rm, flags); \
@@ -118,17 +118,17 @@
GUEST_ASSERT_EQ(v, ex_v); \
__GUEST_ASSERT(v == ex_v, \
"Wanted vector 0x%x for '%s 0x%lx:0x%lx/0x%lx', got 0x%x", \
- ex_v, insn, (uint64_t)_a, (uint64_t)_d, (uint64_t)rm, v); \
+ ex_v, insn, (u64)_a, (u64)_d, (u64)rm, v); \
__GUEST_ASSERT(a == ex_a && d == ex_d, \
"Wanted 0x%lx:0x%lx for '%s 0x%lx:0x%lx/0x%lx', got 0x%lx:0x%lx",\
- (uint64_t)ex_a, (uint64_t)ex_d, insn, (uint64_t)_a, \
- (uint64_t)_d, (uint64_t)rm, (uint64_t)a, (uint64_t)d); \
+ (u64)ex_a, (u64)ex_d, insn, (u64)_a, \
+ (u64)_d, (u64)rm, (u64)a, (u64)d); \
__GUEST_ASSERT(v || ex_v || (flags == ex_flags), \
"Wanted flags 0x%lx for '%s 0x%lx:0x%lx/0x%lx', got 0x%lx", \
- ex_flags, insn, (uint64_t)_a, (uint64_t)_d, (uint64_t)rm, flags);\
+ ex_flags, insn, (u64)_a, (u64)_d, (u64)rm, flags);\
})
-static const uint64_t vals[] = {
+static const u64 vals[] = {
0,
1,
2,
@@ -185,10 +185,10 @@ if (sizeof(type_t) != 1) { \
static void guest_code(void)
{
- guest_test_fastops(uint8_t, "b");
- guest_test_fastops(uint16_t, "w");
- guest_test_fastops(uint32_t, "l");
- guest_test_fastops(uint64_t, "q");
+ guest_test_fastops(u8, "b");
+ guest_test_fastops(u16, "w");
+ guest_test_fastops(u32, "l");
+ guest_test_fastops(u64, "q");
GUEST_DONE();
}
diff --git a/tools/testing/selftests/kvm/x86/feature_msrs_test.c b/tools/testing/selftests/kvm/x86/feature_msrs_test.c
index a72f13a..1585507 100644
--- a/tools/testing/selftests/kvm/x86/feature_msrs_test.c
+++ b/tools/testing/selftests/kvm/x86/feature_msrs_test.c
@@ -12,7 +12,7 @@
#include "kvm_util.h"
#include "processor.h"
-static bool is_kvm_controlled_msr(uint32_t msr)
+static bool is_kvm_controlled_msr(u32 msr)
{
return msr == MSR_IA32_VMX_CR0_FIXED1 || msr == MSR_IA32_VMX_CR4_FIXED1;
}
@@ -21,7 +21,7 @@ static bool is_kvm_controlled_msr(uint32_t msr)
* For VMX MSRs with a "true" variant, KVM requires userspace to set the "true"
* MSR, and doesn't allow setting the hidden version.
*/
-static bool is_hidden_vmx_msr(uint32_t msr)
+static bool is_hidden_vmx_msr(u32 msr)
{
switch (msr) {
case MSR_IA32_VMX_PINBASED_CTLS:
@@ -34,15 +34,15 @@ static bool is_hidden_vmx_msr(uint32_t msr)
}
}
-static bool is_quirked_msr(uint32_t msr)
+static bool is_quirked_msr(u32 msr)
{
return msr != MSR_AMD64_DE_CFG;
}
-static void test_feature_msr(uint32_t msr)
+static void test_feature_msr(u32 msr)
{
- const uint64_t supported_mask = kvm_get_feature_msr(msr);
- uint64_t reset_value = is_quirked_msr(msr) ? supported_mask : 0;
+ const u64 supported_mask = kvm_get_feature_msr(msr);
+ u64 reset_value = is_quirked_msr(msr) ? supported_mask : 0;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c
index 00b6e85..753a0e7 100644
--- a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c
+++ b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c
@@ -26,18 +26,18 @@ static void guest_ud_handler(struct ex_regs *regs)
regs->rip += HYPERCALL_INSN_SIZE;
}
-static const uint8_t vmx_vmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xc1 };
-static const uint8_t svm_vmmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xd9 };
+static const u8 vmx_vmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xc1 };
+static const u8 svm_vmmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xd9 };
-extern uint8_t hypercall_insn[HYPERCALL_INSN_SIZE];
-static uint64_t do_sched_yield(uint8_t apic_id)
+extern u8 hypercall_insn[HYPERCALL_INSN_SIZE];
+static u64 do_sched_yield(u8 apic_id)
{
- uint64_t ret;
+ u64 ret;
asm volatile("hypercall_insn:\n\t"
".byte 0xcc,0xcc,0xcc\n\t"
: "=a"(ret)
- : "a"((uint64_t)KVM_HC_SCHED_YIELD), "b"((uint64_t)apic_id)
+ : "a"((u64)KVM_HC_SCHED_YIELD), "b"((u64)apic_id)
: "memory");
return ret;
@@ -45,9 +45,9 @@ static uint64_t do_sched_yield(uint8_t apic_id)
static void guest_main(void)
{
- const uint8_t *native_hypercall_insn;
- const uint8_t *other_hypercall_insn;
- uint64_t ret;
+ const u8 *native_hypercall_insn;
+ const u8 *other_hypercall_insn;
+ u64 ret;
if (host_cpu_is_intel) {
native_hypercall_insn = vmx_vmcall;
@@ -72,7 +72,7 @@ static void guest_main(void)
* the "right" hypercall.
*/
if (quirk_disabled) {
- GUEST_ASSERT(ret == (uint64_t)-EFAULT);
+ GUEST_ASSERT(ret == (u64)-EFAULT);
GUEST_ASSERT(!memcmp(other_hypercall_insn, hypercall_insn,
HYPERCALL_INSN_SIZE));
} else {
diff --git a/tools/testing/selftests/kvm/x86/flds_emulation.h b/tools/testing/selftests/kvm/x86/flds_emulation.h
index 37b1a9f..fd6b6c6 100644
--- a/tools/testing/selftests/kvm/x86/flds_emulation.h
+++ b/tools/testing/selftests/kvm/x86/flds_emulation.h
@@ -12,7 +12,7 @@
* KVM to emulate the instruction (e.g. by providing an MMIO address) to
* exercise emulation failures.
*/
-static inline void flds(uint64_t address)
+static inline void flds(u64 address)
{
__asm__ __volatile__(FLDS_MEM_EAX :: "a"(address));
}
@@ -21,8 +21,8 @@ static inline void handle_flds_emulation_failure_exit(struct kvm_vcpu *vcpu)
{
struct kvm_run *run = vcpu->run;
struct kvm_regs regs;
- uint8_t *insn_bytes;
- uint64_t flags;
+ u8 *insn_bytes;
+ u64 flags;
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_INTERNAL_ERROR);
diff --git a/tools/testing/selftests/kvm/x86/hwcr_msr_test.c b/tools/testing/selftests/kvm/x86/hwcr_msr_test.c
index 10b1b0b..8e20a03 100644
--- a/tools/testing/selftests/kvm/x86/hwcr_msr_test.c
+++ b/tools/testing/selftests/kvm/x86/hwcr_msr_test.c
@@ -10,11 +10,11 @@
void test_hwcr_bit(struct kvm_vcpu *vcpu, unsigned int bit)
{
- const uint64_t ignored = BIT_ULL(3) | BIT_ULL(6) | BIT_ULL(8);
- const uint64_t valid = BIT_ULL(18) | BIT_ULL(24);
- const uint64_t legal = ignored | valid;
- uint64_t val = BIT_ULL(bit);
- uint64_t actual;
+ const u64 ignored = BIT_ULL(3) | BIT_ULL(6) | BIT_ULL(8);
+ const u64 valid = BIT_ULL(18) | BIT_ULL(24);
+ const u64 legal = ignored | valid;
+ u64 val = BIT_ULL(bit);
+ u64 actual;
int r;
r = _vcpu_set_msr(vcpu, MSR_K7_HWCR, val);
diff --git a/tools/testing/selftests/kvm/x86/hyperv_clock.c b/tools/testing/selftests/kvm/x86/hyperv_clock.c
index e058bc6..c083cea 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_clock.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_clock.c
@@ -98,7 +98,7 @@ static inline void check_tsc_msr_tsc_page(struct ms_hyperv_tsc_page *tsc_page)
GUEST_ASSERT(r2 >= t1 && r2 - t2 < 100000);
}
-static void guest_main(struct ms_hyperv_tsc_page *tsc_page, vm_paddr_t tsc_page_gpa)
+static void guest_main(struct ms_hyperv_tsc_page *tsc_page, gpa_t tsc_page_gpa)
{
u64 tsc_scale, tsc_offset;
@@ -208,7 +208,7 @@ int main(void)
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct ucall uc;
- vm_vaddr_t tsc_page_gva;
+ gva_t tsc_page_gva;
int stage;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_TIME));
@@ -218,7 +218,7 @@ int main(void)
vcpu_set_hv_cpuid(vcpu);
- tsc_page_gva = vm_vaddr_alloc_page(vm);
+ tsc_page_gva = vm_alloc_page(vm);
memset(addr_gva2hva(vm, tsc_page_gva), 0x0, getpagesize());
TEST_ASSERT((addr_gva2gpa(vm, tsc_page_gva) & (getpagesize() - 1)) == 0,
"TSC page has to be page aligned");
diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
index 74cf196..c7fa114 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
@@ -30,7 +30,7 @@ static void guest_nmi_handler(struct ex_regs *regs)
{
}
-static inline void rdmsr_from_l2(uint32_t msr)
+static inline void rdmsr_from_l2(u32 msr)
{
/* Currently, L1 doesn't preserve GPRs during vmexits. */
__asm__ __volatile__ ("rdmsr" : : "c"(msr) :
@@ -76,7 +76,7 @@ void l2_guest_code(void)
}
void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages,
- vm_vaddr_t hv_hcall_page_gpa)
+ gpa_t hv_hcall_page_gpa)
{
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
@@ -231,8 +231,8 @@ static struct kvm_vcpu *save_restore_vm(struct kvm_vm *vm,
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva = 0, hv_pages_gva = 0;
- vm_vaddr_t hcall_page;
+ gva_t vmx_pages_gva = 0, hv_pages_gva = 0;
+ gva_t hcall_page;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -246,7 +246,7 @@ int main(int argc, char *argv[])
vm = vm_create_with_one_vcpu(&vcpu, guest_code);
- hcall_page = vm_vaddr_alloc_pages(vm, 1);
+ hcall_page = vm_alloc_pages(vm, 1);
memset(addr_gva2hva(vm, hcall_page), 0x0, getpagesize());
vcpu_set_hv_cpuid(vcpu);
diff --git a/tools/testing/selftests/kvm/x86/hyperv_extended_hypercalls.c b/tools/testing/selftests/kvm/x86/hyperv_extended_hypercalls.c
index 949e08e..ae047db 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_extended_hypercalls.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_extended_hypercalls.c
@@ -15,19 +15,19 @@
/* Any value is fine */
#define EXT_CAPABILITIES 0xbull
-static void guest_code(vm_paddr_t in_pg_gpa, vm_paddr_t out_pg_gpa,
- vm_vaddr_t out_pg_gva)
+static void guest_code(gpa_t in_pg_gpa, gpa_t out_pg_gpa,
+ gva_t out_pg_gva)
{
- uint64_t *output_gva;
+ u64 *output_gva;
wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID);
wrmsr(HV_X64_MSR_HYPERCALL, in_pg_gpa);
- output_gva = (uint64_t *)out_pg_gva;
+ output_gva = (u64 *)out_pg_gva;
hyperv_hypercall(HV_EXT_CALL_QUERY_CAPABILITIES, in_pg_gpa, out_pg_gpa);
- /* TLFS states output will be a uint64_t value */
+ /* TLFS states output will be a u64 value */
GUEST_ASSERT_EQ(*output_gva, EXT_CAPABILITIES);
GUEST_DONE();
@@ -35,12 +35,12 @@ static void guest_code(vm_paddr_t in_pg_gpa, vm_paddr_t out_pg_gpa,
int main(void)
{
- vm_vaddr_t hcall_out_page;
- vm_vaddr_t hcall_in_page;
+ gva_t hcall_out_page;
+ gva_t hcall_in_page;
struct kvm_vcpu *vcpu;
struct kvm_run *run;
struct kvm_vm *vm;
- uint64_t *outval;
+ u64 *outval;
struct ucall uc;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_HYPERV_CPUID));
@@ -57,11 +57,11 @@ int main(void)
vcpu_set_hv_cpuid(vcpu);
/* Hypercall input */
- hcall_in_page = vm_vaddr_alloc_pages(vm, 1);
+ hcall_in_page = vm_alloc_pages(vm, 1);
memset(addr_gva2hva(vm, hcall_in_page), 0x0, vm->page_size);
/* Hypercall output */
- hcall_out_page = vm_vaddr_alloc_pages(vm, 1);
+ hcall_out_page = vm_alloc_pages(vm, 1);
memset(addr_gva2hva(vm, hcall_out_page), 0x0, vm->page_size);
vcpu_args_set(vcpu, 3, addr_gva2gpa(vm, hcall_in_page),
diff --git a/tools/testing/selftests/kvm/x86/hyperv_features.c b/tools/testing/selftests/kvm/x86/hyperv_features.c
index 130b9ce..7347f1fe 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_features.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_features.c
@@ -22,27 +22,27 @@
KVM_X86_CPU_FEATURE(HYPERV_CPUID_ENLIGHTMENT_INFO, 0, EBX, 0)
struct msr_data {
- uint32_t idx;
+ u32 idx;
bool fault_expected;
bool write;
u64 write_val;
};
struct hcall_data {
- uint64_t control;
- uint64_t expect;
+ u64 control;
+ u64 expect;
bool ud_expected;
};
-static bool is_write_only_msr(uint32_t msr)
+static bool is_write_only_msr(u32 msr)
{
return msr == HV_X64_MSR_EOI;
}
static void guest_msr(struct msr_data *msr)
{
- uint8_t vector = 0;
- uint64_t msr_val = 0;
+ u8 vector = 0;
+ u64 msr_val = 0;
GUEST_ASSERT(msr->idx);
@@ -82,10 +82,10 @@ static void guest_msr(struct msr_data *msr)
GUEST_DONE();
}
-static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall)
+static void guest_hcall(gpa_t pgs_gpa, struct hcall_data *hcall)
{
u64 res, input, output;
- uint8_t vector;
+ u8 vector;
GUEST_ASSERT_NE(hcall->control, 0);
@@ -134,14 +134,14 @@ static void guest_test_msrs_access(void)
struct kvm_vm *vm;
struct ucall uc;
int stage = 0;
- vm_vaddr_t msr_gva;
+ gva_t msr_gva;
struct msr_data *msr;
bool has_invtsc = kvm_cpu_has(X86_FEATURE_INVTSC);
while (true) {
vm = vm_create_with_one_vcpu(&vcpu, guest_msr);
- msr_gva = vm_vaddr_alloc_page(vm);
+ msr_gva = vm_alloc_page(vm);
memset(addr_gva2hva(vm, msr_gva), 0x0, getpagesize());
msr = addr_gva2hva(vm, msr_gva);
@@ -523,17 +523,17 @@ static void guest_test_hcalls_access(void)
struct kvm_vm *vm;
struct ucall uc;
int stage = 0;
- vm_vaddr_t hcall_page, hcall_params;
+ gva_t hcall_page, hcall_params;
struct hcall_data *hcall;
while (true) {
vm = vm_create_with_one_vcpu(&vcpu, guest_hcall);
/* Hypercall input/output */
- hcall_page = vm_vaddr_alloc_pages(vm, 2);
+ hcall_page = vm_alloc_pages(vm, 2);
memset(addr_gva2hva(vm, hcall_page), 0x0, 2 * getpagesize());
- hcall_params = vm_vaddr_alloc_page(vm);
+ hcall_params = vm_alloc_page(vm);
memset(addr_gva2hva(vm, hcall_params), 0x0, getpagesize());
hcall = addr_gva2hva(vm, hcall_params);
diff --git a/tools/testing/selftests/kvm/x86/hyperv_ipi.c b/tools/testing/selftests/kvm/x86/hyperv_ipi.c
index ca61836..771535f 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_ipi.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_ipi.c
@@ -18,7 +18,7 @@
#define IPI_VECTOR 0xfe
-static volatile uint64_t ipis_rcvd[RECEIVER_VCPU_ID_2 + 1];
+static volatile u64 ipis_rcvd[RECEIVER_VCPU_ID_2 + 1];
struct hv_vpset {
u64 format;
@@ -45,13 +45,13 @@ struct hv_send_ipi_ex {
struct hv_vpset vp_set;
};
-static inline void hv_init(vm_vaddr_t pgs_gpa)
+static inline void hv_init(gpa_t pgs_gpa)
{
wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID);
wrmsr(HV_X64_MSR_HYPERCALL, pgs_gpa);
}
-static void receiver_code(void *hcall_page, vm_vaddr_t pgs_gpa)
+static void receiver_code(void *hcall_page, gpa_t pgs_gpa)
{
u32 vcpu_id;
@@ -85,7 +85,7 @@ static inline void nop_loop(void)
asm volatile("nop");
}
-static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa)
+static void sender_guest_code(void *hcall_page, gpa_t pgs_gpa)
{
struct hv_send_ipi *ipi = (struct hv_send_ipi *)hcall_page;
struct hv_send_ipi_ex *ipi_ex = (struct hv_send_ipi_ex *)hcall_page;
@@ -243,7 +243,7 @@ int main(int argc, char *argv[])
{
struct kvm_vm *vm;
struct kvm_vcpu *vcpu[3];
- vm_vaddr_t hcall_page;
+ gva_t hcall_page;
pthread_t threads[2];
int stage = 1, r;
struct ucall uc;
@@ -253,7 +253,7 @@ int main(int argc, char *argv[])
vm = vm_create_with_one_vcpu(&vcpu[0], sender_guest_code);
/* Hypercall input/output */
- hcall_page = vm_vaddr_alloc_pages(vm, 2);
+ hcall_page = vm_alloc_pages(vm, 2);
memset(addr_gva2hva(vm, hcall_page), 0x0, 2 * getpagesize());
diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
index 0ddb632..7a62f6a 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
@@ -21,7 +21,7 @@
#define L2_GUEST_STACK_SIZE 256
/* Exit to L1 from L2 with RDMSR instruction */
-static inline void rdmsr_from_l2(uint32_t msr)
+static inline void rdmsr_from_l2(u32 msr)
{
/* Currently, L1 doesn't preserve GPRs during vmexits. */
__asm__ __volatile__ ("rdmsr" : : "c"(msr) :
@@ -67,7 +67,7 @@ void l2_guest_code(void)
static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm,
struct hyperv_test_pages *hv_pages,
- vm_vaddr_t pgs_gpa)
+ gpa_t pgs_gpa)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
struct vmcb *vmcb = svm->vmcb;
@@ -149,8 +149,8 @@ static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm,
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_gva = 0, hv_pages_gva = 0;
- vm_vaddr_t hcall_page;
+ gva_t nested_gva = 0, hv_pages_gva = 0;
+ gva_t hcall_page;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct ucall uc;
@@ -165,7 +165,7 @@ int main(int argc, char *argv[])
vcpu_alloc_svm(vm, &nested_gva);
vcpu_alloc_hyperv_test_pages(vm, &hv_pages_gva);
- hcall_page = vm_vaddr_alloc_pages(vm, 1);
+ hcall_page = vm_alloc_pages(vm, 1);
memset(addr_gva2hva(vm, hcall_page), 0x0, getpagesize());
vcpu_args_set(vcpu, 3, nested_gva, hv_pages_gva, addr_gva2gpa(vm, hcall_page));
diff --git a/tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c b/tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c
index c542cc4..15ee8b7 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c
@@ -61,14 +61,14 @@ struct hv_tlb_flush_ex {
* - GVAs of the test pages' PTEs
*/
struct test_data {
- vm_vaddr_t hcall_gva;
- vm_paddr_t hcall_gpa;
- vm_vaddr_t test_pages;
- vm_vaddr_t test_pages_pte[NTEST_PAGES];
+ gva_t hcall_gva;
+ gpa_t hcall_gpa;
+ gva_t test_pages;
+ gva_t test_pages_pte[NTEST_PAGES];
};
/* 'Worker' vCPU code checking the contents of the test page */
-static void worker_guest_code(vm_vaddr_t test_data)
+static void worker_guest_code(gva_t test_data)
{
struct test_data *data = (struct test_data *)test_data;
u32 vcpu_id = rdmsr(HV_X64_MSR_VP_INDEX);
@@ -133,12 +133,12 @@ static void set_expected_val(void *addr, u64 val, int vcpu_id)
* Update PTEs swapping two test pages.
* TODO: use swap()/xchg() when these are provided.
*/
-static void swap_two_test_pages(vm_paddr_t pte_gva1, vm_paddr_t pte_gva2)
+static void swap_two_test_pages(gpa_t pte_gva1, gpa_t pte_gva2)
{
- uint64_t tmp = *(uint64_t *)pte_gva1;
+ u64 tmp = *(u64 *)pte_gva1;
- *(uint64_t *)pte_gva1 = *(uint64_t *)pte_gva2;
- *(uint64_t *)pte_gva2 = tmp;
+ *(u64 *)pte_gva1 = *(u64 *)pte_gva2;
+ *(u64 *)pte_gva2 = tmp;
}
/*
@@ -196,12 +196,12 @@ static inline void post_test(struct test_data *data, u64 exp1, u64 exp2)
#define TESTVAL2 0x0202020202020202
/* Main vCPU doing the test */
-static void sender_guest_code(vm_vaddr_t test_data)
+static void sender_guest_code(gva_t test_data)
{
struct test_data *data = (struct test_data *)test_data;
struct hv_tlb_flush *flush = (struct hv_tlb_flush *)data->hcall_gva;
struct hv_tlb_flush_ex *flush_ex = (struct hv_tlb_flush_ex *)data->hcall_gva;
- vm_paddr_t hcall_gpa = data->hcall_gpa;
+ gpa_t hcall_gpa = data->hcall_gpa;
int i, stage = 1;
wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID);
@@ -581,9 +581,9 @@ int main(int argc, char *argv[])
struct kvm_vm *vm;
struct kvm_vcpu *vcpu[3];
pthread_t threads[2];
- vm_vaddr_t test_data_page, gva;
- vm_paddr_t gpa;
- uint64_t *pte;
+ gva_t test_data_page, gva;
+ gpa_t gpa;
+ u64 *pte;
struct test_data *data;
struct ucall uc;
int stage = 1, r, i;
@@ -593,11 +593,11 @@ int main(int argc, char *argv[])
vm = vm_create_with_one_vcpu(&vcpu[0], sender_guest_code);
/* Test data page */
- test_data_page = vm_vaddr_alloc_page(vm);
+ test_data_page = vm_alloc_page(vm);
data = (struct test_data *)addr_gva2hva(vm, test_data_page);
/* Hypercall input/output */
- data->hcall_gva = vm_vaddr_alloc_pages(vm, 2);
+ data->hcall_gva = vm_alloc_pages(vm, 2);
data->hcall_gpa = addr_gva2gpa(vm, data->hcall_gva);
memset(addr_gva2hva(vm, data->hcall_gva), 0x0, 2 * PAGE_SIZE);
@@ -606,7 +606,7 @@ int main(int argc, char *argv[])
* and the test will swap their mappings. The third page keeps the indication
* about the current state of mappings.
*/
- data->test_pages = vm_vaddr_alloc_pages(vm, NTEST_PAGES + 1);
+ data->test_pages = vm_alloc_pages(vm, NTEST_PAGES + 1);
for (i = 0; i < NTEST_PAGES; i++)
memset(addr_gva2hva(vm, data->test_pages + PAGE_SIZE * i),
(u8)(i + 1), PAGE_SIZE);
@@ -617,7 +617,7 @@ int main(int argc, char *argv[])
* Get PTE pointers for test pages and map them inside the guest.
* Use separate page for each PTE for simplicity.
*/
- gva = vm_vaddr_unused_gap(vm, NTEST_PAGES * PAGE_SIZE, KVM_UTIL_MIN_VADDR);
+ gva = vm_unused_gva_gap(vm, NTEST_PAGES * PAGE_SIZE, KVM_UTIL_MIN_VADDR);
for (i = 0; i < NTEST_PAGES; i++) {
pte = vm_get_pte(vm, data->test_pages + i * PAGE_SIZE);
gpa = addr_hva2gpa(vm, pte);
diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
index d88500c..52014a3 100644
--- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
+++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
@@ -73,7 +73,7 @@ static void guest_code(void *test_data)
int main(int argc, char *argv[])
{
const bool has_nested = kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX);
- vm_vaddr_t nested_test_data_gva;
+ gva_t nested_test_data_gva;
struct kvm_vcpu *vcpu;
struct kvm_run *run;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/kvm_clock_test.c b/tools/testing/selftests/kvm/x86/kvm_clock_test.c
index 5bc1222..5ad4aeb 100644
--- a/tools/testing/selftests/kvm/x86/kvm_clock_test.c
+++ b/tools/testing/selftests/kvm/x86/kvm_clock_test.c
@@ -17,8 +17,8 @@
#include "processor.h"
struct test_case {
- uint64_t kvmclock_base;
- int64_t realtime_offset;
+ u64 kvmclock_base;
+ s64 realtime_offset;
};
static struct test_case test_cases[] = {
@@ -31,7 +31,7 @@ static struct test_case test_cases[] = {
#define GUEST_SYNC_CLOCK(__stage, __val) \
GUEST_SYNC_ARGS(__stage, __val, 0, 0, 0)
-static void guest_main(vm_paddr_t pvti_pa, struct pvclock_vcpu_time_info *pvti)
+static void guest_main(gpa_t pvti_pa, struct pvclock_vcpu_time_info *pvti)
{
int i;
@@ -52,7 +52,7 @@ static inline void assert_flags(struct kvm_clock_data *data)
static void handle_sync(struct ucall *uc, struct kvm_clock_data *start,
struct kvm_clock_data *end)
{
- uint64_t obs, exp_lo, exp_hi;
+ u64 obs, exp_lo, exp_hi;
obs = uc->args[2];
exp_lo = start->clock;
@@ -135,8 +135,8 @@ static void enter_guest(struct kvm_vcpu *vcpu)
int main(void)
{
struct kvm_vcpu *vcpu;
- vm_vaddr_t pvti_gva;
- vm_paddr_t pvti_gpa;
+ gva_t pvti_gva;
+ gpa_t pvti_gpa;
struct kvm_vm *vm;
int flags;
@@ -147,7 +147,7 @@ int main(void)
vm = vm_create_with_one_vcpu(&vcpu, guest_main);
- pvti_gva = vm_vaddr_alloc(vm, getpagesize(), 0x10000);
+ pvti_gva = vm_alloc(vm, getpagesize(), 0x10000);
pvti_gpa = addr_gva2gpa(vm, pvti_gva);
vcpu_args_set(vcpu, 2, pvti_gpa, pvti_gva);
diff --git a/tools/testing/selftests/kvm/x86/kvm_pv_test.c b/tools/testing/selftests/kvm/x86/kvm_pv_test.c
index 1b805cb..8ed5fa6 100644
--- a/tools/testing/selftests/kvm/x86/kvm_pv_test.c
+++ b/tools/testing/selftests/kvm/x86/kvm_pv_test.c
@@ -13,7 +13,7 @@
#include "processor.h"
struct msr_data {
- uint32_t idx;
+ u32 idx;
const char *name;
};
@@ -40,8 +40,8 @@ static struct msr_data msrs_to_test[] = {
static void test_msr(struct msr_data *msr)
{
- uint64_t ignored;
- uint8_t vector;
+ u64 ignored;
+ u8 vector;
PR_MSR(msr);
@@ -53,7 +53,7 @@ static void test_msr(struct msr_data *msr)
}
struct hcall_data {
- uint64_t nr;
+ u64 nr;
const char *name;
};
@@ -73,7 +73,7 @@ static struct hcall_data hcalls_to_test[] = {
static void test_hcall(struct hcall_data *hc)
{
- uint64_t r;
+ u64 r;
PR_HCALL(hc);
r = kvm_hypercall(hc->nr, 0, 0, 0, 0);
diff --git a/tools/testing/selftests/kvm/x86/monitor_mwait_test.c b/tools/testing/selftests/kvm/x86/monitor_mwait_test.c
index e45c028..9c156cf 100644
--- a/tools/testing/selftests/kvm/x86/monitor_mwait_test.c
+++ b/tools/testing/selftests/kvm/x86/monitor_mwait_test.c
@@ -67,7 +67,7 @@ static void guest_monitor_wait(void *arg)
int main(int argc, char *argv[])
{
- uint64_t disabled_quirks;
+ u64 disabled_quirks;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct ucall uc;
diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
index f001cb8..761fec2 100644
--- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
@@ -67,7 +67,7 @@ static void l1_guest_code(void *data)
int main(int argc, char *argv[])
{
- vm_vaddr_t guest_gva;
+ gva_t guest_gva;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
index 619229b..0e67cce 100644
--- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
@@ -47,10 +47,10 @@
#define TEST_SYNC_WRITE_FAULT BIT(1)
#define TEST_SYNC_NO_FAULT BIT(2)
-static void l2_guest_code(vm_vaddr_t base)
+static void l2_guest_code(gva_t base)
{
- vm_vaddr_t page0 = TEST_GUEST_ADDR(base, 0);
- vm_vaddr_t page1 = TEST_GUEST_ADDR(base, 1);
+ gva_t page0 = TEST_GUEST_ADDR(base, 0);
+ gva_t page1 = TEST_GUEST_ADDR(base, 1);
READ_ONCE(*(u64 *)page0);
GUEST_SYNC(page0 | TEST_SYNC_READ_FAULT);
@@ -143,7 +143,7 @@ static void l1_guest_code(void *data)
static void test_handle_ucall_sync(struct kvm_vm *vm, u64 arg,
unsigned long *bmap)
{
- vm_vaddr_t gva = arg & ~(PAGE_SIZE - 1);
+ gva_t gva = arg & ~(PAGE_SIZE - 1);
int page_nr, i;
/*
@@ -198,7 +198,7 @@ static void test_handle_ucall_sync(struct kvm_vm *vm, u64 arg,
static void test_dirty_log(bool nested_tdp)
{
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
unsigned long *bmap;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c
index abc824d..fb7dcbe 100644
--- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c
@@ -13,8 +13,8 @@ enum {
struct emulated_instruction {
const char name[32];
- uint8_t opcode[15];
- uint32_t exit_reason[NR_VIRTUALIZATION_FLAVORS];
+ u8 opcode[15];
+ u32 exit_reason[NR_VIRTUALIZATION_FLAVORS];
};
static struct emulated_instruction instructions[] = {
@@ -32,13 +32,13 @@ static struct emulated_instruction instructions[] = {
},
};
-static uint8_t kvm_fep[] = { 0x0f, 0x0b, 0x6b, 0x76, 0x6d }; /* ud2 ; .ascii "kvm" */
-static uint8_t l2_guest_code[sizeof(kvm_fep) + 15];
-static uint8_t *l2_instruction = &l2_guest_code[sizeof(kvm_fep)];
+static u8 kvm_fep[] = { 0x0f, 0x0b, 0x6b, 0x76, 0x6d }; /* ud2 ; .ascii "kvm" */
+static u8 l2_guest_code[sizeof(kvm_fep) + 15];
+static u8 *l2_instruction = &l2_guest_code[sizeof(kvm_fep)];
-static uint32_t get_instruction_length(struct emulated_instruction *insn)
+static u32 get_instruction_length(struct emulated_instruction *insn)
{
- uint32_t i;
+ u32 i;
for (i = 0; i < ARRAY_SIZE(insn->opcode) && insn->opcode[i]; i++)
;
@@ -81,8 +81,8 @@ static void guest_code(void *test_data)
for (i = 0; i < ARRAY_SIZE(instructions); i++) {
struct emulated_instruction *insn = &instructions[i];
- uint32_t insn_len = get_instruction_length(insn);
- uint32_t exit_insn_len;
+ u32 insn_len = get_instruction_length(insn);
+ u32 exit_insn_len;
u32 exit_reason;
/*
@@ -122,7 +122,7 @@ static void guest_code(void *test_data)
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_test_data_gva;
+ gva_t nested_test_data_gva;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
index 3641a42..186e980 100644
--- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
@@ -72,7 +72,7 @@ static void l2_ss_injected_tf_test(void)
}
static void svm_run_l2(struct svm_test_data *svm, void *l2_code, int vector,
- uint32_t error_code)
+ u32 error_code)
{
struct vmcb *vmcb = svm->vmcb;
struct vmcb_control_area *ctrl = &vmcb->control;
@@ -111,7 +111,7 @@ static void l1_svm_code(struct svm_test_data *svm)
GUEST_DONE();
}
-static void vmx_run_l2(void *l2_code, int vector, uint32_t error_code)
+static void vmx_run_l2(void *l2_code, int vector, u32 error_code)
{
GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code));
@@ -216,7 +216,7 @@ static void queue_ss_exception(struct kvm_vcpu *vcpu, bool inject)
*/
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_test_data_gva;
+ gva_t nested_test_data_gva;
struct kvm_vcpu_events events;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
index a6b6da9..11fd246 100644
--- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
@@ -78,7 +78,7 @@ int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_vaddr_t guest_gva = 0;
+ gva_t guest_gva = 0;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
kvm_cpu_has(X86_FEATURE_SVM));
diff --git a/tools/testing/selftests/kvm/x86/nested_set_state_test.c b/tools/testing/selftests/kvm/x86/nested_set_state_test.c
index 0f2102b..8313807 100644
--- a/tools/testing/selftests/kvm/x86/nested_set_state_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_set_state_test.c
@@ -250,14 +250,14 @@ void test_vmx_nested_state(struct kvm_vcpu *vcpu)
static void vcpu_efer_enable_svm(struct kvm_vcpu *vcpu)
{
- uint64_t old_efer = vcpu_get_msr(vcpu, MSR_EFER);
+ u64 old_efer = vcpu_get_msr(vcpu, MSR_EFER);
vcpu_set_msr(vcpu, MSR_EFER, old_efer | EFER_SVME);
}
static void vcpu_efer_disable_svm(struct kvm_vcpu *vcpu)
{
- uint64_t old_efer = vcpu_get_msr(vcpu, MSR_EFER);
+ u64 old_efer = vcpu_get_msr(vcpu, MSR_EFER);
vcpu_set_msr(vcpu, MSR_EFER, old_efer & ~EFER_SVME);
}
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
index 2839f65..f0e4ada 100644
--- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
@@ -53,9 +53,9 @@ enum {
/* The virtual machine object. */
static struct kvm_vm *vm;
-static void check_ia32_tsc_adjust(int64_t max)
+static void check_ia32_tsc_adjust(s64 max)
{
- int64_t adjust;
+ s64 adjust;
adjust = rdmsr(MSR_IA32_TSC_ADJUST);
GUEST_SYNC(adjust);
@@ -64,7 +64,7 @@ static void check_ia32_tsc_adjust(int64_t max)
static void l2_guest_code(void)
{
- uint64_t l1_tsc = rdtsc() - TSC_OFFSET_VALUE;
+ u64 l1_tsc = rdtsc() - TSC_OFFSET_VALUE;
wrmsr(MSR_IA32_TSC, l1_tsc - TSC_ADJUST_VALUE);
check_ia32_tsc_adjust(-2 * TSC_ADJUST_VALUE);
@@ -88,7 +88,7 @@ static void l1_guest_code(void *data)
*/
if (this_cpu_has(X86_FEATURE_VMX)) {
struct vmx_pages *vmx_pages = data;
- uint32_t control;
+ u32 control;
GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
GUEST_ASSERT(load_vmcs(vmx_pages));
@@ -117,7 +117,7 @@ static void l1_guest_code(void *data)
GUEST_DONE();
}
-static void report(int64_t val)
+static void report(s64 val)
{
pr_info("IA32_TSC_ADJUST is %ld (%lld * TSC_ADJUST_VALUE + %lld).\n",
val, val / TSC_ADJUST_VALUE, val % TSC_ADJUST_VALUE);
@@ -125,7 +125,7 @@ static void report(int64_t val)
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_gva;
+ gva_t nested_gva;
struct kvm_vcpu *vcpu;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
index 4260c9e..190e93a 100644
--- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
@@ -19,7 +19,7 @@
/* L2 is scaled up (from L1's perspective) by this factor */
#define L2_SCALE_FACTOR 4ULL
-#define TSC_OFFSET_L2 ((uint64_t) -33125236320908)
+#define TSC_OFFSET_L2 ((u64)-33125236320908)
#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48)
#define L2_GUEST_STACK_SIZE 64
@@ -35,9 +35,9 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 };
* measurements, a difference of 1% between the actual and the expected value
* is tolerated.
*/
-static void compare_tsc_freq(uint64_t actual, uint64_t expected)
+static void compare_tsc_freq(u64 actual, u64 expected)
{
- uint64_t tolerance, thresh_low, thresh_high;
+ u64 tolerance, thresh_low, thresh_high;
tolerance = expected / 100;
thresh_low = expected - tolerance;
@@ -55,7 +55,7 @@ static void compare_tsc_freq(uint64_t actual, uint64_t expected)
static void check_tsc_freq(int level)
{
- uint64_t tsc_start, tsc_end, tsc_freq;
+ u64 tsc_start, tsc_end, tsc_freq;
/*
* Reading the TSC twice with about a second's difference should give
@@ -106,7 +106,7 @@ static void l1_svm_code(struct svm_test_data *svm)
static void l1_vmx_code(struct vmx_pages *vmx_pages)
{
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
- uint32_t control;
+ u32 control;
/* check that L1's frequency looks alright before launching L2 */
check_tsc_freq(UCHECK_L1);
@@ -152,14 +152,14 @@ int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_vaddr_t guest_gva = 0;
+ gva_t guest_gva = 0;
- uint64_t tsc_start, tsc_end;
- uint64_t tsc_khz;
- uint64_t l1_scale_factor;
- uint64_t l0_tsc_freq = 0;
- uint64_t l1_tsc_freq = 0;
- uint64_t l2_tsc_freq = 0;
+ u64 tsc_start, tsc_end;
+ u64 tsc_khz;
+ u64 l1_scale_factor;
+ u64 l0_tsc_freq = 0;
+ u64 l1_tsc_freq = 0;
+ u64 l2_tsc_freq = 0;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
kvm_cpu_has(X86_FEATURE_SVM));
diff --git a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
index 71717118..85d3f4c 100644
--- a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
@@ -128,7 +128,7 @@ static void l1_guest_code(struct svm_test_data *svm)
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
struct vmcb *test_vmcb[2];
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c
index c0d8482..7095006 100644
--- a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c
+++ b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c
@@ -32,7 +32,7 @@
#define RETURN_OPCODE 0xC3
/* Call the specified memory address. */
-static void guest_do_CALL(uint64_t target)
+static void guest_do_CALL(u64 target)
{
((void (*)(void)) target)();
}
@@ -46,14 +46,14 @@ static void guest_do_CALL(uint64_t target)
*/
void guest_code(void)
{
- uint64_t hpage_1 = HPAGE_GVA;
- uint64_t hpage_2 = hpage_1 + (PAGE_SIZE * 512);
- uint64_t hpage_3 = hpage_2 + (PAGE_SIZE * 512);
+ u64 hpage_1 = HPAGE_GVA;
+ u64 hpage_2 = hpage_1 + (PAGE_SIZE * 512);
+ u64 hpage_3 = hpage_2 + (PAGE_SIZE * 512);
- READ_ONCE(*(uint64_t *)hpage_1);
+ READ_ONCE(*(u64 *)hpage_1);
GUEST_SYNC(1);
- READ_ONCE(*(uint64_t *)hpage_2);
+ READ_ONCE(*(u64 *)hpage_2);
GUEST_SYNC(2);
guest_do_CALL(hpage_1);
@@ -62,10 +62,10 @@ void guest_code(void)
guest_do_CALL(hpage_3);
GUEST_SYNC(4);
- READ_ONCE(*(uint64_t *)hpage_1);
+ READ_ONCE(*(u64 *)hpage_1);
GUEST_SYNC(5);
- READ_ONCE(*(uint64_t *)hpage_3);
+ READ_ONCE(*(u64 *)hpage_3);
GUEST_SYNC(6);
}
@@ -107,7 +107,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages,
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint64_t nr_bytes;
+ u64 nr_bytes;
void *hva;
int r;
diff --git a/tools/testing/selftests/kvm/x86/platform_info_test.c b/tools/testing/selftests/kvm/x86/platform_info_test.c
index 9cbf283..80bb07e 100644
--- a/tools/testing/selftests/kvm/x86/platform_info_test.c
+++ b/tools/testing/selftests/kvm/x86/platform_info_test.c
@@ -23,8 +23,8 @@
static void guest_code(void)
{
- uint64_t msr_platform_info;
- uint8_t vector;
+ u64 msr_platform_info;
+ u8 vector;
GUEST_SYNC(true);
msr_platform_info = rdmsr(MSR_PLATFORM_INFO);
@@ -42,7 +42,7 @@ int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint64_t msr_platform_info;
+ u64 msr_platform_info;
struct ucall uc;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_MSR_PLATFORM_INFO));
diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c
index 3eaa216..dc6afac 100644
--- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c
+++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c
@@ -30,9 +30,9 @@
#define NUM_INSNS_RETIRED (NUM_LOOPS * NUM_INSNS_PER_LOOP + NUM_EXTRA_INSNS)
/* Track which architectural events are supported by hardware. */
-static uint32_t hardware_pmu_arch_events;
+static u32 hardware_pmu_arch_events;
-static uint8_t kvm_pmu_version;
+static u8 kvm_pmu_version;
static bool kvm_has_perf_caps;
#define X86_PMU_FEATURE_NULL \
@@ -57,7 +57,7 @@ struct kvm_intel_pmu_event {
* kvm_x86_pmu_feature use syntax that's only valid in function scope, and the
* compiler often thinks the feature definitions aren't compile-time constants.
*/
-static struct kvm_intel_pmu_event intel_event_to_feature(uint8_t idx)
+static struct kvm_intel_pmu_event intel_event_to_feature(u8 idx)
{
const struct kvm_intel_pmu_event __intel_event_to_feature[] = {
[INTEL_ARCH_CPU_CYCLES_INDEX] = { X86_PMU_FEATURE_CPU_CYCLES, X86_PMU_FEATURE_CPU_CYCLES_FIXED },
@@ -89,8 +89,8 @@ static struct kvm_intel_pmu_event intel_event_to_feature(uint8_t idx)
static struct kvm_vm *pmu_vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
void *guest_code,
- uint8_t pmu_version,
- uint64_t perf_capabilities)
+ u8 pmu_version,
+ u64 perf_capabilities)
{
struct kvm_vm *vm;
@@ -132,7 +132,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu)
} while (uc.cmd != UCALL_DONE);
}
-static uint8_t guest_get_pmu_version(void)
+static u8 guest_get_pmu_version(void)
{
/*
* Return the effective PMU version, i.e. the minimum between what KVM
@@ -141,7 +141,7 @@ static uint8_t guest_get_pmu_version(void)
* supported by KVM to verify KVM doesn't freak out and do something
* bizarre with an architecturally valid, but unsupported, version.
*/
- return min_t(uint8_t, kvm_pmu_version, this_cpu_property(X86_PROPERTY_PMU_VERSION));
+ return min_t(u8, kvm_pmu_version, this_cpu_property(X86_PROPERTY_PMU_VERSION));
}
/*
@@ -153,9 +153,9 @@ static uint8_t guest_get_pmu_version(void)
* Sanity check that in all cases, the event doesn't count when it's disabled,
* and that KVM correctly emulates the write of an arbitrary value.
*/
-static void guest_assert_event_count(uint8_t idx, uint32_t pmc, uint32_t pmc_msr)
+static void guest_assert_event_count(u8 idx, u32 pmc, u32 pmc_msr)
{
- uint64_t count;
+ u64 count;
count = _rdpmc(pmc);
if (!(hardware_pmu_arch_events & BIT(idx)))
@@ -236,7 +236,7 @@ do { \
FEP "xor %%eax, %%eax\n\t" \
FEP "xor %%edx, %%edx\n\t" \
"wrmsr\n\t" \
- :: "a"((uint32_t)_value), "d"(_value >> 32), \
+ :: "a"((u32)_value), "d"(_value >> 32), \
"c"(_msr), "D"(_msr), [m]"m"(kvm_pmu_version) \
); \
} while (0)
@@ -255,8 +255,8 @@ do { \
guest_assert_event_count(_idx, _pmc, _pmc_msr); \
} while (0)
-static void __guest_test_arch_event(uint8_t idx, uint32_t pmc, uint32_t pmc_msr,
- uint32_t ctrl_msr, uint64_t ctrl_msr_value)
+static void __guest_test_arch_event(u8 idx, u32 pmc, u32 pmc_msr,
+ u32 ctrl_msr, u64 ctrl_msr_value)
{
GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, "");
@@ -264,14 +264,14 @@ static void __guest_test_arch_event(uint8_t idx, uint32_t pmc, uint32_t pmc_msr,
GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, KVM_FEP);
}
-static void guest_test_arch_event(uint8_t idx)
+static void guest_test_arch_event(u8 idx)
{
- uint32_t nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
- uint32_t pmu_version = guest_get_pmu_version();
+ u32 nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
+ u32 pmu_version = guest_get_pmu_version();
/* PERF_GLOBAL_CTRL exists only for Architectural PMU Version 2+. */
bool guest_has_perf_global_ctrl = pmu_version >= 2;
struct kvm_x86_pmu_feature gp_event, fixed_event;
- uint32_t base_pmc_msr;
+ u32 base_pmc_msr;
unsigned int i;
/* The host side shouldn't invoke this without a guest PMU. */
@@ -289,7 +289,7 @@ static void guest_test_arch_event(uint8_t idx)
GUEST_ASSERT(nr_gp_counters);
for (i = 0; i < nr_gp_counters; i++) {
- uint64_t eventsel = ARCH_PERFMON_EVENTSEL_OS |
+ u64 eventsel = ARCH_PERFMON_EVENTSEL_OS |
ARCH_PERFMON_EVENTSEL_ENABLE |
intel_pmu_arch_events[idx];
@@ -320,7 +320,7 @@ static void guest_test_arch_event(uint8_t idx)
static void guest_test_arch_events(void)
{
- uint8_t i;
+ u8 i;
for (i = 0; i < NR_INTEL_ARCH_EVENTS; i++)
guest_test_arch_event(i);
@@ -328,8 +328,8 @@ static void guest_test_arch_events(void)
GUEST_DONE();
}
-static void test_arch_events(uint8_t pmu_version, uint64_t perf_capabilities,
- uint8_t length, uint32_t unavailable_mask)
+static void test_arch_events(u8 pmu_version, u64 perf_capabilities,
+ u8 length, u32 unavailable_mask)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -373,11 +373,11 @@ __GUEST_ASSERT(expect_gp ? vector == GP_VECTOR : !vector, \
"Expected " #insn "(0x%x) to yield 0x%lx, got 0x%lx", \
msr, expected, val);
-static void guest_test_rdpmc(uint32_t rdpmc_idx, bool expect_success,
- uint64_t expected_val)
+static void guest_test_rdpmc(u32 rdpmc_idx, bool expect_success,
+ u64 expected_val)
{
- uint8_t vector;
- uint64_t val;
+ u8 vector;
+ u64 val;
vector = rdpmc_safe(rdpmc_idx, &val);
GUEST_ASSERT_PMC_MSR_ACCESS(RDPMC, rdpmc_idx, !expect_success, vector);
@@ -393,19 +393,19 @@ static void guest_test_rdpmc(uint32_t rdpmc_idx, bool expect_success,
GUEST_ASSERT_PMC_VALUE(RDPMC, rdpmc_idx, val, expected_val);
}
-static void guest_rd_wr_counters(uint32_t base_msr, uint8_t nr_possible_counters,
- uint8_t nr_counters, uint32_t or_mask)
+static void guest_rd_wr_counters(u32 base_msr, u8 nr_possible_counters,
+ u8 nr_counters, u32 or_mask)
{
const bool pmu_has_fast_mode = !guest_get_pmu_version();
- uint8_t i;
+ u8 i;
for (i = 0; i < nr_possible_counters; i++) {
/*
* TODO: Test a value that validates full-width writes and the
* width of the counters.
*/
- const uint64_t test_val = 0xffff;
- const uint32_t msr = base_msr + i;
+ const u64 test_val = 0xffff;
+ const u32 msr = base_msr + i;
/*
* Fixed counters are supported if the counter is less than the
@@ -418,12 +418,12 @@ static void guest_rd_wr_counters(uint32_t base_msr, uint8_t nr_possible_counters
* KVM drops writes to MSR_P6_PERFCTR[0|1] if the counters are
* unsupported, i.e. doesn't #GP and reads back '0'.
*/
- const uint64_t expected_val = expect_success ? test_val : 0;
+ const u64 expected_val = expect_success ? test_val : 0;
const bool expect_gp = !expect_success && msr != MSR_P6_PERFCTR0 &&
msr != MSR_P6_PERFCTR1;
- uint32_t rdpmc_idx;
- uint8_t vector;
- uint64_t val;
+ u32 rdpmc_idx;
+ u8 vector;
+ u64 val;
vector = wrmsr_safe(msr, test_val);
GUEST_ASSERT_PMC_MSR_ACCESS(WRMSR, msr, expect_gp, vector);
@@ -461,9 +461,9 @@ static void guest_rd_wr_counters(uint32_t base_msr, uint8_t nr_possible_counters
static void guest_test_gp_counters(void)
{
- uint8_t pmu_version = guest_get_pmu_version();
- uint8_t nr_gp_counters = 0;
- uint32_t base_msr;
+ u8 pmu_version = guest_get_pmu_version();
+ u8 nr_gp_counters = 0;
+ u32 base_msr;
if (pmu_version)
nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
@@ -477,7 +477,7 @@ static void guest_test_gp_counters(void)
* counters, of which there are none.
*/
if (pmu_version > 1) {
- uint64_t global_ctrl = rdmsr(MSR_CORE_PERF_GLOBAL_CTRL);
+ u64 global_ctrl = rdmsr(MSR_CORE_PERF_GLOBAL_CTRL);
if (nr_gp_counters)
GUEST_ASSERT_EQ(global_ctrl, GENMASK_ULL(nr_gp_counters - 1, 0));
@@ -495,8 +495,8 @@ static void guest_test_gp_counters(void)
GUEST_DONE();
}
-static void test_gp_counters(uint8_t pmu_version, uint64_t perf_capabilities,
- uint8_t nr_gp_counters)
+static void test_gp_counters(u8 pmu_version, u64 perf_capabilities,
+ u8 nr_gp_counters)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -514,9 +514,9 @@ static void test_gp_counters(uint8_t pmu_version, uint64_t perf_capabilities,
static void guest_test_fixed_counters(void)
{
- uint64_t supported_bitmask = 0;
- uint8_t nr_fixed_counters = 0;
- uint8_t i;
+ u64 supported_bitmask = 0;
+ u8 nr_fixed_counters = 0;
+ u8 i;
/* Fixed counters require Architectural vPMU Version 2+. */
if (guest_get_pmu_version() >= 2)
@@ -533,8 +533,8 @@ static void guest_test_fixed_counters(void)
nr_fixed_counters, supported_bitmask);
for (i = 0; i < MAX_NR_FIXED_COUNTERS; i++) {
- uint8_t vector;
- uint64_t val;
+ u8 vector;
+ u64 val;
if (i >= nr_fixed_counters && !(supported_bitmask & BIT_ULL(i))) {
vector = wrmsr_safe(MSR_CORE_PERF_FIXED_CTR_CTRL,
@@ -561,9 +561,8 @@ static void guest_test_fixed_counters(void)
GUEST_DONE();
}
-static void test_fixed_counters(uint8_t pmu_version, uint64_t perf_capabilities,
- uint8_t nr_fixed_counters,
- uint32_t supported_bitmask)
+static void test_fixed_counters(u8 pmu_version, u64 perf_capabilities,
+ u8 nr_fixed_counters, u32 supported_bitmask)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -583,14 +582,14 @@ static void test_fixed_counters(uint8_t pmu_version, uint64_t perf_capabilities,
static void test_intel_counters(void)
{
- uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
- uint8_t nr_gp_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
- uint8_t pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION);
+ u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
+ u8 nr_gp_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS);
+ u8 pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION);
unsigned int i;
- uint8_t v, j;
- uint32_t k;
+ u8 v, j;
+ u32 k;
- const uint64_t perf_caps[] = {
+ const u64 perf_caps[] = {
0,
PMU_CAP_FW_WRITES,
};
@@ -602,7 +601,7 @@ static void test_intel_counters(void)
* as alternating bit sequencues, e.g. to detect if KVM is checking the
* wrong bit(s).
*/
- const uint32_t unavailable_masks[] = {
+ const u32 unavailable_masks[] = {
0x0,
0xffffffffu,
0xaaaaaaaau,
@@ -620,7 +619,7 @@ static void test_intel_counters(void)
* Intel, i.e. is the last version that is guaranteed to be backwards
* compatible with KVM's existing behavior.
*/
- uint8_t max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5);
+ u8 max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5);
/*
* Detect the existence of events that aren't supported by selftests.
diff --git a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c
index 93b61c0..c123234 100644
--- a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c
+++ b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c
@@ -53,11 +53,11 @@ static const struct __kvm_pmu_event_filter base_event_filter = {
};
struct {
- uint64_t loads;
- uint64_t stores;
- uint64_t loads_stores;
- uint64_t branches_retired;
- uint64_t instructions_retired;
+ u64 loads;
+ u64 stores;
+ u64 loads_stores;
+ u64 branches_retired;
+ u64 instructions_retired;
} pmc_results;
/*
@@ -75,9 +75,9 @@ static void guest_gp_handler(struct ex_regs *regs)
*
* Return on success. GUEST_SYNC(0) on error.
*/
-static void check_msr(uint32_t msr, uint64_t bits_to_flip)
+static void check_msr(u32 msr, u64 bits_to_flip)
{
- uint64_t v = rdmsr(msr) ^ bits_to_flip;
+ u64 v = rdmsr(msr) ^ bits_to_flip;
wrmsr(msr, v);
if (rdmsr(msr) != v)
@@ -89,10 +89,10 @@ static void check_msr(uint32_t msr, uint64_t bits_to_flip)
GUEST_SYNC(-EIO);
}
-static void run_and_measure_loop(uint32_t msr_base)
+static void run_and_measure_loop(u32 msr_base)
{
- const uint64_t branches_retired = rdmsr(msr_base + 0);
- const uint64_t insn_retired = rdmsr(msr_base + 1);
+ const u64 branches_retired = rdmsr(msr_base + 0);
+ const u64 insn_retired = rdmsr(msr_base + 1);
__asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
@@ -147,7 +147,7 @@ static void amd_guest_code(void)
* Run the VM to the next GUEST_SYNC(value), and return the value passed
* to the sync. Any other exit from the guest is fatal.
*/
-static uint64_t run_vcpu_to_sync(struct kvm_vcpu *vcpu)
+static u64 run_vcpu_to_sync(struct kvm_vcpu *vcpu)
{
struct ucall uc;
@@ -161,7 +161,7 @@ static uint64_t run_vcpu_to_sync(struct kvm_vcpu *vcpu)
static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
{
- uint64_t r;
+ u64 r;
memset(&pmc_results, 0, sizeof(pmc_results));
sync_global_to_guest(vcpu->vm, pmc_results);
@@ -182,7 +182,7 @@ static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
*/
static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
{
- uint64_t r;
+ u64 r;
vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler);
r = run_vcpu_to_sync(vcpu);
@@ -195,7 +195,7 @@ static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
* Remove the first occurrence of 'event' (if any) from the filter's
* event list.
*/
-static void remove_event(struct __kvm_pmu_event_filter *f, uint64_t event)
+static void remove_event(struct __kvm_pmu_event_filter *f, u64 event)
{
bool found = false;
int i;
@@ -212,8 +212,8 @@ static void remove_event(struct __kvm_pmu_event_filter *f, uint64_t event)
#define ASSERT_PMC_COUNTING_INSTRUCTIONS() \
do { \
- uint64_t br = pmc_results.branches_retired; \
- uint64_t ir = pmc_results.instructions_retired; \
+ u64 br = pmc_results.branches_retired; \
+ u64 ir = pmc_results.instructions_retired; \
bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ? \
br >= NUM_BRANCHES : br == NUM_BRANCHES; \
\
@@ -228,8 +228,8 @@ do { \
#define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS() \
do { \
- uint64_t br = pmc_results.branches_retired; \
- uint64_t ir = pmc_results.instructions_retired; \
+ u64 br = pmc_results.branches_retired; \
+ u64 ir = pmc_results.instructions_retired; \
\
TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)", \
__func__, br); \
@@ -378,7 +378,7 @@ static bool use_amd_pmu(void)
static bool supports_event_mem_inst_retired(void)
{
- uint32_t eax, ebx, ecx, edx;
+ u32 eax, ebx, ecx, edx;
cpuid(1, &eax, &ebx, &ecx, &edx);
if (x86_family(eax) == 0x6) {
@@ -415,15 +415,15 @@ static bool supports_event_mem_inst_retired(void)
#define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \
KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true)
-static void masked_events_guest_test(uint32_t msr_base)
+static void masked_events_guest_test(u32 msr_base)
{
/*
* The actual value of the counters don't determine the outcome of
* the test. Only that they are zero or non-zero.
*/
- const uint64_t loads = rdmsr(msr_base + 0);
- const uint64_t stores = rdmsr(msr_base + 1);
- const uint64_t loads_stores = rdmsr(msr_base + 2);
+ const u64 loads = rdmsr(msr_base + 0);
+ const u64 stores = rdmsr(msr_base + 1);
+ const u64 loads_stores = rdmsr(msr_base + 2);
int val;
@@ -476,7 +476,7 @@ static void amd_masked_events_guest_code(void)
}
static void run_masked_events_test(struct kvm_vcpu *vcpu,
- const uint64_t masked_events[],
+ const u64 masked_events[],
const int nmasked_events)
{
struct __kvm_pmu_event_filter f = {
@@ -485,7 +485,7 @@ static void run_masked_events_test(struct kvm_vcpu *vcpu,
.flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
};
- memcpy(f.events, masked_events, sizeof(uint64_t) * nmasked_events);
+ memcpy(f.events, masked_events, sizeof(u64) * nmasked_events);
test_with_filter(vcpu, &f);
}
@@ -494,12 +494,12 @@ static void run_masked_events_test(struct kvm_vcpu *vcpu,
#define ALLOW_LOADS_STORES BIT(2)
struct masked_events_test {
- uint64_t intel_events[MAX_TEST_EVENTS];
- uint64_t intel_event_end;
- uint64_t amd_events[MAX_TEST_EVENTS];
- uint64_t amd_event_end;
+ u64 intel_events[MAX_TEST_EVENTS];
+ u64 intel_event_end;
+ u64 amd_events[MAX_TEST_EVENTS];
+ u64 amd_event_end;
const char *msg;
- uint32_t flags;
+ u32 flags;
};
/*
@@ -582,9 +582,9 @@ const struct masked_events_test test_cases[] = {
};
static int append_test_events(const struct masked_events_test *test,
- uint64_t *events, int nevents)
+ u64 *events, int nevents)
{
- const uint64_t *evts;
+ const u64 *evts;
int i;
evts = use_intel_pmu() ? test->intel_events : test->amd_events;
@@ -603,7 +603,7 @@ static bool bool_eq(bool a, bool b)
return a == b;
}
-static void run_masked_events_tests(struct kvm_vcpu *vcpu, uint64_t *events,
+static void run_masked_events_tests(struct kvm_vcpu *vcpu, u64 *events,
int nevents)
{
int ntests = ARRAY_SIZE(test_cases);
@@ -630,7 +630,7 @@ static void run_masked_events_tests(struct kvm_vcpu *vcpu, uint64_t *events,
}
}
-static void add_dummy_events(uint64_t *events, int nevents)
+static void add_dummy_events(u64 *events, int nevents)
{
int i;
@@ -650,7 +650,7 @@ static void add_dummy_events(uint64_t *events, int nevents)
static void test_masked_events(struct kvm_vcpu *vcpu)
{
int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS;
- uint64_t events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
+ u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
/* Run the test cases against a sparse PMU event filter. */
run_masked_events_tests(vcpu, events, 0);
@@ -668,8 +668,8 @@ static int set_pmu_event_filter(struct kvm_vcpu *vcpu,
return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
}
-static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, uint64_t event,
- uint32_t flags, uint32_t action)
+static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, u64 event,
+ u32 flags, u32 action)
{
struct __kvm_pmu_event_filter f = {
.nevents = 1,
@@ -685,9 +685,9 @@ static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, uint64_t event,
static void test_filter_ioctl(struct kvm_vcpu *vcpu)
{
- uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
+ u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
struct __kvm_pmu_event_filter f;
- uint64_t e = ~0ul;
+ u64 e = ~0ul;
int r;
/*
@@ -729,7 +729,7 @@ static void test_filter_ioctl(struct kvm_vcpu *vcpu)
TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed");
}
-static void intel_run_fixed_counter_guest_code(uint8_t idx)
+static void intel_run_fixed_counter_guest_code(u8 idx)
{
for (;;) {
wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
@@ -745,8 +745,8 @@ static void intel_run_fixed_counter_guest_code(uint8_t idx)
}
}
-static uint64_t test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
- uint32_t action, uint32_t bitmap)
+static u64 test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
+ u32 action, u32 bitmap)
{
struct __kvm_pmu_event_filter f = {
.action = action,
@@ -757,9 +757,9 @@ static uint64_t test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
return run_vcpu_to_sync(vcpu);
}
-static uint64_t test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
- uint32_t action,
- uint32_t bitmap)
+static u64 test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
+ u32 action,
+ u32 bitmap)
{
struct __kvm_pmu_event_filter f = base_event_filter;
@@ -770,12 +770,12 @@ static uint64_t test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
return run_vcpu_to_sync(vcpu);
}
-static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, uint8_t idx,
- uint8_t nr_fixed_counters)
+static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, u8 idx,
+ u8 nr_fixed_counters)
{
unsigned int i;
- uint32_t bitmap;
- uint64_t count;
+ u32 bitmap;
+ u64 count;
TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8,
"Invalid nr_fixed_counters");
@@ -815,10 +815,10 @@ static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, uint8_t idx,
static void test_fixed_counter_bitmap(void)
{
- uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
+ u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
- uint8_t idx;
+ u8 idx;
/*
* Check that pmu_event_filter works as expected when it's applied to
diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c
index 1969f4a..1d2f5d4 100644
--- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c
+++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c
@@ -23,13 +23,13 @@
#include <processor.h>
#define BASE_DATA_SLOT 10
-#define BASE_DATA_GPA ((uint64_t)(1ull << 32))
-#define PER_CPU_DATA_SIZE ((uint64_t)(SZ_2M + PAGE_SIZE))
+#define BASE_DATA_GPA ((u64)(1ull << 32))
+#define PER_CPU_DATA_SIZE ((u64)(SZ_2M + PAGE_SIZE))
/* Horrific macro so that the line info is captured accurately :-( */
#define memcmp_g(gpa, pattern, size) \
do { \
- uint8_t *mem = (uint8_t *)gpa; \
+ u8 *mem = (u8 *)gpa; \
size_t i; \
\
for (i = 0; i < size; i++) \
@@ -38,7 +38,7 @@ do { \
pattern, i, gpa + i, mem[i]); \
} while (0)
-static void memcmp_h(uint8_t *mem, uint64_t gpa, uint8_t pattern, size_t size)
+static void memcmp_h(u8 *mem, gpa_t gpa, u8 pattern, size_t size)
{
size_t i;
@@ -70,13 +70,13 @@ enum ucall_syncs {
SYNC_PRIVATE,
};
-static void guest_sync_shared(uint64_t gpa, uint64_t size,
- uint8_t current_pattern, uint8_t new_pattern)
+static void guest_sync_shared(gpa_t gpa, u64 size,
+ u8 current_pattern, u8 new_pattern)
{
GUEST_SYNC5(SYNC_SHARED, gpa, size, current_pattern, new_pattern);
}
-static void guest_sync_private(uint64_t gpa, uint64_t size, uint8_t pattern)
+static void guest_sync_private(gpa_t gpa, u64 size, u8 pattern)
{
GUEST_SYNC4(SYNC_PRIVATE, gpa, size, pattern);
}
@@ -86,10 +86,10 @@ static void guest_sync_private(uint64_t gpa, uint64_t size, uint8_t pattern)
#define MAP_GPA_SHARED BIT(1)
#define MAP_GPA_DO_FALLOCATE BIT(2)
-static void guest_map_mem(uint64_t gpa, uint64_t size, bool map_shared,
+static void guest_map_mem(gpa_t gpa, u64 size, bool map_shared,
bool do_fallocate)
{
- uint64_t flags = MAP_GPA_SET_ATTRIBUTES;
+ u64 flags = MAP_GPA_SET_ATTRIBUTES;
if (map_shared)
flags |= MAP_GPA_SHARED;
@@ -98,19 +98,19 @@ static void guest_map_mem(uint64_t gpa, uint64_t size, bool map_shared,
kvm_hypercall_map_gpa_range(gpa, size, flags);
}
-static void guest_map_shared(uint64_t gpa, uint64_t size, bool do_fallocate)
+static void guest_map_shared(gpa_t gpa, u64 size, bool do_fallocate)
{
guest_map_mem(gpa, size, true, do_fallocate);
}
-static void guest_map_private(uint64_t gpa, uint64_t size, bool do_fallocate)
+static void guest_map_private(gpa_t gpa, u64 size, bool do_fallocate)
{
guest_map_mem(gpa, size, false, do_fallocate);
}
struct {
- uint64_t offset;
- uint64_t size;
+ u64 offset;
+ u64 size;
} static const test_ranges[] = {
GUEST_STAGE(0, PAGE_SIZE),
GUEST_STAGE(0, SZ_2M),
@@ -119,11 +119,11 @@ struct {
GUEST_STAGE(SZ_2M, PAGE_SIZE),
};
-static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate)
+static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate)
{
- const uint8_t def_p = 0xaa;
- const uint8_t init_p = 0xcc;
- uint64_t j;
+ const u8 def_p = 0xaa;
+ const u8 init_p = 0xcc;
+ u64 j;
int i;
/* Memory should be shared by default. */
@@ -134,12 +134,12 @@ static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate)
memcmp_g(base_gpa, init_p, PER_CPU_DATA_SIZE);
for (i = 0; i < ARRAY_SIZE(test_ranges); i++) {
- uint64_t gpa = base_gpa + test_ranges[i].offset;
- uint64_t size = test_ranges[i].size;
- uint8_t p1 = 0x11;
- uint8_t p2 = 0x22;
- uint8_t p3 = 0x33;
- uint8_t p4 = 0x44;
+ gpa_t gpa = base_gpa + test_ranges[i].offset;
+ u64 size = test_ranges[i].size;
+ u8 p1 = 0x11;
+ u8 p2 = 0x22;
+ u8 p3 = 0x33;
+ u8 p4 = 0x44;
/*
* Set the test region to pattern one to differentiate it from
@@ -214,10 +214,10 @@ static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate)
}
}
-static void guest_punch_hole(uint64_t gpa, uint64_t size)
+static void guest_punch_hole(gpa_t gpa, u64 size)
{
/* "Mapping" memory shared via fallocate() is done via PUNCH_HOLE. */
- uint64_t flags = MAP_GPA_SHARED | MAP_GPA_DO_FALLOCATE;
+ u64 flags = MAP_GPA_SHARED | MAP_GPA_DO_FALLOCATE;
kvm_hypercall_map_gpa_range(gpa, size, flags);
}
@@ -227,9 +227,9 @@ static void guest_punch_hole(uint64_t gpa, uint64_t size)
* proper conversion. Freeing (PUNCH_HOLE) should zap SPTEs, and reallocating
* (subsequent fault) should zero memory.
*/
-static void guest_test_punch_hole(uint64_t base_gpa, bool precise)
+static void guest_test_punch_hole(u64 base_gpa, bool precise)
{
- const uint8_t init_p = 0xcc;
+ const u8 init_p = 0xcc;
int i;
/*
@@ -239,8 +239,8 @@ static void guest_test_punch_hole(uint64_t base_gpa, bool precise)
guest_map_private(base_gpa, PER_CPU_DATA_SIZE, false);
for (i = 0; i < ARRAY_SIZE(test_ranges); i++) {
- uint64_t gpa = base_gpa + test_ranges[i].offset;
- uint64_t size = test_ranges[i].size;
+ gpa_t gpa = base_gpa + test_ranges[i].offset;
+ u64 size = test_ranges[i].size;
/*
* Free all memory before each iteration, even for the !precise
@@ -268,7 +268,7 @@ static void guest_test_punch_hole(uint64_t base_gpa, bool precise)
}
}
-static void guest_code(uint64_t base_gpa)
+static void guest_code(u64 base_gpa)
{
/*
* Run the conversion test twice, with and without doing fallocate() on
@@ -289,8 +289,8 @@ static void guest_code(uint64_t base_gpa)
static void handle_exit_hypercall(struct kvm_vcpu *vcpu)
{
struct kvm_run *run = vcpu->run;
- uint64_t gpa = run->hypercall.args[0];
- uint64_t size = run->hypercall.args[1] * PAGE_SIZE;
+ gpa_t gpa = run->hypercall.args[0];
+ u64 size = run->hypercall.args[1] * PAGE_SIZE;
bool set_attributes = run->hypercall.args[2] & MAP_GPA_SET_ATTRIBUTES;
bool map_shared = run->hypercall.args[2] & MAP_GPA_SHARED;
bool do_fallocate = run->hypercall.args[2] & MAP_GPA_DO_FALLOCATE;
@@ -337,7 +337,7 @@ static void *__test_mem_conversions(void *__vcpu)
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
case UCALL_SYNC: {
- uint64_t gpa = uc.args[1];
+ gpa_t gpa = uc.args[1];
size_t size = uc.args[2];
size_t i;
@@ -347,7 +347,7 @@ static void *__test_mem_conversions(void *__vcpu)
for (i = 0; i < size; i += vm->page_size) {
size_t nr_bytes = min_t(size_t, vm->page_size, size - i);
- uint8_t *hva = addr_gpa2hva(vm, gpa + i);
+ u8 *hva = addr_gpa2hva(vm, gpa + i);
/* In all cases, the host should observe the shared data. */
memcmp_h(hva, gpa + i, uc.args[3], nr_bytes);
@@ -366,8 +366,8 @@ static void *__test_mem_conversions(void *__vcpu)
}
}
-static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t nr_vcpus,
- uint32_t nr_memslots)
+static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus,
+ u32 nr_memslots)
{
/*
* Allocate enough memory so that each vCPU's chunk of memory can be
@@ -402,7 +402,7 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t
KVM_MEM_GUEST_MEMFD, memfd, slot_size * i);
for (i = 0; i < nr_vcpus; i++) {
- uint64_t gpa = BASE_DATA_GPA + i * per_cpu_size;
+ gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size;
vcpu_args_set(vcpus[i], 1, gpa);
@@ -450,8 +450,8 @@ static void usage(const char *cmd)
int main(int argc, char *argv[])
{
enum vm_mem_backing_src_type src_type = DEFAULT_VM_MEM_SRC;
- uint32_t nr_memslots = 1;
- uint32_t nr_vcpus = 1;
+ u32 nr_memslots = 1;
+ u32 nr_vcpus = 1;
int opt;
TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));
diff --git a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c
index 13e72fc..10db9fe 100644
--- a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c
+++ b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c
@@ -17,17 +17,17 @@
#define EXITS_TEST_SIZE (EXITS_TEST_NPAGES * PAGE_SIZE)
#define EXITS_TEST_SLOT 10
-static uint64_t guest_repeatedly_read(void)
+static u64 guest_repeatedly_read(void)
{
- volatile uint64_t value;
+ volatile u64 value;
while (true)
- value = *((uint64_t *) EXITS_TEST_GVA);
+ value = *((u64 *)EXITS_TEST_GVA);
return value;
}
-static uint32_t run_vcpu_get_exit_reason(struct kvm_vcpu *vcpu)
+static u32 run_vcpu_get_exit_reason(struct kvm_vcpu *vcpu)
{
int r;
@@ -50,7 +50,7 @@ static void test_private_access_memslot_deleted(void)
struct kvm_vcpu *vcpu;
pthread_t vm_thread;
void *thread_return;
- uint32_t exit_reason;
+ u32 exit_reason;
vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu,
guest_repeatedly_read);
@@ -72,7 +72,7 @@ static void test_private_access_memslot_deleted(void)
vm_mem_region_delete(vm, EXITS_TEST_SLOT);
pthread_join(vm_thread, &thread_return);
- exit_reason = (uint32_t)(uint64_t)thread_return;
+ exit_reason = (u32)(u64)thread_return;
TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT);
TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE);
@@ -86,7 +86,7 @@ static void test_private_access_memslot_not_private(void)
{
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
- uint32_t exit_reason;
+ u32 exit_reason;
vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu,
guest_repeatedly_read);
diff --git a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c
index 49913784..8e38986 100644
--- a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c
+++ b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c
@@ -86,11 +86,11 @@ static void run_vcpu(struct kvm_vcpu *vcpu)
}
}
-static struct kvm_vm *create_vm(uint32_t nr_vcpus, uint32_t bsp_vcpu_id,
+static struct kvm_vm *create_vm(u32 nr_vcpus, u32 bsp_vcpu_id,
struct kvm_vcpu *vcpus[])
{
struct kvm_vm *vm;
- uint32_t i;
+ u32 i;
vm = vm_create(nr_vcpus);
@@ -104,7 +104,7 @@ static struct kvm_vm *create_vm(uint32_t nr_vcpus, uint32_t bsp_vcpu_id,
return vm;
}
-static void run_vm_bsp(uint32_t bsp_vcpu_id)
+static void run_vm_bsp(u32 bsp_vcpu_id)
{
struct kvm_vcpu *vcpus[2];
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c
index f4095a3..8e654cc 100644
--- a/tools/testing/selftests/kvm/x86/set_sregs_test.c
+++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c
@@ -46,9 +46,9 @@ do { \
X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE | \
X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT)
-static uint64_t calc_supported_cr4_feature_bits(void)
+static u64 calc_supported_cr4_feature_bits(void)
{
- uint64_t cr4 = KVM_ALWAYS_ALLOWED_CR4;
+ u64 cr4 = KVM_ALWAYS_ALLOWED_CR4;
if (kvm_cpu_has(X86_FEATURE_UMIP))
cr4 |= X86_CR4_UMIP;
@@ -74,7 +74,7 @@ static uint64_t calc_supported_cr4_feature_bits(void)
return cr4;
}
-static void test_cr_bits(struct kvm_vcpu *vcpu, uint64_t cr4)
+static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4)
{
struct kvm_sregs sregs;
int rc, i;
diff --git a/tools/testing/selftests/kvm/x86/sev_init2_tests.c b/tools/testing/selftests/kvm/x86/sev_init2_tests.c
index b238615..8eeba23 100644
--- a/tools/testing/selftests/kvm/x86/sev_init2_tests.c
+++ b/tools/testing/selftests/kvm/x86/sev_init2_tests.c
@@ -34,7 +34,7 @@ static int __sev_ioctl(int vm_fd, int cmd_id, void *data)
{
struct kvm_sev_cmd cmd = {
.id = cmd_id,
- .data = (uint64_t)data,
+ .data = (u64)data,
.sev_fd = open_sev_dev_path_or_exit(),
};
int ret;
@@ -94,7 +94,7 @@ void test_vm_types(void)
"VM type is KVM_X86_SW_PROTECTED_VM");
}
-void test_flags(uint32_t vm_type)
+void test_flags(u32 vm_type)
{
int i;
@@ -104,7 +104,7 @@ void test_flags(uint32_t vm_type)
"invalid flag");
}
-void test_features(uint32_t vm_type, uint64_t supported_features)
+void test_features(u32 vm_type, u64 supported_features)
{
int i;
diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c
index 8bd37a47..1a49ee3 100644
--- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c
+++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c
@@ -13,9 +13,9 @@
#include "linux/psp-sev.h"
#include "sev.h"
-static void guest_sev_test_msr(uint32_t msr)
+static void guest_sev_test_msr(u32 msr)
{
- uint64_t val = rdmsr(msr);
+ u64 val = rdmsr(msr);
wrmsr(msr, val);
GUEST_ASSERT(val == rdmsr(msr));
@@ -23,7 +23,7 @@ static void guest_sev_test_msr(uint32_t msr)
#define guest_sev_test_reg(reg) \
do { \
- uint64_t val = get_##reg(); \
+ u64 val = get_##reg(); \
\
set_##reg(val); \
GUEST_ASSERT(val == get_##reg()); \
@@ -42,7 +42,7 @@ static void guest_sev_test_regs(void)
static void guest_snp_code(void)
{
- uint64_t sev_msr = rdmsr(MSR_AMD64_SEV);
+ u64 sev_msr = rdmsr(MSR_AMD64_SEV);
GUEST_ASSERT(sev_msr & MSR_AMD64_SEV_ENABLED);
GUEST_ASSERT(sev_msr & MSR_AMD64_SEV_ES_ENABLED);
@@ -104,19 +104,19 @@ static void compare_xsave(u8 *from_host, u8 *from_guest)
abort();
}
-static void test_sync_vmsa(uint32_t type, uint64_t policy)
+static void test_sync_vmsa(u32 type, u64 policy)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_vaddr_t gva;
+ gva_t gva;
void *hva;
double x87val = M_PI;
struct kvm_xsave __attribute__((aligned(64))) xsave = { 0 };
vm = vm_sev_create_with_one_vcpu(type, guest_code_xsave, &vcpu);
- gva = vm_vaddr_alloc_shared(vm, PAGE_SIZE, KVM_UTIL_MIN_VADDR,
- MEM_REGION_TEST_DATA);
+ gva = vm_alloc_shared(vm, PAGE_SIZE, KVM_UTIL_MIN_VADDR,
+ MEM_REGION_TEST_DATA);
hva = addr_gva2hva(vm, gva);
vcpu_args_set(vcpu, 1, gva);
@@ -150,7 +150,7 @@ static void test_sync_vmsa(uint32_t type, uint64_t policy)
kvm_vm_free(vm);
}
-static void test_sev(void *guest_code, uint32_t type, uint64_t policy)
+static void test_sev(void *guest_code, u32 type, u64 policy)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -201,7 +201,7 @@ static void guest_shutdown_code(void)
__asm__ __volatile__("ud2");
}
-static void test_sev_shutdown(uint32_t type, uint64_t policy)
+static void test_sev_shutdown(u32 type, u64 policy)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
@@ -218,7 +218,7 @@ static void test_sev_shutdown(uint32_t type, uint64_t policy)
kvm_vm_free(vm);
}
-static void test_sev_smoke(void *guest, uint32_t type, uint64_t policy)
+static void test_sev_smoke(void *guest, u32 type, u64 policy)
{
const u64 xf_mask = XFEATURE_MASK_X87_AVX;
diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c
index 0e8aec5..3dca85e 100644
--- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c
+++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c
@@ -20,8 +20,8 @@
static void guest_code(bool tdp_enabled)
{
- uint64_t error_code;
- uint64_t vector;
+ u64 error_code;
+ u64 vector;
vector = kvm_asm_safe_ec(FLDS_MEM_EAX, error_code, "a"(MEM_REGION_GVA));
@@ -47,8 +47,8 @@ int main(int argc, char *argv[])
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct ucall uc;
- uint64_t *hva;
- uint64_t gpa;
+ u64 *hva;
+ gpa_t gpa;
int rc;
TEST_REQUIRE(kvm_has_cap(KVM_CAP_SMALLER_MAXPHYADDR));
diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c
index ade8412..7400511 100644
--- a/tools/testing/selftests/kvm/x86/smm_test.c
+++ b/tools/testing/selftests/kvm/x86/smm_test.c
@@ -34,13 +34,13 @@
* independent subset of asm here.
* SMI handler always report back fixed stage SMRAM_STAGE.
*/
-uint8_t smi_handler[] = {
+u8 smi_handler[] = {
0xb0, SMRAM_STAGE, /* mov $SMRAM_STAGE, %al */
0xe4, SYNC_PORT, /* in $SYNC_PORT, %al */
0x0f, 0xaa, /* rsm */
};
-static inline void sync_with_host(uint64_t phase)
+static inline void sync_with_host(u64 phase)
{
asm volatile("in $" XSTR(SYNC_PORT)", %%al \n"
: "+a" (phase));
@@ -65,7 +65,7 @@ static void guest_code(void *arg)
{
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
- uint64_t apicbase = rdmsr(MSR_IA32_APICBASE);
+ u64 apicbase = rdmsr(MSR_IA32_APICBASE);
struct svm_test_data *svm = arg;
struct vmx_pages *vmx_pages = arg;
@@ -113,7 +113,7 @@ static void guest_code(void *arg)
int main(int argc, char *argv[])
{
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
struct kvm_vcpu *vcpu;
struct kvm_regs regs;
diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c
index 992a525..409c6cc 100644
--- a/tools/testing/selftests/kvm/x86/state_test.c
+++ b/tools/testing/selftests/kvm/x86/state_test.c
@@ -144,8 +144,8 @@ static void __attribute__((__flatten__)) guest_code(void *arg)
GUEST_SYNC(1);
if (this_cpu_has(X86_FEATURE_XSAVE)) {
- uint64_t supported_xcr0 = this_cpu_supported_xcr0();
- uint8_t buffer[PAGE_SIZE];
+ u64 supported_xcr0 = this_cpu_supported_xcr0();
+ u8 buffer[PAGE_SIZE];
memset(buffer, 0xcc, sizeof(buffer));
@@ -172,8 +172,8 @@ static void __attribute__((__flatten__)) guest_code(void *arg)
}
if (this_cpu_has(X86_FEATURE_MPX)) {
- uint64_t bounds[2] = { 10, 0xffffffffull };
- uint64_t output[2] = { };
+ u64 bounds[2] = { 10, 0xffffffffull };
+ u64 output[2] = { };
GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_BNDREGS);
GUEST_ASSERT(supported_xcr0 & XFEATURE_MASK_BNDCSR);
@@ -257,8 +257,8 @@ void check_nested_state(int stage, struct kvm_x86_state *state)
int main(int argc, char *argv[])
{
- uint64_t *xstate_bv, saved_xstate_bv;
- vm_vaddr_t nested_gva = 0;
+ u64 *xstate_bv, saved_xstate_bv;
+ gva_t nested_gva = 0;
struct kvm_cpuid2 empty_cpuid = {};
struct kvm_regs regs1, regs2;
struct kvm_vcpu *vcpu, *vcpuN;
@@ -331,7 +331,7 @@ int main(int argc, char *argv[])
* supported features, even if something goes awry in saving
* the original snapshot.
*/
- xstate_bv = (void *)&((uint8_t *)state->xsave->region)[512];
+ xstate_bv = (void *)&((u8 *)state->xsave->region)[512];
saved_xstate_bv = *xstate_bv;
vcpuN = __vm_vcpu_add(vm, vcpu->id + 1);
diff --git a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
index 917b606..d3cc5e4 100644
--- a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
@@ -82,7 +82,7 @@ static void l1_guest_code(struct svm_test_data *svm)
int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
- vm_vaddr_t svm_gva;
+ gva_t svm_gva;
struct kvm_vm *vm;
struct ucall uc;
diff --git a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
index ff99438..7fbfaa0 100644
--- a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
+++ b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
@@ -97,9 +97,9 @@ void test_lbrv_nested_state(bool nested_lbrv)
{
struct kvm_x86_state *state = NULL;
struct kvm_vcpu *vcpu;
- vm_vaddr_t svm_gva;
struct kvm_vm *vm;
struct ucall uc;
+ gva_t svm_gva;
pr_info("Testing with nested LBRV %s\n", nested_lbrv ? "enabled" : "disabled");
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
index a521a9e..6a89eaf 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
@@ -38,7 +38,7 @@ int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM));
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
index 00135cb..c6ea3d6 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
@@ -42,7 +42,7 @@ static void l1_guest_code(struct svm_test_data *svm, struct idt_entry *idt)
int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
- vm_vaddr_t svm_gva;
+ gva_t svm_gva;
struct kvm_vm *vm;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM));
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
index 4bd1655..f72f11d 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
@@ -76,7 +76,7 @@ static void l2_guest_code_nmi(void)
ud2();
}
-static void l1_guest_code(struct svm_test_data *svm, uint64_t is_nmi, uint64_t idt_alt)
+static void l1_guest_code(struct svm_test_data *svm, u64 is_nmi, u64 idt_alt)
{
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
@@ -144,8 +144,8 @@ static void run_test(bool is_nmi)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- vm_vaddr_t svm_gva;
- vm_vaddr_t idt_alt_vm;
+ gva_t svm_gva;
+ gva_t idt_alt_vm;
struct kvm_guest_debug debug;
pr_info("Running %s test\n", is_nmi ? "NMI" : "soft int");
@@ -161,14 +161,14 @@ static void run_test(bool is_nmi)
if (!is_nmi) {
void *idt, *idt_alt;
- idt_alt_vm = vm_vaddr_alloc_page(vm);
+ idt_alt_vm = vm_alloc_page(vm);
idt_alt = addr_gva2hva(vm, idt_alt_vm);
idt = addr_gva2hva(vm, vm->arch.idt);
memcpy(idt_alt, idt, getpagesize());
} else {
idt_alt_vm = 0;
}
- vcpu_args_set(vcpu, 3, svm_gva, (uint64_t)is_nmi, (uint64_t)idt_alt_vm);
+ vcpu_args_set(vcpu, 3, svm_gva, (u64)is_nmi, (u64)idt_alt_vm);
memset(&debug, 0, sizeof(debug));
vcpu_guest_debug_set(vcpu, &debug);
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
index 569869b..a4935ce 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
@@ -28,28 +28,28 @@ static void l2_code(void)
vmcall();
}
-static void l1_vmrun(struct svm_test_data *svm, u64 gpa)
+static void l1_vmrun(struct svm_test_data *svm, gpa_t gpa)
{
generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
asm volatile ("vmrun %[gpa]" : : [gpa] "a" (gpa) : "memory");
}
-static void l1_vmload(struct svm_test_data *svm, u64 gpa)
+static void l1_vmload(struct svm_test_data *svm, gpa_t gpa)
{
generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
asm volatile ("vmload %[gpa]" : : [gpa] "a" (gpa) : "memory");
}
-static void l1_vmsave(struct svm_test_data *svm, u64 gpa)
+static void l1_vmsave(struct svm_test_data *svm, gpa_t gpa)
{
generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
asm volatile ("vmsave %[gpa]" : : [gpa] "a" (gpa) : "memory");
}
-static void l1_vmexit(struct svm_test_data *svm, u64 gpa)
+static void l1_vmexit(struct svm_test_data *svm, gpa_t gpa)
{
generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
@@ -74,7 +74,7 @@ static u64 unmappable_gpa(struct kvm_vcpu *vcpu)
static void test_invalid_vmcb12(struct kvm_vcpu *vcpu)
{
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
struct ucall uc;
@@ -90,7 +90,7 @@ static void test_invalid_vmcb12(struct kvm_vcpu *vcpu)
static void test_unmappable_vmcb12(struct kvm_vcpu *vcpu)
{
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
vcpu_alloc_svm(vcpu->vm, &nested_gva);
vcpu_args_set(vcpu, 2, nested_gva, unmappable_gpa(vcpu));
@@ -103,7 +103,7 @@ static void test_unmappable_vmcb12(struct kvm_vcpu *vcpu)
static void test_unmappable_vmcb12_vmexit(struct kvm_vcpu *vcpu)
{
struct kvm_x86_state *state;
- vm_vaddr_t nested_gva = 0;
+ gva_t nested_gva = 0;
struct ucall uc;
/*
diff --git a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
index 8a62cca..b188724 100644
--- a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
@@ -36,7 +36,7 @@ static void l1_guest_code(struct svm_test_data *svm)
int main(int argc, char *argv[])
{
struct kvm_vcpu *vcpu;
- vm_vaddr_t svm_gva;
+ gva_t svm_gva;
struct kvm_vm *vm;
TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM));
diff --git a/tools/testing/selftests/kvm/x86/sync_regs_test.c b/tools/testing/selftests/kvm/x86/sync_regs_test.c
index 8fa3948..e0c5232 100644
--- a/tools/testing/selftests/kvm/x86/sync_regs_test.c
+++ b/tools/testing/selftests/kvm/x86/sync_regs_test.c
@@ -20,7 +20,7 @@
#include "kvm_util.h"
#include "processor.h"
-#define UCALL_PIO_PORT ((uint16_t)0x1000)
+#define UCALL_PIO_PORT ((u16)0x1000)
struct ucall uc_none = {
.cmd = UCALL_NONE,
diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
index 56306a1..f1c488e 100644
--- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
+++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
@@ -72,13 +72,13 @@ int main(void)
if (has_vmx) {
- vm_vaddr_t vmx_pages_gva;
+ gva_t vmx_pages_gva;
vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code_vmx);
vcpu_alloc_vmx(vm, &vmx_pages_gva);
vcpu_args_set(vcpu, 1, vmx_pages_gva);
} else {
- vm_vaddr_t svm_gva;
+ gva_t svm_gva;
vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code_svm);
vcpu_alloc_svm(vm, &svm_gva);
diff --git a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c
index 12b0964..9158396 100644
--- a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c
+++ b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c
@@ -95,7 +95,7 @@ int main(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
- uint64_t val;
+ u64 val;
ksft_print_header();
ksft_set_plan(5);
diff --git a/tools/testing/selftests/kvm/x86/tsc_scaling_sync.c b/tools/testing/selftests/kvm/x86/tsc_scaling_sync.c
index 59c7304..59da8d4 100644
--- a/tools/testing/selftests/kvm/x86/tsc_scaling_sync.c
+++ b/tools/testing/selftests/kvm/x86/tsc_scaling_sync.c
@@ -21,10 +21,10 @@ pthread_spinlock_t create_lock;
#define TEST_TSC_KHZ 2345678UL
#define TEST_TSC_OFFSET 200000000
-uint64_t tsc_sync;
+u64 tsc_sync;
static void guest_code(void)
{
- uint64_t start_tsc, local_tsc, tmp;
+ u64 start_tsc, local_tsc, tmp;
start_tsc = rdtsc();
do {
diff --git a/tools/testing/selftests/kvm/x86/ucna_injection_test.c b/tools/testing/selftests/kvm/x86/ucna_injection_test.c
index 1e5e564..df1ec82 100644
--- a/tools/testing/selftests/kvm/x86/ucna_injection_test.c
+++ b/tools/testing/selftests/kvm/x86/ucna_injection_test.c
@@ -45,7 +45,7 @@
#define MCI_CTL2_RESERVED_BIT BIT_ULL(29)
-static uint64_t supported_mcg_caps;
+static u64 supported_mcg_caps;
/*
* Record states about the injected UCNA.
@@ -53,30 +53,30 @@ static uint64_t supported_mcg_caps;
* handler. Variables without the 'i_' prefixes are recorded in guest main
* execution thread.
*/
-static volatile uint64_t i_ucna_rcvd;
-static volatile uint64_t i_ucna_addr;
-static volatile uint64_t ucna_addr;
-static volatile uint64_t ucna_addr2;
+static volatile u64 i_ucna_rcvd;
+static volatile u64 i_ucna_addr;
+static volatile u64 ucna_addr;
+static volatile u64 ucna_addr2;
struct thread_params {
struct kvm_vcpu *vcpu;
- uint64_t *p_i_ucna_rcvd;
- uint64_t *p_i_ucna_addr;
- uint64_t *p_ucna_addr;
- uint64_t *p_ucna_addr2;
+ u64 *p_i_ucna_rcvd;
+ u64 *p_i_ucna_addr;
+ u64 *p_ucna_addr;
+ u64 *p_ucna_addr2;
};
static void verify_apic_base_addr(void)
{
- uint64_t msr = rdmsr(MSR_IA32_APICBASE);
- uint64_t base = GET_APIC_BASE(msr);
+ u64 msr = rdmsr(MSR_IA32_APICBASE);
+ u64 base = GET_APIC_BASE(msr);
GUEST_ASSERT(base == APIC_DEFAULT_GPA);
}
static void ucna_injection_guest_code(void)
{
- uint64_t ctl2;
+ u64 ctl2;
verify_apic_base_addr();
xapic_enable();
@@ -106,7 +106,7 @@ static void ucna_injection_guest_code(void)
static void cmci_disabled_guest_code(void)
{
- uint64_t ctl2 = rdmsr(MSR_IA32_MCx_CTL2(UCNA_BANK));
+ u64 ctl2 = rdmsr(MSR_IA32_MCx_CTL2(UCNA_BANK));
wrmsr(MSR_IA32_MCx_CTL2(UCNA_BANK), ctl2 | MCI_CTL2_CMCI_EN);
GUEST_DONE();
@@ -114,7 +114,7 @@ static void cmci_disabled_guest_code(void)
static void cmci_enabled_guest_code(void)
{
- uint64_t ctl2 = rdmsr(MSR_IA32_MCx_CTL2(UCNA_BANK));
+ u64 ctl2 = rdmsr(MSR_IA32_MCx_CTL2(UCNA_BANK));
wrmsr(MSR_IA32_MCx_CTL2(UCNA_BANK), ctl2 | MCI_CTL2_RESERVED_BIT);
GUEST_DONE();
@@ -145,14 +145,15 @@ static void run_vcpu_expect_gp(struct kvm_vcpu *vcpu)
printf("vCPU received GP in guest.\n");
}
-static void inject_ucna(struct kvm_vcpu *vcpu, uint64_t addr) {
+static void inject_ucna(struct kvm_vcpu *vcpu, u64 addr)
+{
/*
* A UCNA error is indicated with VAL=1, UC=1, PCC=0, S=0 and AR=0 in
* the IA32_MCi_STATUS register.
* MSCOD=1 (BIT[16] - MscodDataRdErr).
* MCACOD=0x0090 (Memory controller error format, channel 0)
*/
- uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
+ u64 status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
MCI_STATUS_MISCV | MCI_STATUS_ADDRV | 0x10090;
struct kvm_x86_mce mce = {};
mce.status = status;
@@ -216,10 +217,10 @@ static void test_ucna_injection(struct kvm_vcpu *vcpu, struct thread_params *par
{
struct kvm_vm *vm = vcpu->vm;
params->vcpu = vcpu;
- params->p_i_ucna_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)&i_ucna_rcvd);
- params->p_i_ucna_addr = (uint64_t *)addr_gva2hva(vm, (uint64_t)&i_ucna_addr);
- params->p_ucna_addr = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ucna_addr);
- params->p_ucna_addr2 = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ucna_addr2);
+ params->p_i_ucna_rcvd = (u64 *)addr_gva2hva(vm, (u64)&i_ucna_rcvd);
+ params->p_i_ucna_addr = (u64 *)addr_gva2hva(vm, (u64)&i_ucna_addr);
+ params->p_ucna_addr = (u64 *)addr_gva2hva(vm, (u64)&ucna_addr);
+ params->p_ucna_addr2 = (u64 *)addr_gva2hva(vm, (u64)&ucna_addr2);
run_ucna_injection(params);
@@ -242,7 +243,7 @@ static void test_ucna_injection(struct kvm_vcpu *vcpu, struct thread_params *par
static void setup_mce_cap(struct kvm_vcpu *vcpu, bool enable_cmci_p)
{
- uint64_t mcg_caps = MCG_CTL_P | MCG_SER_P | MCG_LMCE_P | KVM_MAX_MCE_BANKS;
+ u64 mcg_caps = MCG_CTL_P | MCG_SER_P | MCG_LMCE_P | KVM_MAX_MCE_BANKS;
if (enable_cmci_p)
mcg_caps |= MCG_CMCI_P;
@@ -250,7 +251,7 @@ static void setup_mce_cap(struct kvm_vcpu *vcpu, bool enable_cmci_p)
vcpu_ioctl(vcpu, KVM_X86_SETUP_MCE, &mcg_caps);
}
-static struct kvm_vcpu *create_vcpu_with_mce_cap(struct kvm_vm *vm, uint32_t vcpuid,
+static struct kvm_vcpu *create_vcpu_with_mce_cap(struct kvm_vm *vm, u32 vcpuid,
bool enable_cmci_p, void *guest_code)
{
struct kvm_vcpu *vcpu = vm_vcpu_add(vm, vcpuid, guest_code);
diff --git a/tools/testing/selftests/kvm/x86/userspace_io_test.c b/tools/testing/selftests/kvm/x86/userspace_io_test.c
index be7d72f..9c5a875 100644
--- a/tools/testing/selftests/kvm/x86/userspace_io_test.c
+++ b/tools/testing/selftests/kvm/x86/userspace_io_test.c
@@ -10,7 +10,7 @@
#include "kvm_util.h"
#include "processor.h"
-static void guest_ins_port80(uint8_t *buffer, unsigned int count)
+static void guest_ins_port80(u8 *buffer, unsigned int count)
{
unsigned long end;
@@ -26,7 +26,7 @@ static void guest_ins_port80(uint8_t *buffer, unsigned int count)
static void guest_code(void)
{
- uint8_t buffer[8192];
+ u8 buffer[8192];
int i;
/*
diff --git a/tools/testing/selftests/kvm/x86/userspace_msr_exit_test.c b/tools/testing/selftests/kvm/x86/userspace_msr_exit_test.c
index 8463a99..2808ce7 100644
--- a/tools/testing/selftests/kvm/x86/userspace_msr_exit_test.c
+++ b/tools/testing/selftests/kvm/x86/userspace_msr_exit_test.c
@@ -23,21 +23,21 @@ struct kvm_msr_filter filter_allow = {
.nmsrs = 1,
/* Test an MSR the kernel knows about. */
.base = MSR_IA32_XSS,
- .bitmap = (uint8_t*)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
}, {
.flags = KVM_MSR_FILTER_READ |
KVM_MSR_FILTER_WRITE,
.nmsrs = 1,
/* Test an MSR the kernel doesn't know about. */
.base = MSR_IA32_FLUSH_CMD,
- .bitmap = (uint8_t*)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
}, {
.flags = KVM_MSR_FILTER_READ |
KVM_MSR_FILTER_WRITE,
.nmsrs = 1,
/* Test a fabricated MSR that no one knows about. */
.base = MSR_NON_EXISTENT,
- .bitmap = (uint8_t*)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
},
},
};
@@ -49,7 +49,7 @@ struct kvm_msr_filter filter_fs = {
.flags = KVM_MSR_FILTER_READ,
.nmsrs = 1,
.base = MSR_FS_BASE,
- .bitmap = (uint8_t*)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
},
},
};
@@ -61,12 +61,12 @@ struct kvm_msr_filter filter_gs = {
.flags = KVM_MSR_FILTER_READ,
.nmsrs = 1,
.base = MSR_GS_BASE,
- .bitmap = (uint8_t*)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
},
},
};
-static uint64_t msr_non_existent_data;
+static u64 msr_non_existent_data;
static int guest_exception_count;
static u32 msr_reads, msr_writes;
@@ -77,7 +77,7 @@ static u8 bitmap_c0000000[KVM_MSR_FILTER_MAX_BITMAP_SIZE];
static u8 bitmap_c0000000_read[KVM_MSR_FILTER_MAX_BITMAP_SIZE];
static u8 bitmap_deadbeef[1] = { 0x1 };
-static void deny_msr(uint8_t *bitmap, u32 msr)
+static void deny_msr(u8 *bitmap, u32 msr)
{
u32 idx = msr & (KVM_MSR_FILTER_MAX_BITMAP_SIZE - 1);
@@ -142,26 +142,26 @@ struct kvm_msr_filter no_filter_deny = {
* Note: Force test_rdmsr() to not be inlined to prevent the labels,
* rdmsr_start and rdmsr_end, from being defined multiple times.
*/
-static noinline uint64_t test_rdmsr(uint32_t msr)
+static noinline u64 test_rdmsr(u32 msr)
{
- uint32_t a, d;
+ u32 a, d;
guest_exception_count = 0;
__asm__ __volatile__("rdmsr_start: rdmsr; rdmsr_end:" :
"=a"(a), "=d"(d) : "c"(msr) : "memory");
- return a | ((uint64_t) d << 32);
+ return a | ((u64)d << 32);
}
/*
* Note: Force test_wrmsr() to not be inlined to prevent the labels,
* wrmsr_start and wrmsr_end, from being defined multiple times.
*/
-static noinline void test_wrmsr(uint32_t msr, uint64_t value)
+static noinline void test_wrmsr(u32 msr, u64 value)
{
- uint32_t a = value;
- uint32_t d = value >> 32;
+ u32 a = value;
+ u32 d = value >> 32;
guest_exception_count = 0;
@@ -176,26 +176,26 @@ extern char wrmsr_start, wrmsr_end;
* Note: Force test_em_rdmsr() to not be inlined to prevent the labels,
* rdmsr_start and rdmsr_end, from being defined multiple times.
*/
-static noinline uint64_t test_em_rdmsr(uint32_t msr)
+static noinline u64 test_em_rdmsr(u32 msr)
{
- uint32_t a, d;
+ u32 a, d;
guest_exception_count = 0;
__asm__ __volatile__(KVM_FEP "em_rdmsr_start: rdmsr; em_rdmsr_end:" :
"=a"(a), "=d"(d) : "c"(msr) : "memory");
- return a | ((uint64_t) d << 32);
+ return a | ((u64)d << 32);
}
/*
* Note: Force test_em_wrmsr() to not be inlined to prevent the labels,
* wrmsr_start and wrmsr_end, from being defined multiple times.
*/
-static noinline void test_em_wrmsr(uint32_t msr, uint64_t value)
+static noinline void test_em_wrmsr(u32 msr, u64 value)
{
- uint32_t a = value;
- uint32_t d = value >> 32;
+ u32 a = value;
+ u32 d = value >> 32;
guest_exception_count = 0;
@@ -208,7 +208,7 @@ extern char em_wrmsr_start, em_wrmsr_end;
static void guest_code_filter_allow(void)
{
- uint64_t data;
+ u64 data;
/*
* Test userspace intercepting rdmsr / wrmsr for MSR_IA32_XSS.
@@ -328,7 +328,7 @@ static void guest_code_filter_deny(void)
static void guest_code_permission_bitmap(void)
{
- uint64_t data;
+ u64 data;
data = test_rdmsr(MSR_FS_BASE);
GUEST_ASSERT(data == MSR_FS_BASE);
@@ -391,7 +391,7 @@ static void check_for_guest_assert(struct kvm_vcpu *vcpu)
}
}
-static void process_rdmsr(struct kvm_vcpu *vcpu, uint32_t msr_index)
+static void process_rdmsr(struct kvm_vcpu *vcpu, u32 msr_index)
{
struct kvm_run *run = vcpu->run;
@@ -423,7 +423,7 @@ static void process_rdmsr(struct kvm_vcpu *vcpu, uint32_t msr_index)
}
}
-static void process_wrmsr(struct kvm_vcpu *vcpu, uint32_t msr_index)
+static void process_wrmsr(struct kvm_vcpu *vcpu, u32 msr_index)
{
struct kvm_run *run = vcpu->run;
@@ -464,7 +464,7 @@ static void process_ucall_done(struct kvm_vcpu *vcpu)
uc.cmd, UCALL_DONE);
}
-static uint64_t process_ucall(struct kvm_vcpu *vcpu)
+static u64 process_ucall(struct kvm_vcpu *vcpu)
{
struct ucall uc = {};
@@ -489,20 +489,20 @@ static uint64_t process_ucall(struct kvm_vcpu *vcpu)
}
static void run_guest_then_process_rdmsr(struct kvm_vcpu *vcpu,
- uint32_t msr_index)
+ u32 msr_index)
{
vcpu_run(vcpu);
process_rdmsr(vcpu, msr_index);
}
static void run_guest_then_process_wrmsr(struct kvm_vcpu *vcpu,
- uint32_t msr_index)
+ u32 msr_index)
{
vcpu_run(vcpu);
process_wrmsr(vcpu, msr_index);
}
-static uint64_t run_guest_then_process_ucall(struct kvm_vcpu *vcpu)
+static u64 run_guest_then_process_ucall(struct kvm_vcpu *vcpu)
{
vcpu_run(vcpu);
return process_ucall(vcpu);
@@ -519,7 +519,7 @@ KVM_ONE_VCPU_TEST_SUITE(user_msr);
KVM_ONE_VCPU_TEST(user_msr, msr_filter_allow, guest_code_filter_allow)
{
struct kvm_vm *vm = vcpu->vm;
- uint64_t cmd;
+ u64 cmd;
int rc;
rc = kvm_check_cap(KVM_CAP_X86_USER_SPACE_MSR);
@@ -732,7 +732,7 @@ static void run_msr_filter_flag_test(struct kvm_vm *vm)
.flags = KVM_MSR_FILTER_READ,
.nmsrs = 1,
.base = 0,
- .bitmap = (uint8_t *)&deny_bits,
+ .bitmap = (u8 *)&deny_bits,
},
},
};
diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
index a81a247..1720113 100644
--- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
@@ -38,7 +38,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa)
{
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
- uint32_t control;
+ u32 control;
GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
GUEST_ASSERT(load_vmcs(vmx_pages));
@@ -72,7 +72,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa)
int main(int argc, char *argv[])
{
unsigned long apic_access_addr = ~0ul;
- vm_vaddr_t vmx_pages_gva;
+ gva_t vmx_pages_gva;
unsigned long high_gpa;
struct vmx_pages *vmx;
bool done = false;
diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
index 337c53f..80a4fd1 100644
--- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
@@ -33,7 +33,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
{
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
- uint32_t control;
+ u32 control;
GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
GUEST_ASSERT(load_vmcs(vmx_pages));
@@ -110,7 +110,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva;
+ gva_t vmx_pages_gva;
struct vmx_pages *vmx;
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
index a100ee5..a2eacee 100644
--- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
+++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
@@ -52,7 +52,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva;
+ gva_t vmx_pages_gva;
struct kvm_sregs sregs;
struct kvm_vcpu *vcpu;
struct kvm_run *run;
diff --git a/tools/testing/selftests/kvm/x86/vmx_msrs_test.c b/tools/testing/selftests/kvm/x86/vmx_msrs_test.c
index 90720b6..c1e8632 100644
--- a/tools/testing/selftests/kvm/x86/vmx_msrs_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_msrs_test.c
@@ -12,11 +12,10 @@
#include "kvm_util.h"
#include "vmx.h"
-static void vmx_fixed1_msr_test(struct kvm_vcpu *vcpu, uint32_t msr_index,
- uint64_t mask)
+static void vmx_fixed1_msr_test(struct kvm_vcpu *vcpu, u32 msr_index, u64 mask)
{
- uint64_t val = vcpu_get_msr(vcpu, msr_index);
- uint64_t bit;
+ u64 val = vcpu_get_msr(vcpu, msr_index);
+ u64 bit;
mask &= val;
@@ -26,11 +25,10 @@ static void vmx_fixed1_msr_test(struct kvm_vcpu *vcpu, uint32_t msr_index,
}
}
-static void vmx_fixed0_msr_test(struct kvm_vcpu *vcpu, uint32_t msr_index,
- uint64_t mask)
+static void vmx_fixed0_msr_test(struct kvm_vcpu *vcpu, u32 msr_index, u64 mask)
{
- uint64_t val = vcpu_get_msr(vcpu, msr_index);
- uint64_t bit;
+ u64 val = vcpu_get_msr(vcpu, msr_index);
+ u64 bit;
mask = ~mask | val;
@@ -40,7 +38,7 @@ static void vmx_fixed0_msr_test(struct kvm_vcpu *vcpu, uint32_t msr_index,
}
}
-static void vmx_fixed0and1_msr_test(struct kvm_vcpu *vcpu, uint32_t msr_index)
+static void vmx_fixed0and1_msr_test(struct kvm_vcpu *vcpu, u32 msr_index)
{
vmx_fixed0_msr_test(vcpu, msr_index, GENMASK_ULL(31, 0));
vmx_fixed1_msr_test(vcpu, msr_index, GENMASK_ULL(63, 32));
@@ -68,10 +66,10 @@ static void vmx_save_restore_msrs_test(struct kvm_vcpu *vcpu)
}
static void __ia32_feature_control_msr_test(struct kvm_vcpu *vcpu,
- uint64_t msr_bit,
+ u64 msr_bit,
struct kvm_x86_cpu_feature feature)
{
- uint64_t val;
+ u64 val;
vcpu_clear_cpuid_feature(vcpu, feature);
@@ -90,7 +88,7 @@ static void __ia32_feature_control_msr_test(struct kvm_vcpu *vcpu,
static void ia32_feature_control_msr_test(struct kvm_vcpu *vcpu)
{
- uint64_t supported_bits = FEAT_CTL_LOCKED |
+ u64 supported_bits = FEAT_CTL_LOCKED |
FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX |
FEAT_CTL_SGX_LC_ENABLED |
diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
index 915c4200..f13dee3 100644
--- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
@@ -30,7 +30,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
#define L2_GUEST_STACK_SIZE 64
unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
u64 guest_cr4;
- vm_paddr_t pml5_pa, pml4_pa;
+ gpa_t pml5_pa, pml4_pa;
u64 *pml5;
u64 exit_reason;
@@ -73,7 +73,7 @@ void guest_code(struct vmx_pages *vmx_pages)
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva = 0;
+ gva_t vmx_pages_gva = 0;
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
struct kvm_x86_state *state;
diff --git a/tools/testing/selftests/kvm/x86/vmx_pmu_caps_test.c b/tools/testing/selftests/kvm/x86/vmx_pmu_caps_test.c
index 7ff6f62..d004108 100644
--- a/tools/testing/selftests/kvm/x86/vmx_pmu_caps_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_pmu_caps_test.c
@@ -52,16 +52,16 @@ static const union perf_capabilities format_caps = {
.pebs_format = -1,
};
-static void guest_test_perf_capabilities_gp(uint64_t val)
+static void guest_test_perf_capabilities_gp(u64 val)
{
- uint8_t vector = wrmsr_safe(MSR_IA32_PERF_CAPABILITIES, val);
+ u8 vector = wrmsr_safe(MSR_IA32_PERF_CAPABILITIES, val);
__GUEST_ASSERT(vector == GP_VECTOR,
"Expected #GP for value '0x%lx', got %s",
val, ex_str(vector));
}
-static void guest_code(uint64_t current_val)
+static void guest_code(u64 current_val)
{
int i;
@@ -129,7 +129,7 @@ KVM_ONE_VCPU_TEST(vmx_pmu_caps, basic_perf_capabilities, guest_code)
KVM_ONE_VCPU_TEST(vmx_pmu_caps, fungible_perf_capabilities, guest_code)
{
- const uint64_t fungible_caps = host_cap.capabilities & ~immutable_caps.capabilities;
+ const u64 fungible_caps = host_cap.capabilities & ~immutable_caps.capabilities;
int bit;
for_each_set_bit(bit, &fungible_caps, 64) {
@@ -148,7 +148,7 @@ KVM_ONE_VCPU_TEST(vmx_pmu_caps, fungible_perf_capabilities, guest_code)
*/
KVM_ONE_VCPU_TEST(vmx_pmu_caps, immutable_perf_capabilities, guest_code)
{
- const uint64_t reserved_caps = (~host_cap.capabilities |
+ const u64 reserved_caps = (~host_cap.capabilities |
immutable_caps.capabilities) &
~format_caps.capabilities;
union perf_capabilities val = host_cap;
@@ -210,7 +210,7 @@ KVM_ONE_VCPU_TEST(vmx_pmu_caps, lbr_perf_capabilities, guest_code)
KVM_ONE_VCPU_TEST(vmx_pmu_caps, perf_capabilities_unsupported, guest_code)
{
- uint64_t val;
+ u64 val;
int i, r;
vcpu_set_msr(vcpu, MSR_IA32_PERF_CAPABILITIES, host_cap.capabilities);
diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
index 00dd2ac..1b7b6ba 100644
--- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
@@ -152,7 +152,7 @@ void guest_code(struct vmx_pages *vmx_pages)
int main(int argc, char *argv[])
{
- vm_vaddr_t vmx_pages_gva = 0;
+ gva_t vmx_pages_gva = 0;
struct kvm_regs regs1, regs2;
struct kvm_vm *vm;
diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
index ae4a4b6..39ce9a9 100644
--- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
+++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
@@ -48,20 +48,20 @@
* Incremented in the IPI handler. Provides evidence to the sender that the IPI
* arrived at the destination
*/
-static volatile uint64_t ipis_rcvd;
+static volatile u64 ipis_rcvd;
/* Data struct shared between host main thread and vCPUs */
struct test_data_page {
- uint32_t halter_apic_id;
- volatile uint64_t hlt_count;
- volatile uint64_t wake_count;
- uint64_t ipis_sent;
- uint64_t migrations_attempted;
- uint64_t migrations_completed;
- uint32_t icr;
- uint32_t icr2;
- uint32_t halter_tpr;
- uint32_t halter_ppr;
+ u32 halter_apic_id;
+ volatile u64 hlt_count;
+ volatile u64 wake_count;
+ u64 ipis_sent;
+ u64 migrations_attempted;
+ u64 migrations_completed;
+ u32 icr;
+ u32 icr2;
+ u32 halter_tpr;
+ u32 halter_ppr;
/*
* Record local version register as a cross-check that APIC access
@@ -69,19 +69,19 @@ struct test_data_page {
* arch/x86/kvm/lapic.c). If test is failing, check that values match
* to determine whether APIC access exits are working.
*/
- uint32_t halter_lvr;
+ u32 halter_lvr;
};
struct thread_params {
struct test_data_page *data;
struct kvm_vcpu *vcpu;
- uint64_t *pipis_rcvd; /* host address of ipis_rcvd global */
+ u64 *pipis_rcvd; /* host address of ipis_rcvd global */
};
void verify_apic_base_addr(void)
{
- uint64_t msr = rdmsr(MSR_IA32_APICBASE);
- uint64_t base = GET_APIC_BASE(msr);
+ u64 msr = rdmsr(MSR_IA32_APICBASE);
+ u64 base = GET_APIC_BASE(msr);
GUEST_ASSERT(base == APIC_DEFAULT_GPA);
}
@@ -125,12 +125,12 @@ static void guest_ipi_handler(struct ex_regs *regs)
static void sender_guest_code(struct test_data_page *data)
{
- uint64_t last_wake_count;
- uint64_t last_hlt_count;
- uint64_t last_ipis_rcvd_count;
- uint32_t icr_val;
- uint32_t icr2_val;
- uint64_t tsc_start;
+ u64 last_wake_count;
+ u64 last_hlt_count;
+ u64 last_ipis_rcvd_count;
+ u32 icr_val;
+ u32 icr2_val;
+ u64 tsc_start;
verify_apic_base_addr();
xapic_enable();
@@ -248,7 +248,7 @@ static void cancel_join_vcpu_thread(pthread_t thread, struct kvm_vcpu *vcpu)
}
void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
- uint64_t *pipis_rcvd)
+ u64 *pipis_rcvd)
{
long pages_not_moved;
unsigned long nodemask = 0;
@@ -259,9 +259,9 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
int i;
int from, to;
unsigned long bit;
- uint64_t hlt_count;
- uint64_t wake_count;
- uint64_t ipis_sent;
+ u64 hlt_count;
+ u64 wake_count;
+ u64 ipis_sent;
fprintf(stderr, "Calling migrate_pages every %d microseconds\n",
delay_usecs);
@@ -393,12 +393,12 @@ int main(int argc, char *argv[])
int run_secs = 0;
int delay_usecs = 0;
struct test_data_page *data;
- vm_vaddr_t test_data_page_vaddr;
+ gva_t test_data_page_gva;
bool migrate = false;
pthread_t threads[2];
struct thread_params params[2];
struct kvm_vm *vm;
- uint64_t *pipis_rcvd;
+ u64 *pipis_rcvd;
get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs);
if (run_secs <= 0)
@@ -414,16 +414,16 @@ int main(int argc, char *argv[])
params[1].vcpu = vm_vcpu_add(vm, 1, sender_guest_code);
- test_data_page_vaddr = vm_vaddr_alloc_page(vm);
- data = addr_gva2hva(vm, test_data_page_vaddr);
+ test_data_page_gva = vm_alloc_page(vm);
+ data = addr_gva2hva(vm, test_data_page_gva);
memset(data, 0, sizeof(*data));
params[0].data = data;
params[1].data = data;
- vcpu_args_set(params[0].vcpu, 1, test_data_page_vaddr);
- vcpu_args_set(params[1].vcpu, 1, test_data_page_vaddr);
+ vcpu_args_set(params[0].vcpu, 1, test_data_page_gva);
+ vcpu_args_set(params[1].vcpu, 1, test_data_page_gva);
- pipis_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ipis_rcvd);
+ pipis_rcvd = (u64 *)addr_gva2hva(vm, (u64)&ipis_rcvd);
params[0].pipis_rcvd = pipis_rcvd;
params[1].pipis_rcvd = pipis_rcvd;
diff --git a/tools/testing/selftests/kvm/x86/xapic_state_test.c b/tools/testing/selftests/kvm/x86/xapic_state_test.c
index 0c5e12f..637bb90 100644
--- a/tools/testing/selftests/kvm/x86/xapic_state_test.c
+++ b/tools/testing/selftests/kvm/x86/xapic_state_test.c
@@ -23,7 +23,7 @@ static void xapic_guest_code(void)
xapic_enable();
while (1) {
- uint64_t val = (u64)xapic_read_reg(APIC_IRR) |
+ u64 val = (u64)xapic_read_reg(APIC_IRR) |
(u64)xapic_read_reg(APIC_IRR + 0x10) << 32;
xapic_write_reg(APIC_ICR2, val >> 32);
@@ -43,7 +43,7 @@ static void x2apic_guest_code(void)
x2apic_enable();
do {
- uint64_t val = x2apic_read_reg(APIC_IRR) |
+ u64 val = x2apic_read_reg(APIC_IRR) |
x2apic_read_reg(APIC_IRR + 0x10) << 32;
if (val & X2APIC_RSVD_BITS_MASK) {
@@ -56,12 +56,12 @@ static void x2apic_guest_code(void)
} while (1);
}
-static void ____test_icr(struct xapic_vcpu *x, uint64_t val)
+static void ____test_icr(struct xapic_vcpu *x, u64 val)
{
struct kvm_vcpu *vcpu = x->vcpu;
struct kvm_lapic_state xapic;
struct ucall uc;
- uint64_t icr;
+ u64 icr;
/*
* Tell the guest what ICR value to write. Use the IRR to pass info,
@@ -93,7 +93,7 @@ static void ____test_icr(struct xapic_vcpu *x, uint64_t val)
TEST_ASSERT_EQ(icr, val & ~APIC_ICR_BUSY);
}
-static void __test_icr(struct xapic_vcpu *x, uint64_t val)
+static void __test_icr(struct xapic_vcpu *x, u64 val)
{
/*
* The BUSY bit is reserved on both AMD and Intel, but only AMD treats
@@ -109,7 +109,7 @@ static void __test_icr(struct xapic_vcpu *x, uint64_t val)
static void test_icr(struct xapic_vcpu *x)
{
struct kvm_vcpu *vcpu = x->vcpu;
- uint64_t icr, i, j;
+ u64 icr, i, j;
icr = APIC_DEST_SELF | APIC_INT_ASSERT | APIC_DM_FIXED;
for (i = 0; i <= 0xff; i++)
@@ -142,9 +142,9 @@ static void test_icr(struct xapic_vcpu *x)
__test_icr(x, -1ull & ~APIC_DM_FIXED_MASK);
}
-static void __test_apic_id(struct kvm_vcpu *vcpu, uint64_t apic_base)
+static void __test_apic_id(struct kvm_vcpu *vcpu, u64 apic_base)
{
- uint32_t apic_id, expected;
+ u32 apic_id, expected;
struct kvm_lapic_state xapic;
vcpu_set_msr(vcpu, MSR_IA32_APICBASE, apic_base);
@@ -170,9 +170,9 @@ static void __test_apic_id(struct kvm_vcpu *vcpu, uint64_t apic_base)
*/
static void test_apic_id(void)
{
- const uint32_t NR_VCPUS = 3;
+ const u32 NR_VCPUS = 3;
struct kvm_vcpu *vcpus[NR_VCPUS];
- uint64_t apic_base;
+ u64 apic_base;
struct kvm_vm *vm;
int i;
diff --git a/tools/testing/selftests/kvm/x86/xapic_tpr_test.c b/tools/testing/selftests/kvm/x86/xapic_tpr_test.c
index 3862134..ab25db2 100644
--- a/tools/testing/selftests/kvm/x86/xapic_tpr_test.c
+++ b/tools/testing/selftests/kvm/x86/xapic_tpr_test.c
@@ -58,7 +58,7 @@ static void tpr_guest_irq_queue(void)
if (is_x2apic) {
x2apic_write_reg(APIC_SELF_IPI, IRQ_VECTOR);
} else {
- uint32_t icr, icr2;
+ u32 icr, icr2;
icr = APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_FIXED |
IRQ_VECTOR;
@@ -69,9 +69,9 @@ static void tpr_guest_irq_queue(void)
}
}
-static uint8_t tpr_guest_tpr_get(void)
+static u8 tpr_guest_tpr_get(void)
{
- uint32_t taskpri;
+ u32 taskpri;
if (is_x2apic)
taskpri = x2apic_read_reg(APIC_TASKPRI);
@@ -81,9 +81,9 @@ static uint8_t tpr_guest_tpr_get(void)
return GET_APIC_PRI(taskpri);
}
-static uint8_t tpr_guest_ppr_get(void)
+static u8 tpr_guest_ppr_get(void)
{
- uint32_t procpri;
+ u32 procpri;
if (is_x2apic)
procpri = x2apic_read_reg(APIC_PROCPRI);
@@ -93,9 +93,9 @@ static uint8_t tpr_guest_ppr_get(void)
return GET_APIC_PRI(procpri);
}
-static uint8_t tpr_guest_cr8_get(void)
+static u8 tpr_guest_cr8_get(void)
{
- uint64_t cr8;
+ u64 cr8;
asm volatile ("mov %%cr8, %[cr8]\n\t" : [cr8] "=r"(cr8));
@@ -104,7 +104,7 @@ static uint8_t tpr_guest_cr8_get(void)
static void tpr_guest_check_tpr_ppr_cr8_equal(void)
{
- uint8_t tpr;
+ u8 tpr;
tpr = tpr_guest_tpr_get();
@@ -157,19 +157,19 @@ static void tpr_guest_code(void)
GUEST_DONE();
}
-static uint8_t lapic_tpr_get(struct kvm_lapic_state *xapic)
+static u8 lapic_tpr_get(struct kvm_lapic_state *xapic)
{
return GET_APIC_PRI(*((u32 *)&xapic->regs[APIC_TASKPRI]));
}
-static void lapic_tpr_set(struct kvm_lapic_state *xapic, uint8_t val)
+static void lapic_tpr_set(struct kvm_lapic_state *xapic, u8 val)
{
u32 *taskpri = (u32 *)&xapic->regs[APIC_TASKPRI];
*taskpri = SET_APIC_PRI(*taskpri, val);
}
-static uint8_t sregs_tpr(struct kvm_sregs *sregs)
+static u8 sregs_tpr(struct kvm_sregs *sregs)
{
return sregs->cr8 & GENMASK(3, 0);
}
@@ -197,7 +197,7 @@ static void test_tpr_check_tpr_cr8_equal(struct kvm_vcpu *vcpu)
static void test_tpr_set_tpr_for_irq(struct kvm_vcpu *vcpu, bool mask)
{
struct kvm_lapic_state xapic;
- uint8_t tpr;
+ u8 tpr;
static_assert(IRQ_VECTOR >= 16, "invalid IRQ vector number");
tpr = IRQ_VECTOR / 16;
diff --git a/tools/testing/selftests/kvm/x86/xcr0_cpuid_test.c b/tools/testing/selftests/kvm/x86/xcr0_cpuid_test.c
index d038c15..40dc9e6 100644
--- a/tools/testing/selftests/kvm/x86/xcr0_cpuid_test.c
+++ b/tools/testing/selftests/kvm/x86/xcr0_cpuid_test.c
@@ -21,7 +21,7 @@
*/
#define ASSERT_XFEATURE_DEPENDENCIES(supported_xcr0, xfeatures, dependencies) \
do { \
- uint64_t __supported = (supported_xcr0) & ((xfeatures) | (dependencies)); \
+ u64 __supported = (supported_xcr0) & ((xfeatures) | (dependencies)); \
\
__GUEST_ASSERT((__supported & (xfeatures)) != (xfeatures) || \
__supported == ((xfeatures) | (dependencies)), \
@@ -39,7 +39,7 @@ do { \
*/
#define ASSERT_ALL_OR_NONE_XFEATURE(supported_xcr0, xfeatures) \
do { \
- uint64_t __supported = (supported_xcr0) & (xfeatures); \
+ u64 __supported = (supported_xcr0) & (xfeatures); \
\
__GUEST_ASSERT(!__supported || __supported == (xfeatures), \
"supported = 0x%lx, xfeatures = 0x%llx", \
@@ -48,8 +48,8 @@ do { \
static void guest_code(void)
{
- uint64_t initial_xcr0;
- uint64_t supported_xcr0;
+ u64 initial_xcr0;
+ u64 supported_xcr0;
int i, vector;
set_cr4(get_cr4() | X86_CR4_OSXSAVE);
diff --git a/tools/testing/selftests/kvm/x86/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86/xen_shinfo_test.c
index 23909b5..5076f6a 100644
--- a/tools/testing/selftests/kvm/x86/xen_shinfo_test.c
+++ b/tools/testing/selftests/kvm/x86/xen_shinfo_test.c
@@ -116,15 +116,15 @@ struct pvclock_wall_clock {
} __attribute__((__packed__));
struct vcpu_runstate_info {
- uint32_t state;
- uint64_t state_entry_time;
- uint64_t time[5]; /* Extra field for overrun check */
+ u32 state;
+ u64 state_entry_time;
+ u64 time[5]; /* Extra field for overrun check */
};
struct compat_vcpu_runstate_info {
- uint32_t state;
- uint64_t state_entry_time;
- uint64_t time[5];
+ u32 state;
+ u64 state_entry_time;
+ u64 time[5];
} __attribute__((__packed__));
struct arch_vcpu_info {
@@ -133,8 +133,8 @@ struct arch_vcpu_info {
};
struct vcpu_info {
- uint8_t evtchn_upcall_pending;
- uint8_t evtchn_upcall_mask;
+ u8 evtchn_upcall_pending;
+ u8 evtchn_upcall_mask;
unsigned long evtchn_pending_sel;
struct arch_vcpu_info arch;
struct pvclock_vcpu_time_info time;
@@ -145,7 +145,7 @@ struct shared_info {
unsigned long evtchn_pending[64];
unsigned long evtchn_mask[64];
struct pvclock_wall_clock wc;
- uint32_t wc_sec_hi;
+ u32 wc_sec_hi;
/* arch_shared_info here */
};
@@ -658,7 +658,7 @@ int main(int argc, char *argv[])
printf("Testing RUNSTATE_ADJUST\n");
rst.type = KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST;
memset(&rst.u, 0, sizeof(rst.u));
- rst.u.runstate.state = (uint64_t)-1;
+ rst.u.runstate.state = (u64)-1;
rst.u.runstate.time_blocked =
0x5a - rs->time[RUNSTATE_blocked];
rst.u.runstate.time_offline =
@@ -1113,7 +1113,7 @@ int main(int argc, char *argv[])
/* Don't change the address, just trigger a write */
struct kvm_xen_vcpu_attr adj = {
.type = KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST,
- .u.runstate.state = (uint64_t)-1
+ .u.runstate.state = (u64)-1
};
vcpu_ioctl(vcpu, KVM_XEN_VCPU_SET_ATTR, &adj);
diff --git a/tools/testing/selftests/kvm/x86/xss_msr_test.c b/tools/testing/selftests/kvm/x86/xss_msr_test.c
index f331a4e..12c63df 100644
--- a/tools/testing/selftests/kvm/x86/xss_msr_test.c
+++ b/tools/testing/selftests/kvm/x86/xss_msr_test.c
@@ -17,7 +17,7 @@ int main(int argc, char *argv[])
bool xss_in_msr_list;
struct kvm_vm *vm;
struct kvm_vcpu *vcpu;
- uint64_t xss_val;
+ u64 xss_val;
int i, r;
/* Create VM */
diff --git a/tools/testing/selftests/membarrier/Makefile b/tools/testing/selftests/membarrier/Makefile
index fc840e0..829f95c 100644
--- a/tools/testing/selftests/membarrier/Makefile
+++ b/tools/testing/selftests/membarrier/Makefile
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: GPL-2.0-only
-CFLAGS += -g $(KHDR_INCLUDES)
+CFLAGS += -g $(KHDR_INCLUDES) -pthread -I../../../../tools/include
LDLIBS += -lpthread
TEST_GEN_PROGS := membarrier_test_single_thread \
- membarrier_test_multi_thread
+ membarrier_test_multi_thread \
+ membarrier_rseq_stress
include ../lib.mk
diff --git a/tools/testing/selftests/membarrier/membarrier_rseq_stress.c b/tools/testing/selftests/membarrier/membarrier_rseq_stress.c
new file mode 100644
index 0000000..c188d74
--- /dev/null
+++ b/tools/testing/selftests/membarrier/membarrier_rseq_stress.c
@@ -0,0 +1,951 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Membarrier stress test for CFS throttle interactions.
+ *
+ * Reproducer for the interaction between CFS throttle and expedited membarrier.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <syscall.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sched.h>
+#include <time.h>
+#include <signal.h>
+#include <stdatomic.h>
+#include <dirent.h>
+#include <sys/prctl.h>
+#include <sys/mman.h>
+
+#include "../kselftest.h"
+
+/* -- Architecture-specific rseq signature -- */
+#if defined(__x86_64__) || defined(__i386__)
+# define RSEQ_SIG 0x53053053U
+#elif defined(__aarch64__)
+# define RSEQ_SIG 0xd428bc00U
+#elif defined(__powerpc__) || defined(__powerpc64__)
+# define RSEQ_SIG 0x0f000000U
+#elif defined(__s390__) || defined(__s390x__)
+# define RSEQ_SIG 0x0c000000U
+#else
+# define RSEQ_SIG 0
+# define UNSUPPORTED_ARCH 1
+#endif
+
+/* -- rseq ABI (kernel uapi; define locally for portability) -- */
+#define RSEQ_CPU_ID_UNINITIALIZED ((__u32)-1)
+
+#include <linux/compiler.h>
+
+struct rseq_abi {
+ __u32 cpu_id_start;
+ __u32 cpu_id;
+ __u64 rseq_cs;
+ __u32 flags;
+ __u32 node_id;
+ __u32 mm_cid;
+ char end[0];
+} __aligned(32);
+
+/* -- membarrier constants (not in all distro headers) -- */
+#ifndef MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
+# define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ (1 << 7)
+#endif
+#ifndef MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
+# define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ (1 << 8)
+#endif
+#ifndef MEMBARRIER_CMD_FLAG_CPU
+# define MEMBARRIER_CMD_FLAG_CPU (1 << 0)
+#endif
+
+/* -- Test parameters -- */
+#define N_SIBLINGS 2000
+#define NEST_DEPTH 5
+static char g_cgroup_path[4096];
+static int use_cgroup_v2;
+
+#define CFS_QUOTA_US 1000
+#define CFS_PERIOD_US 5000
+#define N_HAMMER_PER_CPU 25
+#define N_BURNER_PER_CPU 50
+#define MAX_STRESS_CPUS 1024
+#define TEST_DURATION_SEC 20
+
+/* Latency thresholds for the sentinel */
+#define LATENCY_WARN_MS 50
+#define LATENCY_CRITICAL_MS 200
+
+/* Sentinel sampling interval */
+#define SENTINEL_INTERVAL_US 500
+
+/* -- Shared globals -- */
+static atomic_int g_stop;
+static atomic_int g_stop_sentinel;
+static atomic_long g_max_latency_us;
+static atomic_long g_interval_max_latency_us;
+static atomic_long g_mb_ok;
+static atomic_long g_mb_err;
+static int g_ncpus_stress;
+static int *g_stress_cpus;
+
+static atomic_int g_test_ready;
+
+/* Per-thread rseq ABI block registered with the kernel */
+static __thread struct rseq_abi tls_rseq
+ __attribute__((tls_model("initial-exec"))) __aligned(32) = {
+ .cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
+};
+
+/* -- Utility -- */
+static int write_file(const char *path, const char *val)
+{
+ int fd = open(path, O_WRONLY | O_CLOEXEC);
+
+ if (fd < 0)
+ return -errno;
+
+ size_t len = strlen(val);
+ ssize_t r = write(fd, val, len);
+
+ close(fd);
+ if (r < 0)
+ return -errno;
+ if ((size_t)r != len)
+ return -EIO;
+ return 0;
+}
+
+static uint64_t monotonic_us(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (uint64_t)ts.tv_sec * 1000000ULL + ts.tv_nsec / 1000ULL;
+}
+
+static void update_max_latency(long lat)
+{
+ long old = atomic_load_explicit(&g_max_latency_us, memory_order_relaxed);
+
+ while (lat > old) {
+ if (atomic_compare_exchange_weak_explicit(&g_max_latency_us, &old, lat,
+ memory_order_relaxed, memory_order_relaxed))
+ break;
+ }
+
+ old = atomic_load_explicit(&g_interval_max_latency_us, memory_order_relaxed);
+ while (lat > old) {
+ if (atomic_compare_exchange_weak_explicit(&g_interval_max_latency_us, &old, lat,
+ memory_order_relaxed, memory_order_relaxed))
+ break;
+ }
+}
+
+static void init_stress_cpus(void)
+{
+ cpu_set_t set;
+ int capacity = MAX_STRESS_CPUS;
+
+ g_stress_cpus = malloc(capacity * sizeof(int));
+ if (!g_stress_cpus)
+ ksft_exit_fail_msg("malloc failed for g_stress_cpus\n");
+
+ if (sched_getaffinity(0, sizeof(set), &set) < 0)
+ ksft_exit_fail_msg("sched_getaffinity failed\n");
+
+ for (int i = 0; i < CPU_SETSIZE && g_ncpus_stress < capacity; i++) {
+ if (CPU_ISSET(i, &set))
+ g_stress_cpus[g_ncpus_stress++] = i;
+ }
+
+ if (g_ncpus_stress == 0)
+ ksft_exit_skip("No CPUs available for stress test\n");
+
+ ksft_print_msg("Stressing %d CPUs discovered via affinity\n", g_ncpus_stress);
+}
+
+/* -- rseq / membarrier helpers -- */
+static int rseq_register_thread(void)
+{
+ int r = syscall(SYS_rseq, &tls_rseq, sizeof(tls_rseq), 0, RSEQ_SIG);
+
+ return (r == 0 || errno == EBUSY || errno == EINVAL) ? 0 : -1;
+}
+
+static int rseq_register_thread_at(struct rseq_abi *rseq)
+{
+ int r = syscall(SYS_rseq, rseq, sizeof(*rseq), 0, RSEQ_SIG);
+
+ return (r == 0 || errno == EBUSY || errno == EINVAL) ? 0 : -1;
+}
+
+static int membarrier_register_rseq_mm(void)
+{
+ return syscall(SYS_membarrier,
+ MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0);
+}
+
+/* -- cgroup helpers -- */
+static void rm_cgroup_recursive(const char *path)
+{
+ DIR *dir = opendir(path);
+
+ if (!dir)
+ return;
+ struct dirent *entry;
+
+ while ((entry = readdir(dir)) != NULL) {
+ if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+ continue;
+ if (entry->d_type == DT_DIR) {
+ char sub_path[4096];
+
+ snprintf(sub_path, sizeof(sub_path), "%s/%s", path, entry->d_name);
+ rm_cgroup_recursive(sub_path);
+ }
+ }
+ closedir(dir);
+ rmdir(path);
+}
+
+static void cgroup_teardown(void);
+
+static int cgroup_setup(void)
+{
+ struct stat st;
+
+ if (stat("/sys/fs/cgroup/cpu", &st) == 0) {
+ use_cgroup_v2 = 0;
+ snprintf(g_cgroup_path, sizeof(g_cgroup_path),
+ "/sys/fs/cgroup/cpu/membarrier_stress_test");
+ } else if (stat("/dev/cgroup/cpu", &st) == 0) {
+ use_cgroup_v2 = 0;
+ snprintf(g_cgroup_path, sizeof(g_cgroup_path),
+ "/dev/cgroup/cpu/membarrier_stress_test");
+ } else if (stat("/cgroup/cpu", &st) == 0) {
+ use_cgroup_v2 = 0;
+ snprintf(g_cgroup_path, sizeof(g_cgroup_path),
+ "/cgroup/cpu/membarrier_stress_test");
+ } else if (stat("/sys/fs/cgroup/cgroup.controllers", &st) == 0) {
+ use_cgroup_v2 = 1;
+ snprintf(g_cgroup_path, sizeof(g_cgroup_path),
+ "/sys/fs/cgroup/membarrier_stress_test");
+ } else {
+ ksft_print_msg("WARN: cgroup mount not found. Using v2 at /sys/fs/cgroup\n");
+ use_cgroup_v2 = 1;
+ snprintf(g_cgroup_path, sizeof(g_cgroup_path),
+ "/sys/fs/cgroup/membarrier_stress_test");
+ }
+
+ /* Robust cleanup before setup */
+ cgroup_teardown();
+
+ if (use_cgroup_v2) {
+ /* Enable cpu controller in root cgroup */
+ if (write_file("/sys/fs/cgroup/cgroup.subtree_control", "+cpu") < 0)
+ ksft_print_msg("WARN: failed to enable cpu controller in /sys/fs/cgroup\n");
+ }
+
+ if (mkdir(g_cgroup_path, 0755) < 0 && errno != EEXIST) {
+ ksft_print_msg("mkdir base %s failed: %s\n", g_cgroup_path, strerror(errno));
+ return -1;
+ }
+
+ if (use_cgroup_v2) {
+ char ctrl_path[4096];
+
+ snprintf(ctrl_path, sizeof(ctrl_path), "%s/cgroup.subtree_control", g_cgroup_path);
+ if (write_file(ctrl_path, "+cpu") < 0)
+ ksft_print_msg("WARN: failed to enable cpu controller in %s\n",
+ g_cgroup_path);
+ }
+
+ for (int i = 0; i < N_SIBLINGS; i++) {
+ char sibling_path[4096];
+
+ snprintf(sibling_path, sizeof(sibling_path), "%s/n%d", g_cgroup_path, i);
+ if (mkdir(sibling_path, 0755) < 0 && errno != EEXIST) {
+ ksft_print_msg("mkdir wide %s failed: %s\n", sibling_path, strerror(errno));
+ return -1;
+ }
+
+ if (use_cgroup_v2) {
+ char ctrl_path[4096];
+
+ snprintf(ctrl_path, sizeof(ctrl_path),
+ "%s/cgroup.subtree_control", sibling_path);
+ if (write_file(ctrl_path, "+cpu") < 0)
+ ksft_print_msg("WARN: failed to enable cpu controller in %s\n",
+ sibling_path);
+ }
+
+ char current_path[4096];
+
+ snprintf(current_path, sizeof(current_path), "%s", sibling_path);
+ for (int j = 0; j < NEST_DEPTH; j++) {
+ snprintf(current_path + strlen(current_path),
+ sizeof(current_path) - strlen(current_path), "/d%d", j);
+ if (mkdir(current_path, 0755) < 0 && errno != EEXIST) {
+ ksft_print_msg("mkdir deep %s failed: %s\n",
+ current_path, strerror(errno));
+ return -1;
+ }
+
+ /* Enable for all but the leaf */
+ if (use_cgroup_v2 && j < NEST_DEPTH - 1) {
+ char ctrl_path[4096];
+
+ snprintf(ctrl_path, sizeof(ctrl_path), "%s/cgroup.subtree_control",
+ current_path);
+ if (write_file(ctrl_path, "+cpu") < 0)
+ ksft_print_msg("WARN: cannot enable cpu controller in %s\n",
+ current_path);
+ }
+ }
+ }
+
+ char quota[64], period[64], max_str[128];
+
+ snprintf(quota, sizeof(quota), "%d", CFS_QUOTA_US);
+ snprintf(period, sizeof(period), "%d", CFS_PERIOD_US);
+ snprintf(max_str, sizeof(max_str), "%d %d", CFS_QUOTA_US, CFS_PERIOD_US);
+
+ if (use_cgroup_v2) {
+ char max_path[4096];
+
+ snprintf(max_path, sizeof(max_path), "%s/cpu.max", g_cgroup_path);
+ if (write_file(max_path, max_str) < 0) {
+ ksft_print_msg("ERROR: cannot write cpu.max at %s\n", max_path);
+ return -1;
+ }
+ ksft_print_msg("cgroup (v2) %s: cpu.max=%s\n", g_cgroup_path, max_str);
+ } else {
+ char quota_path[4096], period_path[4096];
+
+ snprintf(quota_path, sizeof(quota_path), "%s/cpu.cfs_quota_us", g_cgroup_path);
+ snprintf(period_path, sizeof(period_path), "%s/cpu.cfs_period_us", g_cgroup_path);
+
+ if (write_file(period_path, period) < 0) {
+ ksft_print_msg("ERROR: cannot write cpu.cfs_period_us at %s\n",
+ period_path);
+ return -1;
+ }
+ if (write_file(quota_path, quota) < 0) {
+ ksft_print_msg("ERROR: cannot write cpu.cfs_quota_us at %s\n", quota_path);
+ return -1;
+ }
+ ksft_print_msg("cgroup (v1) %s: cpu.cfs_quota_us=%d cpu.cfs_period_us=%d\n",
+ g_cgroup_path, CFS_QUOTA_US, CFS_PERIOD_US);
+ }
+
+ return 0;
+}
+
+static int cgroup_add_pid_to_path(pid_t pid, const char *path)
+{
+ char buf[32], file_path[4096];
+
+ snprintf(buf, sizeof(buf), "%d", (int)pid);
+ if (use_cgroup_v2) {
+ snprintf(file_path, sizeof(file_path), "%s/cgroup.procs", path);
+ return write_file(file_path, buf);
+ }
+ /* In v1, try tasks first, fallback to cgroup.procs */
+ snprintf(file_path, sizeof(file_path), "%s/tasks", path);
+ int r = write_file(file_path, buf);
+
+ if (r < 0) {
+ snprintf(file_path, sizeof(file_path), "%s/cgroup.procs", path);
+ r = write_file(file_path, buf);
+ }
+ return r;
+}
+
+static void cgroup_teardown(void)
+{
+ rm_cgroup_recursive(g_cgroup_path);
+}
+
+static void cgroup_unthrottle(void)
+{
+ if (use_cgroup_v2) {
+ char max_path[4096];
+
+ snprintf(max_path, sizeof(max_path), "%s/cpu.max", g_cgroup_path);
+ write_file(max_path, "max");
+ } else {
+ char quota_path[4096];
+
+ snprintf(quota_path, sizeof(quota_path), "%s/cpu.cfs_quota_us", g_cgroup_path);
+ write_file(quota_path, "-1");
+ }
+}
+
+/* -- CPU burner (inside throttled child process) -- */
+static void *burner_thread_fn(void *arg)
+{
+ struct rseq_abi my_rseq;
+ int cpu = (int)(uintptr_t)arg;
+
+ memset(&my_rseq, 0, sizeof(my_rseq));
+ my_rseq.cpu_id = RSEQ_CPU_ID_UNINITIALIZED;
+
+ if (rseq_register_thread_at(&my_rseq) < 0) {
+ perror("rseq_register (burner)");
+ return NULL;
+ }
+
+ cpu_set_t set;
+
+ CPU_ZERO(&set);
+ CPU_SET(cpu, &set);
+ if (sched_setaffinity(0, sizeof(set), &set) < 0)
+ perror("sched_setaffinity (burner)");
+
+ unsigned long sink = 0;
+
+ while (!atomic_load_explicit(&g_stop, memory_order_relaxed)) {
+ sink++;
+ /* Prevent compiler from optimizing the loop away */
+ asm volatile("" : "+g"(sink));
+ }
+
+ return NULL;
+}
+
+static int burner_thread_fn_wrapper(void *arg)
+{
+ burner_thread_fn(arg);
+ return 0;
+}
+
+static int leaf_child_fn(void *arg)
+{
+ int i = (int)(uintptr_t)arg;
+ int total_burners = g_ncpus_stress * N_BURNER_PER_CPU;
+ int n_threads_per_leaf = total_burners / N_SIBLINGS;
+
+ if (i < (total_burners % N_SIBLINGS))
+ n_threads_per_leaf++;
+
+ prctl(PR_SET_PDEATHSIG, SIGTERM);
+ if (getppid() == 1)
+ _exit(1);
+
+ char leaf_path[4096];
+
+ snprintf(leaf_path, sizeof(leaf_path), "%s/n%d", g_cgroup_path, i);
+ for (int j = 0; j < NEST_DEPTH; j++)
+ snprintf(leaf_path + strlen(leaf_path),
+ sizeof(leaf_path) - strlen(leaf_path), "/d%d", j);
+
+ int r = cgroup_add_pid_to_path(getpid(), leaf_path);
+
+ if (r < 0) {
+ char buf[512];
+ int len = snprintf(buf, sizeof(buf),
+ "[leaf child %d] failed to join cgroup %s: err %d\n",
+ i, leaf_path, -r);
+ (void)!write(2, buf, len);
+ _exit(1);
+ }
+
+ for (int j = 0; j < n_threads_per_leaf; j++) {
+ int cpu = g_stress_cpus[(i * n_threads_per_leaf + j) % g_ncpus_stress];
+
+ /* Allocate stack via mmap (bypasses heap) */
+ size_t stack_size = 64 * 1024;
+ void *stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (stack == MAP_FAILED) {
+ const char *msg = "mmap stack failed\n";
+ (void)!write(2, msg, strlen(msg));
+ _exit(1);
+ }
+
+ /* Use raw clone to create a thread sharing the VM and thread group */
+ pid_t pid = clone(burner_thread_fn_wrapper, stack + stack_size,
+ CLONE_VM | CLONE_THREAD | CLONE_SIGHAND,
+ (void *)(uintptr_t)cpu);
+ if (pid < 0) {
+ const char *msg = "clone burner failed\n";
+ (void)!write(2, msg, strlen(msg));
+ _exit(1);
+ }
+ }
+
+ // Wait for SIGTERM
+ sigset_t mask;
+
+ sigemptyset(&mask);
+ sigaddset(&mask, SIGTERM);
+ int sig;
+
+ sigwait(&mask, &sig);
+
+ _exit(0);
+}
+
+struct leaf_info {
+ pid_t pid;
+ void *stack;
+};
+
+static int run_throttle_child(void *arg)
+{
+ (void)arg;
+ prctl(PR_SET_PDEATHSIG, SIGTERM);
+ if (getppid() == 1)
+ _exit(1);
+
+ int n_leafs = N_SIBLINGS;
+
+ /* Block signals before spawning to avoid missing early failures */
+ sigset_t mask;
+
+ sigemptyset(&mask);
+ sigaddset(&mask, SIGTERM);
+ sigaddset(&mask, SIGCHLD);
+ sigprocmask(SIG_BLOCK, &mask, NULL);
+
+ /* Use mmap for tracking structures to avoid glibc heap usage */
+ struct leaf_info *leaves = mmap(NULL, n_leafs * sizeof(struct leaf_info),
+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (leaves == MAP_FAILED) {
+ const char *msg = "mmap leaves array failed\n";
+ (void)!write(2, msg, strlen(msg));
+ _exit(1);
+ }
+
+ for (int i = 0; i < n_leafs; i++) {
+ size_t stack_size = 64 * 1024;
+ void *stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (stack == MAP_FAILED) {
+ const char *msg = "mmap leaf stack failed\n";
+ (void)!write(2, msg, strlen(msg));
+ _exit(1);
+ }
+
+ leaves[i].stack = stack;
+
+ pid_t pid = clone(leaf_child_fn, stack + stack_size,
+ CLONE_VM | SIGCHLD, (void *)(uintptr_t)i);
+
+ if (pid < 0) {
+ const char *msg = "clone (leaf child) failed\n";
+ (void)!write(2, msg, strlen(msg));
+
+ /* Clean up successfully spawned children */
+ for (int j = 0; j < i; j++) {
+ kill(leaves[j].pid, SIGTERM);
+ waitpid(leaves[j].pid, NULL, 0);
+ munmap(leaves[j].stack, stack_size);
+ }
+ munmap(leaves, n_leafs * sizeof(struct leaf_info));
+
+ if (errno == EAGAIN)
+ _exit(4);
+ else
+ _exit(1);
+ }
+ leaves[i].pid = pid;
+ }
+
+ int failed = 0;
+
+ while (1) {
+ int sig;
+
+ sigwait(&mask, &sig);
+
+ if (sig == SIGTERM) {
+ break;
+ } else if (sig == SIGCHLD) {
+ int status;
+ pid_t pid;
+
+ // Reap all dead children
+ while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
+ for (int i = 0; i < n_leafs; i++) {
+ if (leaves[i].pid == pid) {
+ leaves[i].pid = 0;
+ break;
+ }
+ }
+ if ((WIFEXITED(status) && WEXITSTATUS(status) != 0) ||
+ WIFSIGNALED(status)) {
+ char buf[128];
+ int len = snprintf(buf, sizeof(buf),
+ "[manager] child %d died unexpectedly (status %d)\n",
+ pid, WEXITSTATUS(status));
+ (void)!write(2, buf, len);
+ failed = 1;
+ }
+ }
+ if (failed)
+ break;
+ }
+ }
+
+ // Terminate all leaf kids
+ for (int i = 0; i < n_leafs; i++) {
+ if (leaves[i].pid > 0)
+ kill(leaves[i].pid, SIGTERM);
+ }
+
+ for (int i = 0; i < n_leafs; i++) {
+ if (leaves[i].pid > 0)
+ waitpid(leaves[i].pid, NULL, 0);
+ munmap(leaves[i].stack, 64 * 1024);
+ }
+
+ munmap(leaves, n_leafs * sizeof(struct leaf_info));
+
+ _exit(failed ? 1 : 0);
+}
+
+/* -- Membarrier hammer thread -- */
+static void *hammer_thread_fn(void *arg)
+{
+ int target_cpu = *(int *)arg;
+ long local_ok = 0;
+ long local_err = 0;
+ int count = 0;
+ const int batch_size = 1024;
+
+ if (rseq_register_thread() < 0) {
+ ksft_print_msg("[hammer] rseq_register failed: %s\n", strerror(errno));
+ return NULL;
+ }
+
+ membarrier_register_rseq_mm();
+
+ while (!atomic_load_explicit(&g_stop, memory_order_relaxed)) {
+ int r = syscall(SYS_membarrier,
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+ MEMBARRIER_CMD_FLAG_CPU,
+ target_cpu);
+ if (__builtin_expect(r == 0, 1))
+ local_ok++;
+ else
+ local_err++;
+
+ count++;
+ if (__builtin_expect(count >= batch_size, 0)) {
+ atomic_fetch_add_explicit(&g_mb_ok, local_ok, memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_mb_err, local_err, memory_order_relaxed);
+ local_ok = 0;
+ local_err = 0;
+ count = 0;
+ }
+ }
+
+ /* Flush any remaining counts on exit */
+ if (local_ok > 0)
+ atomic_fetch_add_explicit(&g_mb_ok, local_ok, memory_order_relaxed);
+ if (local_err > 0)
+ atomic_fetch_add_explicit(&g_mb_err, local_err, memory_order_relaxed);
+
+ return NULL;
+}
+
+/* -- Latency sentinel -- */
+static void *sentinel_thread_fn(void *arg)
+{
+ (void)arg;
+ struct sched_param sp = { .sched_priority = 20 };
+
+ if (sched_setscheduler(0, SCHED_FIFO, &sp) < 0)
+ ksft_print_msg("WARN: no SCHED_FIFO for sentinel (less precise)\n");
+
+ while (!atomic_load_explicit(&g_test_ready, memory_order_relaxed) &&
+ !atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) {
+ struct timespec ts = {0, 1000 * 1000}; /* 1ms */
+
+ clock_nanosleep(CLOCK_MONOTONIC, 0, &ts, NULL);
+ }
+
+ uint64_t prev = monotonic_us();
+
+ while (!atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) {
+ struct timespec ts = {
+ .tv_sec = 0,
+ .tv_nsec = SENTINEL_INTERVAL_US * 1000L,
+ };
+ clock_nanosleep(CLOCK_MONOTONIC, 0, &ts, NULL);
+
+ uint64_t now = monotonic_us();
+ long latency_us = (long)(now - prev) - SENTINEL_INTERVAL_US;
+
+ prev = now;
+
+ if (latency_us <= 0)
+ continue;
+
+ update_max_latency(latency_us);
+
+ if (latency_us > LATENCY_CRITICAL_MS * 1000L) {
+ ksft_print_msg("\n[SENTINEL] CRITICAL: %ld ms delay (lockup precursor!)\n",
+ latency_us / 1000);
+ } else if (latency_us > LATENCY_WARN_MS * 1000L) {
+ ksft_print_msg("\n[SENTINEL] WARN: %ld ms latency spike\n",
+ latency_us / 1000);
+ }
+ }
+ return NULL;
+}
+
+/* -- Progress reporter -- */
+static void *reporter_thread_fn(void *arg)
+{
+ (void)arg;
+ int elapsed = 0;
+
+ while (!atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) {
+ for (int i = 0; i < 5; i++) {
+ sleep(1);
+ if (atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed))
+ break;
+ }
+ if (atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed))
+ break;
+ elapsed += 5;
+ long interval_max = atomic_exchange_explicit(&g_interval_max_latency_us,
+ 0, memory_order_relaxed);
+
+ ksft_print_msg("[%3ds] mb: ok=%-10ld err=%-8ld | max_lat=%ld us\n",
+ elapsed,
+ atomic_load(&g_mb_ok),
+ atomic_load(&g_mb_err),
+ interval_max);
+ }
+ return NULL;
+}
+
+/* -- Main -- */
+int main(void)
+{
+ ksft_print_header();
+#ifdef UNSUPPORTED_ARCH
+ ksft_exit_skip("Unsupported architecture\n");
+#endif
+ ksft_set_plan(1);
+
+ if (geteuid() != 0)
+ ksft_exit_skip("Must run as root (cgroup + SCHED_FIFO)\n");
+
+ init_stress_cpus();
+
+ ksft_print_msg("=== membarrier rseq + CFS unthrottle stress ===\n");
+ ksft_print_msg("Stressing CPUs: %d\n", g_ncpus_stress);
+ ksft_print_msg("Quota: %d/%d us (~%d unthrottles/sec/CPU)\n",
+ CFS_QUOTA_US, CFS_PERIOD_US,
+ 1000000 / CFS_PERIOD_US);
+ ksft_print_msg("Hammer threads: %d per CPU (%d total)\n",
+ N_HAMMER_PER_CPU, g_ncpus_stress * N_HAMMER_PER_CPU);
+ ksft_print_msg("Duration: %d seconds\n\n", TEST_DURATION_SEC);
+
+ if (cgroup_setup() < 0) {
+ cgroup_teardown();
+ ksft_exit_skip("cgroup_setup failed (missing permissions or v2 ctrls?)\n");
+ }
+
+ if (rseq_register_thread() < 0) {
+ ksft_print_msg("rseq_register (%s) failed: %s\n", __func__, strerror(errno));
+ cgroup_teardown();
+ ksft_exit_skip("rseq syscall failed or not available\n");
+ }
+ if (membarrier_register_rseq_mm() < 0) {
+ ksft_print_msg("MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ: %s\n"
+ "Kernel >= 5.10 with CONFIG_RSEQ required.\n",
+ strerror(errno));
+ cgroup_teardown();
+ ksft_exit_skip("membarrier register failed\n");
+ }
+ ksft_print_msg("rseq membarrier registered OK\n");
+
+ sigset_t sigmask;
+
+ sigemptyset(&sigmask);
+ sigaddset(&sigmask, SIGTERM);
+ sigprocmask(SIG_BLOCK, &sigmask, NULL);
+
+ void *stack = malloc(1024 * 1024);
+
+ if (!stack) {
+ perror("malloc stack");
+ cgroup_teardown();
+ ksft_exit_fail_msg("Malloc stack failed\n");
+ }
+ pid_t child = clone(run_throttle_child, stack + 1024 * 1024, CLONE_VM | SIGCHLD, NULL);
+
+ if (child < 0) {
+ perror("clone");
+ cgroup_teardown();
+ ksft_exit_fail_msg("Clone failed\n");
+ }
+
+ sigprocmask(SIG_UNBLOCK, &sigmask, NULL);
+ ksft_print_msg("Throttle child PID %d started\n", child);
+
+ int n_threads = g_ncpus_stress * N_HAMMER_PER_CPU + 2;
+ pthread_t *threads = (pthread_t *)calloc(n_threads, sizeof(pthread_t));
+ int *cpuargs = (int *)calloc(g_ncpus_stress * N_HAMMER_PER_CPU, sizeof(int));
+
+ if (!threads || !cpuargs) {
+ perror("calloc");
+ kill(child, SIGTERM);
+ waitpid(child, NULL, 0);
+ cgroup_teardown();
+ ksft_exit_fail_msg("Thread allocation failed\n");
+ }
+
+ int ti = 0, ai = 0;
+ int r;
+
+ ksft_print_msg("Creating sentinel thread...\n");
+ r = pthread_create(&threads[ti], NULL, sentinel_thread_fn, NULL);
+ if (r != 0) {
+ kill(child, SIGTERM);
+ waitpid(child, NULL, 0);
+ cgroup_teardown();
+ free(threads);
+ free(cpuargs);
+ free(g_stress_cpus);
+ ksft_exit_fail_msg("pthread_create (sentinel) failed: %s\n", strerror(r));
+ }
+ ti++;
+
+ ksft_print_msg("Creating reporter thread...\n");
+ r = pthread_create(&threads[ti], NULL, reporter_thread_fn, NULL);
+ if (r != 0) {
+ atomic_store(&g_stop_sentinel, 1);
+ pthread_join(threads[0], NULL);
+ kill(child, SIGTERM);
+ waitpid(child, NULL, 0);
+ cgroup_teardown();
+ free(threads);
+ free(cpuargs);
+ free(g_stress_cpus);
+ ksft_exit_fail_msg("pthread_create (reporter) failed: %s\n", strerror(r));
+ }
+ ti++;
+
+ ksft_print_msg("Creating %d hammer threads...\n", g_ncpus_stress * N_HAMMER_PER_CPU);
+ for (int i = 0; i < g_ncpus_stress; i++) {
+ int cpu = g_stress_cpus[i];
+
+ for (int j = 0; j < N_HAMMER_PER_CPU; j++) {
+ cpuargs[ai] = cpu;
+ r = pthread_create(&threads[ti], NULL, hammer_thread_fn, &cpuargs[ai]);
+ if (r != 0) {
+ ksft_print_msg("pthread_create failed at thread %d: %s\n",
+ ti, strerror(r));
+
+ atomic_store(&g_stop_sentinel, 1);
+ pthread_join(threads[0], NULL);
+ pthread_join(threads[1], NULL);
+
+ atomic_store(&g_stop, 1);
+ for (int k = 2; k < ti; k++)
+ pthread_join(threads[k], NULL);
+
+ kill(child, SIGTERM);
+ waitpid(child, NULL, 0);
+ cgroup_teardown();
+
+ free(threads);
+ free(cpuargs);
+ free(g_stress_cpus);
+
+ if (r == EAGAIN)
+ ksft_exit_skip("Resource limits prevent threads\n");
+ else
+ ksft_exit_fail_msg("Failed to create hammer thread\n");
+ }
+ ti++;
+ ai++;
+ }
+ }
+
+ ksft_print_msg("All threads running. Tip: monitor dmesg for lockups\n\n");
+
+ atomic_store_explicit(&g_test_ready, 1, memory_order_relaxed);
+ int child_failed = 0;
+ int child_status = 0;
+
+ for (int i = 0; i < TEST_DURATION_SEC; i++) {
+ sleep(1);
+ int r = waitpid(child, &child_status, WNOHANG);
+
+ if (r == child) {
+ child_failed = 1;
+ break;
+ }
+ }
+
+ atomic_store(&g_stop_sentinel, 1);
+ pthread_join(threads[0], NULL);
+ pthread_join(threads[1], NULL);
+
+ atomic_store(&g_stop, 1);
+
+ /* Unthrottle to allow children to exit quickly */
+ cgroup_unthrottle();
+
+ if (!child_failed) {
+ kill(child, SIGTERM);
+ waitpid(child, NULL, 0);
+ }
+ for (int i = 2; i < ti; i++)
+ pthread_join(threads[i], NULL);
+
+ long max_lat = atomic_load(&g_max_latency_us);
+ long total_ok = atomic_load(&g_mb_ok);
+ long total_err = atomic_load(&g_mb_err);
+
+ ksft_print_msg("\n=== RESULTS ===\n");
+ ksft_print_msg("membarrier syscalls : %ld ok %ld errors\n", total_ok, total_err);
+ ksft_print_msg("Max scheduler latency: %ld us (%ld ms)\n", max_lat, max_lat / 1000);
+ cgroup_teardown();
+ free(threads);
+ free(cpuargs);
+ free(g_stress_cpus);
+
+ if (child_failed) {
+ if (WIFEXITED(child_status) && WEXITSTATUS(child_status) == 4)
+ ksft_exit_skip("Manager child skipped (resource limits?)\n");
+ ksft_test_result_fail("membarrier_rseq_stress: Manager child died early\n");
+ ksft_exit_fail();
+ } else if (total_ok == 0) {
+ ksft_test_result_fail("membarrier_rseq_stress: No successful membarrier calls\n");
+ ksft_exit_fail();
+ } else if (total_err > 0) {
+ ksft_test_result_fail("membarrier_rseq_stress: syscall errors\n");
+ ksft_exit_fail();
+ } else if (max_lat > LATENCY_CRITICAL_MS * 1000L) {
+ ksft_test_result_fail("membarrier_rseq_stress: LOCKUP PRECURSOR\n");
+ ksft_exit_fail();
+ } else if (max_lat > LATENCY_WARN_MS * 1000L) {
+ ksft_test_result_fail("membarrier_rseq_stress: significant latency spike\n");
+ ksft_exit_fail();
+ } else {
+ ksft_test_result_pass("membarrier_rseq_stress\n");
+ ksft_exit_pass();
+ }
+
+ return 0;
+}
diff --git a/tools/testing/selftests/mm/config b/tools/testing/selftests/mm/config
index 1dbe2b4..06f78bd 100644
--- a/tools/testing/selftests/mm/config
+++ b/tools/testing/selftests/mm/config
@@ -13,3 +13,4 @@
CONFIG_UPROBES=y
CONFIG_MEMORY_FAILURE=y
CONFIG_HWPOISON_INJECT=m
+CONFIG_PROC_MEM_ALWAYS_FORCE=y
diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c
index 7886894..77fb4c5 100644
--- a/tools/testing/selftests/mm/hmm-tests.c
+++ b/tools/testing/selftests/mm/hmm-tests.c
@@ -986,6 +986,56 @@ TEST_F(hmm, migrate)
}
/*
+ * Migrate private file memory to device private memory.
+ */
+TEST_F(hmm, migrate_file_private)
+{
+ struct hmm_buffer *buffer;
+ unsigned long npages;
+ unsigned long size;
+ unsigned long i;
+ int *ptr;
+ int ret;
+ int fd;
+
+ npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
+ ASSERT_NE(npages, 0);
+ size = npages << self->page_shift;
+
+ fd = hmm_create_file(size);
+ ASSERT_GE(fd, 0);
+
+ buffer = malloc(sizeof(*buffer));
+ ASSERT_NE(buffer, NULL);
+
+ buffer->fd = fd;
+ buffer->size = size;
+ buffer->mirror = malloc(size);
+ ASSERT_NE(buffer->mirror, NULL);
+
+ buffer->ptr = mmap(NULL, size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE,
+ buffer->fd, 0);
+ ASSERT_NE(buffer->ptr, MAP_FAILED);
+
+ /* Initialize buffer in system memory. */
+ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
+ ptr[i] = i;
+
+ /* Migrate memory to device. */
+ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(buffer->cpages, npages);
+
+ /* Check what the device read. */
+ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
+ ASSERT_EQ(ptr[i], i);
+
+ hmm_buffer_free(buffer);
+}
+
+/*
* Migrate anonymous memory to device private memory and fault some of it back
* to system memory, then try migrating the resulting mix of system and device
* private memory to the device.
diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh
index d846845..c17b133 100755
--- a/tools/testing/selftests/mm/run_vmtests.sh
+++ b/tools/testing/selftests/mm/run_vmtests.sh
@@ -103,7 +103,7 @@
RUN_DESTRUCTIVE=false
TAP_PREFIX="# "
-while getopts "aht:n" OPT; do
+while getopts "aht:nd" OPT; do
case ${OPT} in
"a") RUN_ALL=true ;;
"h") usage ;;
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index a275ed5..2ed7d80 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -96,6 +96,7 @@
srv6_hl2encap_red_l2vpn_test.sh \
srv6_iptunnel_cache.sh \
stress_reuseport_listen.sh \
+ tcp_ecmp_failover.sh \
tcp_fastopen_backup_key.sh \
test_bpf.sh \
test_bridge_backup_port.sh \
@@ -108,6 +109,7 @@
test_vxlan_nh.sh \
test_vxlan_nolocalbypass.sh \
test_vxlan_under_vrf.sh \
+ test_vxlan_vnifilter_notify.sh \
test_vxlan_vnifiltering.sh \
tfo_passive.sh \
traceroute.sh \
diff --git a/tools/testing/selftests/net/af_unix/scm_inq.c b/tools/testing/selftests/net/af_unix/scm_inq.c
index 3a86be9..6268b5b 100644
--- a/tools/testing/selftests/net/af_unix/scm_inq.c
+++ b/tools/testing/selftests/net/af_unix/scm_inq.c
@@ -8,8 +8,9 @@
#include "kselftest_harness.h"
-#define NR_CHUNKS 100
-#define MSG_LEN 256
+#define NR_CHUNKS 100
+#define MSG_LEN 256
+#define NR_PARTIAL_READS 3
FIXTURE(scm_inq)
{
@@ -120,4 +121,53 @@ TEST_F(scm_inq, basic)
recv_chunks(_metadata, self);
}
+TEST_F(scm_inq, partial_read)
+{
+ char buf[MSG_LEN * NR_PARTIAL_READS] = {};
+ char cmsg_buf[CMSG_SPACE(sizeof(int))];
+ struct msghdr msg = {};
+ struct iovec iov = {};
+ struct cmsghdr *cmsg;
+ int err, inq, ret, i;
+ int remain;
+
+ err = setsockopt(self->fd[1], SOL_SOCKET, SO_INQ, &(int){1}, sizeof(int));
+ if (variant->type != SOCK_STREAM) {
+ ASSERT_EQ(-ENOPROTOOPT, -errno);
+ return;
+ }
+ ASSERT_EQ(0, err);
+
+ ret = send(self->fd[0], buf, sizeof(buf), 0);
+ ASSERT_EQ(sizeof(buf), ret);
+
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ iov.iov_base = buf;
+ iov.iov_len = MSG_LEN;
+
+ for (i = 0; i < NR_PARTIAL_READS; i++) {
+ remain = MSG_LEN * (NR_PARTIAL_READS - 1 - i);
+
+ memset(buf, 0, MSG_LEN);
+ memset(cmsg_buf, 0, sizeof(cmsg_buf));
+ ret = recvmsg(self->fd[1], &msg, 0);
+ ASSERT_EQ(MSG_LEN, ret);
+
+ cmsg = CMSG_FIRSTHDR(&msg);
+ ASSERT_NE(NULL, cmsg);
+ ASSERT_EQ(CMSG_LEN(sizeof(int)), cmsg->cmsg_len);
+ ASSERT_EQ(SOL_SOCKET, cmsg->cmsg_level);
+ ASSERT_EQ(SCM_INQ, cmsg->cmsg_type);
+ ASSERT_EQ(remain, *(int *)CMSG_DATA(cmsg));
+
+ ret = ioctl(self->fd[1], SIOCINQ, &inq);
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(remain, inq);
+ }
+}
+
TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh
index e8031f6..ebdb4c7 100755
--- a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh
+++ b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh
@@ -4,7 +4,7 @@
ALL_TESTS="vlmc_control_test vlmc_querier_test vlmc_igmp_mld_version_test \
vlmc_last_member_test vlmc_startup_query_test vlmc_membership_test \
vlmc_querier_intvl_test vlmc_query_intvl_test vlmc_query_response_intvl_test \
- vlmc_router_port_test vlmc_filtering_test"
+ vlmc_router_port_test vlmc_filtering_test vlmc_mcast_toggle_test"
NUM_NETIFS=4
CHECK_TC="yes"
TEST_GROUP="239.10.10.10"
@@ -537,6 +537,34 @@
log_test "Disable multicast vlan snooping when vlan filtering is disabled"
}
+vlmc_mcast_toggle_test()
+{
+ RET=0
+
+ ip link add name br1-mcast up type bridge mcast_snooping 1 mcast_querier 1 vlan_filtering 1
+ ip link add name dummy1-mcast up master br1-mcast type dummy
+
+ # Enabling per-VLAN multicast snooping should disable the per-port
+ # multicast context on "dummy1-mcast".
+ ip link set dev br1-mcast type bridge mcast_vlan_snooping 1
+
+ # Toggling multicast snooping on the bridge should not affect the
+ # per-port multicast context on "dummy1-mcast" given that per-VLAN
+ # multicast snooping is enabled.
+ ip link set dev br1-mcast type bridge mcast_snooping 0
+ ip link set dev br1-mcast type bridge mcast_snooping 1
+
+ # If both the per-port and per-{port, VLAN} multicast contexts are
+ # enabled on "dummy1-mcast", removing it from the bridge will result
+ # in a splat.
+ ip link set dev dummy1-mcast nomaster
+
+ log_test "Toggling mcast snooping with per-VLAN mcast snooping enabled"
+
+ ip link del dev dummy1-mcast
+ ip link del dev br1-mcast
+}
+
trap cleanup EXIT
setup_prepare
diff --git a/tools/testing/selftests/net/ioam6.sh b/tools/testing/selftests/net/ioam6.sh
index b2b9988..845c26d 100755
--- a/tools/testing/selftests/net/ioam6.sh
+++ b/tools/testing/selftests/net/ioam6.sh
@@ -273,8 +273,8 @@
ip -netns $ioam_node_beta link set ioam-veth-betaR name veth1 &>/dev/null
ip -netns $ioam_node_gamma link set ioam-veth-gamma name veth0 &>/dev/null
- ip -netns $ioam_node_alpha addr add 2001:db8:1::2/64 dev veth0 &>/dev/null
ip -netns $ioam_node_alpha addr add 2001:db8:1::50/64 dev veth0 &>/dev/null
+ ip -netns $ioam_node_alpha addr add 2001:db8:1::2/64 dev veth0 &>/dev/null
ip -netns $ioam_node_alpha link set veth0 up &>/dev/null
ip -netns $ioam_node_alpha link set lo up &>/dev/null
ip -netns $ioam_node_alpha route add 2001:db8:2::/64 \
diff --git a/tools/testing/selftests/net/lib/xdp_native.bpf.c b/tools/testing/selftests/net/lib/xdp_native.bpf.c
index 64f0522..ded3f89 100644
--- a/tools/testing/selftests/net/lib/xdp_native.bpf.c
+++ b/tools/testing/selftests/net/lib/xdp_native.bpf.c
@@ -268,6 +268,17 @@ static int xdp_mode_tx_handler(struct xdp_md *ctx, __u16 port)
return XDP_PASS;
}
+static __always_inline __u16 csum_fold_helper(__u32 csum)
+{
+ csum = (csum & 0xffff) + (csum >> 16);
+ return ~((csum & 0xffff) + (csum >> 16));
+}
+
+static __always_inline __u16 csum_fold_udp_helper(__u32 csum)
+{
+ return csum_fold_helper(csum) ? : 0xffff;
+}
+
static void *update_pkt(struct xdp_md *ctx, __s16 offset, __u32 *udp_csum)
{
void *data_end = (void *)(long)ctx->data_end;
@@ -281,21 +292,22 @@ static void *update_pkt(struct xdp_md *ctx, __s16 offset, __u32 *udp_csum)
if (eth->h_proto == bpf_htons(ETH_P_IP)) {
struct iphdr *iph = data + sizeof(*eth);
- __u16 total_len;
if (iph + 1 > (struct iphdr *)data_end)
return NULL;
- iph->tot_len = bpf_htons(bpf_ntohs(iph->tot_len) + offset);
-
udph = (void *)eth + sizeof(*iph) + sizeof(*eth);
if (!udph || udph + 1 > (struct udphdr *)data_end)
return NULL;
- len_new = bpf_htons(bpf_ntohs(udph->len) + offset);
+ len = iph->tot_len;
+ len_new = bpf_htons(bpf_ntohs(len) + offset);
+ iph->tot_len = len_new;
+ iph->check = csum_fold_helper(
+ bpf_csum_diff(&len, sizeof(len), &len_new,
+ sizeof(len_new), ~((__u32)iph->check)));
} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
struct ipv6hdr *ipv6h = data + sizeof(*eth);
- __u16 payload_len;
if (ipv6h + 1 > (struct ipv6hdr *)data_end)
return NULL;
@@ -304,33 +316,27 @@ static void *update_pkt(struct xdp_md *ctx, __s16 offset, __u32 *udp_csum)
if (!udph || udph + 1 > (struct udphdr *)data_end)
return NULL;
- *udp_csum = ~((__u32)udph->check);
-
len = ipv6h->payload_len;
len_new = bpf_htons(bpf_ntohs(len) + offset);
ipv6h->payload_len = len_new;
-
- *udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
- sizeof(len_new), *udp_csum);
-
- len = udph->len;
- len_new = bpf_htons(bpf_ntohs(udph->len) + offset);
- *udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
- sizeof(len_new), *udp_csum);
} else {
return NULL;
}
+ len = udph->len;
+ len_new = bpf_htons(bpf_ntohs(len) + offset);
+
+ *udp_csum = ~((__u32)udph->check);
+ *udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
+ sizeof(len_new), *udp_csum);
+ *udp_csum = bpf_csum_diff(&len, sizeof(len), &len_new,
+ sizeof(len_new), *udp_csum);
+
udph->len = len_new;
return udph;
}
-static __u16 csum_fold_helper(__u32 csum)
-{
- return ~((csum & 0xffff) + (csum >> 16)) ? : 0xffff;
-}
-
static int xdp_adjst_tail_shrnk_data(struct xdp_md *ctx, __u16 offset,
unsigned long hdr_len)
{
@@ -359,7 +365,7 @@ static int xdp_adjst_tail_shrnk_data(struct xdp_md *ctx, __u16 offset,
return -1;
udp_csum = bpf_csum_diff((__be32 *)tmp_buff, offset, 0, 0, udp_csum);
- udph->check = (__u16)csum_fold_helper(udp_csum);
+ udph->check = (__u16)csum_fold_udp_helper(udp_csum);
if (bpf_xdp_adjust_tail(ctx, 0 - offset) < 0)
return -1;
@@ -403,7 +409,7 @@ static int xdp_adjst_tail_grow_data(struct xdp_md *ctx, __u16 offset)
return -1;
udp_csum = bpf_csum_diff(0, 0, (__be32 *)tmp_buff, offset, udp_csum);
- udph->check = (__u16)csum_fold_helper(udp_csum);
+ udph->check = (__u16)csum_fold_udp_helper(udp_csum);
buff_len = bpf_xdp_get_buff_len(ctx);
@@ -484,8 +490,7 @@ static int xdp_adjst_head_shrnk_data(struct xdp_md *ctx, __u64 hdr_len,
return -1;
udp_csum = bpf_csum_diff((__be32 *)tmp_buff, offset, 0, 0, udp_csum);
-
- udph->check = (__u16)csum_fold_helper(udp_csum);
+ udph->check = (__u16)csum_fold_udp_helper(udp_csum);
if (bpf_xdp_load_bytes(ctx, 0, tmp_buff, MAX_ADJST_OFFSET) < 0)
return -1;
@@ -542,7 +547,7 @@ static int xdp_adjst_head_grow_data(struct xdp_md *ctx, __u64 hdr_len,
return -1;
udp_csum = bpf_csum_diff(0, 0, (__be32 *)data_buff, offset, udp_csum);
- udph->check = (__u16)csum_fold_helper(udp_csum);
+ udph->check = (__u16)csum_fold_udp_helper(udp_csum);
if (hdr_len > MAX_ADJST_OFFSET || hdr_len == 0)
return -1;
diff --git a/tools/testing/selftests/net/link_netns.py b/tools/testing/selftests/net/link_netns.py
index aab043c..6d1f863 100755
--- a/tools/testing/selftests/net/link_netns.py
+++ b/tools/testing/selftests/net/link_netns.py
@@ -3,13 +3,14 @@
import time
-from lib.py import ksft_run, ksft_exit, ksft_true
+from lib.py import ksft_run, ksft_exit, ksft_eq, ksft_true
from lib.py import ip
from lib.py import NetNS, NetNSEnter
from lib.py import RtnlFamily
LINK_NETNSID = 100
+LINK_NETNSID2 = 200
def test_event() -> None:
@@ -32,6 +33,57 @@
"Received unexpected link notification")
+def test_event_all_nsid() -> None:
+ """NETLINK_LISTEN_ALL_NSID notifications: local events must not
+ carry nsid even with a self-referential mapping. Remote events
+ must carry the correct nsid."""
+
+ with NetNS() as ns1, NetNS() as ns2:
+ net1, net2 = str(ns1), str(ns2)
+
+ with NetNSEnter(net1):
+ rtnl = RtnlFamily()
+ rtnl.ntf_listen_all_nsid()
+ rtnl.ntf_subscribe("rtnlgrp-link")
+
+ # Case 1: no nsid assigned, local event, no nsid expected.
+ ip("link add dummy-lo type dummy", ns=net1)
+
+ # Case 2: self-referential nsid, local event, still no nsid.
+ ip(f"netns set {net1} {LINK_NETNSID}", ns=net1)
+ ip("link add dummy-sr type dummy", ns=net1)
+
+ # Case 3: remote event, nsid present.
+ ip(f"netns set {net2} {LINK_NETNSID2}", ns=net1)
+ ip("link add dummy-re type dummy", ns=net2)
+
+ # Collect the three newlink events, ignoring unrelated noise.
+ events = {}
+ for msg in rtnl.poll_ntf(duration=1):
+ if msg['name'] == 'getlink':
+ ifname = msg['msg'].get('ifname')
+ if ifname in ('dummy-lo', 'dummy-sr', 'dummy-re'):
+ events[ifname] = msg
+ if len(events) == 3:
+ break
+
+ ksft_true('dummy-lo' in events, "missing local event")
+ ksft_true(events['dummy-lo'].get('nsid') is None,
+ "local event without nsid should not carry nsid")
+
+ ksft_true('dummy-sr' in events, "missing self-ref event")
+ ksft_true(events['dummy-sr'].get('nsid') is None,
+ "local event with self-ref nsid should not carry nsid")
+
+ ksft_true('dummy-re' in events, "missing remote event")
+ ksft_eq(events['dummy-re'].get('nsid'), LINK_NETNSID2,
+ "remote event should carry nsid")
+
+ ip("link del dummy-lo", ns=net1)
+ ip("link del dummy-sr", ns=net1)
+ ip("link del dummy-re", ns=net2)
+
+
def validate_link_netns(netns, ifname, link_netnsid) -> bool:
link_info = ip(f"-d link show dev {ifname}", ns=netns, json=True)
if not link_info:
@@ -133,7 +185,12 @@
def main() -> None:
- ksft_run([test_event, test_link_net, test_peer_net])
+ ksft_run([
+ test_event,
+ test_event_all_nsid,
+ test_link_net,
+ test_peer_net,
+ ])
ksft_exit()
diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
index a6447f7..d158678 100755
--- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
@@ -401,7 +401,7 @@
mptcp_lib_wait_local_port_listen "${listener_ns}" "${port}"
local start
- start=$(date +%s%3N)
+ start=$(date +%s%N)
ip netns exec ${connector_ns} \
./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
$extra_args $connect_addr < "$cin" > "$cout" &
@@ -423,7 +423,7 @@
fi
local stop
- stop=$(date +%s%3N)
+ stop=$(date +%s%N)
if $capture; then
sleep 1
@@ -439,7 +439,7 @@
fi
local duration
- duration=$((stop-start))
+ duration=$(((stop-start) / 1000000))
printf "(duration %05sms) " "${duration}"
if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ] || [ ${timeout_pid} -ne 0 ]; then
mptcp_lib_pr_fail "client exit code $retc, server $rets"
diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh
index beec41f66..4b3f71e 100755
--- a/tools/testing/selftests/net/mptcp/mptcp_join.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh
@@ -1828,6 +1828,22 @@
fi
}
+chk_add_drop_tx_nr()
+{
+ local drop_tx_nr=$1
+ local count
+
+ print_check "add addr tx drop"
+ count=$(mptcp_lib_get_counter ${ns1} "MPTcpExtAddAddrTxDrop")
+ if [ -z "$count" ]; then
+ print_skip
+ elif [ "$count" != "$drop_tx_nr" ]; then
+ fail_test "got $count ADD_ADDR drop[s] TX, expected $drop_tx_nr"
+ else
+ print_ok
+ fi
+}
+
chk_rm_nr()
{
local rm_addr_nr=$1
@@ -3278,6 +3294,21 @@
chk_mpc_endp_attempt ${retl} 1
fi
+
+ # first signal address drops, second one still progresses
+ if reset "signal addr list progresses after tx drop"; then
+ pm_nl_set_limits $ns1 0 2
+ pm_nl_set_limits $ns2 1 0
+ ip netns exec $ns1 sysctl -q net.ipv4.tcp_timestamps=1
+ ip netns exec $ns2 sysctl -q net.ipv4.tcp_timestamps=1
+
+ pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal port 10100
+ pm_nl_add_endpoint $ns1 dead:beef:3::1 flags signal
+ run_tests $ns1 $ns2 dead:beef:1::1
+ chk_add_drop_tx_nr 1
+ chk_add_tx_nr 1 1
+ chk_add_nr 1 1 0
+ fi
}
bind_tests()
@@ -4069,6 +4100,10 @@
chk_rm_nr 0 1
chk_mptcp_info subflows 0 subflows 0
chk_subflows_total 1 1
+ # check counters are not affected by errors at creation time
+ userspace_pm_add_sf $ns2 10.0.12.2 10 2>/dev/null
+ chk_mptcp_info subflows 0 subflows 0
+ chk_subflows_total 1 1
kill_events_pids
mptcp_lib_kill_group_wait $tests_pid
fi
diff --git a/tools/testing/selftests/net/mptcp/mptcp_lib.sh b/tools/testing/selftests/net/mptcp/mptcp_lib.sh
index 5fea7e7..5ef6033 100644
--- a/tools/testing/selftests/net/mptcp/mptcp_lib.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_lib.sh
@@ -28,7 +28,7 @@
MPTCP_LIB_SUBTESTS=()
MPTCP_LIB_SUBTESTS_DUPLICATED=0
MPTCP_LIB_SUBTEST_FLAKY=0
-MPTCP_LIB_SUBTESTS_LAST_TS_MS=
+MPTCP_LIB_SUBTESTS_LAST_TS_NS=
MPTCP_LIB_TEST_COUNTER=0
MPTCP_LIB_TEST_FORMAT="%02u %-50s"
MPTCP_LIB_IP_MPTCP=0
@@ -236,7 +236,7 @@
}
mptcp_lib_subtests_last_ts_reset() {
- MPTCP_LIB_SUBTESTS_LAST_TS_MS="$(date +%s%3N)"
+ MPTCP_LIB_SUBTESTS_LAST_TS_NS="$(date +%s%N)"
}
mptcp_lib_subtests_last_ts_reset
@@ -255,7 +255,7 @@
__mptcp_lib_result_add() {
local result="${1}"
local time="time="
- local ts_prev_ms
+ local ts_prev_ns
shift
local id=$((${#MPTCP_LIB_SUBTESTS[@]} + 1))
@@ -265,9 +265,9 @@
# not to add two '#'
[[ "${*}" != *"#"* ]] && time="# ${time}"
- ts_prev_ms="${MPTCP_LIB_SUBTESTS_LAST_TS_MS}"
+ ts_prev_ns="${MPTCP_LIB_SUBTESTS_LAST_TS_NS}"
mptcp_lib_subtests_last_ts_reset
- time+="$((MPTCP_LIB_SUBTESTS_LAST_TS_MS - ts_prev_ms))ms"
+ time+="$(((MPTCP_LIB_SUBTESTS_LAST_TS_NS - ts_prev_ns) / 1000000))ms"
MPTCP_LIB_SUBTESTS+=("${result} ${id} - ${KSFT_TEST}: ${*} ${time}")
}
@@ -474,20 +474,24 @@
wait_local_port_listen "${@}" "tcp"
}
+# $1: error file, $2: cmd, $3: expected msg, [$4: expected error]
mptcp_lib_check_output() {
local err="${1}"
local cmd="${2}"
local expected="${3}"
+ local exp_error="${4:-0}"
local cmd_ret=0
local out
- if ! out=$(${cmd} 2>"${err}"); then
- cmd_ret=${?}
- fi
+ out=$(${cmd} 2>"${err}") || cmd_ret=1
- if [ ${cmd_ret} -ne 0 ]; then
- mptcp_lib_pr_fail "command execution '${cmd}' stderr"
- cat "${err}"
+ if [ "${cmd_ret}" != "${exp_error}" ]; then
+ mptcp_lib_pr_fail "unexpected returned code for '${cmd}', info:"
+ if [ "${exp_error}" = 0 ]; then
+ cat "${err}"
+ else
+ echo "${out}"
+ fi
return 2
elif [ "${out}" = "${expected}" ]; then
return 0
diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh
index 123d9d7..04594df 100755
--- a/tools/testing/selftests/net/mptcp/pm_netlink.sh
+++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh
@@ -122,10 +122,12 @@
local cmd="$1"
local expected="$2"
local msg="$3"
+ local exp_error="$4"
local rc=0
mptcp_lib_print_title "$msg"
- mptcp_lib_check_output "${err}" "${cmd}" "${expected}" || rc=${?}
+ mptcp_lib_check_output "${err}" "${cmd}" "${expected}" "${exp_error}" ||
+ rc=${?}
if [ ${rc} -eq 2 ]; then
mptcp_lib_result_fail "${msg} # error ${rc}"
ret=${KSFT_FAIL}
@@ -158,13 +160,13 @@
"3,10.0.1.3,signal backup")" "dump addrs"
del_endpoint 2
-check "get_endpoint 2" "" "simple del addr"
+check "get_endpoint 2" "" "simple del addr" 1
check "show_endpoints" \
"$(format_endpoints "1,10.0.1.1" \
"3,10.0.1.3,signal backup")" "dump addrs after del"
add_endpoint 10.0.1.3 2>/dev/null
-check "get_endpoint 4" "" "duplicate addr"
+check "get_endpoint 4" "" "duplicate addr" 1
add_endpoint 10.0.1.4 flags signal
check "get_endpoint 4" "$(format_endpoints "4,10.0.1.4,signal")" "id addr increment"
@@ -173,7 +175,7 @@
add_endpoint "10.0.1.${i}" flags signal >/dev/null 2>&1
done
check "get_endpoint 9" "$(format_endpoints "9,10.0.1.9,signal")" "hard addr limit"
-check "get_endpoint 10" "" "above hard addr limit"
+check "get_endpoint 10" "" "above hard addr limit" 1
del_endpoint 9
for i in $(seq 10 255); do
@@ -192,9 +194,13 @@
flush_endpoint
check "show_endpoints" "" "flush addrs"
-add_endpoint 10.0.1.1 flags unknown
-check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" "ignore unknown flags"
-flush_endpoint
+# "unknown" flag is only supported by pm_nl_ctl
+if ! mptcp_lib_is_ip_mptcp; then
+ add_endpoint 10.0.1.1 flags unknown
+ check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" \
+ "ignore unknown flags"
+ flush_endpoint
+fi
set_limits 9 1 2>/dev/null
check "get_limits" "${default_limits}" "rcv addrs above hard limit"
diff --git a/tools/testing/selftests/net/netfilter/Makefile b/tools/testing/selftests/net/netfilter/Makefile
index ee2d1a5..d953ee2 100644
--- a/tools/testing/selftests/net/netfilter/Makefile
+++ b/tools/testing/selftests/net/netfilter/Makefile
@@ -26,6 +26,7 @@
nft_concat_range.sh \
nft_conntrack_helper.sh \
nft_fib.sh \
+ nft_fib_nexthop.sh \
nft_flowtable.sh \
nft_interface_stress.sh \
nft_meta.sh \
diff --git a/tools/testing/selftests/net/netfilter/nft_fib_nexthop.sh b/tools/testing/selftests/net/netfilter/nft_fib_nexthop.sh
new file mode 100755
index 0000000..c4f2030
--- /dev/null
+++ b/tools/testing/selftests/net/netfilter/nft_fib_nexthop.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# shellcheck disable=SC2154
+#
+# Exercise nft_fib6_eval()'s sibling/nh enumeration on three route shapes:
+# 1) route via a single external nexthop (nhid)
+# 2) route via an external nexthop group (nhid -> group, two members)
+# 3) route via old-style multipath (nexthop ... nexthop ...)
+#
+# In each scenario the route's nexthop set contains veth0 (the iif of the
+# test packet). nft_fib6_info_nh_uses_dev() must walk the set and report
+# veth0 as a valid oif. For (2) and (3) the matching nexthop is the second
+# member, so the walk has to traverse beyond the primary nh.
+#
+# After sending $PKTS ICMPv6 echo requests from ns1, check two counters on
+# nsrouter:
+# nf_ok -- `fib daddr . iif oif eq "veth0"` must equal $PKTS
+# nf_bad -- `fib daddr . iif oif missing` must stay at 0
+# Both rules also match on iif veth0 and ip6 daddr dead:dead::/64 so that
+# kernel-generated ND/MLD/RA traffic cannot pollute the counters.
+#
+# Topology similar to nft_fib.sh, without ns2; two dummy interfaces on
+# nsrouter host extra nh devices:
+#
+# dead:1::99 dead:1::1
+# ns1 <----veth----> nsrouter --- dummy0 dead:2::1
+# \-- dummy1 dead:9::1
+
+source lib.sh
+
+ret=0
+PKTS=3
+
+checktool "nft --version" "run test without nft"
+checktool "ip -V" "run test without iproute2"
+
+setup_ns nsrouter ns1
+trap cleanup_all_ns EXIT
+
+if ! ip link add veth0 netns "$nsrouter" type veth peer name eth0 netns "$ns1" \
+ > /dev/null 2>&1; then
+ echo "SKIP: No virtual ethernet pair device support in kernel"
+ exit $ksft_skip
+fi
+
+ip -net "$ns1" link set lo up
+ip -net "$ns1" link set eth0 up
+ip -net "$ns1" -6 addr add dead:1::99/64 dev eth0 nodad
+ip -net "$ns1" -6 route add default via dead:1::1
+
+ip -net "$nsrouter" link set lo up
+ip -net "$nsrouter" link set veth0 up
+ip -net "$nsrouter" -6 addr add dead:1::1/64 dev veth0 nodad
+
+if ! ip -net "$nsrouter" link add dummy0 type dummy 2>/dev/null; then
+ echo "SKIP: dummy netdev not available"
+ exit $ksft_skip
+fi
+ip -net "$nsrouter" link set dummy0 up
+ip -net "$nsrouter" -6 addr add dead:2::1/64 dev dummy0 nodad
+
+ip -net "$nsrouter" link add dummy1 type dummy
+ip -net "$nsrouter" link set dummy1 up
+ip -net "$nsrouter" -6 addr add dead:9::1/64 dev dummy1 nodad
+
+ip netns exec "$nsrouter" sysctl -q net.ipv6.conf.all.forwarding=1
+
+load_fib_rule() {
+ # filter on iif + daddr so the counters only see our test packets
+ ip netns exec "$nsrouter" nft -f /dev/stdin <<EOF
+flush ruleset
+table ip6 t {
+ counter nf_ok { }
+ counter nf_bad { }
+ chain c {
+ type filter hook prerouting priority 0; policy accept;
+ iif "veth0" ip6 daddr dead:dead::/64 fib daddr . iif oif eq "veth0" counter name nf_ok
+ iif "veth0" ip6 daddr dead:dead::/64 fib daddr . iif oif missing counter name nf_bad
+ }
+}
+EOF
+}
+
+bad_counter() {
+ local counter=$1
+ local expect=$2
+ local tag=$3
+
+ echo "FAIL ($tag): counter $counter has unexpected value (expected \"$expect\")" 1>&2
+ ip netns exec "$nsrouter" nft list counter ip6 t "$counter" 1>&2
+}
+
+run_scenario() {
+ local what="$1"; shift
+ # counter output format is "packets PACKET_NUM bytes BYTES_NUM";
+ # we only care about the packet count
+ local expect_ok="packets $PKTS bytes"
+ local expect_bad="packets 0 bytes"
+ local lret=0
+
+ # reset route + nexthop state between scenarios
+ ip -net "$nsrouter" -6 route del dead:dead::/64 > /dev/null 2>&1 || true
+ ip -net "$nsrouter" nexthop flush > /dev/null 2>&1 || true
+
+ # run the scenario function passed by the caller
+ "$@" || echo "WARN ($what): scenario setup returned non-zero"
+
+ load_fib_rule || { echo "FAIL ($what): nft load"; ret=1; return; }
+
+ # ping a daddr inside dead:dead::/64 so fib has to walk the nh set
+ ip netns exec "$ns1" ping -6 -c "$PKTS" -i 0.1 -W 1 dead:dead::1 \
+ > /dev/null 2>&1 || true
+
+ # verify the packets went through the expected fib path
+ if ! ip netns exec "$nsrouter" nft list counter ip6 t nf_ok | grep -q "$expect_ok"; then
+ bad_counter nf_ok "$expect_ok" "$what"
+ lret=1
+ fi
+ if ! ip netns exec "$nsrouter" nft list counter ip6 t nf_bad | grep -q "$expect_bad"; then
+ bad_counter nf_bad "$expect_bad" "$what"
+ lret=1
+ fi
+
+ if [ $lret -eq 0 ]; then
+ echo "PASS: $what"
+ else
+ ret=1
+ fi
+}
+
+scenario_single_nh() {
+ ip -net "$nsrouter" nexthop add id 1 via dead:1::99 dev veth0
+ ip -net "$nsrouter" -6 route add dead:dead::/64 nhid 1
+}
+run_scenario "single external nexthop (nhid -> veth0)" scenario_single_nh
+
+scenario_nh_group() {
+ ip -net "$nsrouter" nexthop add id 1 via dead:2::2 dev dummy0
+ ip -net "$nsrouter" nexthop add id 2 via dead:1::99 dev veth0
+ ip -net "$nsrouter" nexthop add id 100 group 1/2
+ ip -net "$nsrouter" -6 route add dead:dead::/64 nhid 100
+}
+run_scenario "nexthop group (dummy0 + veth0)" scenario_nh_group
+
+scenario_old_multipath() {
+ ip -net "$nsrouter" -6 route add dead:dead::/64 \
+ nexthop via dead:2::2 dev dummy0 \
+ nexthop via dead:1::99 dev veth0
+}
+run_scenario "old-style multipath (sibling on veth0)" scenario_old_multipath
+
+exit $ret
diff --git a/tools/testing/selftests/net/openvswitch/openvswitch.sh b/tools/testing/selftests/net/openvswitch/openvswitch.sh
index b327d30..3cdd953 100755
--- a/tools/testing/selftests/net/openvswitch/openvswitch.sh
+++ b/tools/testing/selftests/net/openvswitch/openvswitch.sh
@@ -26,6 +26,7 @@
netlink_checks ovsnl: validate netlink attrs and settings
upcall_interfaces ovs: test the upcall interfaces
tunnel_metadata ovs: test extraction of tunnel metadata
+ tunnel_refcount ovs: test tunnel vport reference cleanup
drop_reason drop: test drop reasons are emitted
psample psample: Sampling packets with psample"
@@ -830,6 +831,42 @@
return 0
}
+test_tunnel_refcount() {
+ sbxname="test_tunnel_refcount"
+ sbx_add "${sbxname}" || return 1
+
+ ovs_sbx "${sbxname}" ip netns add trefns || return 1
+ on_exit "ovs_sbx ${sbxname} ip netns del trefns"
+
+ for tun_type in gre vxlan geneve; do
+ info "testing ${tun_type} tunnel vport refcount"
+
+ ovs_sbx "${sbxname}" ip netns exec trefns \
+ python3 $ovs_base/ovs-dpctl.py \
+ add-dp dp-${tun_type} || return 1
+
+ ovs_sbx "${sbxname}" ip netns exec trefns \
+ python3 $ovs_base/ovs-dpctl.py \
+ add-if --no-lwt -t ${tun_type} \
+ dp-${tun_type} ovs-${tun_type}0 || return 1
+
+ ovs_wait ip -netns trefns link show \
+ ovs-${tun_type}0 >/dev/null 2>&1 || return 1
+
+ info "deleting dp - may hang if reference counting is broken"
+ ovs_sbx "${sbxname}" ip netns exec trefns \
+ python3 $ovs_base/ovs-dpctl.py \
+ del-dp dp-${tun_type} &
+
+ dev_removed() {
+ ! ip -netns trefns link show "$1" >/dev/null 2>&1
+ }
+ ovs_wait dev_removed dp-${tun_type} || return 1
+ ovs_wait dev_removed ovs-${tun_type}0 || return 1
+ done
+ return 0
+}
+
run_test() {
(
tname="$1"
diff --git a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
index 848f61f..bbe35e2 100644
--- a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
+++ b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
@@ -11,7 +11,6 @@
import math
import multiprocessing
import re
-import socket
import struct
import sys
import time
@@ -2069,7 +2068,7 @@
elif vport_type == "internal":
return OvsVport.OVS_VPORT_TYPE_INTERNAL
elif vport_type == "gre":
- return OvsVport.OVS_VPORT_TYPE_INTERNAL
+ return OvsVport.OVS_VPORT_TYPE_GRE
elif vport_type == "vxlan":
return OvsVport.OVS_VPORT_TYPE_VXLAN
elif vport_type == "geneve":
@@ -2121,6 +2120,7 @@
)
TUNNEL_DEFAULTS = [("geneve", 6081),
+ ("gre", 0),
("vxlan", 4789)]
for tnl in TUNNEL_DEFAULTS:
@@ -2129,9 +2129,13 @@
dport = tnl[1]
if not lwt:
+ if tnl[0] == "gre":
+ # GRE tunnels have no options.
+ break
+
vportopt = OvsVport.ovs_vport_msg.vportopts()
vportopt["attrs"].append(
- ["OVS_TUNNEL_ATTR_DST_PORT", socket.htons(dport)]
+ ["OVS_TUNNEL_ATTR_DST_PORT", dport]
)
msg["attrs"].append(
["OVS_VPORT_ATTR_OPTIONS", vportopt]
@@ -2145,6 +2149,9 @@
geneve_port=dport,
geneve_collect_metadata=True,
geneve_udp_zero_csum6_rx=1)
+ elif tnl[0] == "gre":
+ ipr.link("add", ifname=vport_ifname, kind="gretap",
+ gre_collect_metadata=True)
elif tnl[0] == "vxlan":
ipr.link("add", ifname=vport_ifname, kind=tnl[0],
vxlan_learning=0, vxlan_collect_metadata=1,
@@ -2563,7 +2570,7 @@
if vpo:
dpo = vpo.get_attr("OVS_TUNNEL_ATTR_DST_PORT")
if dpo:
- opts += " tnl-dport:%s" % socket.ntohs(dpo)
+ opts += " tnl-dport:%s" % dpo
print(
" port %d: %s (%s%s)"
% (
@@ -2632,7 +2639,7 @@
"--ptype",
type=str,
default="netdev",
- choices=["netdev", "internal", "geneve", "vxlan"],
+ choices=["netdev", "internal", "gre", "geneve", "vxlan"],
help="Interface type (default netdev)",
)
addifcmd.add_argument(
@@ -2645,7 +2652,7 @@
addifcmd.add_argument(
"-l",
"--lwt",
- type=bool,
+ action=argparse.BooleanOptionalAction,
default=True,
help="Use LWT infrastructure instead of vport (default true)."
)
diff --git a/tools/testing/selftests/net/ovpn/test-close-socket.sh b/tools/testing/selftests/net/ovpn/test-close-socket.sh
index af1532b..ec9a51b 100755
--- a/tools/testing/selftests/net/ovpn/test-close-socket.sh
+++ b/tools/testing/selftests/net/ovpn/test-close-socket.sh
@@ -53,7 +53,7 @@
for p in $(seq 1 ${OVPN_NUM_PEERS}); do
ovpn_cmd_ok "send ping traffic to peer ${p}" \
- ip netns exec ovpn_peer0 ping -qfc 500 -w 3 \
+ ip netns exec ovpn_peer0 ping -qfc 100 -w 3 \
5.5.5.$((p + 1))
done
}
diff --git a/tools/testing/selftests/net/ovpn/test-mark.sh b/tools/testing/selftests/net/ovpn/test-mark.sh
index 5a8f475..7c1d56e 100755
--- a/tools/testing/selftests/net/ovpn/test-mark.sh
+++ b/tools/testing/selftests/net/ovpn/test-mark.sh
@@ -66,7 +66,7 @@
for p in $(seq 1 3); do
ovpn_cmd_ok "send baseline traffic to peer ${p}" \
- ip netns exec ovpn_peer0 ping -qfc 500 -w 3 \
+ ip netns exec ovpn_peer0 ping -qfc 100 -w 3 \
5.5.5.$((p + 1))
done
}
@@ -101,7 +101,7 @@
local total_count
for p in $(seq 1 3); do
- if ping_output=$(ip netns exec ovpn_peer0 ping -qfc 500 -w 1 \
+ if ping_output=$(ip netns exec ovpn_peer0 ping -qfc 100 -w 1 \
5.5.5.$((p + 1)) 2>&1); then
printf '%s\n' "expected ping to peer ${p} to fail \
after nft drop rule"
@@ -144,7 +144,7 @@
sleep 1
for p in $(seq 1 3); do
ovpn_cmd_ok "send recovery traffic to peer ${p}" \
- ip netns exec ovpn_peer0 ping -qfc 500 -w 3 \
+ ip netns exec ovpn_peer0 ping -qfc 100 -w 3 \
5.5.5.$((p + 1))
done
}
diff --git a/tools/testing/selftests/net/ovpn/test.sh b/tools/testing/selftests/net/ovpn/test.sh
index b50dbe4..9b56108 100755
--- a/tools/testing/selftests/net/ovpn/test.sh
+++ b/tools/testing/selftests/net/ovpn/test.sh
@@ -98,10 +98,10 @@
sleep 0.3
ovpn_cmd_ok "send baseline traffic to peer ${p}" \
ip netns exec ovpn_peer0 \
- ping -qfc 500 -w 3 5.5.5.$((p + 1))
+ ping -qfc 100 -w 3 5.5.5.$((p + 1))
ovpn_cmd_ok "send large-payload traffic to peer ${p}" \
ip netns exec ovpn_peer0 \
- ping -qfc 500 -s 3000 -w 3 5.5.5.$((p + 1))
+ ping -qfc 100 -s 3000 -w 3 5.5.5.$((p + 1))
wait "${tcpdump_pid1}" || return 1
wait "${tcpdump_pid2}" || return 1
@@ -110,7 +110,7 @@
ovpn_run_lan_traffic() {
ovpn_cmd_ok "ping LAN behind peer1" \
- ip netns exec ovpn_peer0 ping -qfc 500 -w 3 "${OVPN_LAN_IP}"
+ ip netns exec ovpn_peer0 ping -qfc 100 -w 3 "${OVPN_LAN_IP}"
}
ovpn_run_float_mode() {
@@ -127,7 +127,7 @@
for p in $(seq 1 ${OVPN_NUM_PEERS}); do
peer_ns="ovpn_peer${p}"
ovpn_cmd_ok "ping tunnel after float peer ${p}" \
- ip netns exec "${peer_ns}" ping -qfc 500 -w 3 5.5.5.1
+ ip netns exec "${peer_ns}" ping -qfc 100 -w 3 5.5.5.1
done
}
diff --git a/tools/testing/selftests/net/rds/config b/tools/testing/selftests/net/rds/config
index 97db7ec..3d62d0c 100644
--- a/tools/testing/selftests/net/rds/config
+++ b/tools/testing/selftests/net/rds/config
@@ -1,3 +1,4 @@
+CONFIG_MODULES=n
CONFIG_NET_NS=y
CONFIG_NET_SCH_NETEM=y
CONFIG_RDS=y
diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh
index c499953..ace3a99 100755
--- a/tools/testing/selftests/net/rtnetlink.sh
+++ b/tools/testing/selftests/net/rtnetlink.sh
@@ -24,6 +24,8 @@
kci_test_macsec
kci_test_macsec_vlan
kci_test_team_bridge_macvlan
+ kci_test_bridge_promisc_netlink
+ kci_test_bridge_promisc_sysfs
kci_test_ipsec
kci_test_ipsec_offload
kci_test_fdb_get
@@ -61,6 +63,14 @@
fi
}
+sysfs_write()
+{
+ local val="$1"
+ local path="$2"
+
+ echo "$val" > "$path"
+}
+
run_cmd_common()
{
local cmd="$*"
@@ -680,6 +690,59 @@
end_test "PASS: team_bridge_macvlan"
}
+# Test that changing bridge port flags via the netlink path does not sleep with
+# the bridge spin lock held.
+kci_test_bridge_promisc_netlink()
+{
+ local dummy="test_dummy1"
+ local bridge="test_br1"
+ local team="test_team1"
+ local ret=0
+
+ run_cmd ip link add $team up type team
+ run_cmd ip link add $bridge up type bridge vlan_filtering 1
+ run_cmd ip link add $dummy up type dummy
+ run_cmd ip link set $dummy master $bridge
+ run_cmd ip link set $team master $bridge
+
+ # This causes the bridge driver to sync all the static FDB entries to
+ # the team device (which supports unicast filtering) and remove it from
+ # promiscuous mode. The call to dev_set_promiscuity() can sleep due to
+ # Rx mode inlining, which is a problem if the bridge spin lock is held.
+ run_cmd bridge link set dev $dummy flood off learning off
+
+ run_cmd ip link del $dummy
+ run_cmd ip link del $bridge
+ run_cmd ip link del $team
+
+ end_test "PASS: bridge_promisc_netlink"
+}
+
+# Same as kci_test_bridge_promisc_netlink(), but the flags are changed via the
+# sysfs path.
+kci_test_bridge_promisc_sysfs()
+{
+ local dummy="test_dummy1"
+ local bridge="test_br1"
+ local team="test_team1"
+ local ret=0
+
+ run_cmd ip link add $team up type team
+ run_cmd ip link add $bridge up type bridge vlan_filtering 1
+ run_cmd ip link add $dummy up type dummy
+ run_cmd ip link set $dummy master $bridge
+ run_cmd ip link set $team master $bridge
+
+ run_cmd sysfs_write 0 /sys/class/net/$dummy/brport/unicast_flood
+ run_cmd sysfs_write 0 /sys/class/net/$dummy/brport/learning
+
+ run_cmd ip link del $dummy
+ run_cmd ip link del $bridge
+ run_cmd ip link del $team
+
+ end_test "PASS: bridge_promisc_sysfs"
+}
+
#-------------------------------------------------------------------
# Example commands
# ip x s add proto esp src 14.0.0.52 dst 14.0.0.70 \
diff --git a/tools/testing/selftests/net/tcp_ecmp_failover.sh b/tools/testing/selftests/net/tcp_ecmp_failover.sh
new file mode 100755
index 0000000..5768aa8
--- /dev/null
+++ b/tools/testing/selftests/net/tcp_ecmp_failover.sh
@@ -0,0 +1,216 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright 2026 Google LLC.
+#
+# This test verifies TCP flow failover between ECMP routes
+# upon carrier loss on the active device.
+#
+# socat -----------------------------> socat
+# |
+# .-- veth-c1 -|- veth-s1 --.
+# dummy0 -| | |-- dummy0
+# '-- veth-c2 -|- veth-s2 --'
+# |
+#
+
+REQUIRE_JQ=no
+REQUIRE_MZ=no
+NUM_NETIFS=0
+
+source forwarding/lib.sh
+
+CLIENT_IP="10.0.59.1"
+SERVER_IP="10.0.92.1"
+CLIENT_IP6="2001:db8:5a9a::1"
+SERVER_IP6="2001:db8:9292::1"
+
+setup_server()
+{
+ IP="ip -n $server"
+ NS_EXEC="ip netns exec $server"
+
+ $IP link add dummy0 type dummy
+ $IP link set dummy0 up
+
+ $IP -4 addr add $SERVER_IP/32 dev dummy0
+ $IP -6 addr add $SERVER_IP6/128 dev dummy0 nodad
+
+ $IP link set veth-s1 up
+ $IP link set veth-s2 up
+
+ $IP -4 addr add 192.168.1.2/24 dev veth-s1
+ $IP -4 addr add 192.168.2.2/24 dev veth-s2
+
+ $IP -4 route add $CLIENT_IP/32 \
+ nexthop via 192.168.1.1 dev veth-s1 weight 1 \
+ nexthop via 192.168.2.1 dev veth-s2 weight 1
+
+ $IP -6 addr add 2001:db8:1::2/64 dev veth-s1 nodad
+ $IP -6 addr add 2001:db8:2::2/64 dev veth-s2 nodad
+
+ $IP -6 route add $CLIENT_IP6/128 \
+ nexthop via 2001:db8:1::1 dev veth-s1 weight 1 \
+ nexthop via 2001:db8:2::1 dev veth-s2 weight 1
+}
+
+setup_client()
+{
+ IP="ip -n $client"
+ NS_EXEC="ip netns exec $client"
+
+ $IP link add dummy0 type dummy
+ $IP link set dummy0 up
+
+ $IP -4 addr add $CLIENT_IP/32 dev dummy0
+ $IP -6 addr add $CLIENT_IP6/128 dev dummy0 nodad
+
+ $IP link set veth-c1 up
+ $IP link set veth-c2 up
+
+ $IP -4 addr add 192.168.1.1/24 dev veth-c1
+ $IP -4 addr add 192.168.2.1/24 dev veth-c2
+
+ $IP -4 route add $SERVER_IP/32 \
+ nexthop via 192.168.1.2 dev veth-c1 weight 1 \
+ nexthop via 192.168.2.2 dev veth-c2 weight 1
+
+ $IP -6 addr add 2001:db8:1::1/64 dev veth-c1 nodad
+ $IP -6 addr add 2001:db8:2::1/64 dev veth-c2 nodad
+
+ $IP -6 route add $SERVER_IP6/128 \
+ nexthop via 2001:db8:1::2 dev veth-c1 weight 1 \
+ nexthop via 2001:db8:2::2 dev veth-c2 weight 1
+
+ # By default, tcp_retries1=3 triggers a route refresh
+ # after 3 retransmits (~5s). Ensure this never occurs
+ # for test stability.
+ $NS_EXEC sysctl -qw net.ipv4.tcp_retries1=100
+
+ # When NETDEV_CHANGE is issued for a dev tied to an ECMP
+ # route, RTNH_F_LINKDOWN is flagged and the sernum is
+ # bumped to invalidate the route via sk_dst_check().
+ #
+ # Without ignore_routes_with_linkdown=1, subsequent
+ # lookups may still select the same RTNH_F_LINKDOWN route.
+ $NS_EXEC sysctl -qw net.ipv4.conf.veth-c1.ignore_routes_with_linkdown=1
+ $NS_EXEC sysctl -qw net.ipv4.conf.veth-c2.ignore_routes_with_linkdown=1
+
+ $NS_EXEC sysctl -qw net.ipv6.conf.veth-c1.ignore_routes_with_linkdown=1
+ $NS_EXEC sysctl -qw net.ipv6.conf.veth-c2.ignore_routes_with_linkdown=1
+}
+
+setup()
+{
+ setup_ns client server
+
+ ip -n "$client" link add veth-c1 type veth peer veth-s1 netns "$server"
+ ip -n "$client" link add veth-c2 type veth peer veth-s2 netns "$server"
+
+ setup_server
+ setup_client
+}
+
+cleanup()
+{
+ cleanup_all_ns > /dev/null 2>&1
+}
+
+tcp_ecmp_failover()
+{
+ local pf=$1; shift
+ local server_ip=$1; shift
+ local client_ip=$1; shift
+
+ RET=0
+
+ tcpdump_start veth-s1 "$server"
+ tcpdump_start veth-s2 "$server"
+
+ ip netns exec "$server" \
+ socat -u TCP-LISTEN:8080,pf="$pf",bind="$server_ip",reuseaddr /dev/null &
+ server_pid=$!
+
+ # Wait for server to start listening.
+ # Sometimes client fails without this sleep.
+ sleep 1
+
+ ip netns exec "$client" \
+ socat -u /dev/zero TCP:"$server_ip":8080,pf="$pf",bind="$client_ip" &
+ client_pid=$!
+
+ # To capture enough packets.
+ sleep 3
+
+ tcpdump_stop veth-s1
+ tcpdump_stop veth-s2
+
+ pkts_s1=$(tcpdump_show veth-s1 | wc -l)
+ pkts_s2=$(tcpdump_show veth-s2 | wc -l)
+
+ tcpdump_cleanup veth-s1
+ tcpdump_cleanup veth-s2
+
+ # Detect the device chosen by the client
+ if [ "$pkts_s1" -gt "$pkts_s2" ]; then
+ veth_down=veth-s1
+ veth_up=veth-s2
+ else
+ veth_down=veth-s2
+ veth_up=veth-s1
+ fi
+
+ # Taking down $veth_down causes its peer to lose carrier,
+ # triggering NETDEV_CHANGE. This flags RTNH_F_LINKDOWN
+ # and bumps the sernum for the route associated with that
+ # peer, invalidating the cached dst in the TCP socket.
+ #
+ # Consequently, sk_dst_check() fails, forcing the subsequent
+ # lookup to select the remaining healthy route via $veth_up.
+ ip -n "$server" link set "$veth_down" down
+
+ tcpdump_start "$veth_up" "$server"
+
+ # To capture enough packets.
+ sleep 3
+
+ tcpdump_stop "$veth_up"
+
+ kill -9 "$client_pid" > /dev/null 2>&1
+ kill -9 "$server_pid" > /dev/null 2>&1
+ wait 2> /dev/null
+
+ pkts=$(tcpdump_show $veth_up | wc -l)
+
+ tcpdump_cleanup "$veth_up"
+
+ if [ "$pkts" -lt 1000 ]; then
+ RET=$ksft_fail
+ fi
+}
+
+test_ipv4()
+{
+ setup
+ tcp_ecmp_failover IPv4 $SERVER_IP $CLIENT_IP
+ log_test "TCP IPv4 failover"
+ cleanup
+}
+
+test_ipv6()
+{
+ setup
+ tcp_ecmp_failover IPv6 "[$SERVER_IP6]" "[$CLIENT_IP6]"
+ log_test "TCP IPv6 failover"
+ cleanup
+}
+
+require_command socat
+require_command tcpdump
+
+trap cleanup EXIT
+
+test_ipv4
+test_ipv6
+
+exit "$EXIT_STATUS"
diff --git a/tools/testing/selftests/net/test_vxlan_vnifilter_notify.sh b/tools/testing/selftests/net/test_vxlan_vnifilter_notify.sh
new file mode 100755
index 0000000..9d51a9e
--- /dev/null
+++ b/tools/testing/selftests/net/test_vxlan_vnifilter_notify.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# shellcheck disable=SC2034,SC2154,SC2317,SC2329
+#
+# Test for VXLAN vnifilter netlink notifications (RTM_NEWTUNNEL /
+# RTM_DELTUNNEL).
+#
+# Verifies that:
+# - Adding a new VNI sends a notification
+# - Adding a new VNI with a remote sends a notification
+# - Deleting a VNI sends a notification
+# - Re-adding an existing VNI with the same attributes does not send
+# a spurious notification
+# - Updating an existing VNI's remote sends a notification
+# - Deleting a non-existent VNI does not send a notification
+
+source lib.sh
+
+require_command bridge
+
+VXLAN_DEV=vxlan100
+
+ALL_TESTS="
+ test_vni_add_notify
+ test_vni_add_remote_notify
+ test_vni_del_notify
+ test_vni_readd_no_notify
+ test_vni_update_remote_notify
+ test_vni_del_nonexistent_no_notify
+"
+
+setup_prepare()
+{
+ setup_ns NS1
+ defer cleanup_all_ns
+
+ ip -n "$NS1" link add $VXLAN_DEV type vxlan dstport 4789 \
+ local 10.0.0.1 nolearning external vnifilter
+ ip -n "$NS1" link set $VXLAN_DEV up
+}
+
+# Run bridge monitor in the background, execute a command, then count
+# the notification lines.
+# Usage: vni_notify_check <command> [args...]
+# Sets: NOTIFY_COUNT with the number of notifications observed.
+vni_notify_check()
+{
+ local tmpf cmd_ret monitor_pid
+
+ tmpf=$(mktemp)
+ defer rm "$tmpf"
+
+ defer_scope_push
+ ip netns exec "$NS1" bridge monitor vni > "$tmpf" 2>/dev/null &
+ monitor_pid=$!
+ defer kill_process "$monitor_pid"
+
+ sleep 0.5
+ if [ ! -e "/proc/$monitor_pid" ]; then
+ RET=$ksft_skip
+ log_test "iproute2 'bridge monitor vni' not supported"
+ return "$RET"
+ fi
+
+ "$@"
+ cmd_ret=$?
+ sleep 0.2
+ defer_scope_pop
+
+ NOTIFY_COUNT=$(grep -c "$VXLAN_DEV" "$tmpf")
+ NOTIFY_COUNT=${NOTIFY_COUNT:-0}
+ return "$cmd_ret"
+}
+
+# Adding a brand new VNI should produce a notification.
+test_vni_add_notify()
+{
+ RET=0
+
+ vni_notify_check \
+ bridge -n "$NS1" vni add vni 1000 dev "$VXLAN_DEV"
+ check_err $? "Failed to add VNI"
+
+ [ "$NOTIFY_COUNT" -eq 1 ]
+ check_err $? "Expected 1 notification for VNI add, got $NOTIFY_COUNT"
+
+ bridge -n "$NS1" vni delete vni 1000 dev "$VXLAN_DEV" 2>/dev/null
+
+ log_test "VNI add sends notification"
+}
+
+# Adding a VNI with a remote should produce a notification.
+test_vni_add_remote_notify()
+{
+ RET=0
+
+ vni_notify_check \
+ bridge -n "$NS1" vni add vni 4000 remote 10.0.0.2 dev "$VXLAN_DEV"
+ check_err $? "Failed to add VNI with remote"
+
+ [ "$NOTIFY_COUNT" -eq 1 ]
+ check_err $? "Expected 1 notification for VNI add with remote, got $NOTIFY_COUNT"
+
+ bridge -n "$NS1" vni delete vni 4000 dev "$VXLAN_DEV"
+
+ log_test "VNI add with remote sends notification"
+}
+
+# Deleting a VNI should produce a notification.
+test_vni_del_notify()
+{
+ RET=0
+
+ bridge -n "$NS1" vni add vni 2000 dev "$VXLAN_DEV"
+
+ vni_notify_check \
+ bridge -n "$NS1" vni delete vni 2000 dev "$VXLAN_DEV"
+ check_err $? "Failed to delete VNI"
+
+ [ "$NOTIFY_COUNT" -eq 1 ]
+ check_err $? "Expected 1 notification for VNI del, got $NOTIFY_COUNT"
+
+ log_test "VNI delete sends notification"
+}
+
+# Re-adding an existing VNI with the same attributes should not produce
+# a notification.
+test_vni_readd_no_notify()
+{
+ RET=0
+
+ bridge -n "$NS1" vni add vni 3000 dev "$VXLAN_DEV"
+
+ vni_notify_check \
+ bridge -n "$NS1" vni add vni 3000 dev "$VXLAN_DEV"
+ check_err $? "Failed to re-add VNI"
+
+ [ "$NOTIFY_COUNT" -eq 0 ]
+ check_err $? "Expected 0 notifications for VNI re-add, got $NOTIFY_COUNT"
+
+ bridge -n "$NS1" vni delete vni 3000 dev "$VXLAN_DEV"
+
+ log_test "VNI re-add does not send spurious notification"
+}
+
+# Updating an existing VNI's remote should produce a notification.
+test_vni_update_remote_notify()
+{
+ RET=0
+
+ bridge -n "$NS1" vni add vni 5000 remote 10.0.0.2 dev "$VXLAN_DEV"
+
+ vni_notify_check \
+ bridge -n "$NS1" vni add vni 5000 remote 10.0.0.3 dev "$VXLAN_DEV"
+ check_err $? "Failed to update VNI remote"
+
+ [ "$NOTIFY_COUNT" -eq 1 ]
+ check_err $? "Expected 1 notification for VNI remote update, got $NOTIFY_COUNT"
+
+ bridge -n "$NS1" vni delete vni 5000 dev "$VXLAN_DEV"
+
+ log_test "VNI remote update sends notification"
+}
+
+# Deleting a non-existent VNI should not produce a notification.
+test_vni_del_nonexistent_no_notify()
+{
+ RET=0
+
+ vni_notify_check \
+ bridge -n "$NS1" vni delete vni 9999 dev "$VXLAN_DEV" 2>/dev/null
+
+ [ "$NOTIFY_COUNT" -eq 0 ]
+ check_err $? "Expected 0 notifications for non-existent VNI del, got $NOTIFY_COUNT"
+
+ log_test "Non-existent VNI delete does not send notification"
+}
+
+trap defer_scopes_cleanup EXIT
+
+setup_prepare
+tests_run
+
+exit "$EXIT_STATUS"
diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c
index 9e2ccea..30a236b 100644
--- a/tools/testing/selftests/net/tls.c
+++ b/tools/testing/selftests/net/tls.c
@@ -946,6 +946,49 @@ TEST_F(tls, peek_and_splice)
EXPECT_EQ(memcmp(mem_send, mem_recv, send_len), 0);
}
+TEST_F(tls, splice_to_pipe_small)
+{
+ int send_len = TLS_PAYLOAD_MAX_LEN;
+ char mem_send[TLS_PAYLOAD_MAX_LEN];
+ char mem_recv[TLS_PAYLOAD_MAX_LEN];
+ size_t total = 0;
+ int p[2];
+
+ memrnd(mem_send, sizeof(mem_send));
+
+ ASSERT_GE(pipe(p), 0);
+
+ /* Shrink pipe to 1 page (typically 4096 bytes) to force multiple
+ * splice iterations for a 16384-byte TLS record.
+ */
+ EXPECT_GE(fcntl(p[1], F_SETPIPE_SZ, 4096), 4096);
+
+ EXPECT_EQ(send(self->fd, mem_send, send_len, 0), send_len);
+
+ while (total < (size_t)send_len) {
+ ssize_t spliced, drained;
+
+ spliced = splice(self->cfd, NULL, p[1], NULL,
+ send_len - total, 0);
+ EXPECT_GT(spliced, 0);
+ if (spliced <= 0)
+ break;
+
+ drained = read(p[0], mem_recv + total, spliced);
+ EXPECT_EQ(drained, spliced);
+ if (drained <= 0)
+ break;
+
+ total += drained;
+ }
+
+ EXPECT_EQ(total, (size_t)send_len);
+ EXPECT_EQ(memcmp(mem_send, mem_recv, send_len), 0);
+
+ close(p[0]);
+ close(p[1]);
+}
+
#define MAX_FRAGS 48
TEST_F(tls, splice_short)
{
diff --git a/tools/testing/selftests/rdma/rxe_ipv6.sh b/tools/testing/selftests/rdma/rxe_ipv6.sh
index b7059bf..32dad68 100755
--- a/tools/testing/selftests/rdma/rxe_ipv6.sh
+++ b/tools/testing/selftests/rdma/rxe_ipv6.sh
@@ -8,6 +8,8 @@
PORT=4791
IP6_ADDR="2001:db8::1/64"
+source "$(dirname "$0")/../kselftest/ktap_helpers.sh"
+
exec > /dev/null
# Cleanup function to run on exit (even on failure)
@@ -21,8 +23,8 @@
# 1. Prerequisites check
for mod in tun veth rdma_rxe; do
if ! modinfo "$mod" >/dev/null 2>&1; then
- echo "Error: Kernel module '$mod' not found."
- exit 1
+ echo "SKIP: Kernel module '$mod' not found." >&2
+ exit $KSFT_SKIP
fi
done
diff --git a/tools/testing/selftests/rdma/rxe_rping_between_netns.sh b/tools/testing/selftests/rdma/rxe_rping_between_netns.sh
index e5b876f..e7554fb 100755
--- a/tools/testing/selftests/rdma/rxe_rping_between_netns.sh
+++ b/tools/testing/selftests/rdma/rxe_rping_between_netns.sh
@@ -8,6 +8,8 @@
IP_B="1.1.1.2"
PORT=4791
+source "$(dirname "$0")/../kselftest/ktap_helpers.sh"
+
exec > /dev/null
# --- Cleanup Routine ---
@@ -27,6 +29,11 @@
exit 1
fi
+if ! modinfo rdma_rxe >/dev/null 2>&1; then
+ echo "SKIP: Kernel module 'rdma_rxe' not found." >&2
+ exit $KSFT_SKIP
+fi
+
modprobe rdma_rxe || { echo "Failed to load rdma_rxe"; exit 1; }
# --- Setup Network Topology ---
diff --git a/tools/testing/selftests/rdma/rxe_socket_with_netns.sh b/tools/testing/selftests/rdma/rxe_socket_with_netns.sh
index 002e509..9478657c 100755
--- a/tools/testing/selftests/rdma/rxe_socket_with_netns.sh
+++ b/tools/testing/selftests/rdma/rxe_socket_with_netns.sh
@@ -4,6 +4,8 @@
PORT=4791
MODS=("tun" "rdma_rxe")
+source "$(dirname "$0")/../kselftest/ktap_helpers.sh"
+
exec > /dev/null
# --- Helper: Cleanup Routine ---
@@ -26,6 +28,10 @@
fi
for m in "${MODS[@]}"; do
+ if ! modinfo "$m" >/dev/null 2>&1; then
+ echo "SKIP: Kernel module '$m' not found." >&2
+ exit $KSFT_SKIP
+ fi
modprobe "$m" || { echo "Error: Failed to load $m"; exit 1; }
done
diff --git a/tools/testing/selftests/rdma/rxe_test_NETDEV_UNREGISTER.sh b/tools/testing/selftests/rdma/rxe_test_NETDEV_UNREGISTER.sh
index 021ca45..8c18cea 100755
--- a/tools/testing/selftests/rdma/rxe_test_NETDEV_UNREGISTER.sh
+++ b/tools/testing/selftests/rdma/rxe_test_NETDEV_UNREGISTER.sh
@@ -5,6 +5,8 @@
RXE_NAME="rxe0"
RDMA_PORT=4791
+source "$(dirname "$0")/../kselftest/ktap_helpers.sh"
+
exec > /dev/null
# --- Cleanup Routine ---
@@ -19,8 +21,8 @@
# 1. Dependency Check
if ! modinfo rdma_rxe >/dev/null 2>&1; then
- echo "Error: rdma_rxe module not found."
- exit 1
+ echo "SKIP: rdma_rxe module not found." >&2
+ exit $KSFT_SKIP
fi
modprobe rdma_rxe
diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile
index 4ef9082..50d69e2 100644
--- a/tools/testing/selftests/rseq/Makefile
+++ b/tools/testing/selftests/rseq/Makefile
@@ -14,14 +14,20 @@
# still track changes to header files and depend on shared object.
OVERRIDE_TARGETS = 1
-TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test param_test \
- param_test_benchmark param_test_compare_twice param_test_mm_cid \
- param_test_mm_cid_benchmark param_test_mm_cid_compare_twice \
- syscall_errors_test slice_test
+TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test \
+ param_test_benchmark param_test_mm_cid_benchmark
-TEST_GEN_PROGS_EXTENDED = librseq.so
+TEST_GEN_PROGS_EXTENDED = librseq.so \
+ param_test \
+ param_test_compare_twice \
+ param_test_mm_cid \
+ param_test_mm_cid_compare_twice \
+ syscall_errors_test \
+ legacy_check \
+ slice_test \
+ check_optimized
-TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh
+TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh run_legacy_check.sh run_timeslice_test.sh
TEST_FILES := settings
@@ -62,3 +68,6 @@
$(OUTPUT)/slice_test: slice_test.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h
$(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@
+
+$(OUTPUT)/check_optimized: check_optimized.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h
+ $(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@
diff --git a/tools/testing/selftests/rseq/check_optimized.c b/tools/testing/selftests/rseq/check_optimized.c
new file mode 100644
index 0000000..a13e3f2
--- /dev/null
+++ b/tools/testing/selftests/rseq/check_optimized.c
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: LGPL-2.1
+#define _GNU_SOURCE
+#include <assert.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+
+#include "rseq.h"
+
+int main(int argc, char **argv)
+{
+ if (__rseq_register_current_thread(true, false))
+ return -1;
+ return 0;
+}
diff --git a/tools/testing/selftests/rseq/config b/tools/testing/selftests/rseq/config
new file mode 100644
index 0000000..a646080
--- /dev/null
+++ b/tools/testing/selftests/rseq/config
@@ -0,0 +1,3 @@
+CONFIG_EXPERT=y
+CONFIG_RSEQ=y
+CONFIG_RSEQ_SLICE_EXTENSION=y
diff --git a/tools/testing/selftests/rseq/legacy_check.c b/tools/testing/selftests/rseq/legacy_check.c
new file mode 100644
index 0000000..3f7de4e
--- /dev/null
+++ b/tools/testing/selftests/rseq/legacy_check.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <errno.h>
+#include <signal.h>
+#include <stdint.h>
+#include <unistd.h>
+
+#include "rseq.h"
+
+#include "../kselftest_harness.h"
+
+FIXTURE(legacy)
+{
+};
+
+static int cpu_id_in_sigfn = -1;
+
+static void sigfn(int sig)
+{
+ struct rseq_abi *rs = rseq_get_abi();
+
+ cpu_id_in_sigfn = rs->cpu_id_start;
+}
+
+FIXTURE_SETUP(legacy)
+{
+ int res = __rseq_register_current_thread(true, true);
+
+ switch (res) {
+ case -ENOSYS:
+ SKIP(return, "RSEQ not enabled\n");
+ case -EBUSY:
+ SKIP(return, "GLIBC owns RSEQ. Disable GLIBC RSEQ registration\n");
+ default:
+ ASSERT_EQ(res, 0);
+ }
+
+ ASSERT_NE(signal(SIGUSR1, sigfn), SIG_ERR);
+}
+
+FIXTURE_TEARDOWN(legacy)
+{
+}
+
+TEST_F(legacy, legacy_test)
+{
+ struct rseq_abi *rs = rseq_get_abi();
+
+ ASSERT_NE(rs, NULL);
+
+ /* Overwrite rs::cpu_id_start */
+ rs->cpu_id_start = -1;
+ sleep(1);
+ ASSERT_NE(rs->cpu_id_start, -1);
+
+ rs->cpu_id_start = -1;
+ ASSERT_EQ(raise(SIGUSR1), 0);
+ ASSERT_NE(rs->cpu_id_start, -1);
+ ASSERT_NE(cpu_id_in_sigfn, -1);
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c
index 05d03e6..e1e98db 100644
--- a/tools/testing/selftests/rseq/param_test.c
+++ b/tools/testing/selftests/rseq/param_test.c
@@ -38,7 +38,7 @@ static int opt_modulo, verbose;
static int opt_yield, opt_signal, opt_sleep,
opt_disable_rseq, opt_threads = 200,
opt_disable_mod = 0, opt_test = 's';
-
+static bool opt_rseq_legacy;
static long long opt_reps = 5000;
static __thread __attribute__((tls_model("initial-exec")))
@@ -281,9 +281,12 @@ unsigned int yield_mod_cnt, nr_abort;
} \
}
+#define rseq_no_glibc true
+
#else
#define printf_verbose(fmt, ...)
+#define rseq_no_glibc false
#endif /* BENCHMARK */
@@ -481,7 +484,7 @@ void *test_percpu_spinlock_thread(void *arg)
long long i, reps;
if (!opt_disable_rseq && thread_data->reg &&
- rseq_register_current_thread())
+ __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
abort();
reps = thread_data->reps;
for (i = 0; i < reps; i++) {
@@ -558,7 +561,7 @@ void *test_percpu_inc_thread(void *arg)
long long i, reps;
if (!opt_disable_rseq && thread_data->reg &&
- rseq_register_current_thread())
+ __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
abort();
reps = thread_data->reps;
for (i = 0; i < reps; i++) {
@@ -712,7 +715,7 @@ void *test_percpu_list_thread(void *arg)
long long i, reps;
struct percpu_list *list = (struct percpu_list *)arg;
- if (!opt_disable_rseq && rseq_register_current_thread())
+ if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
abort();
reps = opt_reps;
@@ -895,7 +898,7 @@ void *test_percpu_buffer_thread(void *arg)
long long i, reps;
struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
- if (!opt_disable_rseq && rseq_register_current_thread())
+ if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
abort();
reps = opt_reps;
@@ -1105,7 +1108,7 @@ void *test_percpu_memcpy_buffer_thread(void *arg)
long long i, reps;
struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
- if (!opt_disable_rseq && rseq_register_current_thread())
+ if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
abort();
reps = opt_reps;
@@ -1258,7 +1261,7 @@ void *test_membarrier_worker_thread(void *arg)
const int iters = opt_reps;
int i;
- if (rseq_register_current_thread()) {
+ if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) {
fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
errno, strerror(errno));
abort();
@@ -1323,7 +1326,7 @@ void *test_membarrier_manager_thread(void *arg)
intptr_t expect_a = 0, expect_b = 0;
int cpu_a = 0, cpu_b = 0;
- if (rseq_register_current_thread()) {
+ if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) {
fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
errno, strerror(errno));
abort();
@@ -1475,6 +1478,7 @@ static void show_usage(int argc, char **argv)
printf(" [-D M] Disable rseq for each M threads\n");
printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
+ printf(" [-O] Test with optimized RSEQ\n");
printf(" [-v] Verbose output.\n");
printf(" [-h] Show this help.\n");
printf("\n");
@@ -1602,6 +1606,9 @@ int main(int argc, char **argv)
case 'M':
opt_mo = RSEQ_MO_RELEASE;
break;
+ case 'L':
+ opt_rseq_legacy = true;
+ break;
default:
show_usage(argc, argv);
goto error;
@@ -1618,7 +1625,7 @@ int main(int argc, char **argv)
if (set_signal_handler())
goto error;
- if (!opt_disable_rseq && rseq_register_current_thread())
+ if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))
goto error;
if (!opt_disable_rseq && !rseq_validate_cpu_id()) {
fprintf(stderr, "Error: cpu id getter unavailable\n");
diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h
index ecef315..5f4ea21 100644
--- a/tools/testing/selftests/rseq/rseq-abi.h
+++ b/tools/testing/selftests/rseq/rseq-abi.h
@@ -192,9 +192,14 @@ struct rseq_abi {
struct rseq_abi_slice_ctrl slice_ctrl;
/*
+ * Place holder to push the size above 32 bytes.
+ */
+ __u8 __reserved;
+
+ /*
* Flexible array member at end of structure, after last feature field.
*/
char end[];
-} __attribute__((aligned(4 * sizeof(__u64))));
+} __attribute__((aligned(256)));
#endif /* _RSEQ_ABI_H */
diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c
index a736727..be0d0a9 100644
--- a/tools/testing/selftests/rseq/rseq.c
+++ b/tools/testing/selftests/rseq/rseq.c
@@ -56,6 +56,7 @@ ptrdiff_t rseq_offset;
* unsuccessful.
*/
unsigned int rseq_size = -1U;
+static unsigned int rseq_alloc_size;
/* Flags used during rseq registration. */
unsigned int rseq_flags;
@@ -115,29 +116,17 @@ bool rseq_available(void)
}
}
-/* The rseq areas need to be at least 32 bytes. */
-static
-unsigned int get_rseq_min_alloc_size(void)
-{
- unsigned int alloc_size = rseq_size;
-
- if (alloc_size < ORIG_RSEQ_ALLOC_SIZE)
- alloc_size = ORIG_RSEQ_ALLOC_SIZE;
- return alloc_size;
-}
-
/*
* Return the feature size supported by the kernel.
*
* Depending on the value returned by getauxval(AT_RSEQ_FEATURE_SIZE):
*
- * 0: Return ORIG_RSEQ_FEATURE_SIZE (20)
+ * 0: Return ORIG_RSEQ_FEATURE_SIZE (20)
* > 0: Return the value from getauxval(AT_RSEQ_FEATURE_SIZE).
*
* It should never return a value below ORIG_RSEQ_FEATURE_SIZE.
*/
-static
-unsigned int get_rseq_kernel_feature_size(void)
+static unsigned int get_rseq_kernel_feature_size(void)
{
unsigned long auxv_rseq_feature_size, auxv_rseq_align;
@@ -152,15 +141,24 @@ unsigned int get_rseq_kernel_feature_size(void)
return ORIG_RSEQ_FEATURE_SIZE;
}
-int rseq_register_current_thread(void)
+int __rseq_register_current_thread(bool nolibc, bool legacy)
{
+ unsigned int size;
int rc;
if (!rseq_ownership) {
/* Treat libc's ownership as a successful registration. */
- return 0;
+ return nolibc ? -EBUSY : 0;
}
- rc = sys_rseq(&__rseq.abi, get_rseq_min_alloc_size(), 0, RSEQ_SIG);
+
+ /* The minimal allocation size is 32, which is the legacy allocation size */
+ size = get_rseq_kernel_feature_size();
+ if (legacy || size < ORIG_RSEQ_ALLOC_SIZE)
+ rseq_alloc_size = ORIG_RSEQ_ALLOC_SIZE;
+ else
+ rseq_alloc_size = size;
+
+ rc = sys_rseq(&__rseq.abi, rseq_alloc_size, 0, RSEQ_SIG);
if (rc) {
/*
* After at least one thread has registered successfully
@@ -179,9 +177,8 @@ int rseq_register_current_thread(void)
* The first thread to register sets the rseq_size to mimic the libc
* behavior.
*/
- if (RSEQ_READ_ONCE(rseq_size) == 0) {
- RSEQ_WRITE_ONCE(rseq_size, get_rseq_kernel_feature_size());
- }
+ if (RSEQ_READ_ONCE(rseq_size) == 0)
+ RSEQ_WRITE_ONCE(rseq_size, size);
return 0;
}
@@ -194,7 +191,7 @@ int rseq_unregister_current_thread(void)
/* Treat libc's ownership as a successful unregistration. */
return 0;
}
- rc = sys_rseq(&__rseq.abi, get_rseq_min_alloc_size(), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
+ rc = sys_rseq(&__rseq.abi, rseq_alloc_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
if (rc)
return -1;
return 0;
diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h
index f51a5fd..c62ebb9 100644
--- a/tools/testing/selftests/rseq/rseq.h
+++ b/tools/testing/selftests/rseq/rseq.h
@@ -8,6 +8,7 @@
#ifndef RSEQ_H
#define RSEQ_H
+#include <assert.h>
#include <stdint.h>
#include <stdbool.h>
#include <pthread.h>
@@ -142,7 +143,12 @@ static inline struct rseq_abi *rseq_get_abi(void)
* succeed. A restartable sequence executed from a non-registered
* thread will always fail.
*/
-int rseq_register_current_thread(void);
+int __rseq_register_current_thread(bool nolibc, bool legacy);
+
+static inline int rseq_register_current_thread(void)
+{
+ return __rseq_register_current_thread(false, false);
+}
/*
* Unregister rseq for current thread.
diff --git a/tools/testing/selftests/rseq/run_legacy_check.sh b/tools/testing/selftests/rseq/run_legacy_check.sh
new file mode 100755
index 0000000..5577b46
--- /dev/null
+++ b/tools/testing/selftests/rseq/run_legacy_check.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0" ./legacy_check
diff --git a/tools/testing/selftests/rseq/run_param_test.sh b/tools/testing/selftests/rseq/run_param_test.sh
index 8d31426..69a3fa0 100755
--- a/tools/testing/selftests/rseq/run_param_test.sh
+++ b/tools/testing/selftests/rseq/run_param_test.sh
@@ -34,6 +34,11 @@
SLOW_REPS=100
NR_THREADS=$((6*${NR_CPUS}))
+# Prevent GLIBC from registering RSEQ so the selftest can run in legacy and
+# performance optimized mode.
+GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0"
+export GLIBC_TUNABLES
+
function do_tests()
{
local i=0
@@ -103,6 +108,40 @@
NR_LOOPS=
}
+echo "Testing in legacy RSEQ mode"
+echo "Yield injection (25%)"
+inject_blocking -m 4 -y -L
+
+echo "Yield injection (50%)"
+inject_blocking -m 2 -y -L
+
+echo "Yield injection (100%)"
+inject_blocking -m 1 -y -L
+
+echo "Kill injection (25%)"
+inject_blocking -m 4 -k -L
+
+echo "Kill injection (50%)"
+inject_blocking -m 2 -k -L
+
+echo "Kill injection (100%)"
+inject_blocking -m 1 -k -L
+
+echo "Sleep injection (1ms, 25%)"
+inject_blocking -m 4 -s 1 -L
+
+echo "Sleep injection (1ms, 50%)"
+inject_blocking -m 2 -s 1 -L
+
+echo "Sleep injection (1ms, 100%)"
+inject_blocking -m 1 -s 1 -L
+
+./check_optimized || {
+ echo "Skipping optimized RSEQ mode test. Not supported";
+ exit 0
+}
+
+echo "Testing in optimized RSEQ mode"
echo "Yield injection (25%)"
inject_blocking -m 4 -y
diff --git a/tools/testing/selftests/rseq/run_timeslice_test.sh b/tools/testing/selftests/rseq/run_timeslice_test.sh
new file mode 100755
index 0000000..551ebed
--- /dev/null
+++ b/tools/testing/selftests/rseq/run_timeslice_test.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+
+# Prevent GLIBC from registering RSEQ so the selftest can run in legacy
+# and performance optimized mode.
+GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0"
+export GLIBC_TUNABLES
+
+./check_optimized || {
+ echo "Skipping optimized RSEQ mode test. Not supported";
+ exit 0
+}
+
+./slice_test
diff --git a/tools/testing/selftests/rseq/slice_test.c b/tools/testing/selftests/rseq/slice_test.c
index 357122d..e402d44 100644
--- a/tools/testing/selftests/rseq/slice_test.c
+++ b/tools/testing/selftests/rseq/slice_test.c
@@ -124,6 +124,13 @@ FIXTURE_SETUP(slice_ext)
{
cpu_set_t affinity;
+ if (__rseq_register_current_thread(true, false))
+ SKIP(return, "RSEQ not supported\n");
+
+ if (prctl(PR_RSEQ_SLICE_EXTENSION, PR_RSEQ_SLICE_EXTENSION_SET,
+ PR_RSEQ_SLICE_EXT_ENABLE, 0, 0))
+ SKIP(return, "Time slice extension not supported\n");
+
ASSERT_EQ(sched_getaffinity(0, sizeof(affinity), &affinity), 0);
/* Pin it on a single CPU. Avoid CPU 0 */
@@ -137,11 +144,6 @@ FIXTURE_SETUP(slice_ext)
break;
}
- ASSERT_EQ(rseq_register_current_thread(), 0);
-
- ASSERT_EQ(prctl(PR_RSEQ_SLICE_EXTENSION, PR_RSEQ_SLICE_EXTENSION_SET,
- PR_RSEQ_SLICE_EXT_ENABLE, 0, 0), 0);
-
self->noise_params.noise_nsecs = variant->noise_nsecs;
self->noise_params.sleep_nsecs = variant->sleep_nsecs;
self->noise_params.run = 1;
diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index 789037b..5d2dffc 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -175,6 +175,7 @@
maximal \
maybe_null \
minimal \
+ non_scx_kfunc_deny \
numa \
allowed_cpus \
peek_dsq \
diff --git a/tools/testing/selftests/sched_ext/dequeue.c b/tools/testing/selftests/sched_ext/dequeue.c
index 4e93262..383d06e 100644
--- a/tools/testing/selftests/sched_ext/dequeue.c
+++ b/tools/testing/selftests/sched_ext/dequeue.c
@@ -33,6 +33,7 @@ static void worker_fn(int id)
/* Do some work to trigger scheduling events */
for (j = 0; j < 10000; j++)
sum += j;
+ asm volatile("" : : "r"(sum));
/* Sleep to trigger dequeue */
usleep(1000 + (id * 100));
diff --git a/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.bpf.c b/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.bpf.c
new file mode 100644
index 0000000..9f16d39
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.bpf.c
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Verify that context-sensitive SCX kfuncs (even "unlocked" ones) are
+ * restricted to only SCX struct_ops programs. Non-SCX struct_ops programs,
+ * such as TCP congestion control programs, should be rejected by the BPF
+ * verifier when attempting to call these kfuncs.
+ *
+ * Copyright (C) 2026 Ching-Chun (Jim) Huang <jserv@ccns.ncku.edu.tw>
+ * Copyright (C) 2026 Cheng-Yang Chou <yphbchou0911@gmail.com>
+ */
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+/* SCX kfunc from scx_kfunc_ids_any set */
+void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
+
+SEC("struct_ops/ssthresh")
+__u32 BPF_PROG(tcp_ca_ssthresh, struct sock *sk)
+{
+ /*
+ * This call should be rejected by the verifier because this is a
+ * TCP congestion control program (non-SCX struct_ops).
+ */
+ scx_bpf_kick_cpu(0, 0);
+ return 2;
+}
+
+SEC("struct_ops/cong_avoid")
+void BPF_PROG(tcp_ca_cong_avoid, struct sock *sk, __u32 ack, __u32 acked) {}
+
+SEC("struct_ops/undo_cwnd")
+__u32 BPF_PROG(tcp_ca_undo_cwnd, struct sock *sk) { return 2; }
+
+SEC(".struct_ops")
+struct tcp_congestion_ops tcp_non_scx_ca = {
+ .ssthresh = (void *)tcp_ca_ssthresh,
+ .cong_avoid = (void *)tcp_ca_cong_avoid,
+ .undo_cwnd = (void *)tcp_ca_undo_cwnd,
+ .name = "tcp_kfunc_deny",
+};
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.c b/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.c
new file mode 100644
index 0000000..1c03157
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/non_scx_kfunc_deny.c
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Verify that context-sensitive SCX kfuncs (even "unlocked" ones) are
+ * restricted to only SCX struct_ops programs. Non-SCX struct_ops programs,
+ * such as TCP congestion control programs, should be rejected by the BPF
+ * verifier when attempting to call these kfuncs.
+ *
+ * Copyright (C) 2026 Ching-Chun (Jim) Huang <jserv@ccns.ncku.edu.tw>
+ * Copyright (C) 2026 Cheng-Yang Chou <yphbchou0911@gmail.com>
+ */
+
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include "non_scx_kfunc_deny.bpf.skel.h"
+#include "scx_test.h"
+
+static enum scx_test_status run(void *ctx)
+{
+ struct non_scx_kfunc_deny *skel;
+ int err;
+
+ skel = non_scx_kfunc_deny__open();
+ if (!skel) {
+ SCX_ERR("Failed to open skel");
+ return SCX_TEST_FAIL;
+ }
+
+ err = non_scx_kfunc_deny__load(skel);
+ non_scx_kfunc_deny__destroy(skel);
+
+ if (err == 0) {
+ SCX_ERR("non-SCX BPF program loaded when it should have been rejected");
+ return SCX_TEST_FAIL;
+ }
+
+ return SCX_TEST_PASS;
+}
+
+struct scx_test non_scx_kfunc_deny = {
+ .name = "non_scx_kfunc_deny",
+ .description = "Verify that non-SCX struct_ops programs cannot call SCX kfuncs",
+ .run = run,
+};
+REGISTER_SCX_TEST(&non_scx_kfunc_deny)
diff --git a/tools/testing/selftests/sched_ext/total_bw.c b/tools/testing/selftests/sched_ext/total_bw.c
index 5b0a619..2af01ce 100644
--- a/tools/testing/selftests/sched_ext/total_bw.c
+++ b/tools/testing/selftests/sched_ext/total_bw.c
@@ -100,6 +100,98 @@ static int read_total_bw_values(long *bw_values, int max_cpus)
return cpu_count;
}
+/*
+ * Read a per-CPU dl_server param (runtime or period) from debugfs.
+ * Returns the value in nanoseconds, or -1 on failure.
+ */
+static long read_server_param(const char *server, const char *param, int cpu)
+{
+ char path[128];
+ long value = -1;
+ FILE *fp;
+
+ snprintf(path, sizeof(path),
+ "/sys/kernel/debug/sched/%s_server/cpu%d/%s",
+ server, cpu, param);
+ fp = fopen(path, "r");
+ if (!fp)
+ return -1;
+ if (fscanf(fp, "%ld", &value) != 1)
+ value = -1;
+ fclose(fp);
+
+ return value;
+}
+
+/*
+ * Write a per-CPU dl_server param to debugfs. Returns 0 on success.
+ */
+static int write_server_param(const char *server, const char *param,
+ int cpu, long value)
+{
+ char path[128];
+ FILE *fp;
+ int ret = 0;
+
+ snprintf(path, sizeof(path),
+ "/sys/kernel/debug/sched/%s_server/cpu%d/%s",
+ server, cpu, param);
+ fp = fopen(path, "w");
+ if (!fp)
+ return -1;
+ if (fprintf(fp, "%ld", value) < 0)
+ ret = -1;
+ if (fclose(fp) != 0)
+ ret = -1;
+
+ return ret;
+}
+
+static int read_fair_runtime_all(int nr_cpus, long *runtimes)
+{
+ int i;
+
+ for (i = 0; i < nr_cpus; i++) {
+ runtimes[i] = read_server_param("fair", "runtime", i);
+ if (runtimes[i] <= 0)
+ return -1;
+ }
+
+ return 0;
+}
+
+static int write_fair_runtime_all(int nr_cpus, long value)
+{
+ int i;
+
+ for (i = 0; i < nr_cpus; i++) {
+ if (write_server_param("fair", "runtime", i, value) < 0) {
+ SCX_ERR("Failed to write fair_server runtime on CPU %d", i);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Restore per-CPU fair_server runtimes.
+ */
+static int restore_fair_runtime_all(int nr_cpus, const long *runtimes)
+{
+ int ret = 0;
+ int i;
+
+ for (i = 0; i < nr_cpus; i++) {
+ if (write_server_param("fair", "runtime", i, runtimes[i]) < 0) {
+ SCX_ERR("Failed to restore fair_server runtime on CPU %d", i);
+ ret = -1;
+ }
+ }
+
+ return ret;
+}
+
static bool verify_total_bw_consistency(long *bw_values, int count)
{
int i;
@@ -217,6 +309,9 @@ static enum scx_test_status run(void *ctx)
struct bpf_link *link;
long loaded_bw[MAX_CPUS];
long unloaded_bw[MAX_CPUS];
+ long doubled_bw[MAX_CPUS];
+ long original_runtime[MAX_CPUS], doubled_runtime;
+ enum scx_test_status ret;
int i;
/* Test scenario 2: BPF program loaded */
@@ -257,7 +352,111 @@ static enum scx_test_status run(void *ctx)
}
fprintf(stderr, "All total_bw values are consistent across all scenarios\n");
- return SCX_TEST_PASS;
+
+ /*
+ * Validate auto-register/unregister of dl_server bandwidth reservations.
+ *
+ * Doubling fair_server's runtime doubles its bw contribution. With a
+ * full-mode BPF scheduler (minimal_ops), the kernel should detach
+ * fair_server and attach ext_server, dropping total_bw back to its
+ * pre-customization (default ext_server-only) value. On unload, the
+ * fair_server reservation should come back with its customized runtime
+ * preserved, so total_bw doubles again.
+ */
+ if (read_fair_runtime_all(test_ctx->nr_cpus, original_runtime) < 0) {
+ fprintf(stderr, "Skipping attach/detach validation: debugfs not accessible\n");
+ return SCX_TEST_PASS;
+ }
+ doubled_runtime = original_runtime[0] * 2;
+
+ fprintf(stderr,
+ "Setting fair_server runtime to %ld ns on all CPUs (orig %ld)\n",
+ doubled_runtime, original_runtime[0]);
+
+ if (write_fair_runtime_all(test_ctx->nr_cpus, doubled_runtime) < 0) {
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+
+ if (fetch_verify_total_bw(doubled_bw, test_ctx->nr_cpus) < 0) {
+ SCX_ERR("Failed to get stable values after doubling fair runtime");
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+
+ /*
+ * After doubling the runtime, fair_server's bw contribution must grow.
+ * We don't assert exactly 2x, because the kernel's to_ratio() truncates
+ * the value, so 2 * to_ratio(period, runtime) and
+ * to_ratio(period, 2 * runtime) can differ.
+ */
+ for (i = 0; i < test_ctx->nr_cpus; i++) {
+ if (doubled_bw[i] <= test_ctx->baseline_bw[i]) {
+ SCX_ERR("CPU%d: fair did not increase total_bw (baseline=%ld, doubled=%ld)",
+ i, test_ctx->baseline_bw[i], doubled_bw[i]);
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+ }
+
+ link = bpf_map__attach_struct_ops(test_ctx->skel->maps.minimal_ops);
+ if (!link) {
+ SCX_ERR("Failed to attach scheduler for detach test");
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+
+ if (fetch_verify_total_bw(loaded_bw, test_ctx->nr_cpus) < 0) {
+ SCX_ERR("Failed to get stable values with BPF loaded (detach test)");
+ bpf_link__destroy(link);
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+
+ /*
+ * In full mode the customized fair_server is detached and ext_server is
+ * attached at its default runtime, total_bw must match baseline.
+ */
+ for (i = 0; i < test_ctx->nr_cpus; i++) {
+ if (loaded_bw[i] != test_ctx->baseline_bw[i]) {
+ SCX_ERR("CPU%d: expected bw %ld (fair detached, ext default), got %ld",
+ i, test_ctx->baseline_bw[i], loaded_bw[i]);
+ bpf_link__destroy(link);
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+ }
+
+ bpf_link__destroy(link);
+
+ if (fetch_verify_total_bw(unloaded_bw, test_ctx->nr_cpus) < 0) {
+ SCX_ERR("Failed to get stable values after BPF unload (detach test)");
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+
+ /*
+ * After unload, fair_server is re-attached with its preserved 2x
+ * runtime, so total_bw should return to the doubled value.
+ */
+ for (i = 0; i < test_ctx->nr_cpus; i++) {
+ if (unloaded_bw[i] != doubled_bw[i]) {
+ SCX_ERR("CPU%d: BPF unloaded: expected %ld (fair restored at 2x), got %ld",
+ i, doubled_bw[i], unloaded_bw[i]);
+ ret = SCX_TEST_FAIL;
+ goto restore;
+ }
+ }
+
+ fprintf(stderr,
+ "dl_server attach/detach with customized fair runtime verified\n");
+ ret = SCX_TEST_PASS;
+
+restore:
+ if (restore_fair_runtime_all(test_ctx->nr_cpus, original_runtime) < 0)
+ SCX_ERR("Failed to fully restore per-CPU fair_server runtimes");
+
+ return ret;
}
static void cleanup(void *ctx)
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json b/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
index b056eb9..d0cad65 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
@@ -1144,6 +1144,620 @@
"teardown": [
"$TC qdisc del dev $DUMMY clsact"
]
+ },
+ {
+ "id": "531c",
+ "name": "Redirect multiport: dummy egress -> dev1 ingress -> dummy egress (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin"
+ ]
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.10.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY clsact",
+ "$TC filter add dev $DUMMY egress protocol ip prio 10 matchall action mirred ingress redirect dev $DEV1 index 1",
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred egress redirect dev $DUMMY index 2"
+ ],
+ "cmdUnderTest": "ping -c1 -W0.01 -I $DUMMY 10.10.10.1",
+ "expExitCode": "1",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 3
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY clsact",
+ "$TC qdisc del dev $DEV1 clsact"
+ ]
+ },
+ {
+ "id": "b1d7",
+ "name": "Redirect singleport: dev1 ingress -> dev1 egress -> dev1 ingress (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred egress redirect dev $DEV1 index 1"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DEV1 egress protocol ip prio 11 matchall action mirred ingress redirect dev $DEV1 index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "egress",
+ "index": 1,
+ "stats": {
+ "packets": 3
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact"
+ ]
+ },
+ {
+ "id": "c66d",
+ "name": "Redirect multiport: dev1 ingress -> dummy ingress -> dev1 egress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred ingress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY ingress protocol ip prio 11 matchall action mirred egress redirect dev $DEV1 index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "aa99",
+ "name": "Redirect multiport: dev1 ingress -> dummy ingress -> dev1 ingress (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred ingress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY ingress protocol ip prio 11 matchall action mirred ingress redirect dev $DEV1 index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 2,
+ "overlimits": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "37d7",
+ "name": "Redirect multiport: dev1 ingress -> dummy egress -> dev1 ingress (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred egress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY egress protocol ip prio 11 matchall action mirred ingress redirect dev $DEV1 index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "egress",
+ "index": 1,
+ "stats": {
+ "packets": 3
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "6d02",
+ "name": "Redirect multiport: dummy egress -> dev1 ingress -> dummy egress, different prios (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin"
+ ]
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.10.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY clsact",
+ "$TC filter add dev $DUMMY egress protocol ip prio 10 matchall action mirred ingress redirect dev $DEV1 index 1",
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 11 matchall action mirred egress redirect dev $DUMMY index 2"
+ ],
+ "cmdUnderTest": "ping -c1 -W0.01 -I $DUMMY 10.10.10.1",
+ "expExitCode": "1",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 3
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY clsact",
+ "$TC qdisc del dev $DEV1 clsact"
+ ]
+ },
+ {
+ "id": "8115",
+ "name": "Redirect multiport: dev1 ingress -> dummy ingress -> dummy egress -> dev1 egress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred ingress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact",
+ "$TC filter add dev $DUMMY ingress protocol ip prio 11 matchall action mirred egress redirect dev $DUMMY index 2"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY egress protocol ip prio 12 matchall action mirred egress redirect dev $DEV1 index 3",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "9eb3",
+ "name": "Redirect multiport: dev1 ingress -> dummy egress -> dev1 egress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred egress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY egress protocol ip prio 11 matchall action mirred egress redirect dev $DEV1 index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "egress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "d837",
+ "name": "Redirect multiport: dev1 ingress -> dummy egress -> dummy ingress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred egress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY egress protocol ip prio 11 matchall action mirred ingress redirect dev $DUMMY index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "egress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "2071",
+ "name": "Redirect singleport: dev1 ingress -> dev1 ingress (Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred ingress redirect dev $DEV1 index 1",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 1,
+ "overlimits": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact"
+ ]
+ },
+ {
+ "id": "0101",
+ "name": "Redirect singleport: dummy egress -> dummy ingress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin"
+ ]
+ },
+ "setup": [
+ "$IP addr add 10.10.10.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY clsact",
+ "$TC filter add dev $DUMMY egress protocol ip prio 11 matchall action mirred ingress redirect dev $DUMMY index 1"
+ ],
+ "cmdUnderTest": "ping -c1 -W0.01 -I $DUMMY 10.10.10.1",
+ "expExitCode": "1",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
+ },
+ {
+ "id": "cf97",
+ "name": "Redirect multiport: dev1 ingress -> dummy ingress -> dummy egress (No Loop)",
+ "category": [
+ "filter",
+ "mirred"
+ ],
+ "plugins": {
+ "requires": [
+ "nsPlugin",
+ "scapyPlugin"
+ ]
+ },
+ "setup": [
+ "$TC qdisc add dev $DEV1 clsact",
+ "$TC filter add dev $DEV1 ingress protocol ip prio 10 matchall action mirred ingress redirect dev $DUMMY index 1",
+ "$TC qdisc add dev $DUMMY clsact"
+ ],
+ "cmdUnderTest": "$TC filter add dev $DUMMY ingress protocol ip prio 11 matchall action mirred egress redirect dev $DUMMY index 2",
+ "scapy": [
+ {
+ "iface": "$DEV0",
+ "count": 1,
+ "packet": "Ether()/IP(dst='10.10.10.1', src='10.10.10.10')/ICMP()"
+ }
+ ],
+ "expExitCode": "0",
+ "verifyCmd": "$TC -j -s actions get action mirred index 1",
+ "matchJSON": [
+ {
+ "total acts": 0
+ },
+ {
+ "actions": [
+ {
+ "order": 1,
+ "kind": "mirred",
+ "mirred_action": "redirect",
+ "direction": "ingress",
+ "index": 1,
+ "stats": {
+ "packets": 1
+ },
+ "not_in_hw": true
+ }
+ ]
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DEV1 clsact",
+ "$TC qdisc del dev $DUMMY clsact"
+ ]
}
-
]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
index eefadd0..82c38a1 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
@@ -702,6 +702,7 @@
"$TC qdisc add dev $DUMMY parent 1:1 handle 2:0 netem duplicate 100%",
"$TC filter add dev $DUMMY parent 1:0 protocol ip prio 1 u32 match ip dst 10.10.10.1/32 flowid 1:1",
"$TC class add dev $DUMMY parent 1:0 classid 1:2 hfsc ls m2 10Mbit",
+ "$TC qdisc add dev $DUMMY parent 1:2 handle 3:0 netem duplicate 100%",
"$TC filter add dev $DUMMY parent 1:0 protocol ip prio 2 u32 match ip dst 10.10.10.2/32 flowid 1:2",
"ping -c 1 10.10.10.1 -I$DUMMY > /dev/null || true",
"$TC filter del dev $DUMMY parent 1:0 protocol ip prio 1",
@@ -714,8 +715,8 @@
{
"kind": "hfsc",
"handle": "1:",
- "bytes": 294,
- "packets": 3
+ "bytes": 392,
+ "packets": 4
}
],
"matchCount": "1",
@@ -1136,5 +1137,194 @@
"teardown": [
"$TC qdisc del dev $DUMMY handle 1: root"
]
+ },
+ {
+ "id": "7a5f",
+ "name": "Force red to dequeue from its child's gso_skb with qfq leaf",
+ "category": [
+ "qdisc",
+ "tbf",
+ "red",
+ "qfq"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+ "$TC qdisc add dev $DUMMY parent 1: handle 2: red limit 757 min 16 max 24 avpkt 16",
+ "$TC qdisc add dev $DUMMY parent 2: handle 3: qfq",
+ "$TC class add dev $DUMMY classid 3:1 parent 3: qfq maxpkt 512 weight 1",
+ "$TC filter add dev $DUMMY parent 3: protocol ip prio 1 matchall classid 3:1 action ok"
+ ],
+ "cmdUnderTest": "ping -c 1 10.10.10.1 -W0.01 -I$DUMMY || true",
+ "expExitCode": "0",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY parent 1:",
+ "matchJSON": [
+ {
+ "kind": "red",
+ "handle": "2:",
+ "bytes": 98,
+ "packets": 1,
+ "backlog": 0,
+ "qlen": 0
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY handle 1: root"
+ ]
+ },
+ {
+ "id": "cdae",
+ "name": "Force sfb to dequeue from its child's gso_skb with qfq leaf",
+ "category": [
+ "qdisc",
+ "tbf",
+ "sfb",
+ "qfq"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+ "$TC qdisc add dev $DUMMY parent 1: handle 2: sfb",
+ "$TC qdisc add dev $DUMMY parent 2: handle 3: qfq",
+ "$TC class add dev $DUMMY classid 3:1 parent 3: qfq maxpkt 512 weight 1",
+ "$TC filter add dev $DUMMY parent 3: protocol ip prio 1 matchall classid 3:1 action ok"
+ ],
+ "cmdUnderTest": "ping -c 1 10.10.10.1 -W0.01 -I$DUMMY || true",
+ "expExitCode": "0",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY parent 1:",
+ "matchJSON": [
+ {
+ "kind": "sfb",
+ "handle": "2:",
+ "bytes": 98,
+ "packets": 1,
+ "backlog": 0,
+ "qlen": 0
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY handle 1: root"
+ ]
+ },
+ {
+ "id": "291d",
+ "name": "Force red to dequeue from its child's gso_skb with dualpi2 leaf",
+ "category": [
+ "qdisc",
+ "tbf",
+ "red",
+ "dualpi2"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+ "$TC qdisc add dev $DUMMY parent 1: handle 2: red limit 757 min 16 max 24 avpkt 16",
+ "$TC qdisc add dev $DUMMY parent 2: handle 3: dualpi2"
+ ],
+ "cmdUnderTest": "ping -c 1 10.10.10.1 -W0.01 -I$DUMMY || true",
+ "expExitCode": "0",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY parent 1:",
+ "matchJSON": [
+ {
+ "kind": "red",
+ "handle": "2:",
+ "bytes": 98,
+ "packets": 1,
+ "backlog": 0,
+ "qlen": 0
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY handle 1: root"
+ ]
+ },
+ {
+ "id": "9c6d",
+ "name": "Force sfb to dequeue from its child's gso_skb with dualpi2 leaf",
+ "category": [
+ "qdisc",
+ "tbf",
+ "sfb",
+ "dualpi2"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+ "$TC qdisc add dev $DUMMY parent 1: handle 2: sfb",
+ "$TC qdisc add dev $DUMMY parent 2: handle 3: dualpi2"
+ ],
+ "cmdUnderTest": "ping -c 1 10.10.10.1 -W0.01 -I$DUMMY || true",
+ "expExitCode": "0",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY parent 1:",
+ "matchJSON": [
+ {
+ "kind": "sfb",
+ "handle": "2:",
+ "bytes": 98,
+ "packets": 1,
+ "backlog": 0,
+ "qlen": 0
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY handle 1: root"
+ ]
+ },
+ {
+ "id": "3a62",
+ "name": "Try to create a qlen underflow with QFQ/CBS",
+ "category": [
+ "qdisc",
+ "qfq",
+ "cbs"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.10.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: qfq",
+ "$TC class add dev $DUMMY classid 1:1 parent 1: qfq",
+ "$TC class add dev $DUMMY classid 1:2 parent 1: qfq",
+ "$TC qdisc add dev $DUMMY handle 2: parent 1:1 cbs",
+ "$TC qdisc add dev $DUMMY handle 3: parent 2: netem delay 5000000000",
+ "$TC filter add dev $DUMMY parent 1: prio 1 u32 match ip dst 10.10.10.1 classid 1:1 action ok",
+ "$TC filter add dev $DUMMY parent 1: prio 2 u32 match ip dst 10.10.10.2 classid 1:2 action ok",
+ "ping -c 1 10.10.10.1 -W0.01 -I$DUMMY || true",
+ "$IP l set $DUMMY down",
+ "$IP l set $DUMMY up",
+ "$TC qdisc replace dev $DUMMY handle 4: parent 2: pfifo"
+ ],
+ "cmdUnderTest": "ping -c 1 10.10.10.2 -W0.01 -I$DUMMY",
+ "expExitCode": "1",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY parent 1:1",
+ "matchJSON": [
+ {
+ "kind": "cbs",
+ "handle": "2:",
+ "bytes": 0,
+ "packets": 0
+ }
+ ],
+ "teardown": [
+ "$TC qdisc del dev $DUMMY handle 1: root"
+ ]
}
]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json
index 718d2df..472b672 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json
@@ -338,84 +338,34 @@
]
},
{
- "id": "d34d",
- "name": "NETEM test qdisc duplication restriction in qdisc tree in netem_change root",
- "category": ["qdisc", "netem"],
+ "id": "8c17",
+ "name": "Test netem's recursive duplicate",
+ "category": [
+ "qdisc",
+ "netem"
+ ],
"plugins": {
"requires": "nsPlugin"
},
"setup": [
- "$TC qdisc add dev $DUMMY root handle 1: netem limit 1",
- "$TC qdisc add dev $DUMMY parent 1: handle 2: netem limit 1"
+ "$IP link set dev $DUMMY up || true",
+ "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+ "$TC qdisc add dev $DUMMY root handle 1: netem limit 1000 duplicate 100%",
+ "$TC qdisc add dev $DUMMY parent 1: handle 2: netem limit 1000 duplicate 100%"
],
- "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: netem duplicate 50%",
- "expExitCode": "2",
- "verifyCmd": "$TC -s qdisc show dev $DUMMY",
- "matchPattern": "qdisc netem",
- "matchCount": "2",
- "teardown": [
- "$TC qdisc del dev $DUMMY handle 1:0 root"
- ]
- },
- {
- "id": "b33f",
- "name": "NETEM test qdisc duplication restriction in qdisc tree in netem_change non-root",
- "category": ["qdisc", "netem"],
- "plugins": {
- "requires": "nsPlugin"
- },
- "setup": [
- "$TC qdisc add dev $DUMMY root handle 1: netem limit 1",
- "$TC qdisc add dev $DUMMY parent 1: handle 2: netem limit 1"
+ "cmdUnderTest": "ping -c 1 10.10.11.11 -W 0.01",
+ "expExitCode": "1",
+ "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY root",
+ "matchJSON": [
+ {
+ "kind": "netem",
+ "handle": "1:",
+ "bytes": 294,
+ "packets": 3
+ }
],
- "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 2: netem duplicate 50%",
- "expExitCode": "2",
- "verifyCmd": "$TC -s qdisc show dev $DUMMY",
- "matchPattern": "qdisc netem",
- "matchCount": "2",
"teardown": [
- "$TC qdisc del dev $DUMMY handle 1:0 root"
+ "$TC qdisc del dev $DUMMY handle 1: root"
]
- },
- {
- "id": "cafe",
- "name": "NETEM test qdisc duplication restriction in qdisc tree",
- "category": ["qdisc", "netem"],
- "plugins": {
- "requires": "nsPlugin"
- },
- "setup": [
- "$TC qdisc add dev $DUMMY root handle 1: netem limit 1 duplicate 100%"
- ],
- "cmdUnderTest": "$TC qdisc add dev $DUMMY parent 1: handle 2: netem duplicate 100%",
- "expExitCode": "2",
- "verifyCmd": "$TC -s qdisc show dev $DUMMY",
- "matchPattern": "qdisc netem",
- "matchCount": "1",
- "teardown": [
- "$TC qdisc del dev $DUMMY handle 1:0 root"
- ]
- },
- {
- "id": "1337",
- "name": "NETEM test qdisc duplication restriction in qdisc tree across branches",
- "category": ["qdisc", "netem"],
- "plugins": {
- "requires": "nsPlugin"
- },
- "setup": [
- "$TC qdisc add dev $DUMMY parent root handle 1:0 hfsc",
- "$TC class add dev $DUMMY parent 1:0 classid 1:1 hfsc rt m2 10Mbit",
- "$TC qdisc add dev $DUMMY parent 1:1 handle 2:0 netem",
- "$TC class add dev $DUMMY parent 1:0 classid 1:2 hfsc rt m2 10Mbit"
- ],
- "cmdUnderTest": "$TC qdisc add dev $DUMMY parent 1:2 handle 3:0 netem duplicate 100%",
- "expExitCode": "2",
- "verifyCmd": "$TC -s qdisc show dev $DUMMY",
- "matchPattern": "qdisc netem",
- "matchCount": "1",
- "teardown": [
- "$TC qdisc del dev $DUMMY handle 1:0 root"
- ]
- }
+ }
]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json
index 557fb07..cd19d05 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json
@@ -302,5 +302,31 @@
"$TC qdisc del dev $ETH root",
"echo \"1\" > /sys/bus/netdevsim/del_device"
]
+ },
+ {
+ "id": "c7e1",
+ "name": "Class dump after graft and delete of explicit child qdisc",
+ "category": [
+ "qdisc",
+ "taprio"
+ ],
+ "plugins": {
+ "requires": "nsPlugin"
+ },
+ "setup": [
+ "echo \"1 1 8\" > /sys/bus/netdevsim/new_device",
+ "$TC qdisc replace dev $ETH handle 8001: parent root taprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 base-time 0 sched-entry S ff 20000000 clockid CLOCK_TAI",
+ "$TC qdisc add dev $ETH parent 8001:1 handle 8002: pfifo",
+ "$TC qdisc del dev $ETH parent 8001:1 handle 8002:"
+ ],
+ "cmdUnderTest": "$TC class show dev $ETH",
+ "expExitCode": "0",
+ "verifyCmd": "$TC class show dev $ETH",
+ "matchPattern": "class taprio 8001:[0-9]+ root",
+ "matchCount": "8",
+ "teardown": [
+ "$TC qdisc del dev $ETH root",
+ "echo \"1\" > /sys/bus/netdevsim/del_device"
+ ]
}
]
diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c
index 3851262..2f3bac9 100644
--- a/tools/testing/selftests/timers/posix_timers.c
+++ b/tools/testing/selftests/timers/posix_timers.c
@@ -78,19 +78,25 @@ static void sig_handler(int nr)
done = 1;
}
+static inline int64_t calcdiff_ns(struct timespec t1, struct timespec t2)
+{
+ int64_t diff;
+
+ diff = NSEC_PER_SEC * (int64_t)((int) t1.tv_sec - (int) t2.tv_sec);
+ diff += ((int) t1.tv_nsec - (int) t2.tv_nsec);
+ return diff;
+}
+
/*
* Check the expected timer expiration matches the GTOD elapsed delta since
* we armed the timer. Keep a 0.5 sec error margin due to various jitter.
*/
-static int check_diff(struct timeval start, struct timeval end)
+static int check_diff(struct timespec start, struct timespec end)
{
- long long diff;
+ long long diff = calcdiff_ns(end, start);
- diff = end.tv_usec - start.tv_usec;
- diff += (end.tv_sec - start.tv_sec) * USEC_PER_SEC;
-
- if (llabs(diff - DELAY * USEC_PER_SEC) > USEC_PER_SEC / 2) {
- printf("Diff too high: %lld..", diff);
+ if (llabs(diff - DELAY * NSEC_PER_SEC) > NSEC_PER_SEC / 2) {
+ printf("Diff too high: %lld ns..", diff);
return -1;
}
@@ -99,22 +105,25 @@ static int check_diff(struct timeval start, struct timeval end)
static void check_itimer(int which, const char *name)
{
- struct timeval start, end;
+ struct timespec start, end;
struct itimerval val = {
.it_value.tv_sec = DELAY,
};
+ int clock_id = CLOCK_REALTIME;
done = 0;
if (which == ITIMER_VIRTUAL)
signal(SIGVTALRM, sig_handler);
- else if (which == ITIMER_PROF)
+ else if (which == ITIMER_PROF) {
+ clock_id = CLOCK_THREAD_CPUTIME_ID;
signal(SIGPROF, sig_handler);
+ }
else if (which == ITIMER_REAL)
signal(SIGALRM, sig_handler);
- if (gettimeofday(&start, NULL) < 0)
- fatal_error(name, "gettimeofday()");
+ if (clock_gettime(clock_id, &start))
+ fatal_error(name, "clock_gettime()");
if (setitimer(which, &val, NULL) < 0)
fatal_error(name, "setitimer()");
@@ -126,18 +135,19 @@ static void check_itimer(int which, const char *name)
else if (which == ITIMER_REAL)
idle_loop();
- if (gettimeofday(&end, NULL) < 0)
- fatal_error(name, "gettimeofday()");
+ if (clock_gettime(clock_id, &end))
+ fatal_error(name, "clock_gettime()");
ksft_test_result(check_diff(start, end) == 0, "%s\n", name);
}
static void check_timer_create(int which, const char *name)
{
- struct timeval start, end;
+ struct timespec start, end;
struct itimerspec val = {
.it_value.tv_sec = DELAY,
};
+ int clock_id = CLOCK_REALTIME;
timer_t id;
done = 0;
@@ -148,16 +158,16 @@ static void check_timer_create(int which, const char *name)
if (signal(SIGALRM, sig_handler) == SIG_ERR)
fatal_error(name, "signal()");
- if (gettimeofday(&start, NULL) < 0)
- fatal_error(name, "gettimeofday()");
+ if (clock_gettime(clock_id, &start))
+ fatal_error(name, "clock_gettime()");
if (timer_settime(id, 0, &val, NULL) < 0)
fatal_error(name, "timer_settime()");
user_loop();
- if (gettimeofday(&end, NULL) < 0)
- fatal_error(name, "gettimeofday()");
+ if (clock_gettime(clock_id, &end))
+ fatal_error(name, "clock_gettime()");
ksft_test_result(check_diff(start, end) == 0,
"timer_create() per %s\n", name);
@@ -445,15 +455,6 @@ static void check_delete(void)
ksft_test_result(!tsig.signals, "check_delete\n");
}
-static inline int64_t calcdiff_ns(struct timespec t1, struct timespec t2)
-{
- int64_t diff;
-
- diff = NSEC_PER_SEC * (int64_t)((int) t1.tv_sec - (int) t2.tv_sec);
- diff += ((int) t1.tv_nsec - (int) t2.tv_nsec);
- return diff;
-}
-
static void check_sigev_none(int which, const char *name)
{
struct timespec start, now;
diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c
index fbd9b1e..0b23c09 100644
--- a/tools/testing/selftests/ublk/kublk.c
+++ b/tools/testing/selftests/ublk/kublk.c
@@ -1735,6 +1735,17 @@ static int __cmd_dev_add(const struct dev_ctx *ctx)
goto fail;
}
+ /*
+ * The kernel may reduce nr_hw_queues (e.g. capped to nr_cpu_ids).
+ * Cap nthreads to the actual queue count to avoid creating extra
+ * handler threads that will hang during device removal.
+ *
+ * per_io_tasks mode is excluded: threads interleave across all
+ * queues so nthreads > nr_hw_queues is valid and intentional.
+ */
+ if (!ctx->per_io_tasks && dev->nthreads > info->nr_hw_queues)
+ dev->nthreads = info->nr_hw_queues;
+
ret = ublk_start_daemon(ctx, dev);
ublk_dbg(UBLK_DBG_DEV, "%s: daemon exit %d\n", __func__, ret);
if (ret < 0)
diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h
index b4864aa..9e0dfd3 100644
--- a/tools/testing/vma/include/dup.h
+++ b/tools/testing/vma/include/dup.h
@@ -1330,7 +1330,7 @@ static inline int __compat_vma_mmap(struct vm_area_desc *desc,
/* Update the VMA from the descriptor. */
compat_set_vma_from_desc(vma, desc);
/* Complete any specified mmap actions. */
- return mmap_action_complete(vma, &desc->action);
+ return mmap_action_complete(vma, &desc->action, /*is_compat=*/true);
}
static inline int compat_vma_mmap(struct file *file, struct vm_area_struct *vma)
diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h
index a30b8bc..64164e2 100644
--- a/tools/testing/vma/include/stubs.h
+++ b/tools/testing/vma/include/stubs.h
@@ -87,7 +87,8 @@ static inline int mmap_action_prepare(struct vm_area_desc *desc)
}
static inline int mmap_action_complete(struct vm_area_struct *vma,
- struct mmap_action *action)
+ struct mmap_action *action,
+ bool is_compat)
{
return 0;
}
diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c
index 02bc6b0..572b854 100644
--- a/virt/kvm/dirty_ring.c
+++ b/virt/kvm/dirty_ring.c
@@ -63,7 +63,8 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
- if (!memslot || (offset + __fls(mask)) >= memslot->npages)
+ if (!memslot || offset >= memslot->npages ||
+ offset + __fls(mask) >= memslot->npages)
return;
KVM_MMU_LOCK(kvm);
